utils_data_prep.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/klarteno/0eee61b980f8feca34c731338f03b5b7/copy-of-aml-final-project.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### use the .py version of this for import in notebooks\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "gather": {
          "logged": 1663659864903
        },
        "id": "r9YGgf9KND8p"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import sys\n",
        "import os\n",
        "\n",
        "import glob\n",
        "from torchvision.transforms import ToTensor\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset,DataLoader\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder #, OneHotEncoder\n",
        "\n",
        "from pprint import pprint\n",
        "import time\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "gather": {
          "logged": 1663659865238
        },
        "id": "-ehlkCF4yoTM",
        "outputId": "52a5b938-c1aa-4e2f-8bb6-94c2c2b772bf"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "\"\\nfrom google.colab import drive\\ndrive.mount('/content/drive', force_remount=True)\\n%cd '/content/drive/My Drive/1_MalmoUni/AdvML/Project'\\n\""
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "'''\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "%cd '/content/drive/My Drive/1_MalmoUni/AdvML/Project'\n",
        "'''"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# CICIDS2017 dataset\n",
        "------------------------------------------------\n",
        "\n",
        "<div>\n",
        "    <b>Dataset description</b>: The CICIDS2017 dataset, which has been created by the <a href=\"https://www.unb.ca/cic/datasets/ids-2017.html\">Canadian Institute for Cyber-security (CIC)</a>, consists of labeled network flows. The CICIDS2017 contains benign and the most up-to-date common attacks. It is made up of 2,830,743 records with a total of 78 features.\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "gather": {
          "logged": 1663659865556
        }
      },
      "outputs": [],
      "source": [
        "def clean_column_name(column):\n",
        "    column = column.strip(' ')\n",
        "    column = column.replace('/', '_')\n",
        "    column = column.replace(' ', '_')\n",
        "    column = column.lower()\n",
        "    return column"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "gather": {
          "logged": 1663659885567
        }
      },
      "outputs": [],
      "source": [
        "# DATA_DIR  = os.path.join(os.path.abspath(\".\"), \"datasets\\MachineLearningCSV\\MachineLearningCVE\")\n",
        "DATA_DIR  = os.path.join(os.path.abspath(\".\"), \"datasets/MachineLearningCSV/MachineLearningCVE\")\n",
        "\n",
        "\n",
        "# Read all the .csv files\n",
        "filenames = glob.glob(os.path.join(DATA_DIR,  '*.csv'))\n",
        "datasets = [pd.read_csv(filename) for filename in filenames]\n",
        "\n",
        "# Remove white spaces and rename the columns\n",
        "for dataset in datasets:\n",
        "    dataset.columns = [clean_column_name(column) for column in dataset.columns]\n",
        "\n",
        "# Concatenate the datasets\n",
        "dataset = pd.concat(datasets, axis=0, ignore_index=True)\n",
        "#dataset.drop(labels=['fwd_header_length.1'], axis= 1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "gather": {
          "logged": 1663659885962
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 2830743 entries, 0 to 2830742\n",
            "Data columns (total 79 columns):\n",
            " #   Column                       Dtype  \n",
            "---  ------                       -----  \n",
            " 0   destination_port             int64  \n",
            " 1   flow_duration                int64  \n",
            " 2   total_fwd_packets            int64  \n",
            " 3   total_backward_packets       int64  \n",
            " 4   total_length_of_fwd_packets  int64  \n",
            " 5   total_length_of_bwd_packets  int64  \n",
            " 6   fwd_packet_length_max        int64  \n",
            " 7   fwd_packet_length_min        int64  \n",
            " 8   fwd_packet_length_mean       float64\n",
            " 9   fwd_packet_length_std        float64\n",
            " 10  bwd_packet_length_max        int64  \n",
            " 11  bwd_packet_length_min        int64  \n",
            " 12  bwd_packet_length_mean       float64\n",
            " 13  bwd_packet_length_std        float64\n",
            " 14  flow_bytes_s                 float64\n",
            " 15  flow_packets_s               float64\n",
            " 16  flow_iat_mean                float64\n",
            " 17  flow_iat_std                 float64\n",
            " 18  flow_iat_max                 int64  \n",
            " 19  flow_iat_min                 int64  \n",
            " 20  fwd_iat_total                int64  \n",
            " 21  fwd_iat_mean                 float64\n",
            " 22  fwd_iat_std                  float64\n",
            " 23  fwd_iat_max                  int64  \n",
            " 24  fwd_iat_min                  int64  \n",
            " 25  bwd_iat_total                int64  \n",
            " 26  bwd_iat_mean                 float64\n",
            " 27  bwd_iat_std                  float64\n",
            " 28  bwd_iat_max                  int64  \n",
            " 29  bwd_iat_min                  int64  \n",
            " 30  fwd_psh_flags                int64  \n",
            " 31  bwd_psh_flags                int64  \n",
            " 32  fwd_urg_flags                int64  \n",
            " 33  bwd_urg_flags                int64  \n",
            " 34  fwd_header_length            int64  \n",
            " 35  bwd_header_length            int64  \n",
            " 36  fwd_packets_s                float64\n",
            " 37  bwd_packets_s                float64\n",
            " 38  min_packet_length            int64  \n",
            " 39  max_packet_length            int64  \n",
            " 40  packet_length_mean           float64\n",
            " 41  packet_length_std            float64\n",
            " 42  packet_length_variance       float64\n",
            " 43  fin_flag_count               int64  \n",
            " 44  syn_flag_count               int64  \n",
            " 45  rst_flag_count               int64  \n",
            " 46  psh_flag_count               int64  \n",
            " 47  ack_flag_count               int64  \n",
            " 48  urg_flag_count               int64  \n",
            " 49  cwe_flag_count               int64  \n",
            " 50  ece_flag_count               int64  \n",
            " 51  down_up_ratio                int64  \n",
            " 52  average_packet_size          float64\n",
            " 53  avg_fwd_segment_size         float64\n",
            " 54  avg_bwd_segment_size         float64\n",
            " 55  fwd_header_length.1          int64  \n",
            " 56  fwd_avg_bytes_bulk           int64  \n",
            " 57  fwd_avg_packets_bulk         int64  \n",
            " 58  fwd_avg_bulk_rate            int64  \n",
            " 59  bwd_avg_bytes_bulk           int64  \n",
            " 60  bwd_avg_packets_bulk         int64  \n",
            " 61  bwd_avg_bulk_rate            int64  \n",
            " 62  subflow_fwd_packets          int64  \n",
            " 63  subflow_fwd_bytes            int64  \n",
            " 64  subflow_bwd_packets          int64  \n",
            " 65  subflow_bwd_bytes            int64  \n",
            " 66  init_win_bytes_forward       int64  \n",
            " 67  init_win_bytes_backward      int64  \n",
            " 68  act_data_pkt_fwd             int64  \n",
            " 69  min_seg_size_forward         int64  \n",
            " 70  active_mean                  float64\n",
            " 71  active_std                   float64\n",
            " 72  active_max                   int64  \n",
            " 73  active_min                   int64  \n",
            " 74  idle_mean                    float64\n",
            " 75  idle_std                     float64\n",
            " 76  idle_max                     int64  \n",
            " 77  idle_min                     int64  \n",
            " 78  label                        object \n",
            "dtypes: float64(24), int64(54), object(1)\n",
            "memory usage: 1.7+ GB\n"
          ]
        }
      ],
      "source": [
        "dataset.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "gather": {
          "logged": 1663659886319
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>destination_port</th>\n",
              "      <th>flow_duration</th>\n",
              "      <th>total_fwd_packets</th>\n",
              "      <th>total_backward_packets</th>\n",
              "      <th>total_length_of_fwd_packets</th>\n",
              "      <th>total_length_of_bwd_packets</th>\n",
              "      <th>fwd_packet_length_max</th>\n",
              "      <th>fwd_packet_length_min</th>\n",
              "      <th>fwd_packet_length_mean</th>\n",
              "      <th>fwd_packet_length_std</th>\n",
              "      <th>...</th>\n",
              "      <th>min_seg_size_forward</th>\n",
              "      <th>active_mean</th>\n",
              "      <th>active_std</th>\n",
              "      <th>active_max</th>\n",
              "      <th>active_min</th>\n",
              "      <th>idle_mean</th>\n",
              "      <th>idle_std</th>\n",
              "      <th>idle_max</th>\n",
              "      <th>idle_min</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>54865</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>20</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>BENIGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>55054</td>\n",
              "      <td>109</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>20</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>BENIGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>55055</td>\n",
              "      <td>52</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>20</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>BENIGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>46236</td>\n",
              "      <td>34</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>20</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>BENIGN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>54863</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>20</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>BENIGN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 79 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   destination_port  flow_duration  total_fwd_packets  total_backward_packets  \\\n",
              "0             54865              3                  2                       0   \n",
              "1             55054            109                  1                       1   \n",
              "2             55055             52                  1                       1   \n",
              "3             46236             34                  1                       1   \n",
              "4             54863              3                  2                       0   \n",
              "\n",
              "   total_length_of_fwd_packets  total_length_of_bwd_packets  \\\n",
              "0                           12                            0   \n",
              "1                            6                            6   \n",
              "2                            6                            6   \n",
              "3                            6                            6   \n",
              "4                           12                            0   \n",
              "\n",
              "   fwd_packet_length_max  fwd_packet_length_min  fwd_packet_length_mean  \\\n",
              "0                      6                      6                     6.0   \n",
              "1                      6                      6                     6.0   \n",
              "2                      6                      6                     6.0   \n",
              "3                      6                      6                     6.0   \n",
              "4                      6                      6                     6.0   \n",
              "\n",
              "   fwd_packet_length_std  ...  min_seg_size_forward  active_mean  active_std  \\\n",
              "0                    0.0  ...                    20          0.0         0.0   \n",
              "1                    0.0  ...                    20          0.0         0.0   \n",
              "2                    0.0  ...                    20          0.0         0.0   \n",
              "3                    0.0  ...                    20          0.0         0.0   \n",
              "4                    0.0  ...                    20          0.0         0.0   \n",
              "\n",
              "   active_max  active_min  idle_mean  idle_std  idle_max  idle_min   label  \n",
              "0           0           0        0.0       0.0         0         0  BENIGN  \n",
              "1           0           0        0.0       0.0         0         0  BENIGN  \n",
              "2           0           0        0.0       0.0         0         0  BENIGN  \n",
              "3           0           0        0.0       0.0         0         0  BENIGN  \n",
              "4           0           0        0.0       0.0         0         0  BENIGN  \n",
              "\n",
              "[5 rows x 79 columns]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "gather": {
          "logged": 1663659887286
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "BENIGN                        2273097\n",
              "DoS Hulk                       231073\n",
              "PortScan                       158930\n",
              "DDoS                           128027\n",
              "DoS GoldenEye                   10293\n",
              "FTP-Patator                      7938\n",
              "SSH-Patator                      5897\n",
              "DoS slowloris                    5796\n",
              "DoS Slowhttptest                 5499\n",
              "Bot                              1966\n",
              "Web Attack � Brute Force         1507\n",
              "Web Attack � XSS                  652\n",
              "Infiltration                       36\n",
              "Web Attack � Sql Injection         21\n",
              "Heartbleed                         11\n",
              "Name: label, dtype: int64"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset.label.value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "gather": {
          "logged": 1663659896188
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "403550"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset.duplicated(keep=False).sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "gather": {
          "logged": 1663659918426
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Datase duplicates : True\n",
            "count of duplicate values dropped:  403550\n"
          ]
        }
      ],
      "source": [
        "print('Datase duplicates :', dataset.duplicated().any())\n",
        "lenght_data = len(dataset)\n",
        "dataset.drop_duplicates(inplace=True, keep=False, ignore_index=True)\n",
        "# Remove duplicate rows\n",
        "dups_count = lenght_data-len(dataset)\n",
        "\n",
        "print('count of duplicate values dropped: ', dups_count)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Check and drop if there are missing values in each feature.\n",
        "All the missing values come from the `flow_bytes_s`."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "gather": {
          "logged": 1663659920019
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "flow_bytes_s  has NAN values : 334\n",
            "count of NANs values dropped:  334\n"
          ]
        }
      ],
      "source": [
        "def print_nans():\n",
        "    result = dataset.isna().sum()\n",
        "    \n",
        "    for idx in result.index:\n",
        "        if result[idx] > 0:\n",
        "            print(idx,' has NAN values :' ,result[idx])      \n",
        "\n",
        "print_nans()  \n",
        "\n",
        "lenght_data = len(dataset)\n",
        "dataset.dropna(axis=0, inplace=True, how=\"any\")\n",
        "# Remove duplicate rows\n",
        "dups_count = lenght_data-len(dataset)\n",
        "\n",
        "print('count of NANs values dropped: ', dups_count)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Checking if all values are finite."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "gather": {
          "logged": 1663659922946
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "are all values finite:  False\n",
            "flow_bytes_s  has NAN values : 1132\n",
            "flow_packets_s  has NAN values : 1132\n"
          ]
        }
      ],
      "source": [
        "print('are all values finite: ',np.all(np.isfinite(dataset.drop(['label'], axis=1))))\n",
        "# Replace infinite values to NaN\n",
        "dataset.replace([-np.inf, np.inf], np.nan, inplace=True)\n",
        "print_nans()  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Check which labels are related to infinte values  <br/>\n",
        "BENIGN labels are the majority and have the highest count of infinte values  <br/>\n",
        "The number of infinte values are small comparative with the number of values for each label  <br/>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "gather": {
          "logged": 1663659923312
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "BENIGN         1001\n",
              "PortScan        124\n",
              "Bot               4\n",
              "DDoS              2\n",
              "FTP-Patator       1\n",
              "Name: label, dtype: int64"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset[(dataset['flow_bytes_s'].isna()) & (dataset['flow_packets_s'].isna())].label.value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "gather": {
          "logged": 1663659924000
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "count of infinte values dropped:  1132\n"
          ]
        }
      ],
      "source": [
        "lenght_data = len(dataset)\n",
        "\n",
        "# Remove infinte values\n",
        "dataset.dropna(axis=0, how='any', inplace=True)\n",
        "\n",
        "dups_count = lenght_data-len(dataset)\n",
        "\n",
        "print('count of infinte values dropped: ', dups_count)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Outliers\n",
        "There are outliers but is assummed the neural networks will handle it for the moment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "gather": {
          "logged": 1663659931768
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ack_flag_count                      0\n",
            "act_data_pkt_fwd               193207\n",
            "active_max                     558821\n",
            "active_mean                    558821\n",
            "active_min                     558821\n",
            "                                ...  \n",
            "total_backward_packets         239223\n",
            "total_fwd_packets              252467\n",
            "total_length_of_bwd_packets    492852\n",
            "total_length_of_fwd_packets    309634\n",
            "urg_flag_count                 252178\n",
            "Length: 79, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "Q1 = dataset.quantile(0.25)\n",
        "Q3 = dataset.quantile(0.75)\n",
        "IQR = Q3 - Q1\n",
        "\n",
        "# Identifying outliers with interquartile range\n",
        "filt = (dataset < (Q1 - 1.5 * IQR)) | (dataset > (Q3 + 1.5 * IQR))\n",
        "print(filt.sum())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "gather": {
          "logged": 1663659932884
        }
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+AAAAHoCAYAAADJ64U5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+yklEQVR4nO3deZgdVYE3/m+TJoRNCSiDhEQII0dBHdxmcWFY3X7g7BpFEUdRQV8ZJRpQX+WdcQEM6jiCor6yOCoz7jsYNpd3HMdlGEBmjg5BQZQRTVAQBQL9+6Pqhts3tzudrTpJfz7P00/3rXvq1Kmlu++3zqmqkbGxsQAAAACb1jbT3QAAAACYCQRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgADCJUsohpZR/LaXcVkoZK6UcO91tmilKKae223zv9Zz/2Hb+g6dY/opSyg/XZ1ncp93m523A/Bu037dkw9Z9XY9jYPM2Ot0NAJjpSikLk5yc5KAkC5LcmeSnSb6V5Lxa6+XT2LwZrZQyN8knk/w4yUlJ7kjyL9PaKNgE2sB3bJJP11qvnNbGbOVKKX+a5MBa66nT3BRgGgjgANOolPLYJF9JcneSC5J8L8n2SfZLclSS25II4NPncUl2SfLCWusnp7ktM9GbkpyW5qQUm9beSd6Y5IdJrtzAurZPcs8GzL+17/c/TfL8JKdOsfyHklyY5K5N1B6gQwI4wPR6Y5IdkjxqsNeplPLyJHtMR6M2N6WUbZPMqrX+tuNF97b/io6XO6OVUnautd5Wa12VZNV0t4d1s6G/p/b7eLXWe7JhJzTWUEoZSbJjrfX2jVkvsHYCOMD0ekiSXwwb8llrvTfJT3qv2yGi1yf5P4NDF0spp6YJ8/vUWn/YTjsvTS/LA5KcmeTIJNsluTTJS2utN5dSXpzklUn2SdPztaTW+plhy0xybZLXJilJbkry5lrruaWUBUnenuTQJNsm+UyS42utt/XV89Akr0jyx2mG2c9K8p9J3ltrff8E6/LwJC9M8swkD0rytFLKh5tNU584uL1KKa9JcnqSg2qtXxt8f6DsI9t1OijJjkmWJzkvyZnth9201wI/uJ3l8lJK0ix8ZJJ657f1HpYmvP8yyX8nOafWen5fuR2TvL5dt72SrEzy5ST/u9b6o75yB6cZAfGCNCdqTmzb9IMkJ9dav1BKeUSStyV5fJqRFB9OclKt9e6Btj0kyRuSHJ5ktzTH1seSnFpr/fVattc30/SQzmvDUf97T0lyUZJX1lrfWUrZJskpSZ6SZiTHrkluTvKFJK+vtf6ib969c9/x9Z9JXpNk/yT/lOTYCY7rPdNcDnBYuy22T7P/zk+ytLf/Boy2db0gzX6pSd5Sa71wsvXua+eUtt1U9/8ky7lfkiVJ/jzN7+Sv2+3y7v62TuX4bcudl+ZvwC5pepT/Isn9knwnyatqrd9syx2b5Nx2tnNLKb2fv1JrPXhd9mlb31iS82utxw5OS3JO25bHJvlNkk8n+Zv+IDjBfu9Ne2i7TsckeWCS/0pySq31iwNt2CFNT/qzk9w/yVVJXpfkeUmeP9nv8UA9L0pyQpKHpemB/maSv621fr2vzN6Z4t/mUsoVaf4O9rZJzwtqredN0IZj0+yfQ2qtV/RN3y7N78LRSfZN8tskX0vyhlrrv/eVOzj3/R3ZMcnL2vJvTXJqKeWANL3xj0/z/2JlmuNuaa31C2vdSMA6cRM2gOl1XZLdSil/vgmXcVGaD6BvSPL+NEH8U6WUVyd5dZoPxScnmZ3k46WUfYbUcWSSd6W5HvrVaYbGf7CUcnSaD3y3pQnn/5zmw+A7BuY/OE1Y+Hw7//9OExbfV0o5ZYJ2fzjJH6U5eXBSkh+1bX1CG+gHvSDJ96cQvh+b5BtJDkny3rY9P04T3i/oK/o3Sd7X/vyWNB/cnzdJvaNJliX5qzTDRU9IEzS+n+RJA+UuTrPNv5vmBMhH04Txb5ZS9hpS/cuSvCrNh/CT03yI/nR7Lell7TJek+Zyhv+VJsT1t+0xSb6dZh+c09b3+TQnRZa1Iwwmc36S3ZM8dch7x6TprfxI+3p2mm36gzQnBl7RbpcXJrmilDJ7SB1/muQ9aY7VVyT50iRteWSagHpZmpMYJye5Mc22PnuCeU5PsqhdxhvaNn50KjfUm+q2m+r+n2Q5u6S5v8Brk1yTZn++KU24PrKv3FSP334XpznR87dpQtfDk3yxlLJz+/5X0xzjSXPM9471N7fT1mefDnNgmm33rTTHc6+Ot09x/qQ5Fp+UZGmavyMPTPO7sPdAuY+l+d3617btX0/yqSSPmuqCSimnp/mbeXea/XJmmhNEl5dSnr4Obe735jR/M5P7tvPz0uyDKWuPu4vShPtvpFnX09r2/b/2OBn0N2l+Xy5M83fim6WU3dL8Lh2U5ANJjk+zP25J8gfr0iZgavSAA0yvNyU5IsknSik/SPMh8VtJrqi1/udGWsa/1Vpf1nvR9uS+Msm8JA+vtf6qnX5Zkv9I8uI0vV39HpZk/17vbCnln9KEng8lWVxr7X2Afm9747JjSin9vVofqrW+t7/CUso70nzwO7mUsnSwxzbJrUkO7+9xLaW8L00Y/+s0AaU3/QlpesaWZO3+Ps1IgD+qtV7Vzv/uNL2uzymlfLDWemmt9dNtKHpxkmX9PU8T2D/N6IAltdYzJin3giRPSPK2Wmv/OlySJpy8NWsG/T3TbP9ftmV7++qTSf6y7/r095ZSvpMmJL6pb/4Pprmx3+MGRiZc2tZxdJoe1IlcmOakyjFtG3vz75wmPH+p1vqzdvKdSR5Ua/1N3/zvLaX8S5oP+H+a5kRNvwOSPHKKx/xXkiystfb3Hr6zlPKhJC8qpZxaa/3pwDwPaOvvbb/3pukRfXsp5Z8G2jpoqttuqvt/Im9Jsx1eUmt9X/8bbQ90z5SO34G6v1trPaGvvmvT7IPnpOmdX15KWZYmZH6j1vqPA/Ovzz4d5pFJHl9r/df29Tltr/8LSimvmuJw6J8nOaq3/0splyf5tyQvSft3qw3HT0/ygVrrcX3rfVmaXvu1Ks0fylcn+X9JDq213tVO/0Ca0UBnl1L2nWDExYRqrcvaE5dPGrKd18XL05zYfGqt9eK+dp+d5gTO0vb9fguSPLTvdzWllGekObn2rFrrVPYhsIH0gANMo1rrN5I8Jk2vzv3ThLOzk1xbSvlae4f0DfXOgde93pcLeuG7bctVSX6VZlj8oE/3D42utd6SZhjvvUnOGlL/tmmGLPfK9w/TndP2uuyaZtj1/dKE5zXaPTjcudb6/TQB7Ji2x7HnhWl6YScd5ltK2T3NMMvP9sJLW+9Y7usB/LPJ6pjEL9vvh7TLmcifpdlub+2f2A71vDLJnwwErqS5G/4v+8r29tVPhtwc7utJ9iil7JQk7RD1R6bpod6ulPKA3ldb9tdJnjzZitVaVyT5XJJntCclev4yzdD48/vKjvWCWillVilll3ZZl7VFhvWqfWGqJ5xqrb/pC1+zSym7tvVfnOZzzbCev/cMbL9fpuk9nps1Q8pq67jtprr/hy1nmzQ99P+Zpsd1nPZylA05fgdHpPT2xbDf9TWs5z4d5ht94bu/LaPp+3uxFn/ff/Kl1vqtNCNw+tflqPb7uJ71dpj6VE9s/kmSkSRn9MJ3W8dP0pxweXDWoTd9E3humuH33xk4LmenGVnwxFLK9gPzXNAfvlu94/Zp7ckQYBPTAw4wzWqtV6d5/E9KKQ9Oc33gi9IMs/xMKeUx/R8A18Pygdcr2+/XDym7Ms01rmuro1f2p7XWwTsV9+pfXU8bBk9NM8x6/pC65g6Z9v0h05JmiOyH0wzL/XRb9zOTfL7W+j8TzNPTG17/vSHvXZsmGK/XSY9a649KKW9O0wv301LKlWmut/9YGxL62/CTWuvKIdV8L80w3Qck6f+gPNH2v3GC6Umz/W9PM3ohaa4Z/j8TNP93Jpje74I01xA/M/cNzT+mXd7n+wuWUp6ZZqTCo9KcjOm3Lvt6De2Jl5PbZf9umpC0tvqHha5r2++T7e8pb7t12P/DPKBt90UDPfuD1vf4HXf81Fp/0Y6EGfa7PtR67NNhhh3HvevHp9qWYXWsGJh/nzTb4r+HlK25b79OZrJtfU37fWGayxOmw8PS3P/glknKPCDj/0as8XtWa/1KKeWCNP+Dji6lfCvJJUn+qdZ67WB5YMPpAQfYjNRaf1RrvSBNCP9/aa7V/P327ck+mE94QnWSIZITTR92c6J1rWOwno+kuebzi2mG7D4tzdD7Xs/csP9Hd0xQ7yfSfGh/Yft6UZproj8wSVuGtWmjq7W+Pk1P3N+kub7/RUn+rb2WdEPasCHbv/f9zDTbfNjXa9aYe01fTPNh/5gkKc3N9/44yYX9J2Ha+xn8U/vyxDS9kUfkvuvH12VfD/P2JH+X5vr5F6QZanxE7rv8YFj9w353prIf1mnbTXH/T7acyX7Hp9rmNUzyN2CqNyJbn306zFT/XqxPHSNDfl7b9pzMumzr9frbvIFGklydiY/LI7JmOB/6e1ZrfX6SR6S5p8Iv0pxouao0T+IANjI94ACboVrrWGnuPP2ENNdqJ/c9CmvXIbNsjKHqm0Q7ZPnINNeBv3TgvcPXtb5a651tj80rSnNH7BemuSv7RVOYvdd7dsCQ9x6aJkgM62Fbl/YtT/IPSf6hlDInzdDo15RSzmyHf16X5KmllF1qrbcOzL5/mqHlP9+QNgz4Qfv9nlrrJetbSa11VSnlI0lObC+NeHaaEDA47P95ae7GfEitdfUH/jL8xnnr43lJvlprXdQ/sZTyu5PMs3+Szw5M6/WCTra/13nbTWH/D3NLmpEEB66l+k15/E4WIjf1Pt3Yrk+zLR6SNUc/lCnWcV37/YC+n3v2b7/3tvW6/m3ekBMDPT9IcwO6y3qXKGyIWus1aXr2z2j/Zn8zyWmllLPWMioDWEd6wAGmUSnliIFrmXvTt89915ZemyTtDaBuTnJoaZ7h2iu7MM1NkDZXvR6rcT1KpZQHpekhXB/vT/Mos9OT/GGaa6TXejOkNgD9S5KjSikP72vLSO678dyn1qdBpZT7l4G7idfmeci9ANAbpvvpNP9/Tx6Y/2lphvd+dmN8oO7z72k+WL902D0FSimjpZRhwWGYXtg+Jk0oq7V9lFWfe9IEjNWfMdrt+/p1bfgE7smax9KOaW4sOJHjSyn37yt//yQvTXOjv69MMt+Ut9067P81tPv7o0n2L6W8cPD93u/7pjx+01yukAwPkZt6n25sn2u/jzsm2puzTWX4edKcsBlL8ur+/dr+3XpBmqcy/HuyXn+bb2/fn+rv3TAXpHnU3auGvVlKmcplJWnvoTAuD7QnBq9Pc3+HORvQRmAIPeAA0+sdaR5D9tk0wwnvSHON9HPSPG/3gvYa8Z53p7m79ZdKKZ9Oc3fsl6YJCY/rsN1TVmu9rZTy5STPLaX8Js1d3h+c5q7F12cdrkPtq/M/SylfT3MjorE0d6qeqhPThK6vlVLOSvPB+cg0zzj+yJA7SE/VIWkeq/aJNNeZ3p7mBnsvSvLNWmtty52X5jnGS0rz6KSvprmW+YQk/5PmTtQbTTua4nlpbnZ1VSnlg2mua92hXe6fpwlv502hrn8vpVydJtjcb4K2fjzNteKXtSMVtk0TQnbY4JW5r/6XlOZO/JekuQb7r3PftcTD/DzNI5c+mCa8vyDNHaFf1N+jO2gdt91U9/9EXp/k0CQfKKU8Oc1N3kbSnJQZzX13xt9Ux++1aW5mdkIp5Y40Jyd+Vmu9LJt+n25sX0wz8uC49sZkl6S5pvvFae5+/8i1VVBrraWUt6W5xOCr7fG2c1vHTkmOHjjpty5/m/81zV3Mzy6lfCHNY86+WWsddl+Oifx9mmHmbyulHJrmGP1VmuP6sLQjFqZQzzFJXllK+VSaa+bvTnNpyVOS/HOd/AkBwHoQwAGm16vS3G33iWk+4O6S5q60V6Xp3T1voPzpae6W/rw0d2++Ns0Q7MdkMw3greemeUbtUWnC5w+SvC7Nh71z17PO96XZbpe3w36npNb67VLK49PcVOuENNePL09zDfGZ69mW5L7Hgh2c5jr3WUluSHN36tX11lrvLqU8JU3gelaaEHdrmucWv77WOuzGahuk1nplKeVRacLiM9IEg9uS/DDNMbYuoe38NI84ujfJGo9RqrVeWJrHk72yLbcyTY/kyZk8JE/Vq9K0/ZlpfnduTHMs9G4eNcySNDc1fHmawP6DNAHqIxOUX20dtt2U9v8ky1lZSvmjNCc1/jzN3cxvS/M7/g995TbJ8Vtr/U0pZVGaEPnONI86+0qaIc6bep9uVO2Jk79I88ztZ6e558RVabbpCZn63d+XlFL+O/c90/2uNEOzn1Nr/dpA8XX52/zRNCdWFqV5bvw2aU4KTTmAt39H/r+2bc/LfTcJ/Emax7JN+kSIPle0bTkyyYPSjHa4PsniNCcVgI1sZGzMZR0AbHnauzL/U5oPwx+d7vYAm792BMe2tdbN9fp1YCvnGnAAtlQvSzO0ePA52MAMN+QZ2Gl7jB+e5jnZANPCEHQAthillN3TXN/4pCQHJTllyHPIAd7QXjpweZrLeg7MffcKWNtj4QA2GQEcgC3J/mmeKX5rkvdmw67ZBrZeX0vzGMdXp7k2e0WSTyT537XWH09nw4CZzTXgbBSllO3S3GTkp7nvkUMAAAAzxaw0NzT81kQj9PSAs7E8Ls3ZZgAAgJnsSWkeJ7kGAZyN5adJ8uEPfzh77LHHdLcFAACgUzfffHOOPvropM1GwwjgbCz3JMkee+yRvfbaa7rbAgAAMF0mvCTXY8gAAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRgdLobAJva+9///ixfvnyt5VauXJkkmTt37jrVv3Dhwhx33HHr1TYAAGDmEMDZ6i1fvjzXXFsza84uk5a757e3JkluXnnXlOvuzQMAALA2Ajgzwqw5u2SHBx82aZk7fnRpkqy13LB5AAAA1sY14AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQGcrdpll12WlStXTnczOnHZZZflsssum+5mAAAAExDA2aotW7ZsxgTwZcuWZdmyZdPdDAAAYAICOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6MDodDcA2DiuueaaJMlRRx01zS2BiY2OjmabbbbJXXfdNaXyD3zgA3PLLbesMX3WrFm55557NnbzNorBts2bNy8333xz7rnnnjz60Y/O9773vdx5551Jkkc84hG5+uqrx80/MjKS2bNn521ve1u++tWv5uMf/3iSZLvttsuJJ56Yd7/73Xnta1+bj370o1m0aFHe+ta35rTTTsvY2FhOOeWUnHbaadlnn32SJCtWrMgZZ5yRJUuWZGxsLGeccUZe8pKX5KyzzsqqVasya9asvOxlL8vZZ5+dsbGxvPzlL88555yTJUuWZO7cuePadf755+fjH/94jjrqqCxfvnxomWH627C28msru3z58jXWcZiPfexjueCCC3LsscfmL/7iL9baxg1td1e++tWv5m1ve1uWLFmSJz7xiZ0sc2Nshy635ea43wD66QEHoDOrVq2acvhOMjR8J9lsw3eyZttuuumm1dO++93vrg7fSdYI30kyNjaWO++8M0uXLl0dvpPkzjvvzNvf/vbccccdOe2003Lttdfm9NNPzx133JGlS5dm6dKlq3/uufDCC3PttdfmwgsvXP3z0qVLU2vNddddl+9///s588wzU2vN97///SxdunR1+UG9tnzuc5+bsMww/W3Y0LLD1nGYCy64IEly3nnnTamN69OW6fCOd7wjSXLmmWd2tsyNsR263Jab434D6CeAT4NSysGllG+v4zwHllKeuQHL/EAp5UnrOz+bN73esPW54YYb1pi2atWqJMntt9+esbGx3H777avL3njjjat/vv7667NixYpceumlGRsby7Jly3LJJZdkbGxsjXr7X99www0ZGxvLJZdckpUrV66efv7554+bZ1iZYfrbsLbyayu7fPnyNdZxmI997GPjXn/iE5+YtI0b2u6ufPWrX129/1etWpWvf/3rm3yZG2M7dLktN8f9BjBoix2CXkoZrbWumu52dOjAJEcm+ef1mbnW+qKN2potxK233prf/va3GRudtUnqv3fVb1cPiQTYXCxdujQHHHBA7r333iT3Bfepuvfee3PhhRfm+OOPT5JxPfETlRnmwgsvXN2GtZVfW9nBXu+lS5fmrLPOWqOeXu93z3nnnbfOw9DXpd1d6fV+95x55pmbfBj6xtgOXW7LzXG/AQza5AG8lPLhJCXJdkn+O8lfJ/l4knfVWj/TljkqyatqrYeUUh6U5B+SLEiyfZKP1lrf0pb7YZL/m+TQJMtLKa9L8tEk90syJ8kXaq2vacveP8kHkxyQ5Kb262e11sWllNlJ3pzkj5PMTnJ1kuNrrbdPsA57J/l2kvOSHNS264Ra69dKKaNJvpBkt3b6vyV5Sa31rnbeU5I8J8m9SX6d5IkDde+S5JNJPltrfWcp5flJTkizb36Z5PgkP0/yt0nuV0q5MslXa62vmKCtf5LkTUnuaet4ea31ilLKFUmW1lo/3/a+9/b9Hkm+UWv9s1JKSfLOJA9ot8s7a63nDlsOAJuvG264Ibfccsvq4D02NrZO869atSqXX375pOFlKmWuuOKKcb22k5VfW9le73fPsBECG8u6tLsrgydR1vWkyvrYGNuhy225Oe43gEFd9ICfWGv9eZKUUt6UZEmaIPv8JJ9pyxybpBf0Lkjyd7XWr7ZB+dJSyrdqrcva9x9Uaz2krW9OkqNqrbeXUrZNcnEp5am11ouSvCHJylrrQ0spuyb5TpLeOLTXJPllrfX323pOT3JKktdNsh67JbmqDfB/nOSjpZR9k9yV5Dm11l+UUkaSnJ/mJMN72zD9jCRPqLX+qpSyW6313ibnJqWUB6cJ32+ttX68HSL+zCQH1VrvLKU8LckHa61PKKW8IcmRtda/XMv2/tvcd3JgVpIdBwvUWh/bLn9BksuTnN6eSPhIkqNrrf9VStk5ybdLKd+otf7XWpa52dpll12yYsWK3Jk5m6T+bUbnZOHC38lb3/rWTVL/VBmCDvRbsGBBDjjggCxbtiyrVq3KyMhIkqkH8dHR0RxyyCEbXObggw9e3Ya1lV9b2fnz548L4QsWLJjCmqyfdWl3V0ZHR8eF7tHRTf8RbmNshy635ea43wAGdXEN+DGllO+UUq5O0xN8YJogfFAp5QGllN3S9ER/opSyY5KDk7yr7en9tyR7JnlYX339Y8tmJXlbKeU/0gTsh7f1J8khaUN9rXVFkk/3zfeMJM8tpVzZLucZSfZdy3rcleQf2/q+kuQ3aXr2t0myuK3nqjS98702HJnkPbXWX7Xz/aKvvgelCb8n1lp7Y/uOSvJ7Sb7Z1ndakvlradegy5KcWUp5dZKH9ZY9qB0h8Lkkr6m1/muS/dJs5wvbZX8tzaiFhw2bH4DN1+LFi7No0aJss03zb350dHSdAts222yTRYsWrX79l3+55rnfwTLD9LdhbeXXVnbx4sWTvu455phjxr0+9thjJ23j+rRlOrzyla8c9/qkk07a5MvcGNuhy225Oe43gEGbNIC3PbrHJ3lqrfURSV6fZE6t9Y40vd/PThPKP1Nr/XXbnrEkj6u1Hth+7VtrfVdftf3DxF+VZG6SP6i1PjJNyO51dY60dQ0zkqaXuLeMh9Va1/WvdK/+56QZVv6kdh3PHmjDRFYmqUmePlDnB/va9Xu11nU6xV9rfWWSF6Y5YfCxUspxg2Xa0QKfSHJ+rbU3KmAkyc/7ln1grXXvWuun1mX5TI/Pfe5z090EYCMb1sPbC9E77bRTRkZGstNOO60uO3/+/NU/77PPPtl1111z2GGHZWRkJEcccUQOP/zwjIyMrFFv/+sFCxZkZGQkhx9++LhHOD3/+c8fN8+wMsP0t2Ft5ddWduHChWus4zB/9Vd/Ne71+jyGbF3a3ZWDDjpo9f4fHR3t5DFkG2M7dLktN8f9BjBoU/eA75LmOuZflFK2SzM0u+e8NEPPj819PdW3pel5PblXqJQyv5SyxyT1/7TW+ttSyrwkf9L33uVphrmnlDJ34L3PJnlVKWX79v2dSylr6+mdnSZs904szEkToHdJE1xva3uVn9M3z+eSHN8O507b29/z27ZNDyul/H07fP1zaUYM7NWWn1VKeUxb/ldJ7r+WNqaUUmqtV9da/z5Nj/3jhhQ7J8l/1Vrf3jetJrmjlPK8vroeWkq539qWCTBVo6OjmT179pTLP/CBDxw6fdasTXNjxY1hsG3z5s1bPe3Rj350tttuu9XvPeIRj1hj/pGRkWy33XZZvHjxuJ7n7bbbLq961auyww475OSTT87++++fJUuWZIcddsjixYuzePHi1T/3LFq0KPvvv38WLVq0+ufFixenlJJ99903++23X0466aSUUrLffvtl8eLFq8sP6rXlqKOOmrDMMP1t2NCyw9ZxmF4v+Pr0fk+1LdOh1wveRe93z8bYDl1uy81xvwH0G1nXG7Osi/a64g8neVSSH6e5kdnv11oPbt//QZLUWh/SN88eSd6RZjh5ktyW5K/b65J/mOY66Gvasg9O8rEk2ya5MU3v+Pdrrae2Nzc7N83Q6h8mWdG+93dtD/CpaQLwvWl6sv9PrfWTE6zH3m3bz0ry1CQ75L7rrO+fpjf5QWlu9HZdku1rrce2ofqUJEcnWdWuy0Ht19Ja62PbbfSP7XsvSTMq4KQ0w+tnJ/lYrfUN7XK+lOaa7q9MchO2TyV5SLu8W5O8sNZ6Xe8mbGluOPfDJNekuVFbklxea31lKeUhaW7CNr9d/v8keWbvGv7JtNvo+ksvvTR77bXX2op35pRTTsny5ctzZ3bMDg8+bNKyd/zo0iRZa7nBeR62GVwDnmT1ndg3h7YAAMBM8+Mf/ziHHXZYkuxTa/3hsDKb9A4e7WPCnjXJ+w8ZMu3mNCF0WPm9B17/KMnvT1D9r5M8u+0dv1+Sr6e5QVpqrXenueHaZDddG7b8NyZ548C0XyY5fILyY0ne0n71uyLJY9syq5L0n6b9cPs1WNcvkzx+Cm38swmmH9z3cujQ+FrrD5L8f2tbBgAAAOtui30O+BTMTfKl9k7gc5J8pNZ6yTS3CQAAgBlqqw3gtdafJXnMWgv2KaW8N8kfDkxe1T626wEbq20bqpSye5IvD3nrk7XWv+26PQAAAKzdVhvA10et9aXT3YapaE8uHDjd7QAAAGDqungOOAAAAMx4AjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAGerdsQRR2Tu3LnT3YxOHHHEETniiCOmuxkAAMAEBHC2aoceeuiMCeCHHnpoDj300OluBgAAMAEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANCB0eluAHThnt/emjt+dOlayyRZa7k15/md9W8YAAAwYwjgbPUWLlw4pXIrV85OksydO3cdav+dKdcPAADMbAI4W73jjjtuupsAAADgGnAAAADoggAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6MDodDcANqX3v//9Wb58+ZTLr1y5Mkkyd+7cjbL8hQsX5rjjjtsodQEAAFs2AZyt2vLly/OD//xe9thpaof6z29flSQZvf1nG7zsm9u6AAAAEgGcGWCPnUbzgkfuOqWy5161IkmmXH4qdQEAACSuAQcAAIBOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0YHS6GwCb0sqVK/Pbu+6d7mZsMS677LIkyaGHHjrNLQEAgK2PAM5WbeXKlblHAJ+yZcuWJRHAAQBgUzAEHQAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA6MTncDgM3HNddckyQ56qijprklwFSMjIxkdHQ099xzT+69997V03beeef86le/SpLsvvvuufXWW3PXXXdl2223TZLcfffdE9a5++6757bbbstrX/vanHXWWbn55ptXvzd79uzsuuuuq6fttttuedaznpWzzz47SbLDDjvkjjvuWF1+2223Xb2sPfbYI7NmzcpPfvKTjI2NJUkOOuigfOtb38puu+2Wn//855k3b16e/OQn5z3veU+SZP78+TnyyCPznve8JyeccEI+85nP5Kabblrdlgc+8IH5xS9+kXnz5uW6664btx5z5szJ6173upx77rn5yU9+kj333DMnnnhizjjjjNx0001ZsmRJnvjEJ2bFihV5wxvekBtuuCHz5s3LDjvskNe//vWZO3fu6rpWrFiRM844I0uWLMncuXPHzfN3f/d3mT9/ft785jfnzjvvzP/8z/9kjz32yOzZs/P6178+Y2NjOeOMM/LiF784Z599du6+++6MjIwkSWbNmjVuWcuXL88pp5yS0047Lfe///3zlre8JXfffXe23XbbvO51r8vcuXPHldlnn33Gte8lL3lJzjnnnLzkJS/JWWedlbGxsTXWpd+KFSvylre8Jb/5zW9yyy235PTTT19d52Tl++tdvnx5Tj755Oy555554xvfuMayem178YtfnHe961256aabMm/evJx44ok566yzsmrVqsyaNSsvf/nLc84554wrt+eee+bUU09dvQ17239Q//+sOXPm5H3ve9+k+29rNVPWE5It+3gf6f0TnE6llIOTLK21PnZzrG9z0K7T7Frrl9dz/i8m+V+11uvWWnj96t87yfWXXnpp9tprr02xiPXyrGc9K/fc+Zuc8vjdp1T+3KtWJEle8MhdN3jZ5161IjvPL3nrW9+6wXV1RfAGenbaaafcfvvtay03MjKSjflZYrC+3uv1Wc7gOixYsCA33HBDkmR0dDSf+tSncvbZZ+dLX/rSuPme/vSn5/jjj1/9+uyzz85FF12Upz3taTn++OPHzbPTTjvlSU960hp19OoZGxvLRRddlPnz569e9kTLOuGEE3LjjTdmwYIFOeCAA8bV2SvXX+ass84a17758+fnxhtvHLeswXXpN7ju/XWurfxgeyZaVn/b+te/f1/0Xg+2fXAb9rb/oMH/XWvbf1urmbKekGy+x/uPf/zjHHbYYUmyT631h8PKGIK+5Tg4yZPXd+Za69M3Vfhm6yB8A/2mEr6TbNTwPay+3uv1Wc7gOvQHu1WrVuWiiy7Kl7+85nntZcuWZeXKlUmaXpZLL700Y2NjueSSS7J8+fJx89x+++25+OKLhy7/y1/+ci655JKMjY0NDd/9y1q+fPnqIHvDDTdk2bJla5S78sorx5W5/vrrx7XvhhtuWGNZ/evSb8WKFbnkkkvW2D7XX3/90HYOlh9sz7BlDbZtcFmDr4eVu/jii1dvw0suuWSNdRn2v+uLX/zihPtv2LbYGsyU9YRkyz/epzQEvZTy4SQlyXZJ/jvJXyf5eJJ31Vo/05Y5Ksmraq2HlFL2T3Jukh2TXJnkd5O8qdb6+UkWs20p5dwkv5dkVZJja63XllI+muSTtdaPlVJek+R1SXattd5TSrk2yZ/WWr9fSnlTkkVJbkryb1NYpzcmeXaS3yYZS3JIrfXWUsofJDktyf3aom+otX6hneflSU5McmuSLyZ5Wa31AW3v77eTvD/JU5Nsn+ToJC9N8gdJfpPkT2qtN7f1vCbJX6bZ/jclOa7WenMp5dR2O98/ycIk1yX5qyT7tnVtU0o5PMmFtdbTJlivFyd5ZZI705xgeWat9b9KKT9McmSSnyXp/7SxIMn5tdZXTrburLvb77o3t7RDBQHY/Jx99tlDg/2qVaty4YUX5vjjj8+FF164enj/vffemzPPPDP33HPPuPK994fV0xtuPpHesq6++uo1pg++Pu208f/6ly5dmgMOOGDC5Q+uS78LL7xwjWX06hzWCz5YftWqVTn99NPHlbn77rvHLat/262v/ssr7r333qHrMsxE+2+q829pZsp6QrLlH+9T7QE/sdb62FrrI5J8L8mSJOcleX5fmWPThO4k+VCSf6i1PjzJO5M8bgrLeGSS82qtj05yVpIL2umXJjms/fmwdvmPK6U8KMnObfg+KskzkhyY5NAkD51sQaWUuUkWJ3lUrfXAJAclub2UskuS9yZ5Tq31MWkC6zmllF1KKY9MckqSx9daH5cmJPfbLcnXa62PSvJ/23afVWt9ZJLvJHl5u+znpjkh8Yftun4xyZl99Tw2yXOSPCzJtkmOrrVe3bbrglrrgROF79bbkjy5Xa/HJRl3KrnW+rO2jgOTvCDNyYR3T7bukywLALZYE/Wqj42N5fLLL0+SXHHFFauD56pVqybsyV7XZQwuq78neaJyv/71r8dNu+GGG8a1b7L6B11xxRVD2zbR+g2WHxsbGzpKon9Za2vbVPWWu2rVqqHrMsxE+2+q829pZsp6QrLlH+9TvQnbMaWUo5PMTtOr/f0kf5vkHaWUB6TpQf7jttz9kjw8yUeSpNb67VLKVVNYxn/XWr/S/vyhJO9r67o0ycmllNlJ9koTMA9P8qP2vSQ5JMk/1VpvT5JSyv9N8vpJlvWrJDXJP5ZSLkry+VrrbaWUxyfZJ8mXSim9smNpAvPjk3yx1npLO/3cJM/tq/P2vt7i7yb5ca31yvb1d5Ic0f78jDQh+7vtMkaT/LKvnotrrbe26/HNNL3f6+KyJOeWUj6T5Au11uXDCpVS5qcZxXB0rfW6UsrTM/G6f3sd20CSnWZvkwfNX7jFXANuCDow00x0XfnIyEgOOeSQJMnBBx+cZcuWZdWqVRkdHc2ee+65TiF8bdeu95Z19dVXTxrCR0ZGssMOO4wL4b1rxXvtm6z+QQcffHAuuuiiNdq2YMGCofUMlh8ZGcmOO+64RgjvX1b/ttsQvW04Ojo6dF2GmWj/TXX+Lc1MWU9Itvzjfa094KWUJyU5PslT2x7w1yeZU2u9I8ln0gzjfk6Sz9Raf51kJE1w2ygXhdVar2/b+Zwk38h9PeKHpQmbaZe5LnXek+QPk7wrTaj/TtvDPZLkql4Pcfs1v9b67dy3XhO5s+/ne9IMbe9/3TvZMZJmOH6v/ofXWp/QV3ai+abqz5O8Ns2JkstLKU8bLNCe2Ph8kiW11m/0tWuidQeArc4JJ5yQWbNmrTF9dHQ0ixYtSpIsWrQo22zTfFzaZpttctJJJ60xT+/9YfWMjk7+b7y3rMWLF68xffD1ySefPG7a4sWLx7VvsvoHLVq0aGjbBtsxUfnR0dEsWbJkXJltt9123LLW1rapmDVr1urlbrPNNkPXZaL2DrZhXebf0syU9YRkyz/ep/JXcZc0PbS/KKVsl+b6757z0gw9Pzbt8PNa6y+TXJsmmKeU8ugkj5jCcn63DftJE7avrrX+qn19WZJTk1xSa70xzXDvJ+e+AH5pkmeWUnYspcxKM7R6QqWUnZM8sNb6lVrrG5Nck6bX/l+SPKSUckhf2ceVUkaSXJHk6W2PfzJ++P26+GySE9ph8CmlbFdK+b0pzPerrDnsfZxSymiShbXWf2uHqX85yaOGlPl4kg/VWj/e99Zk684M8LnPfW66mwBsRnbaaacplVvbNc7rarC+3uv1Wc7gOvT37o6OjuapT31qnvzkNe9vesQRR6x+rM2uu+6aww47LCMjIzn88MOzcOHCcfPstNNOecpTnjJ0+U9+8pNz+OGHZ2RkZMKe5d6yFi5cmPnz569u5xFHHLFGuQMPPHBcmX322Wdc+xYsWLDGsvrXpd+uu+6aww8/fI3tM9FjyAbLD7Zn2LIG2za4rMHXw8o95SlPWb0NDz/88DXWZdj/rqc//ekT7r8t7XFFUzVT1hOSLf94n0oA/1Kam4H9V/vzd3tv1Fq/luaGXfertX69b55jkvxNKeU7aW4e9h8ZP8x6mCuTPLud5xVtHT2XprlZWC9wfz3JbbXWH7ft+HyaHt0r2zL/vpZl3T/Jp0spV5VSrklyc5obva1MM0T8jaWU/yil/Gea4D9Sa/2PJGck+UYp5Wvt+qxtndZQa/1Qkg8n+Uo7NP87SZ4w+VxJkk8leWwp5cpSyskTlJmV5LxSytWllP9I8qAk5wyUeUKa0QPPbeu6spTy6snWfV3XEYBujIyMZNtttx3XyzgyMpL73e9+q1/vvvvumT17dpKmh7L3LPCJ7L777tl+++2zZMmS7LHHHuPemz179rhpu+2227gb3+ywww7jyvcva4899si8efPGBemDDjoo22+/ffbaa6/MmTMn++67b1760peufn/+/PmrXx9//PGZN2/euLbMmzdv9XyD5syZkyVLlmThwoWZM2dOFi5cmJNOOml1HSeddFKSpiflwQ9+cEZGRrLXXntlv/32W6M3ZdGiRdl///3H9ar25jn55JOzaNGi7Lfffnnwgx+cOXPmZO+9915dT2/ek046KaWULFy4MPvuu2/23XffNZa1ePHi7LDDDqt7tnvlSymry/WXGWzf4sWLV38vpQxdl8H1KqVkwYIF2X777Sfs/R4s31/v4sWLs/3222ffffedsKe9t/777rvv6v3Va2NvO/Ta3l9u4cKF47bhVHq55syZs9b9t7WaKesJyZZ9vG+S54CXUnZMcketday9I/oVSUob8rZYpZSda623tT+fmuR3a63PnXyumcFzwIfXtaU9B7x3x/Ytqc0AALA5mMpzwNf1+uKpekKSt/UNXz5uSw/frdNKKU9IczO65UlePM3tAQAAYAuxSQJ4rfXLGf+s6SRJKeWzaYaS97uh1vqMTdGO9s7ebxny1mtrrV9c1/pqrS/b8FZtuFLKgWmuvx/07lrrB7ptDQAAAFOxqXrAh9pUQXuS5X0xzXO2tyrt480OnOZmAAAAsA427NkQAAAAwJQI4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0IHR6W4AbEpz587Nb1fcOd3N2GIcccQR090EAADYagngbNXmzp2b227/2XQ3Y4tx6KGHTncTAABgq2UIOgAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOiCAAwAAQAcEcAAAAOiAAA4AAAAdEMABAACgAwI4AAAAdEAABwAAgA4I4AAAANABARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0YHS6GwCb2s23r8q5V62YctkkUy6/trp23uBaAACArYUAzlZt4cKF61R+1cqVSZKd587d4GXvvB7LBwAAtl4COFu14447brqbAAAAkMQ14AAAANAJARwAAAA6IIADAABABwRwAAAA6IAADgAAAB0QwAEAAKADAjgAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOCOAAAADQAQEcAAAAOjA63Q1gqzErSW6++ebpbgcAAEDn+rLQrInKCOBsLA9KkqOPPnq62wEAADCdHpTkumFvCOBsLN9K8qQkP01yzzS3BQAAoGuz0oTvb01UYGRsbKy75gAAAMAM5SZsAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHRDAAQAAoAMCOAAAAHRAAAcAAIAOjE53A2BTKaXsl+T8JLsl+UWSY2qtP5jeVrGxlFKWJvmLJHsneUSt9Zp2+oT7fX3fY/NWStktyYeS7JvkziT/neQltdZbHA8zTynl00n2SXJvktuT/K9a65WOhZmrlPLGJKem/V/hWJh5Sik/TPLb9itJltRaL3YszDyllDlJ3pHk8DTHwzdqrS/u8ljQA87W7L1Jzqq17pfkrCTnTHN72Lg+neSgJD8amD7Zfl/f99i8jSU5o9Zaaq2PTHJdktPa9xwPM8/za62/V2t9VJKlST7YTncszECllEcn+cMkN/RNdizMTH9Zaz2w/bq4neZYmHnOSBO896u1PiLJ/26nd3YsjIyNjW3wWsDmppSye5LvJ9mt1npPKWVWmrNSD6m13jK9rWNjas9qH9n2aky435OMrM97jpctTynlL5Icn+Q5cTzMaKWUY5K8IsnT41iYcUop2yW5Is3fgsuTHJnkZ3EszDj9nxX6pvnMMMOUUnZK8uMke9Vab++b3umxoAecrdX8JDfVWu9Jkvb7T9rpbL0m2+/r+x5bkFLKNmnC92fjeJixSikfKKXckOTNSZ4fx8JM9bdJ/rHWen3fNMfCzPXhUspVpZSzSym7xLEwE+2bJiC/sZTy7VLKFaWUJ6bjY0EAB2Br8g9prvt993Q3hOlTa31RrXVBktcmedt0t4fulVL+KMnjkpw93W1hs/CkWuvvpTkmRuJ/xEw1mmRhkn+vtT42yZIkn0yyU5eNEMDZWt2YZF47FCTt9z3b6Wy9Jtvv6/seW4j2xnwPSfKsWuu9cTzMeLXWDyU5JM2QQ8fCzPLHSR6a5Pp2+PFeSS5O0wPmWJhhaq03tt/vTHNS5gnxP2Im+lGSVUk+miS11m8m+XmS36TDY0EAZ6tUa/1ZkiuTPLud9Ow0Z7tcm7MVm2y/r+97HTWdDVRKeXOSxyT50/YDluNhBiql7FRKmd/3+qgkK9Jc93tlHAszRq31tFrrnrXWvWute6c5CfOUWus/x7Ewo5RSdiyl3L/9eSTJoiRX+h8x89Raf57mfhBHJKvvYN67/vvKdHQsuAkbW61SykPTPBZgbpKVaR4LUKe3VWwspZR3JfnzJHukOXv5i1rrAZPt9/V9j81bKeWAJNek+Qf6m3by9bXWP3M8zCyllN9J8pkkOya5J034Xlxr/a5jYWYbuGGnY2EGKaUsTPKJJLPar2uTvKLW+lPHwszTHg8fTPPYsLuTvK7W+qUujwUBHAAAADpgCDoAAAB0QAAHAACADgjgAAAA0AEBHAAAADoggAMAAEAHBHAAAADogAAOAAAAHfj/AZw/LYdT5jcHAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 1080x576 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "import seaborn as sns\n",
        "sns.set_theme(style=\"white\", color_codes=True)\n",
        "\n",
        "fig = plt.figure(figsize=(15, 8))\n",
        "sns.boxplot(data=dataset[[\"average_packet_size\", \"avg_bwd_segment_size\"]], orient=\"h\")\n",
        "\n",
        "plt.title('Summary of some variables containing outliers', fontsize=18)\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "gather": {
          "logged": 1663659933278
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'float64': Index(['fwd_packet_length_mean', 'fwd_packet_length_std',\n",
            "       'bwd_packet_length_mean', 'bwd_packet_length_std', 'flow_bytes_s',\n",
            "       'flow_packets_s', 'flow_iat_mean', 'flow_iat_std', 'fwd_iat_mean',\n",
            "       'fwd_iat_std', 'bwd_iat_mean', 'bwd_iat_std', 'fwd_packets_s',\n",
            "       'bwd_packets_s', 'packet_length_mean', 'packet_length_std',\n",
            "       'packet_length_variance', 'average_packet_size', 'avg_fwd_segment_size',\n",
            "       'avg_bwd_segment_size', 'active_mean', 'active_std', 'idle_mean',\n",
            "       'idle_std'],\n",
            "      dtype='object'),\n",
            " 'int64': Index(['destination_port', 'flow_duration', 'total_fwd_packets',\n",
            "       'total_backward_packets', 'total_length_of_fwd_packets',\n",
            "       'total_length_of_bwd_packets', 'fwd_packet_length_max',\n",
            "       'fwd_packet_length_min', 'bwd_packet_length_max',\n",
            "       'bwd_packet_length_min', 'flow_iat_max', 'flow_iat_min',\n",
            "       'fwd_iat_total', 'fwd_iat_max', 'fwd_iat_min', 'bwd_iat_total',\n",
            "       'bwd_iat_max', 'bwd_iat_min', 'fwd_psh_flags', 'bwd_psh_flags',\n",
            "       'fwd_urg_flags', 'bwd_urg_flags', 'fwd_header_length',\n",
            "       'bwd_header_length', 'min_packet_length', 'max_packet_length',\n",
            "       'fin_flag_count', 'syn_flag_count', 'rst_flag_count', 'psh_flag_count',\n",
            "       'ack_flag_count', 'urg_flag_count', 'cwe_flag_count', 'ece_flag_count',\n",
            "       'down_up_ratio', 'fwd_header_length.1', 'fwd_avg_bytes_bulk',\n",
            "       'fwd_avg_packets_bulk', 'fwd_avg_bulk_rate', 'bwd_avg_bytes_bulk',\n",
            "       'bwd_avg_packets_bulk', 'bwd_avg_bulk_rate', 'subflow_fwd_packets',\n",
            "       'subflow_fwd_bytes', 'subflow_bwd_packets', 'subflow_bwd_bytes',\n",
            "       'init_win_bytes_forward', 'init_win_bytes_backward', 'act_data_pkt_fwd',\n",
            "       'min_seg_size_forward', 'active_max', 'active_min', 'idle_max',\n",
            "       'idle_min'],\n",
            "      dtype='object'),\n",
            " 'object': Index(['label'], dtype='object')}\n",
            "         count unique     top     freq\n",
            "label  2425727     15  BENIGN  2035505\n"
          ]
        }
      ],
      "source": [
        "dataset_datatypes = dataset.columns.to_series().groupby(dataset.dtypes).groups\n",
        "datatypes_info = {k.name: v for k, v in dataset_datatypes.items()}\n",
        "\n",
        "pprint(datatypes_info)\n",
        "pprint(dataset.describe(include=[object]).transpose())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "gather": {
          "logged": 1663659933621
        }
      },
      "outputs": [],
      "source": [
        "dataset[['flow_bytes_s', 'flow_packets_s']] = dataset[['flow_bytes_s', 'flow_packets_s']].apply(pd.to_numeric)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZfyxB9Xrc0vo"
      },
      "source": [
        "## Inputs-Output Data Split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "gather": {
          "logged": 1663659933941
        },
        "id": "9FxgbkWAXIZv",
        "outputId": "0947e91b-1cdd-48a3-90f8-af120cf87a05"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "shape of X:  (2425727, 78)\n",
            "number of labels of y:  15\n",
            "Class labels:  ['BENIGN' 'DDoS' 'PortScan' 'Bot' 'Infiltration'\n",
            " 'Web Attack � Brute Force' 'Web Attack � XSS'\n",
            " 'Web Attack � Sql Injection' 'FTP-Patator' 'SSH-Patator' 'DoS slowloris'\n",
            " 'DoS Slowhttptest' 'DoS Hulk' 'DoS GoldenEye' 'Heartbleed']\n"
          ]
        }
      ],
      "source": [
        "Y = dataset.label\n",
        "X = dataset.drop(columns='label')\n",
        "\n",
        "class_labels = Y.unique()\n",
        "\n",
        "num_classes = Y.nunique()     # number of unique values\n",
        "print(\"shape of X: \",X.shape)\n",
        "print(\"number of labels of y: \", num_classes)\n",
        "print(\"Class labels: \", class_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "gather": {
          "logged": 1663659934290
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "\"\\n\\nfrom sklearn.impute import SimpleImputer\\n\\nmean_imp = SimpleImputer(missing_values=-1, strategy='mean')\\ncat_imp = SimpleImputer(missing_values=-1, strategy='most_frequent')\\nfor c in X.columns:\\n  X[c] = mean_imp.fit_transform(X[[c]]).ravel()\\n  \\n\""
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#  are there missing values\n",
        "'''\n",
        "\n",
        "from sklearn.impute import SimpleImputer\n",
        "\n",
        "mean_imp = SimpleImputer(missing_values=-1, strategy='mean')\n",
        "cat_imp = SimpleImputer(missing_values=-1, strategy='most_frequent')\n",
        "for c in X.columns:\n",
        "  X[c] = mean_imp.fit_transform(X[[c]]).ravel()\n",
        "  \n",
        "'''\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "gather": {
          "logged": 1663659934616
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['destination_port', 'flow_duration', 'total_fwd_packets',\n",
              "       'total_backward_packets', 'total_length_of_fwd_packets',\n",
              "       'total_length_of_bwd_packets', 'fwd_packet_length_max',\n",
              "       'fwd_packet_length_min', 'fwd_packet_length_mean',\n",
              "       'fwd_packet_length_std', 'bwd_packet_length_max',\n",
              "       'bwd_packet_length_min', 'bwd_packet_length_mean',\n",
              "       'bwd_packet_length_std', 'flow_bytes_s', 'flow_packets_s',\n",
              "       'flow_iat_mean', 'flow_iat_std', 'flow_iat_max', 'flow_iat_min',\n",
              "       'fwd_iat_total', 'fwd_iat_mean', 'fwd_iat_std', 'fwd_iat_max',\n",
              "       'fwd_iat_min', 'bwd_iat_total', 'bwd_iat_mean', 'bwd_iat_std',\n",
              "       'bwd_iat_max', 'bwd_iat_min', 'fwd_psh_flags', 'bwd_psh_flags',\n",
              "       'fwd_urg_flags', 'bwd_urg_flags', 'fwd_header_length',\n",
              "       'bwd_header_length', 'fwd_packets_s', 'bwd_packets_s',\n",
              "       'min_packet_length', 'max_packet_length', 'packet_length_mean',\n",
              "       'packet_length_std', 'packet_length_variance', 'fin_flag_count',\n",
              "       'syn_flag_count', 'rst_flag_count', 'psh_flag_count', 'ack_flag_count',\n",
              "       'urg_flag_count', 'cwe_flag_count', 'ece_flag_count', 'down_up_ratio',\n",
              "       'average_packet_size', 'avg_fwd_segment_size', 'avg_bwd_segment_size',\n",
              "       'fwd_header_length.1', 'fwd_avg_bytes_bulk', 'fwd_avg_packets_bulk',\n",
              "       'fwd_avg_bulk_rate', 'bwd_avg_bytes_bulk', 'bwd_avg_packets_bulk',\n",
              "       'bwd_avg_bulk_rate', 'subflow_fwd_packets', 'subflow_fwd_bytes',\n",
              "       'subflow_bwd_packets', 'subflow_bwd_bytes', 'init_win_bytes_forward',\n",
              "       'init_win_bytes_backward', 'act_data_pkt_fwd', 'min_seg_size_forward',\n",
              "       'active_mean', 'active_std', 'active_max', 'active_min', 'idle_mean',\n",
              "       'idle_std', 'idle_max', 'idle_min'],\n",
              "      dtype='object')"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "X.columns"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Select the number of relevant features out of total of 78"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "gather": {
          "logged": 1663659935014
        },
        "id": "lN9xTgzr_EQ9",
        "outputId": "aafeb6b2-0a3c-41db-eea4-c800c0499c4f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "X_select_78_features (2425727, 78)\n",
            "X_select_23_features (2425727, 23)\n"
          ]
        }
      ],
      "source": [
        "X_select_78_features = X\n",
        "\n",
        "selected_features = [\"bwd_packet_length_min\", \"subflow_fwd_bytes\", \"total_length_of_fwd_packets\", \"fwd_packet_length_mean\", \"bwd_packet_length_std\", \"flow_iat_min\", \"fwd_iat_min\", \"flow_iat_mean\", \"flow_duration\", \"flow_iat_std\", \"init_win_bytes_forward\", \"active_min\", \"active_mean\", \"bwd_packets_s\", \"bwd_iat_mean\", \"fwd_iat_mean\", \"ack_flag_count\", \"fwd_psh_flags\", \"syn_flag_count\", \"fwd_packets_s\", \"init_win_bytes_backward\", \"psh_flag_count\", \"average_packet_size\"]\n",
        "X_select_23_features = X[np.intersect1d(X.columns, selected_features)]\n",
        "\n",
        "print('X_select_78_features', X_select_78_features.shape)\n",
        "print('X_select_23_features', X_select_23_features.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "gather": {
          "logged": 1663659940723
        },
        "id": "RbXJPB5FbyTt",
        "outputId": "5fbc7846-d038-4190-906f-e793ab8d55d5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "after spliting the data:\n",
            "x training data shape: (1819295, 78)\n",
            "x test data shape: (606432, 78)\n",
            "x training data shape: (1819295, 23)\n",
            "x test data shape: (606432, 23)\n",
            "y training data shape: (1819295,)\n",
            "y test data shape: (606432,)\n"
          ]
        }
      ],
      "source": [
        "X_train_78_features, X_test_78_features, Y_train, Y_test = train_test_split(X_select_78_features, Y, random_state=42, stratify=Y)\n",
        "X_train_23_features, X_test_23_features, Y_train, Y_test = train_test_split(X_select_23_features, Y, random_state=42, stratify=Y)\n",
        "\n",
        "\n",
        "print(\"\\nafter spliting the data:\")\n",
        "print(\"x training data shape:\", X_train_78_features.shape)\n",
        "print(\"x test data shape:\", X_test_78_features.shape)\n",
        "\n",
        "print(\"x training data shape:\", X_train_23_features.shape)\n",
        "print(\"x test data shape:\", X_test_23_features.shape)\n",
        "\n",
        "print(\"y training data shape:\", Y_train.shape)\n",
        "print(\"y test data shape:\", Y_test.shape)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Encode the text of the labels to numbers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "gather": {
          "logged": 1663659941144
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'BENIGN': 0,\n",
            " 'Bot': 1,\n",
            " 'DDoS': 2,\n",
            " 'DoS GoldenEye': 3,\n",
            " 'DoS Hulk': 4,\n",
            " 'DoS Slowhttptest': 5,\n",
            " 'DoS slowloris': 6,\n",
            " 'FTP-Patator': 7,\n",
            " 'Heartbleed': 8,\n",
            " 'Infiltration': 9,\n",
            " 'PortScan': 10,\n",
            " 'SSH-Patator': 11,\n",
            " 'Web Attack � Brute Force': 12,\n",
            " 'Web Attack � Sql Injection': 13,\n",
            " 'Web Attack � XSS': 14}\n"
          ]
        }
      ],
      "source": [
        "le = LabelEncoder()       # Encode target labels with value between 0 and n_classes-1\n",
        "\n",
        "Y_train_binary = le.fit_transform(Y_train)\n",
        "\n",
        "#print(\"instances per label in test set\\n\", y_test_binary.value_counts())\n",
        "# transform -\tTransform labels to normalized encoding.\n",
        "Y_test_binary = le.transform(Y_test)\n",
        "\n",
        "#we use fit_transform() on training data but transform() on the test data\n",
        "\n",
        "# classes_ - ndarray of shape (n_classes,) - Holds the label for each class.\n",
        "# To create a dictionary from two sequences, use dict(zip(keys, values))\n",
        "# The zip(fields, values) method returns an iterator that generates two-items tuples \n",
        "labels_dict = dict(zip(le.classes_, range(len(le.classes_))))\n",
        "\n",
        "pprint(labels_dict)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "import pickle\n",
        "\n",
        "_file = open(\"datasets/labels_dict_file.pkl\",\"wb\")\n",
        "pickle.dump(labels_dict, _file)\n",
        "_file.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "gather": {
          "logged": 1663569663983
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/mnt/batch/tasks/shared/LS_root/mounts/clusters/mycomputenotebook1/code/Users/bredsoby\r\n"
          ]
        }
      ],
      "source": [
        "!pwd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Sample only trainning data an leave the test or validation data as is  </br>\n",
        " run the code in the cell only if saved sampled data is missing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "gather": {
          "logged": 1663659941928
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'\\niht = InstanceHardnessThreshold(random_state=42, sampling_strategy=\\'majority\\', n_jobs=os.cpu_count())\\n\\nX_train_78_features_sampled, Y_train_78_features_sampled = iht.fit_resample(X_train_78_features, Y_train_binary)\\n\\n# save what have been sampled to files \\nX_train_78_features_sampled.to_csv(\\'datasets/X_train_78_features_sampled.csv\\')\\nnp.savetxt(\"datasets/y_train_binary_78_features_sampled.csv\", Y_train_78_features_sampled, delimiter=\",\")\\nX_train_78_features_sampled = None\\nY_train_78_features_sampled = None\\n\\n\\nX_train_23_features_sampled, Y_train_23_features_sampled = iht.fit_resample(X_train_23_features, Y_train_binary)\\n\\n# save what have been sampled to files\\nX_train_23_features_sampled.to_csv(\\'datasets/X_train_23_features_sampled.csv\\')\\nnp.savetxt(\"datasets/y_train_binary_23_features_sampled.csv\", Y_train_23_features_sampled, delimiter=\",\")\\n\\n\\nX_test_78_features.to_csv(\\'datasets/X_test_78_features.csv\\')\\nX_test_23_features.to_csv(\\'datasets/X_test_23_features.csv\\')\\n\\nnp.savetxt(\"datasets/y_test_binary.csv\", Y_test_binary, delimiter=\",\")\\n'"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# from imblearn.under_sampling import InstanceHardnessThreshold\n",
        "\n",
        "\n",
        "'''\n",
        "iht = InstanceHardnessThreshold(random_state=42, sampling_strategy='majority', n_jobs=os.cpu_count())\n",
        "\n",
        "X_train_78_features_sampled, Y_train_78_features_sampled = iht.fit_resample(X_train_78_features, Y_train_binary)\n",
        "\n",
        "# save what have been sampled to files \n",
        "X_train_78_features_sampled.to_csv('datasets/X_train_78_features_sampled.csv')\n",
        "np.savetxt(\"datasets/y_train_binary_78_features_sampled.csv\", Y_train_78_features_sampled, delimiter=\",\")\n",
        "X_train_78_features_sampled = None\n",
        "Y_train_78_features_sampled = None\n",
        "\n",
        "\n",
        "X_train_23_features_sampled, Y_train_23_features_sampled = iht.fit_resample(X_train_23_features, Y_train_binary)\n",
        "\n",
        "# save what have been sampled to files\n",
        "X_train_23_features_sampled.to_csv('datasets/X_train_23_features_sampled.csv')\n",
        "np.savetxt(\"datasets/y_train_binary_23_features_sampled.csv\", Y_train_23_features_sampled, delimiter=\",\")\n",
        "\n",
        "\n",
        "X_test_78_features.to_csv('datasets/X_test_78_features.csv')\n",
        "X_test_23_features.to_csv('datasets/X_test_23_features.csv')\n",
        "\n",
        "np.savetxt(\"datasets/y_test_binary.csv\", Y_test_binary, delimiter=\",\")\n",
        "'''\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "gather": {
          "logged": 1663659942395
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'\\n# zip the generated files\\nfrom zipfile import ZipFile\\nimport os\\nfrom os.path import basename\\n\\nDATA_DIR5  = os.path.join(os.path.abspath(\".\"), \"datasets\")\\n# create a ZipFile object\\nwith ZipFile(\\'sampleDir.zip\\', \\'w\\') as zipObj:\\n   # Iterate over all the files in directory\\n   for folderName, subfolders, filenames in os.walk(DATA_DIR5):\\n       for filename in filenames:\\n           print(filename)\\n           #create complete filepath of file in directory\\n           filePath = os.path.join(folderName, filename)\\n           # Add file to zip\\n           zipObj.write(filePath, basename(filePath))\\n'"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "'''\n",
        "# zip the generated files\n",
        "from zipfile import ZipFile\n",
        "import os\n",
        "from os.path import basename\n",
        "\n",
        "DATA_DIR5  = os.path.join(os.path.abspath(\".\"), \"datasets\")\n",
        "# create a ZipFile object\n",
        "with ZipFile('sampleDir.zip', 'w') as zipObj:\n",
        "   # Iterate over all the files in directory\n",
        "   for folderName, subfolders, filenames in os.walk(DATA_DIR5):\n",
        "       for filename in filenames:\n",
        "           print(filename)\n",
        "           #create complete filepath of file in directory\n",
        "           filePath = os.path.join(folderName, filename)\n",
        "           # Add file to zip\n",
        "           zipObj.write(filePath, basename(filePath))\n",
        "'''"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "gather": {
          "logged": 1663659942932
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'\\nfrom imblearn.combine import SMOTETomek\\n\\nsmt = SMOTETomek(random_state=42, n_jobs= 1)\\n# X_sampled, y_sampled = smt.fit_resample(X.to_numpy(), y.to_numpy())\\nX_sampled, y_sampled = smt.fit_resample(X, y)\\n'"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# it run for 483 minutes then I interapted\n",
        "\n",
        "'''\n",
        "from imblearn.combine import SMOTETomek\n",
        "\n",
        "smt = SMOTETomek(random_state=42, n_jobs= 1)\n",
        "# X_sampled, y_sampled = smt.fit_resample(X.to_numpy(), y.to_numpy())\n",
        "X_sampled, y_sampled = smt.fit_resample(X, y)\n",
        "'''"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "gather": {
          "logged": 1663659943237
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "load_78_features = True"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Load 78 features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "gather": {
          "logged": 1663659971175
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "X_test__78_features (606432, 78)\n",
            "X_train_78_features (1796034, 78)\n",
            "Y_train_binary__78_features (1796034,)\n",
            "0.0     1503367\n",
            "4.0      128632\n",
            "2.0       96004\n",
            "10.0      42979\n",
            "3.0        7709\n",
            "7.0        4110\n",
            "6.0        3967\n",
            "5.0        3882\n",
            "11.0       2303\n",
            "1.0        1457\n",
            "12.0       1084\n",
            "14.0        489\n",
            "9.0          27\n",
            "13.0         16\n",
            "8.0           8\n",
            "dtype: int64\n",
            "Y_test_binary (606432,)\n"
          ]
        }
      ],
      "source": [
        "if load_78_features is True:   \n",
        "     \n",
        "    X_train__78_features = pd.read_csv('datasets/X_train_78_features_sampled.csv')\n",
        "    if len(X_train__78_features.columns) == 79: # Check the files loaded because extra columns gets added\n",
        "        X_train__78_features = X_train__78_features.drop(columns='Unnamed: 0')\n",
        "\n",
        "    X_test__78_features = pd.read_csv('datasets/X_test_78_features.csv')\n",
        "    if len(X_test__78_features.columns) == 79:\n",
        "        X_test__78_features = X_test__78_features.drop(columns='Unnamed: 0')\n",
        "    print('X_test__78_features', X_test__78_features.shape)\n",
        "\n",
        "\n",
        "    Y_train_binary__78_features = np.loadtxt('datasets/y_train_binary_78_features_sampled.csv', delimiter=',')\n",
        "    print('X_train_78_features', X_train__78_features.shape)\n",
        "    print('Y_train_binary__78_features', Y_train_binary__78_features.shape)\n",
        "    pprint(pd.DataFrame(Y_train_binary__78_features).value_counts())\n",
        "\n",
        "    Y_test__binary = np.loadtxt('datasets/y_test_binary.csv', delimiter=',')\n",
        "    print('Y_test_binary', Y_test__binary.shape)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Load 23 features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "gather": {
          "logged": 1663659971558
        }
      },
      "outputs": [],
      "source": [
        "if load_78_features is False:\n",
        "\n",
        "    X_train__23_features = pd.read_csv('datasets/X_train_23_features_sampled.csv')\n",
        "    if len(X_train__23_features.columns) == 24:\n",
        "        X_train__23_features = X_train__23_features.drop(columns='Unnamed: 0')\n",
        "\n",
        "\n",
        "    X_test__23_features = pd.read_csv('datasets/X_test_23_features.csv')\n",
        "    if len(X_test__23_features.columns) == 24:\n",
        "        X_test__23_features = X_test__23_features.drop(columns='Unnamed: 0')\n",
        "    print('X_select_23_features', X_test__23_features.shape)\n",
        "\n",
        "\n",
        "    Y_train_binary__23_features = np.loadtxt('datasets/y_train_binary_23_features_sampled.csv', delimiter=',')\n",
        "    print('X_train__23_features', X_train__23_features.shape)\n",
        "    print('y_train_binary__23_features', Y_train_binary__23_features.shape)\n",
        "    pprint(pd.DataFrame(Y_train_binary__23_features).value_counts())\n",
        "\n",
        "    Y_test__binary = np.loadtxt('datasets/y_test_binary.csv', delimiter=',')\n",
        "    print('Y_test__binary',Y_test__binary.shape)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "gather": {
          "logged": 1663659971902
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "if load_78_features is True:\n",
        "    X_train = X_train__78_features\n",
        "    X_test = X_test__78_features\n",
        "    Y_train = Y_train_binary__78_features\n",
        "    Y_test = Y_test__binary\n",
        "else :\n",
        "    X_train = X_train__23_features\n",
        "    X_test = X_test__23_features\n",
        "    Y_train = Y_train_binary__23_features\n",
        "    Y_test = Y_test__binary\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 188,
      "metadata": {
        "gather": {
          "logged": 1663544103250
        },
        "id": "oUiskJ-rybZ3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/anaconda/envs/azureml_py38/lib/python3.8/site-packages/sklearn/base.py:450: UserWarning: X does not have valid feature names, but MinMaxScaler was fitted with feature names\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "\n",
        "# scaler = StandardScaler()\n",
        "scaler = MinMaxScaler()\n",
        "\n",
        "X_train = scaler.fit_transform(X_train)\n",
        "X_test = scaler.transform(X_test.to_numpy())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 189,
      "metadata": {
        "gather": {
          "logged": 1663544103955
        }
      },
      "outputs": [],
      "source": [
        "class SecurityDataset(torch.utils.data.Dataset):\n",
        "\n",
        "  def __init__(self, X_train, y_train, transform=torch.tensor, target_transform=torch.tensor):\n",
        "    self.X_train = torch.tensor(X_train, dtype = torch.float32)\n",
        "    self.Y_train = torch.tensor(y_train)  \n",
        "    \n",
        "    self.transform = transform\n",
        "    self.target_transform = target_transform\n",
        "    \n",
        "    if self.transform:\n",
        "        self.X_train = self.transform(X_train, dtype=torch.float32)\n",
        "    if self.target_transform:\n",
        "        self.Y_train = self.target_transform(y_train, dtype=torch.int64)\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.Y_train)\n",
        "  \n",
        "  def __getitem__(self, index):\n",
        "    feature = torch.index_select(self.X_train, 0, torch.tensor([index]))\n",
        "    label = torch.index_select(self.Y_train, 0, torch.tensor([index]))\n",
        "    \n",
        "    return feature, label "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 190,
      "metadata": {
        "gather": {
          "logged": 1663544104606
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "\"## test :\\ndataiter = iter(train_dataset)\\nx_data, y_data = dataiter.next()\\n\\nprint('Type of x_data: ', type(x_data))\\nprint('shape of x_data: ', x_data.shape)\\n\\nprint('Type of y_data: ', type(y_data))\\nprint('Shape of y_data: ', y_data.shape)\""
            ]
          },
          "execution_count": 190,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# train_dataset = SecurityDataset(torch.tensor(X_train_sampled.values), y_train_binary_sampled)\n",
        "train_dataset = SecurityDataset(X_train, Y_train)\n",
        "test_dataset = SecurityDataset(X_test, Y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Class Weights"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 191,
      "metadata": {
        "gather": {
          "logged": 1663544105212
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "classes_y:  [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14]\n",
            "class_weights:  [7.96449570e-02 8.21795470e+01 1.24719387e+00 1.55319237e+01\n",
            " 9.30838361e-01 3.08437919e+01 3.01829090e+01 2.91327494e+01\n",
            " 1.49669500e+04 4.43465185e+03 2.78590940e+00 5.19911420e+01\n",
            " 1.10457196e+02 7.48347500e+03 2.44858078e+02]\n",
            "\n",
            "classes_class_weights:  {0: 0.08, 1: 82.18, 2: 1.247, 3: 15.532, 4: 0.931, 5: 30.844, 6: 30.183, 7: 29.133, 8: 14966.95, 9: 4434.652, 10: 2.786, 11: 51.991, 12: 110.457, 13: 7483.475, 14: 244.858}\n"
          ]
        }
      ],
      "source": [
        "from sklearn.utils import class_weight\n",
        "\n",
        "classes_y = np.array(list(labels_dict.values()))\n",
        "print('classes_y: ',classes_y)\n",
        "\n",
        "#calculate the class weights\n",
        "class_weights = class_weight.compute_class_weight(class_weight = 'balanced',\n",
        "                                                 classes = classes_y, # np.unique(y_train_binary),\n",
        "                                                 y = Y_train)\n",
        "print('class_weights: ',class_weights)\n",
        "print()\n",
        "\n",
        "\n",
        "# class_weights.round(decimals=3, out=None)\n",
        "class_weights = np.around(class_weights, decimals=3)\n",
        "classes_class_weights = dict(zip(classes_y, class_weights))\n",
        "print('classes_class_weights: ',classes_class_weights)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Apply WeightedRandomSampler only to train data and leave test or validate data untouched because is treated as unseen"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 193,
      "metadata": {
        "gather": {
          "logged": 1663544137400
        }
      },
      "outputs": [],
      "source": [
        "weights_sampler = 1. / class_weights\n",
        "sample_weights = [0] * len(train_dataset)\n",
        "# weights_sampler =np.around(weights_sampler, decimals=5)\n",
        "\n",
        "for idx, (data, label) in enumerate(train_dataset):\n",
        "        class_weight = class_weights[ int(label.item()) ]\n",
        "        sample_weights[idx] = class_weight\n",
        "        \n",
        "sampler = torch.utils.data.WeightedRandomSampler(sample_weights, num_samples=\n",
        "                                    len(sample_weights), replacement=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 194,
      "metadata": {
        "gather": {
          "logged": 1663544138110
        }
      },
      "outputs": [],
      "source": [
        "batch_size = 32\n",
        "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, sampler=sampler, drop_last=True, num_workers=os.cpu_count()-1)\n",
        "test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True, num_workers=os.cpu_count()-1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Do not run because it takes a lot of time , run only if the output of the cell is not displayed"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 195,
      "metadata": {
        "gather": {
          "logged": 1663545095819
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[tensor(1203265), tensor(1196363), tensor(1197605), tensor(1195857), tensor(1196577), tensor(1195953), tensor(1196841), tensor(1197033), tensor(1199512), tensor(1194588), tensor(1197486), tensor(1198282), tensor(1197379), tensor(1197214), tensor(1196470)]\n",
            "\n",
            "\n",
            "tensor(1203265)\n",
            "tensor(1194588)\n"
          ]
        }
      ],
      "source": [
        "\n",
        "#labels_count = [0] * num_classes\n",
        "\n",
        "#for i in range(15):\n",
        "#        labels_count[i]=i\n",
        "\n",
        "#for epoch in range(10):\n",
        "#    for data, labels in train_loader:\n",
        "#        for i in range(num_classes):\n",
        "#                labels_count[i] += torch.sum(labels==i)\n",
        "        \n",
        "#print(labels_count)\n",
        "#print()\n",
        "#print()\n",
        "#print('max of labels count',max(labels_count))\n",
        "#print('min of labels count',min(labels_count))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Run only for test if nan are present in data "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 196,
      "metadata": {
        "gather": {
          "logged": 1663545180546
        }
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 56126/56126 [01:24<00:00, 662.65it/s]\n"
          ]
        }
      ],
      "source": [
        "if torch.cuda.is_available():\n",
        "   torch.cuda.empty_cache() \n",
        "DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "print('DEVICE', DEVICE) \n",
        "\n",
        "\n",
        "import tqdm           \n",
        "\n",
        "for step, (batch_x, batch_y) in enumerate(tqdm.tqdm(train_loader),0):\n",
        "            # zero the parameter gradients\n",
        "            batch_x, batch_y = batch_x.to(DEVICE), batch_y.to(DEVICE)\n",
        "            \n",
        "            # batch_x = batch_x.view(-1, input_size)\n",
        "            # batch_x = batch_x.float()\n",
        "            assert not torch.isnan(batch_x).any()\n",
        "            assert  torch.isfinite(batch_x).all()\n",
        "            \n",
        "            assert not torch.isnan(batch_y).any()\n",
        "            assert torch.isfinite(batch_y).all()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 197,
      "metadata": {
        "gather": {
          "logged": 1663545181004
        }
      },
      "outputs": [],
      "source": [
        "def get_dataloaders():\n",
        "    \n",
        "    return train_loader,test_loader"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "include_colab_link": true,
      "name": "Copy of AML Final Project.ipynb",
      "provenance": []
    },
    "kernel_info": {
      "name": "python3"
    },
    "kernelspec": {
      "display_name": "Python 3.10.6 ('pytorch_env')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      }
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    },
    "vscode": {
      "interpreter": {
        "hash": "201b6068cbed112f118c29a6393440d4ce4dc02105f31305c61d838eb972bb93"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}