irc_model_XGBoost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IRC Malware Detection Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model 1 features:\n",
    "- periodicity\n",
    "- duration\n",
    "- pkt_size\n",
    "- msg_count\n",
    "- src_ports_count\n",
    "- dst_port\n",
    "\n",
    "Model 2:\n",
    "- added more features - new features:\n",
    "    - username special chars mean\n",
    "    - msg special chars mean\n",
    "    - msg word entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "library imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-23T21:15:53.803849Z",
     "start_time": "2020-02-23T21:15:52.786008Z"
    },
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from irc_utils import compute_score, exponential_moving_average\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import plot_tree\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from matplotlib.pylab import rcParams\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " #### Data processing\n",
    "- Each row in csv file represent one session\n",
    "- Each session is represented by\n",
    "    - periodicity, duration, total packet size, message count, number of source ports, destination port"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-23T21:15:53.848195Z",
     "start_time": "2020-02-23T21:15:53.821198Z"
    },
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>periodicity</th>\n",
       "      <th>duration</th>\n",
       "      <th>pkt_size</th>\n",
       "      <th>msg_count</th>\n",
       "      <th>src_ports_count</th>\n",
       "      <th>dst_port</th>\n",
       "      <th>username_spec_chars</th>\n",
       "      <th>msg_spec_chars</th>\n",
       "      <th>msg_word_entropy</th>\n",
       "      <th>malicious</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.9980</td>\n",
       "      <td>64833.0</td>\n",
       "      <td>2529.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>41124.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.9081</td>\n",
       "      <td>3.3992</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>104400.0</td>\n",
       "      <td>772.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>58356.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.9084</td>\n",
       "      <td>3.1766</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>79200.0</td>\n",
       "      <td>2955.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>38992.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.9080</td>\n",
       "      <td>3.4412</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.2772</td>\n",
       "      <td>3600.0</td>\n",
       "      <td>257.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>50444.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.9080</td>\n",
       "      <td>2.9502</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>75600.0</td>\n",
       "      <td>2831.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>57990.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.9084</td>\n",
       "      <td>3.4055</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  periodicity  duration  pkt_size  msg_count  src_ports_count  \\\n",
       "0           0       0.9980   64833.0    2529.0       19.0              1.0   \n",
       "1           1       1.0000  104400.0     772.0        6.0              1.0   \n",
       "2           2       0.9999   79200.0    2955.0       23.0              1.0   \n",
       "3           3       0.2772    3600.0     257.0        2.0              1.0   \n",
       "4           4       0.9999   75600.0    2831.0       22.0              1.0   \n",
       "\n",
       "   dst_port  username_spec_chars  msg_spec_chars  msg_word_entropy  malicious  \n",
       "0   41124.0                  0.5          0.9081            3.3992        1.0  \n",
       "1   58356.0                  0.5          0.9084            3.1766        1.0  \n",
       "2   38992.0                  0.5          0.9080            3.4412        1.0  \n",
       "3   50444.0                  0.5          0.9080            2.9502        1.0  \n",
       "4   57990.0                  0.5          0.9084            3.4055        1.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import data\n",
    "data = pd.read_csv('input/trn_data_small.csv', delimiter=';')\n",
    "data = data.rename(columns={\"src_spec_chars\": \"username_spec_chars\"})\n",
    "plt.rcParams[\"figure.figsize\"] = (10,7)\n",
    "# Prepare data for training\n",
    "# Divide the data into attributes and labels\n",
    "X = data.iloc[1:, 1:-1].values\n",
    "y = data.iloc[1:, -1].values\n",
    "\n",
    "# Splitting the dataset into the Training set and Test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Detection Model \n",
    "- using random forest classifier to classify malicious/non-malicous communication\n",
    "\n",
    "#### Hyperparameter Tuning\n",
    "\n",
    "What are the hyperparameters?\n",
    "- n_estimators: The n_estimators parameter specifies the number of trees in the forest of the model \n",
    "- max_depth: The max_depth parameter specifies the maximum depth of each tree\n",
    "- min_samples_split: The min_samples_split parameter specifies the minimum number of samples required to split an internal leaf node.\n",
    "- min_samples_leaf: The min_samples_leaf parameter specifies the minimum number of samples required to be at a leaf node\n",
    "\n",
    "#### Validation curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-23T21:15:54.020846Z",
     "start_time": "2020-02-23T21:15:53.891547Z"
    },
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import validation_curve\n",
    "param_range = range(1,10)\n",
    "train_scores, test_scores = validation_curve(\n",
    "                                xgb.XGBClassifier(),\n",
    "                                X = X_train, y = y_train, \n",
    "                                param_name = 'n_estimators', \n",
    "                                param_range = param_range, cv = 3)\n",
    "train_scores_mean = np.mean(train_scores, axis=1)\n",
    "train_scores_std = np.std(train_scores, axis=1)\n",
    "test_scores_mean = np.mean(test_scores, axis=1)\n",
    "test_scores_std = np.std(test_scores, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-23T21:15:54.372857Z",
     "start_time": "2020-02-23T21:15:54.077266Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAG5CAYAAADGcOOUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de7xe453//9cnB+Ich/QkCIZWBEESxzjXBKlTO1N+jOq3pdpS2jJ0qtqmM0o7o2gNNdRpWqqM1rQ6SoVGHSOCiqhQrcQhEUSCIMnn98dae7uzs5PsRO59bXu/no/H/dj3Wuu61/qs+76531nXtdaKzESSJEmdq1fpAiRJknoiQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTmiwiBkVERkSfevq3EfGpjrRdjm39S0Rc8m7q7a4iYsOImBMRvZfQJiPi7zqzrveqiHg6IvZp0rpHRsTjDdMfjoiJETE7Ir4UERdFxDeasW2pMxnCpKWIiP+LiDHtzD8oIp5f1sCUmftl5hUroK49ImJqm3WfmZmffbfrXsz2PhgRl0bEc/WP4eSI+HZErNaM7a1omfm3zFw9M+cDRMTtEbHc71VE/EdE/K7NvHMj4tcN02tExDl1YHktIv4WEddFxA4NbbJeNiciXoyIqyOi//LW1cHalxqgImLNen/+Vtf2ZD29XjNrA8jMcZn54YZZ/wyMzcw1MvP8zDwuM7/T7DqkZjOESUt3BXBkRESb+f8E/DQz5xWoqVNFxDrA3cAqwE6ZuQbwUaA/sOlyrG+5jvR1Md8ANomITwNExE7Ap4Dj6umVgduArYDRwJrAFsA1wH5t1rVNZq4ObAKsDXyrE+pfrIhYCfg9sCUwiqr2nYCZwIgCJW0EPPpuV9JNvnfqTjLThw8fS3hQBY9ZwG4N89YG5lL9eAIcADwIvAo8A3yroe0gIIE+9fTtwGfr572BfwdeBJ4Cvtim7aeBx4DZ9fLP1fNXA94AFgBz6seHqH68/7th2wdS/Xi9Um93i4ZlTwMnAw/X+/dzoN9i3oN/BR4Bei1m+UL72M5+Hg38EfgB1Q/5d+uahjS0H1Dv0/vq6dHAxLrdXcDWi9n2t4Ef1s/7Aq8B32/47OYC6zTWCPwbML9eNgf4Ud0+qULUE/V2LwBiCd+NPev92aj+nD7XsOyzwHPAakv5fiXwdw3TXwB+1zD9IeBG4CVgCnBMw7KVgXOBZ+vHucDK9bL1gF/X+/ESMI7qH95X1d+bN+p9/+d2avos8AKw+hLqfhrYp34+giqkv1Lv84+AleplUX/u06n++3ik5XMH9gcmUX2/pwEn1/P3AKbWz29r81ltDlwO/GtDLYv9rtR1nkr1PX+Thu+oDx+lHx4Jk5YiM98ArgWOapj9j8DkzHyonn6tXt6fKpB9PiIO7sDqj6H6AdkWGAZ8os3y6bxzFOXTwA8iYrvMfI3qaMqzWXWxrZ6Zzza+MCI2B64GTqIKODcB/1sf5Wjcj1HAxsDWVGGpPfsA/5OZCzqwT4uzA1WQfD8wBvgf4PA2tdyRmdMjYlvgJ8DngHWBHwM31keX2rqD6kcbYDjwPLBbPb0T8HhmvtT4gsz8OlUoOb5+745vWDy6Xs/WdU1/v7gdysyxwHXAA/V2L25YvA9wc/1ZdUhErA0cDNzTMPsaYCpVGPsEcGZE7FUv+zqwIzAU2IYqDJ1eL/tq/boBVO/5v1Ql5z8BfwM+Vu/799opZR/g/zJzTgdLnw98mSr47QTsTRUmAfal+jw2B9aiek9n1ssupQquawBDqALXQjJzLxb+rP7cuLyD35XDqf677J894Mi13jsMYVLHXAF8IiL61dNH1fMAyMzbM/ORzFyQmQ9ThZ/dO7DefwTOzcxn6qDw3caFmfmbzHwyK3cAvwNGdrDmTwK/ycxbMvNtqiNuqwA7N7Q5PzOfrbf9v1Q/5u1Zl+oIx7vxbGb+MDPn1cH2Z8BhDcv/v3oewLHAjzPz3sycn9UYujepAkdbdwObRcS6VD/2lwLrR8TqVJ/BHctY51mZ+Upm/g0Yy+LfkxbjqN6fn2Vm481416MKZgBExNCIeCUiXm0cdF6bEBGvUB0R3ZAqSBARGwC7AKdm5tzMnAhcwjv/IDgCGJOZ0zNzBtVRwX+ql70NfBDYKDPfzmqcVUdvFrxMn3dmPpCZ99Sf7dN1/S3f/7eBNYCPUB1VfCwzn2tYNjgi1szMlzNzQke32aAj35Xz6//G3liO9UtNYwiTOiAz76T6gTw4IjalOuLQEhiIiB0iYmxEzIiIWVRdWh0ZwPwhqu7LFn9tXBgR+0XEPRHxUv0jvX8H19uy7tb11UexngHWb2jzfMPz14HVF7OumVQ/6O/GM22mxwKr1u/dIKqwc0O9bCPgq3VoeaXe9w2o9mkh9Q/reKof/d2oQtddVOFleUJYR98T6uD371TdgGPaDKhf6D3LzImZ2R84lKobsdF29bJ+wIXAuDrwfwh4KTNnN7T9K+98hgt9xvXzlvfo+1Tdl7+LiKci4rTF7/IilunzjojNI+LX9YkqrwJnUn9PM/M2qu7JC4DpEXFxRKxZv/TjVN/pv0bEHfW4umXVke9K2++e1CUYwqSOu5LqCMSRVN1MLzQs+xnVuJ0NMnMt4CKqsTBL8xzVD0aLDVue1N0p11P9yL+//pG+qWG9Szuq8SzVD1TL+qLe1rQO1NXWrcAhEbG4/2e0dLmt2jDvA23aLFRvVmcpXkvVVXQ48OuGsPEM8G+Z2b/hsWpmXr2Y7d8B7EXVrXt/Pf33VGH5D4t5TUePCi3JuVTddl+ut/PvDct+D+y7LGeP1kcsL6HqHh5C9RmuExFrNDTbkHc+w4U+43rZs/W6ZmfmVzNzE6qxgV+JiL1bNrWUUm4F/n4Zar8QmAxslplrUnV9tn7/szqjcXtgMFW35Cn1/Psz8yDgfcAvqb4Py6oj35UV8VlLK5whTOq4K6nGyhxDQ1dkbQ2qIxZzI2IEVddaR1wLfCkiBtbjgRqPVqxEdcRkBjAvIvajGl/T4gVg3YhYawnrPiAi9o6IvlRjhN6kOkq0rM6hGpd2RURsBBAR69eXX9i67gqbRnUWae+I+H907KzJn1F1mx5Bw5FF4L+A4+qjZBERq0XEAW3CSKM7qALypMx8i/qkAOAvdW3teYHqbMTlEhH7U50h+pV61glUR0r3rKevpArZN0TEkPp96Uc19m9x6+xNNfbvDeCpzHyG6vP6bkT0i4itgc8A/12/5Grg9IgYUF864oyWZRExOiL+rg7fs6jGbbWM6Vvavl9FFW6uj4iPRESviFg3quvQ7d9O+zWoBt3PiYiPAJ9v2Kfh9efYctLEXGBBRKwUEUdExFp1+Hy1ob5lsazfFanLMIRJHVSPdbmL6szEG9ss/gJVd9Rsqh/Cjv6L/r+Am4GHgAlUg9Vbtjcb+FK9rpepgt2NDcsnU/0IP1V3wyzUVZeZj1MdtfshVVfqx6gGY7/Vwdoa1/US1Viyt4F76/38PdWP+5S62TFURzhmUl3aYKlhLzPvpfph/hDw24b54+v1/aje9yks/qQB6m2twjtHvSZR/dgv7igYwHlU4/xejojzl1Zro/oH/iLgSy2D/jNzOlXQvTgiVsnMuVRnT04CfkMVMh6nGvT/j21W+VBEzKHa108BhzScTHA41Zmdz1J1134zM2+tl/0rVVfsw1RnHU6o5wFsRnVEaw7VuLn/rE8kgGrs4en19+bktvuXmW9S/YNjMnBLXft9VF2M97bzlpxM9f2cTfWd/nnDsjXreS9TdZfOpOoqhWr82tN1F+ZxVGF8mSzHd0XqMqLj4zQlSZK0ongkTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQW8525mut566+WgQYNKlyFJkrRUDzzwwIuZOaC9Ze+5EDZo0CDGjx9fugxJkqSlioi/Lm6Z3ZGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVEDTQlhE/CQipkfEnxazPCLi/IiYEhEPR8R2zapFkiSpq2nmkbDLgVFLWL4fsFn9OBa4sIm1SJIkdSl9mrXizPxDRAxaQpODgCszM4F7IqJ/RHwwM59rVk3LoipL6nkionQJktQjNC2EdcD6wDMN01PreYuEsIg4lupoGRtuuGFTi3r6oYncc8/feOON+U3djtQV9e4d9O4d9Okd9OnTq/4b78zvUy3r26dX63SvXtCrVxXcGuNbS5hryXTv/F14fkfatP5t2EJHX9Oh9Ro8pR5p5dX703/Q4GLbLxnCOiwzLwYuBhg2bFhTD1Gde8LXOG/cjs3chNTt9Ok1n769F9CnV8Ojnu7be/7C83stWLht78b5i7bt03sBfdu0a7dty/aW1rZ3+3X0iiTCI+BST7LGym/R74zp9OtXJg6VDGHTgA0apgfW84raaO1X2GHDqaXLkDpdJizIYN6CXsxb0Iu35/duff7OvPr5/HfmLchezFvQm3kLepfeBUlaJvt95Al+8fUFxbZfMoTdCBwfEdcAOwCzusJ4sBNvuIuRE57jhRfmlC5FKiIzyaThkQ3zqiNFCxa802bBggXMm/fOY8GCbP1btcv6edV2/vyWeTB//oLWNvPnZ+trWto0rqdlfss62k5XjwWt81uet6yjZd78+Y3zFjBv3jttJfUcb701n35bfKxoDU0LYRFxNbAHsF5ETAW+CfQFyMyLgJuA/YEpwOvAp5tVy7Lo1SvYZJO1ef/7VytditTp3gld7/xtCUlLmvdOKMuFgtuCBdU6YeG21bZa1rXw8kWDX/thsO1yWLiOlm20raGl9pbl74wTCxwaJvUcCxYkG220VtEamnl25OFLWZ7AF5u1/XdjnXVWYZ11VildhvSetrhw1nZeR0JeR4Pf8m2vbZiT1FOsvHIvVl65XKfge2JgvqT3nojqjMqurjGcSepZeveOomdHG8Ik9Wgt3ZAtl9mQpM7ivSMlSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgFNDWERMSoiHo+IKRFxWjvLN4yIsRHxYEQ8HBH7N7MeSZKkrqJpISwiegMXAPsBg4HDI2Jwm2anA9dm5rbAYcB/NqseSZKkrqSZR8JGAFMy86nMfAu4BjioTZsE1qyfrwU828R6JEmSuoxmhrD1gWcapqfW8xp9CzgyIqYCNwEntLeiiDg2IsZHxPgZM2Y0o1ZJkqROVXpg/uHA5Zk5ENgfuCoiFqkpMy/OzGGZOWzAgAGdXqQkSdKK1swQNg3YoGF6YD2v0WeAawEy826gH7BeE2uSJEnqEpoZwu4HNouIjSNiJaqB9ze2afM3YG+AiNiCKoTZ3yhJkrq9poWwzJwHHA/cDDxGdRbkoxExJiIOrJt9FTgmIh4CrgaOzsxsVk2SJEldRZ9mrjwzb6IacN8474yG55OAXZpZgyRJUldUemC+JElSj2QIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFNDWERcSoiHg8IqZExGmLafOPETEpIh6NiJ81sx5JkqSuok+zVhwRvYELgI8CU4H7I+LGzJzU0GYz4GvALpn5ckS8r1n1SJIkdSXNPBI2ApiSmU9l5lvANcBBbdocA1yQmS8DZOb0JtYjSZLUZTQzhK0PPNMwPbWe12hzYPOI+GNE3BMRo9pbUUQcGxHjI2L8jBkzmlSuJElS5yk9ML8PsBmwB3A48F8R0b9to8y8ODOHZeawAQMGdHKJkiRJK14zQ9g0YIOG6YH1vEZTgRsz8+3M/AvwZ6pQJkmS1K01M4TdD2wWERtHxErAYcCNbdr8kuooGBGxHlX35FNNrEmSJKlLaFoIy8x5wPHAzcBjwLWZ+WhEjImIA+tmNwMzI2ISMBY4JTNnNqsmSZKkriIys3QNy2TYsGE5fvz40mVIkiQtVUQ8kJnD2ltWemC+JElSj7TUEBYRJ0TE2p1RjCRJUk/RkSNh76e62v219W2IotlFSZIkdXdLDWGZeTrVZSMuBY4GnoiIMyNi0ybXJkmS1G11aExYVqP3n68f84C1gesi4ntNrE2SJKnbWuoNvCPiROAo4EXgEqrLSLwdEb2AJ4B/bm6JkiRJ3c9SQxiwDnBoZv61cWZmLoiI0c0pS5IkqXvrSHfkb4GXWiYiYs2I2AEgMx9rVmGSJEndWUdC2IXAnIbpOfU8SZIkLaeOhLDIhsvqZ+YCOtaNKUmSpMXoSAh7KiK+FBF968eJeJNtSZKkd6UjIew4YGdgGjAV2AE4tplFSZIkdXdL7VbMzOnAYZ1QiyRJUo/RkeuE9QM+A2wJ9GuZn5n/r4l1SZIkdWsd6Y68CvgA8PfAHcBAYHYzi5IkSeruOhLC/i4zvwG8lplXAAdQjQuTJEnScupICHu7/vtKRAwB1gLe17ySJEmSur+OXO/r4ohYGzgduBFYHfhGU6uSJEnq5pYYwuqbdL+amS8DfwA26ZSqJEmSurkldkfWV8f/506qRZIkqcfoyJiwWyPi5IjYICLWaXk0vTJJkqRurCNjwj5Z//1iw7zErklJkqTl1pEr5m/cGYVIkiT1JB25Yv5R7c3PzCtXfDmSJEk9Q0e6I4c3PO8H7A1MAAxhkiRJy6kj3ZEnNE5HRH/gmqZVJEmS1AN05OzItl4DHCcmSZL0LnRkTNj/Up0NCVVoGwxc28yiJEmSuruOjAn794bn84C/ZubUJtUjSZLUI3QkhP0NeC4z5wJExCoRMSgzn25qZZIkSd1YR8aE/QJY0DA9v54nSZKk5dSRENYnM99qmaifr9S8kiRJkrq/joSwGRFxYMtERBwEvNi8kiRJkrq/jowJOw74aUT8qJ6eCrR7FX1JkiR1TEcu1voksGNErF5Pz2l6VZIkSd3cUrsjI+LMiOifmXMyc05ErB0R/9oZxUmSJHVXHRkTtl9mvtIykZkvA/s3ryRJkqTuryMhrHdErNwyERGrACsvob0kSZKWoiMD838K/D4iLgMCOBq4oplFSZIkdXcdGZh/dkQ8BOxDdQ/Jm4GNml2YJElSd9aR7kiAF6gC2D8AewGPNa0iSZKkHmCxR8IiYnPg8PrxIvBzIDJzz06qTZIkqdtaUnfkZGAcMDozpwBExJc7pSpJkqRubkndkYcCzwFjI+K/ImJvqoH5kiRJepcWG8Iy85eZeRjwEWAscBLwvoi4MCL27awCJUmSuqOlDszPzNcy82eZ+TFgIPAgcGrTK5MkSerGOnp2JFBdLT8zL87MvZtVkCRJUk+wTCFMkiRJK4YhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUQFNDWESMiojHI2JKRJy2hHYfj4iMiGHNrEeSJKmraFoIi4jewAXAfsBg4PCIGNxOuzWAE4F7m1WLJElSV9PMI2EjgCmZ+VRmvgVcAxzUTrvvAGcDc5tYiyRJUpfSzBC2PvBMw/TUel6riNgO2CAzf7OkFUXEsRExPiLGz5gxY8VXKkmS1MmKDcyPiF7AOcBXl9Y2My/OzGGZOWzAgAHNL06SJKnJmhnCpgEbNEwPrOe1WAMYAtweEU8DOwI3OjhfkiT1BM0MYfcDm0XExhGxEnAYcGPLwsyclZnrZeagzBwE3AMcmJnjm1iTJElSl9C0EJaZ84DjgZuBx4BrM/PRiBgTEQc2a7uSJEnvBX2aufLMvAm4qc28MxbTdo9m1iJJktSVeMV8SZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKmApoawiBgVEY9HxJSIOK2d5V+JiEkR8XBE/D4iNmpmPZIkSV1F00JYRPQGLgD2AwYDh0fE4DbNHgSGZebWwHXA95pVjyRJUlfSzCNhI4ApmflUZr4FXAMc1NggM8dm5uv15D3AwCbWI0mS1GU0M4StDzzTMD21nrc4nwF+296CiDg2IsZHxPgZM2aswBIlSZLK6BID8yPiSGAY8P32lmfmxZk5LDOHDRgwoHOLkyRJaoI+TVz3NGCDhumB9byFRMQ+wNeB3TPzzSbWI0mS1GU080jY/cBmEbFxRKwEHAbc2NggIrYFfgwcmJnTm1iLJElSl9K0EJaZ84DjgZuBx4BrM/PRiBgTEQfWzb4PrA78IiImRsSNi1mdJElSt9LM7kgy8ybgpjbzzmh4vk8zty9JktRVdYmB+ZIkST2NIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIRJkiQVYAiTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCjCESZIkFWAIkyRJKsAQJkmSVIAhTJIkqQBDmCRJUgGGMEmSpAIMYZIkSQUYwiRJkgowhEmSJBVgCJMkSSrAECZJklSAIUySJKkAQ5gkSVIBhjBJkqQCDGGSJEkF9CldwIrw9ttvM3XqVObOnVu6FL2H9evXj4EDB9K3b9/SpUiSeoBuEcKmTp3KGmuswaBBg4iI0uXoPSgzmTlzJlOnTmXjjTcuXY4kqQfoFt2Rc+fOZd111zWAablFBOuuu65HUyVJnaZbhDDAAKZ3ze+QJKkzdZsQJkmS9F5iCFsBZs6cydChQxk6dCgf+MAHWH/99Vun33rrrQ6t49Of/jSPP/74EttccMEF/PSnP10RJUuSpMK6xcD80tZdd10mTpwIwLe+9S1WX311Tj755IXaZCaZSa9e7efeyy67bKnb+eIXv/jui22Cpe2bJElaVPcLYf/RpHE9X81lfsmUKVM48MAD2XbbbXnwwQe55ZZb+Pa3v82ECRN44403+OQnP8kZZ5wBwK677sqPfvQjhgwZwnrrrcdxxx3Hb3/7W1ZddVV+9atf8b73vY/TTz+d9dZbj5NOOoldd92VXXfdldtuu41Zs2Zx2WWXsfPOO/Paa69x1FFH8dhjjzF48GCefvppLrnkEoYOHbpQbaeccgq/+c1v6NOnD/vttx9nn302zz//PJ/73Of4y1/+QkRw8cUXs8MOO/C9732PK6+8EoDPfe5znHDCCe3u28MPP8yYMWN488032WyzzfjJT37Caqut9u7fe0mSuiEPXTTZ5MmT+fKXv8ykSZNYf/31Oeussxg/fjwPPfQQt9xyC5MmTVrkNbNmzWL33XfnoYceYqedduInP/lJu+vOTO677z6+//3vM2bMGAB++MMf8oEPfIBJkybxjW98gwcffHCR173wwgvcdNNNPProozz88MN87WtfA6ojbR/96Ed5+OGHeeCBB9hiiy249957+elPf8r999/P3XffzX/+53/yyCOPLLJvffv25ayzzuL3v/89EyZMYOutt+a8885bUW+jJEndTvc7ErYcR6yaadNNN2XYsGGt01dffTWXXnop8+bN49lnn2XSpEkMHjx4odesssoq7LfffgBsv/32jBs3rt11H3rooa1tnn76aQDuvPNOTj31VAC22WYbttxyy0Vet84669CrVy+OOeYYDjjgAEaPHg3A7bffzjXXXANAnz59WHPNNbnzzjv5+Mc/ziqrrALAwQcfzLhx49h3330X2re77rqLSZMmsfPOOwPw1ltvseuuuy77GyZJUg/R/UJYFwLAZOQAABKmSURBVNPYHffEE09w3nnncd9999G/f3+OPPLIdq9LtdJKK7U+7927N/PmzWt33SuvvPJS27Snb9++jB8/nltuuYVf/OIXXHjhhfzud78Dlu0yDY37lpmMGjWKq666qsOvlySpJ7M7shO9+uqrrLHGGqy55po899xz3HzzzSt8G7vssgvXXnstAI888ki73Z2zZ8/m1VdfZfTo0fzgBz9o7bLcc889ueiiiwCYP38+r776KiNHjuSGG27gjTfeYM6cOfzqV79i5MiRi6xz55135o477uCpp54C4LXXXuOJJ55Y4fsnSVJ34ZGwTrTddtsxePBgPvKRj7DRRhuxyy67rPBtnHDCCRx11FEMHjy49bHWWmst1GbWrFkceuihvPnmmyxYsIBzzjkHgB/96Eccc8wx/PjHP6ZPnz78+Mc/ZsSIERx++OEMHz4cgM9//vNstdVWTJkyZaF1vv/97+fSSy/lk5/8ZOtlOc4880w222yzFb6PkiR1B5HZtcZQLc2wYcNy/PjxC8177LHH2GKLLQpV1LXMmzePefPm0a9fP5544gn23XdfnnjiCfr0MW93hN8lSdKKFBEPZOaw9pb5y9zNzJkzh7333pt58+aRma1HtSRJUtfir3M3079/fx544IHSZUiSpKVwYL4kSVIBhjBJkqQCDGGSJEkFGMIkSZIKMIStIM8//zyHHXYYm266Kdtvvz37778/f/7zn0uX1a5Bgwbx4osvArTeZqito48+muuuu26J67n88st59tlnW6c/+9nPtntxWEmStChD2AqQmRxyyCHssccePPnkkzzwwAN897vf5YUXXlio3bLcWqiz3HXXXcv92rYh7JJLLlnkPphdQVd83yVJ6nYhLOLbTXksydixY+nbty/HHXdc67xtttmGkSNHcvvttzNy5EgOPPDA1oByzjnnMGTIEIYMGcK5554LVLf5OeCAA9hmm20YMmQIP//5zwE47bTTGDx4MFtvvTUnn3zyItu+6KKLOOWUU1qnL7/8co4//nigutn29ttvz5ZbbsnFF1/cbu2rr746UAXJ448/ng9/+MPss88+TJ8+vbXNmDFjGD58OEOGDOHYY48lM7nuuusYP348RxxxBEOHDuWNN95gjz32oOVCuldffTVbbbUVQ4YMab2heMv2vv71r7PNNtuw4447LhJUAe644w6GDh3K0KFD2XbbbZk9ezYAZ599NltttRXbbLMNp512GgATJ05kxx13ZOutt+aQQw7h5ZdfBmCPPfbgpJNOYtiwYZx33nnMmDGDj3/84wwfPpzhw4fzxz/+cfEfqCRJnSEz31OP7bffPtuaNGlS63P4VlMeS3LeeeflSSed1O6ysWPH5qqrrppPPfVUZmaOHz8+hwwZknPmzMnZs2fn4MGDc8KECXndddflZz/72dbXvfLKK/niiy/m5ptvngsWLMjMzJdffnmR9U+fPj033XTT1ulRo0bluHHjMjNz5syZmZn5+uuv55ZbbpkvvvhiZmZutNFGOWPGjMzMXG211TIz8/rrr8999tkn582bl9OmTcu11lorf/GLXyy0nszMI488Mm+88cbMzNx9993z/vvvb13WMj1t2rTcYIMNcvr06fn222/nnnvumTfccEP9+dD6+lNOOSW/853vLLJPo0ePzjvvvDMzM2fPnp1vv/123nTTTbnTTjvla6+9tlBNW221Vd5+++2ZmfmNb3wjTzzxxNZaPv/5z7eu8/DDD299X/7617/mRz7ykUW2m7nwd0mSpHcLGJ+LyTTd7mKtmd8sXcIiRowYwcYbbwzAnXfeySGHHMJqq60GwKGHHsq4ceMYNWoUX/3qVzn11FMZPXo0I0eObL390Gc+8xlGjx7N6NGjF1n3gAED2GSTTbjnnnvYbLPNmDx5cus9Kc8//3xuuOEGAJ555hmeeOIJ1l133XZr/MMf/sDhhx9O7969+dCHPsRee+3Vumzs2LF873vf4/XXX+ell15iyy235GMf+9hi9/f+++9njz32YMCAAQAcccQR/OEPf+Dggw9mpZVWat2P7bffnltuuWWR1++yyy585Stf4YgjjuDQQw9l4MCB3HrrrXz6059m1VVXBWCdddZh1qxZvPLKK+y+++4AfOpTn+If/uEfWtfzyU9+svX5rbfeutB4tVdffZU5c+a0HgmUJKmzdbvuyBK23HLLJV6lviVwLcnmm2/OhAkT2GqrrTj99NMZM2YMffr04b777uMTn/gEv/71rxk1ahTz589v7ao744wzADjssMO49tpruf766znkkEOICG6//XZuvfVW7r77bh566CG23XZb5s6du8z7NnfuXL7whS9w3XXX8cgjj3DMMccs13pa9O3bl4gAoHfv3u2O1zrttNO45JJLeOONN9hll12YPHnycm2r8X1fsGAB99xzDxMnTmTixIlMmzbNACZJKsoQtgLstddevPnmmwuNu3r44YcZN27cIm1HjhzJL3/5S15//XVee+01brjhBkaOHMmzzz7LqquuypFHHskpp5zChAkTmDNnDrNmzWL//ffnBz/4AQ899BC9e/duDRJjxowB4JBDDuFXv/oVV199NYcddhgAs2bNYu2112bVVVdl8uTJ3HPPPUvch912242f//znzJ8/n+eee46xY8cCtAau9dZbjzlz5ix0xuQaa6zROl6r0YgRI7jjjjt48cUXmT9/PldffXXr0aqOePLJJ9lqq6049dRTGT58OJMnT+ajH/0ol112Ga+//joAL730EmuttRZrr7126/t81VVXLXY7++67Lz/84Q9bpydOnNjheiRJaoZu1x1ZQkRwww03cNJJJ3H22WfTr18/Bg0axLnnnsu0adMWarvddttx9NFHM2LECKC6rMO2227LzTffzCmnnEKvXr3o27cvF154IbNnz+aggw5i7ty5ZCbnnHNOu9tfe+212WKLLZg0aVLrekeNGsVFF13EFltswYc//GF23HHHJe7DIYccwm233cbgwYPZcMMN2WmnnYDqXpTHHHMMQ4YM4QMf+ADDhw9vfc3RRx/NcccdxyqrrMLdd9/dOv+DH/wgZ511FnvuuSeZyQEHHMBBBx3U4ffz3HPPZezYsfTq1Ystt9yS/fbbj5VXXpmJEycybNgwVlppJfbff3/OPPNMrrjiCo477jhef/11NtlkEy677LJ213n++efzxS9+ka233pp58+ax2267cdFFF3W4JkmSVrSoxoy9dwwbNixbzsBr8dhjj7HFFlsUqkjdid8lSdKKFBEPZOaw9pbZHSlJklSAIUySJKmAbhPC3mvdqup6/A5JkjpTtwhh/fr1Y+bMmf6IarllJjNnzqRfv36lS5Ek9RDd4uzIgQMHMnXqVGbMmFG6FL2H9evXj4EDB5YuQ5LUQ3SLENa3b9/WK9JLkiS9FzS1OzIiRkXE4xExJSJOa2f5yhHx83r5vRExqJn1SJIkdRVNC2ER0Ru4ANgPGAwcHhGD2zT7DPByZv4d8APg7GbVI0mS1JU080jYCGBKZj6VmW8B1wBtL5t+EHBF/fw6YO9oubGgJElSN9bMMWHrA880TE8Fdlhcm8ycFxGzgHWBFxsbRcSxwLH15JyIeLwpFb9jvbY19DA9ef978r5Dz95/973n6sn735P3HTpn/zda3IL3xMD8zLwYuHipDVeQiBi/uFsM9AQ9ef978r5Dz95/971n7jv07P3vyfsO5fe/md2R04ANGqYH1vPabRMRfYC1gJlNrEmSJKlLaGYIux/YLCI2joiVgMOAG9u0uRH4VP38E8Bt6RVXJUlSD9C07sh6jNfxwM1Ab+AnmfloRIwBxmfmjcClwFURMQV4iSqodQWd1vXZRfXk/e/J+w49e//d956rJ+9/T953KLz/4YEnSZKkztct7h0pSZL0XmMIkyRJKsAQ1iAifhIR0yPiT6Vr6WwRsUFEjI2ISRHxaEScWLqmzhQR/SLivoh4qN7/b5euqbNFRO+IeDAifl26ls4WEU9HxCMRMTEixpeupzNFRP+IuC4iJkfEYxGxU+maOktEfLj+zFser0bESaXr6iwR8eX6/3d/ioirI6Jf6Zo6S0ScWO/3oyU/c8eENYiI3YA5wJWZOaR0PZ0pIj4IfDAzJ0TEGsADwMGZOalwaZ2ivlPDapk5JyL6AncCJ2bmPYVL6zQR8RVgGLBmZo4uXU9nioingWGZ2eMuWhkRVwDjMvOS+kz2VTPzldJ1dbb6VnvTgB0y86+l62m2iFif6v9zgzPzjYi4FrgpMy8vW1nzRcQQqrv4jADeAv4POC4zp3R2LR4Ja5CZf6A6S7PHycznMnNC/Xw28BjVHQ16hKzMqSf71o8e8y+UiBgIHABcUroWdZ6IWAvYjepMdTLzrZ4YwGp7A0/2hADWoA+wSn2dzlWBZwvX01m2AO7NzNczcx5wB3BoiUIMYVpERAwCtgXuLVtJ56q74yYC04FbMrMn7f+5wD8DC0oXUkgCv4uIB+rbpPUUGwMzgMvqruhLImK10kUVchhwdekiOktmTgP+Hfgb8BwwKzN/V7aqTvMnYGRErBsRqwL7s/DF5TuNIUwLiYjVgeuBkzLz1dL1dKbMnJ+ZQ6nu7jCiPmTd7UXEaGB6Zj5QupaCds3M7YD9gC/WQxN6gj7AdsCFmbkt8BpwWtmSOl/dDXsg8IvStXSWiFgbOIgqiH8IWC0ijixbVefIzMeAs4HfUXVFTgTml6jFEKZW9Vio64GfZub/lK6nlLo7ZiwwqnQtnWQX4MB6XNQ1wF4R8d9lS+pc9VEBMnM6cAPVWJGeYCowteGo73VUoayn2Q+YkJkvlC6kE+0D/CUzZ2Tm28D/ADsXrqnTZOalmbl9Zu4GvAz8uUQdhjABrQPTLwUey8xzStfT2SJiQET0r5+vAnwUmFy2qs6RmV/LzIGZOYiqS+a2zOwR/yIGiIjV6pNRqLvi9qXqruj2MvN54JmI+HA9a2+gR5yM08bh9KCuyNrfgB0jYtX6//97U40F7hEi4n313w2pxoP9rEQdTbtt0XtRRFwN7AGsFxFTgW9m5qVlq+o0uwD/BDxSj4sC+JfMvKlgTZ3pg8AV9RlSvYBrM7PHXaqhh3o/cEP1O0Qf4GeZ+X9lS+pUJwA/rbvkngI+XbieTlUH748CnytdS2fKzHsj4jpgAjAPeJCedQuj6yNiXeBt4IulTkjxEhWSJEkF2B0pSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJHWaiMiI+I+G6ZMj4lsFSyIi+kfEFxqmP1Sfur8i1n1wRAxeEeuS1P0YwiR1pjeBQyNivdKFNOgPtIawzHw2Mz+xgtZ9MLBMIay+mbKkHsAQJqkzzaO6IOSX2y6IiMsj4hMN03Pqv3tExB0R8auIeCoizoqIIyLivoh4JCI2bWddq0XET+o2D0bEQfX8Let5EyPi4YjYDDgL2LSe9/2IGBQRf6rbHx0Rv4yIWyLi6Yg4PiK+Uq/znohYp253TETcHxEPRcT19VXId6a6H+H363VvGhFD69c9HBE31PfvIyJuj4hzI2I8cGJE/ENE/Kle3x9W8GcgqYswhEnqbBcAR0TEWsvwmm2A44AtqO7ssHlmjgAuobrie1tfp7r90ghgT6ogtFq9jvPqG7UPo7p34mnAk5k5NDNPaWddQ6huazIc+Dfg9fpm13cDR9Vt/iczh2fmNlS3fvlMZt4F3AicUq/7SeBK4NTM3Bp4BPhmw3ZWysxhmfkfwBnA39frO3AZ3idJ7yGGMEmdKjNfpQojX1qGl92fmc9l5pvAk8Dv6vmPAIPaab8vcFp9C67bgX7AhlTB6V8i4lRgo8x8owPbHpuZszNzBjAL+N92tj0kIsZFxCPAEcCWbVdSh87+mXlHPesKYLeGJj9veP5H4PKIOAbo3YEaJb0HGcIklXAu8BlgtYZ586j/nxQRvYCVGpa92fB8QcP0Atq/B24AH6+PQA3NzA0z87HM/BnVkaU3gJsiYq8O1NqRbV8OHJ+ZWwHfpgp9y+q1lieZeRxwOrAB8EB9jztJ3YwhTFKny8yXgGupgliLp4Ht6+cHAn3fxSZuBk6I+q7cEbFt/XcT4KnMPB/4FbA1MBtY411si/r1z0VEX6ojYS1a152Zs4CXI2JkveyfgDtoR0Rsmpn3ZuYZwAyqMCapmzGESSrlP4DGsyT/C9g9Ih4CdqLhyNBy+A5ViHs4Ih6tpwH+EfhT3U05BLgyM2cCf6wHwn9/Obf3DeBeqm7EyQ3zrwFOqQfybwp8imp82sPAUGDMYtb3/fqkgz8BdwEPLWddkrqwyMzSNUiSJPU4HgmTJEkqwBAmSZJUgCFMkiSpAEOYJElSAYYwSZKkAgxhkiRJBRjCJEmSCvj/AY8XeVvNkb+PAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"Validation Curve with XGBoost Classifier\")\n",
    "plt.xlabel('Num estimators')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.ylim(0, 1.05)\n",
    "lw = 2\n",
    "plt.plot(param_range, train_scores_mean, label=\"Training score\",\n",
    "              color=\"darkorange\", lw=lw)\n",
    "plt.fill_between(param_range, train_scores_mean - train_scores_std,\n",
    "                 train_scores_mean + train_scores_std, alpha=0.2,\n",
    "                 color=\"darkorange\", lw=lw)\n",
    "plt.plot(param_range, test_scores_mean, label=\"Cross-validation score\",\n",
    "              color=\"navy\", lw=lw)\n",
    "plt.fill_between(param_range, test_scores_mean - test_scores_std,\n",
    "                 test_scores_mean + test_scores_std, alpha=0.2,\n",
    "                 color=\"navy\", lw=lw)\n",
    "plt.legend(loc=\"best\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exhaustive Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-23T21:16:23.507803Z",
     "start_time": "2020-02-23T21:15:54.419419Z"
    },
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5184 candidates, totalling 15552 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    1.6s\n",
      "[Parallel(n_jobs=-1)]: Done 2832 tasks      | elapsed:    6.1s\n",
      "[Parallel(n_jobs=-1)]: Done 10832 tasks      | elapsed:   20.1s\n",
      "[Parallel(n_jobs=-1)]: Done 15552 out of 15552 | elapsed:   29.0s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score=nan,\n",
       "             estimator=XGBClassifier(base_score=0.5, booster='gbtree',\n",
       "                                     colsample_bylevel=1, colsample_bynode=1,\n",
       "                                     colsample_bytree=1, gamma=0,\n",
       "                                     learning_rate=0.1, max_delta_step=0,\n",
       "                                     max_depth=3, min_child_weight=1,\n",
       "                                     missing=None, n_estimators=100, n_jobs=1,\n",
       "                                     nthread=None, objective='binary:logistic',\n",
       "                                     random_state=0, reg_alpha=0, reg_lambda=1,\n",
       "                                     scale_po...silent=None,\n",
       "                                     subsample=1, verbosity=1),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'colsample_bytree': [0.6, 0.8, 1],\n",
       "                         'gamma': [0, 0.1, 0.2], 'max_depth': [2, 3, 4, 5],\n",
       "                         'min_child_weight': [0, 0.5, 1],\n",
       "                         'n_estimators': [5, 10, 25],\n",
       "                         'reg_alpha': [0.01, 0.1, 10.0, 10],\n",
       "                         'reg_lambda': [0.01, 0.1, 10.0, 10]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1', verbose=1)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "parameters =    {\n",
    "            'max_depth': [2, 3, 4, 5],\n",
    "            'n_estimators': [5, 10, 25],\n",
    "            'gamma': [0, 0.1, 0.2],\n",
    "            'min_child_weight': [0, 0.5, 1],\n",
    "            'colsample_bytree': [0.6, 0.8, 1],\n",
    "            'reg_alpha': [1e-2, 1e-1, 1e1,10],\n",
    "            'reg_lambda': [1e-2, 1e-1, 1e1,10],\n",
    "            }\n",
    "\n",
    "clf = GridSearchCV(xgb.XGBClassifier(), parameters, scoring = 'f1', cv = 3, verbose = 1, n_jobs=-1)\n",
    "\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-23T21:17:26.594598Z",
     "start_time": "2020-02-23T21:17:26.589205Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.6,\n",
       " 'gamma': 0,\n",
       " 'max_depth': 2,\n",
       " 'min_child_weight': 0,\n",
       " 'n_estimators': 5,\n",
       " 'reg_alpha': 0.01,\n",
       " 'reg_lambda': 0.01}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-23T13:26:07.045427Z",
     "start_time": "2020-02-23T13:26:07.036049Z"
    },
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix:\n",
      "[[429   0]\n",
      " [  3  18]]\n",
      "\n",
      "Sensitivity(=Recall) TPR = TP / (TP + FN):\t\t0.8571\n",
      "Specificity SPC = TN / (FP + TN):\t\t\t1.0\n",
      "Precision PPV = TP / (TP + FP):\t\t\t\t1.0\n",
      "Negative Predictive Value NPV = TN / (TN + FN):\t\t0.9931\n",
      "False Positive Rate FPR = FP / (FP + TN)):\t\t0.0\n",
      "False Discovery rate FDR = FP / (FP + TP):\t\t0.0\n",
      "False Negative rate FNR = FN / (FN + TP):\t\t0.1429\n",
      "Accuraccy ACC = (TP + TN) / (P + N):\t\t\t0.9933\n",
      "F1-score F1 = 2TP / (2TP + FP + FN):\t\t\t0.9231\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.8571, 1.0, 1.0, 0.9931, 0.0, 0.0, 0.1429, 0.9933, 0.9231]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # Extract single tree to visualize it\n",
    "# estimator = model.estimators_[np.random.randint(0, n_estimators)]\n",
    "# plot_tree(estimator, feature_names=data.columns[1:-1], class_names=['malicious', 'nonmalicious'],filled=True)\n",
    "# plt.show()\n",
    "\n",
    "compute_score(y_test,y_pred, True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}