prepare_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import os\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_rows = 10\n",
    "pd.options.display.max_columns = 999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_cols(df, cols_dict, sample=None):\n",
    "    \n",
    "    if sample is not None:\n",
    "        n_examples = int(df.shape[0] * sample)\n",
    "        df = df.sample(n_examples, random_state=42).reset_index(drop=True)\n",
    "    \n",
    "    d = {}\n",
    "    cat_cols = []\n",
    "    for ind, col in enumerate(df.columns): \n",
    "        if col != \"target\":\n",
    "            if col in cols_dict[\"cat_cols\"]:\n",
    "                d[col] = f\"cat_{ind}\"\n",
    "                cat_cols.append(f\"cat_{ind}\")\n",
    "            else:\n",
    "                d[col] = f\"num_{ind}\"\n",
    "    df = df.rename(d, axis=1)\n",
    "    for col in cat_cols:\n",
    "        df[col] = df[col].fillna(\"new_category_nan\")\n",
    "        df[col] = df[col].astype(\"str\")\n",
    "    print(\"Tot cat columns : \", len(cat_cols))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# telecom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/blastchar/telco-customer-churn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7043, 21)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerID</th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>MultipleLines</th>\n",
       "      <th>InternetService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>OnlineBackup</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>Contract</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>PaymentMethod</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7590-VHVEG</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5575-GNVDE</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>34</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3668-QPYBK</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7795-CFOCW</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>45</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9237-HQITU</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   customerID  gender  SeniorCitizen Partner Dependents  tenure PhoneService  \\\n",
       "0  7590-VHVEG  Female              0     Yes         No       1           No   \n",
       "1  5575-GNVDE    Male              0      No         No      34          Yes   \n",
       "2  3668-QPYBK    Male              0      No         No       2          Yes   \n",
       "3  7795-CFOCW    Male              0      No         No      45           No   \n",
       "4  9237-HQITU  Female              0      No         No       2          Yes   \n",
       "\n",
       "      MultipleLines InternetService OnlineSecurity OnlineBackup  \\\n",
       "0  No phone service             DSL             No          Yes   \n",
       "1                No             DSL            Yes           No   \n",
       "2                No             DSL            Yes          Yes   \n",
       "3  No phone service             DSL            Yes           No   \n",
       "4                No     Fiber optic             No           No   \n",
       "\n",
       "  DeviceProtection TechSupport StreamingTV StreamingMovies        Contract  \\\n",
       "0               No          No          No              No  Month-to-month   \n",
       "1              Yes          No          No              No        One year   \n",
       "2               No          No          No              No  Month-to-month   \n",
       "3              Yes         Yes          No              No        One year   \n",
       "4               No          No          No              No  Month-to-month   \n",
       "\n",
       "  PaperlessBilling              PaymentMethod  MonthlyCharges TotalCharges  \\\n",
       "0              Yes           Electronic check           29.85        29.85   \n",
       "1               No               Mailed check           56.95       1889.5   \n",
       "2              Yes               Mailed check           53.85       108.15   \n",
       "3               No  Bank transfer (automatic)           42.30      1840.75   \n",
       "4              Yes           Electronic check           70.70       151.65   \n",
       "\n",
       "  Churn  \n",
       "0    No  \n",
       "1    No  \n",
       "2   Yes  \n",
       "3    No  \n",
       "4   Yes  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"telecom\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}WA_Fn-UseC_-Telco-Customer-Churn.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"customerID\"],\n",
    "    \"cat_cols\": [\n",
    "        \"gender\", \"SeniorCitizen\", \"Partner\", \"Dependents\",\n",
    "        \"PhoneService\", \"MultipleLines\", \"InternetService\", \"OnlineSecurity\", \"OnlineBackup\", \"DeviceProtection\", \n",
    "        \"TechSupport\", \"StreamingTV\", \"StreamingMovies\", \"Contract\",\n",
    "        \"PaperlessBilling\", \"PaymentMethod\"\n",
    "    ], \n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"Churn\"] == \"Yes\"\n",
    "data.drop([\"Churn\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>num_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>cat_6</th>\n",
       "      <th>cat_7</th>\n",
       "      <th>cat_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>cat_10</th>\n",
       "      <th>cat_11</th>\n",
       "      <th>cat_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>cat_14</th>\n",
       "      <th>cat_15</th>\n",
       "      <th>cat_16</th>\n",
       "      <th>num_17</th>\n",
       "      <th>num_18</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>34</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>45</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    cat_0 cat_1 cat_2 cat_3  num_4 cat_5             cat_6        cat_7 cat_8  \\\n",
       "0  Female     0   Yes    No      1    No  No phone service          DSL    No   \n",
       "1    Male     0    No    No     34   Yes                No          DSL   Yes   \n",
       "2    Male     0    No    No      2   Yes                No          DSL   Yes   \n",
       "3    Male     0    No    No     45    No  No phone service          DSL   Yes   \n",
       "4  Female     0    No    No      2   Yes                No  Fiber optic    No   \n",
       "\n",
       "  cat_9 cat_10 cat_11 cat_12 cat_13          cat_14 cat_15  \\\n",
       "0   Yes     No     No     No     No  Month-to-month    Yes   \n",
       "1    No    Yes     No     No     No        One year     No   \n",
       "2   Yes     No     No     No     No  Month-to-month    Yes   \n",
       "3    No    Yes    Yes     No     No        One year     No   \n",
       "4    No     No     No     No     No  Month-to-month    Yes   \n",
       "\n",
       "                      cat_16  num_17   num_18  target  \n",
       "0           Electronic check   29.85    29.85   False  \n",
       "1               Mailed check   56.95   1889.5   False  \n",
       "2               Mailed check   53.85   108.15    True  \n",
       "3  Bank transfer (automatic)   42.30  1840.75   False  \n",
       "4           Electronic check   70.70   151.65    True  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "cat_3 object\n",
      "num_4 int64\n",
      "cat_5 object\n",
      "cat_6 object\n",
      "cat_7 object\n",
      "cat_8 object\n",
      "cat_9 object\n",
      "cat_10 object\n",
      "cat_11 object\n",
      "cat_12 object\n",
      "cat_13 object\n",
      "cat_14 object\n",
      "cat_15 object\n",
      "cat_16 object\n",
      "num_17 float64\n",
      "num_18 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.loc[data['num_18'] == ' ', 'num_18'] = None\n",
    "data['num_18'] = data['num_18'].astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# mortgages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.crowdanalytix.com/contests/propensity-to-fund-mortgages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(45642, 22)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unique_ID</th>\n",
       "      <th>MORTGAGE NUMBER</th>\n",
       "      <th>PROPERTY VALUE</th>\n",
       "      <th>MORTGAGE PAYMENT</th>\n",
       "      <th>GDS</th>\n",
       "      <th>LTV</th>\n",
       "      <th>TDS</th>\n",
       "      <th>AMORTIZATION</th>\n",
       "      <th>MORTGAGE AMOUNT</th>\n",
       "      <th>RATE</th>\n",
       "      <th>MORTGAGE PURPOSE</th>\n",
       "      <th>PAYMENT FREQUENCY</th>\n",
       "      <th>PROPERTY TYPE</th>\n",
       "      <th>TERM</th>\n",
       "      <th>FSA</th>\n",
       "      <th>AGE RANGE</th>\n",
       "      <th>GENDER</th>\n",
       "      <th>INCOME</th>\n",
       "      <th>INCOME TYPE</th>\n",
       "      <th>NAICS CODE</th>\n",
       "      <th>CREDIT SCORE</th>\n",
       "      <th>RESULT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CAX_Train_1</td>\n",
       "      <td>1</td>\n",
       "      <td>900000</td>\n",
       "      <td>5429</td>\n",
       "      <td>61.98</td>\n",
       "      <td>65.00</td>\n",
       "      <td>71.63</td>\n",
       "      <td>360</td>\n",
       "      <td>1040000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Refinance</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>L4C</td>\n",
       "      <td>Under 25</td>\n",
       "      <td>Male</td>\n",
       "      <td>108000</td>\n",
       "      <td>8</td>\n",
       "      <td>44-45</td>\n",
       "      <td>681</td>\n",
       "      <td>FUNDED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CAX_Train_2</td>\n",
       "      <td>2</td>\n",
       "      <td>386000</td>\n",
       "      <td>2179</td>\n",
       "      <td>35.22</td>\n",
       "      <td>74.29</td>\n",
       "      <td>40.65</td>\n",
       "      <td>360</td>\n",
       "      <td>390000.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>Refinance</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>L9T</td>\n",
       "      <td>70 and over</td>\n",
       "      <td>Male</td>\n",
       "      <td>78000</td>\n",
       "      <td>2</td>\n",
       "      <td>56</td>\n",
       "      <td>710</td>\n",
       "      <td>FUNDED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CAX_Train_3</td>\n",
       "      <td>3</td>\n",
       "      <td>531000</td>\n",
       "      <td>2152</td>\n",
       "      <td>30.97</td>\n",
       "      <td>80.00</td>\n",
       "      <td>35.41</td>\n",
       "      <td>360</td>\n",
       "      <td>424800.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>Purchase</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>6</td>\n",
       "      <td>M1N</td>\n",
       "      <td>35-39</td>\n",
       "      <td>Female</td>\n",
       "      <td>87000</td>\n",
       "      <td>2</td>\n",
       "      <td>72</td>\n",
       "      <td>709</td>\n",
       "      <td>FUNDED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CAX_Train_4</td>\n",
       "      <td>4</td>\n",
       "      <td>1200000</td>\n",
       "      <td>5410</td>\n",
       "      <td>19.04</td>\n",
       "      <td>75.00</td>\n",
       "      <td>34.14</td>\n",
       "      <td>360</td>\n",
       "      <td>960000.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>Purchase</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>M2M</td>\n",
       "      <td>45-49</td>\n",
       "      <td>Male</td>\n",
       "      <td>300000</td>\n",
       "      <td>8</td>\n",
       "      <td>54</td>\n",
       "      <td>761</td>\n",
       "      <td>FUNDED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CAX_Train_5</td>\n",
       "      <td>5</td>\n",
       "      <td>350000</td>\n",
       "      <td>3342</td>\n",
       "      <td>29.59</td>\n",
       "      <td>80.00</td>\n",
       "      <td>34.85</td>\n",
       "      <td>360</td>\n",
       "      <td>592000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Refinance</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>L7G</td>\n",
       "      <td>50-54</td>\n",
       "      <td>Male</td>\n",
       "      <td>147000</td>\n",
       "      <td>8</td>\n",
       "      <td>62</td>\n",
       "      <td>762</td>\n",
       "      <td>FUNDED</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unique_ID  MORTGAGE NUMBER  PROPERTY VALUE  MORTGAGE PAYMENT    GDS  \\\n",
       "0  CAX_Train_1                1          900000              5429  61.98   \n",
       "1  CAX_Train_2                2          386000              2179  35.22   \n",
       "2  CAX_Train_3                3          531000              2152  30.97   \n",
       "3  CAX_Train_4                4         1200000              5410  19.04   \n",
       "4  CAX_Train_5                5          350000              3342  29.59   \n",
       "\n",
       "     LTV    TDS  AMORTIZATION  MORTGAGE AMOUNT  RATE MORTGAGE PURPOSE  \\\n",
       "0  65.00  71.63           360        1040000.0   4.0        Refinance   \n",
       "1  74.29  40.65           360         390000.0   4.5        Refinance   \n",
       "2  80.00  35.41           360         424800.0   3.5         Purchase   \n",
       "3  75.00  34.14           360         960000.0   5.5         Purchase   \n",
       "4  80.00  34.85           360         592000.0   5.0        Refinance   \n",
       "\n",
       "  PAYMENT FREQUENCY    PROPERTY TYPE  TERM  FSA    AGE RANGE  GENDER  INCOME  \\\n",
       "0           Monthly  Single Detached    12  L4C     Under 25    Male  108000   \n",
       "1           Monthly  Single Detached    12  L9T  70 and over    Male   78000   \n",
       "2           Monthly  Single Detached     6  M1N        35-39  Female   87000   \n",
       "3           Monthly  Single Detached    12  M2M        45-49    Male  300000   \n",
       "4           Monthly  Single Detached    12  L7G        50-54    Male  147000   \n",
       "\n",
       "   INCOME TYPE NAICS CODE  CREDIT SCORE  RESULT  \n",
       "0            8      44-45           681  FUNDED  \n",
       "1            2         56           710  FUNDED  \n",
       "2            2         72           709  FUNDED  \n",
       "3            8         54           761  FUNDED  \n",
       "4            8         62           762  FUNDED  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"mortgages\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}CAX_MortgageModeling_Train.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"Unique_ID\", \"MORTGAGE NUMBER\"],\n",
    "    \"cat_cols\": [\n",
    "        \"MORTGAGE PURPOSE\", \"PAYMENT FREQUENCY\", \"PROPERTY TYPE\", \"TERM\", \"FSA\", \"AGE RANGE\", \"GENDER\", \n",
    "        \"INCOME TYPE\", \"NAICS CODE\"\n",
    "    ], \n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"RESULT\"] == \"FUNDED\"\n",
    "data.drop([\"RESULT\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  9\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_0</th>\n",
       "      <th>num_1</th>\n",
       "      <th>num_2</th>\n",
       "      <th>num_3</th>\n",
       "      <th>num_4</th>\n",
       "      <th>num_5</th>\n",
       "      <th>num_6</th>\n",
       "      <th>num_7</th>\n",
       "      <th>cat_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>cat_10</th>\n",
       "      <th>cat_11</th>\n",
       "      <th>cat_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>cat_14</th>\n",
       "      <th>num_15</th>\n",
       "      <th>cat_16</th>\n",
       "      <th>cat_17</th>\n",
       "      <th>num_18</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>900000</td>\n",
       "      <td>5429</td>\n",
       "      <td>61.98</td>\n",
       "      <td>65.00</td>\n",
       "      <td>71.63</td>\n",
       "      <td>360</td>\n",
       "      <td>1040000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Refinance</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>L4C</td>\n",
       "      <td>Under 25</td>\n",
       "      <td>Male</td>\n",
       "      <td>108000</td>\n",
       "      <td>8</td>\n",
       "      <td>44-45</td>\n",
       "      <td>681</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>386000</td>\n",
       "      <td>2179</td>\n",
       "      <td>35.22</td>\n",
       "      <td>74.29</td>\n",
       "      <td>40.65</td>\n",
       "      <td>360</td>\n",
       "      <td>390000.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>Refinance</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>L9T</td>\n",
       "      <td>70 and over</td>\n",
       "      <td>Male</td>\n",
       "      <td>78000</td>\n",
       "      <td>2</td>\n",
       "      <td>56</td>\n",
       "      <td>710</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>531000</td>\n",
       "      <td>2152</td>\n",
       "      <td>30.97</td>\n",
       "      <td>80.00</td>\n",
       "      <td>35.41</td>\n",
       "      <td>360</td>\n",
       "      <td>424800.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>Purchase</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>6</td>\n",
       "      <td>M1N</td>\n",
       "      <td>35-39</td>\n",
       "      <td>Female</td>\n",
       "      <td>87000</td>\n",
       "      <td>2</td>\n",
       "      <td>72</td>\n",
       "      <td>709</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1200000</td>\n",
       "      <td>5410</td>\n",
       "      <td>19.04</td>\n",
       "      <td>75.00</td>\n",
       "      <td>34.14</td>\n",
       "      <td>360</td>\n",
       "      <td>960000.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>Purchase</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>M2M</td>\n",
       "      <td>45-49</td>\n",
       "      <td>Male</td>\n",
       "      <td>300000</td>\n",
       "      <td>8</td>\n",
       "      <td>54</td>\n",
       "      <td>761</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>350000</td>\n",
       "      <td>3342</td>\n",
       "      <td>29.59</td>\n",
       "      <td>80.00</td>\n",
       "      <td>34.85</td>\n",
       "      <td>360</td>\n",
       "      <td>592000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Refinance</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>Single Detached</td>\n",
       "      <td>12</td>\n",
       "      <td>L7G</td>\n",
       "      <td>50-54</td>\n",
       "      <td>Male</td>\n",
       "      <td>147000</td>\n",
       "      <td>8</td>\n",
       "      <td>62</td>\n",
       "      <td>762</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     num_0  num_1  num_2  num_3  num_4  num_5      num_6  num_7      cat_8  \\\n",
       "0   900000   5429  61.98  65.00  71.63    360  1040000.0    4.0  Refinance   \n",
       "1   386000   2179  35.22  74.29  40.65    360   390000.0    4.5  Refinance   \n",
       "2   531000   2152  30.97  80.00  35.41    360   424800.0    3.5   Purchase   \n",
       "3  1200000   5410  19.04  75.00  34.14    360   960000.0    5.5   Purchase   \n",
       "4   350000   3342  29.59  80.00  34.85    360   592000.0    5.0  Refinance   \n",
       "\n",
       "     cat_9           cat_10 cat_11 cat_12       cat_13  cat_14  num_15 cat_16  \\\n",
       "0  Monthly  Single Detached     12    L4C     Under 25    Male  108000      8   \n",
       "1  Monthly  Single Detached     12    L9T  70 and over    Male   78000      2   \n",
       "2  Monthly  Single Detached      6    M1N        35-39  Female   87000      2   \n",
       "3  Monthly  Single Detached     12    M2M        45-49    Male  300000      8   \n",
       "4  Monthly  Single Detached     12    L7G        50-54    Male  147000      8   \n",
       "\n",
       "  cat_17  num_18  target  \n",
       "0  44-45     681    True  \n",
       "1     56     710    True  \n",
       "2     72     709    True  \n",
       "3     54     761    True  \n",
       "4     62     762    True  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_0 int64\n",
      "num_1 int64\n",
      "num_2 float64\n",
      "num_3 float64\n",
      "num_4 float64\n",
      "num_5 int64\n",
      "num_6 float64\n",
      "num_7 float64\n",
      "cat_8 object\n",
      "cat_9 object\n",
      "cat_10 object\n",
      "cat_11 object\n",
      "cat_12 object\n",
      "cat_13 object\n",
      "cat_14 object\n",
      "num_15 int64\n",
      "cat_16 object\n",
      "cat_17 object\n",
      "num_18 int64\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Taxi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.crowdanalytix.com/contests/mckinsey-big-data-hackathon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(892557, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>offer_gk</th>\n",
       "      <th>weekday_key</th>\n",
       "      <th>hour_key</th>\n",
       "      <th>driver_gk</th>\n",
       "      <th>order_gk</th>\n",
       "      <th>driver_latitude</th>\n",
       "      <th>driver_longitude</th>\n",
       "      <th>origin_order_latitude</th>\n",
       "      <th>origin_order_longitude</th>\n",
       "      <th>distance_km</th>\n",
       "      <th>duration_min</th>\n",
       "      <th>offer_class_group</th>\n",
       "      <th>ride_type_desc</th>\n",
       "      <th>driver_response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1105373</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>6080</td>\n",
       "      <td>174182</td>\n",
       "      <td>55.818842</td>\n",
       "      <td>37.334562</td>\n",
       "      <td>55.814567</td>\n",
       "      <td>37.355010</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>759733</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>6080</td>\n",
       "      <td>358774</td>\n",
       "      <td>55.805342</td>\n",
       "      <td>37.515023</td>\n",
       "      <td>55.819329</td>\n",
       "      <td>37.466398</td>\n",
       "      <td>18.802</td>\n",
       "      <td>25.217</td>\n",
       "      <td>Standard</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>416977</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>6080</td>\n",
       "      <td>866260</td>\n",
       "      <td>55.813978</td>\n",
       "      <td>37.347688</td>\n",
       "      <td>55.814827</td>\n",
       "      <td>37.354074</td>\n",
       "      <td>6.747</td>\n",
       "      <td>9.800</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>889660</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>6080</td>\n",
       "      <td>163522</td>\n",
       "      <td>55.745922</td>\n",
       "      <td>37.421748</td>\n",
       "      <td>55.743469</td>\n",
       "      <td>37.431130</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1120055</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>6080</td>\n",
       "      <td>506710</td>\n",
       "      <td>55.803578</td>\n",
       "      <td>37.521602</td>\n",
       "      <td>55.812559</td>\n",
       "      <td>37.527407</td>\n",
       "      <td>12.383</td>\n",
       "      <td>19.250</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   offer_gk  weekday_key  hour_key  driver_gk  order_gk  driver_latitude  \\\n",
       "0   1105373            5        20       6080    174182        55.818842   \n",
       "1    759733            5        14       6080    358774        55.805342   \n",
       "2    416977            6        14       6080    866260        55.813978   \n",
       "3    889660            2         6       6080    163522        55.745922   \n",
       "4   1120055            4        16       6080    506710        55.803578   \n",
       "\n",
       "   driver_longitude  origin_order_latitude  origin_order_longitude  \\\n",
       "0         37.334562              55.814567               37.355010   \n",
       "1         37.515023              55.819329               37.466398   \n",
       "2         37.347688              55.814827               37.354074   \n",
       "3         37.421748              55.743469               37.431130   \n",
       "4         37.521602              55.812559               37.527407   \n",
       "\n",
       "   distance_km  duration_min offer_class_group ride_type_desc  driver_response  \n",
       "0       -1.000        -1.000           Economy        private                0  \n",
       "1       18.802        25.217          Standard        private                1  \n",
       "2        6.747         9.800           Economy        private                0  \n",
       "3       -1.000        -1.000           Economy        private                1  \n",
       "4       12.383        19.250           Economy        private                1  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"taxi\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}CAX_TrainingData_McK.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"offer_gk\",\"order_gk\"],\n",
    "    \"cat_cols\": [\n",
    "        \"weekday_key\", \"hour_key\", \"driver_gk\", \"offer_class_group\", \"ride_type_desc\"\n",
    "    ], \n",
    "    \"time_cols\": [\"driver_latitude\", \"driver_longitude\", \"origin_order_latitude\", \"origin_order_longitude\"\n",
    "]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"driver_response\"] == 1\n",
    "data.drop([\"driver_response\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>num_3</th>\n",
       "      <th>num_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>cat_6</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>6080</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>6080</td>\n",
       "      <td>18.802</td>\n",
       "      <td>25.217</td>\n",
       "      <td>Standard</td>\n",
       "      <td>private</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>6080</td>\n",
       "      <td>6.747</td>\n",
       "      <td>9.800</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>6080</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>-1.000</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>6080</td>\n",
       "      <td>12.383</td>\n",
       "      <td>19.250</td>\n",
       "      <td>Economy</td>\n",
       "      <td>private</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  cat_0 cat_1 cat_2   num_3   num_4     cat_5    cat_6  target\n",
       "0     5    20  6080  -1.000  -1.000   Economy  private   False\n",
       "1     5    14  6080  18.802  25.217  Standard  private    True\n",
       "2     6    14  6080   6.747   9.800   Economy  private   False\n",
       "3     2     6  6080  -1.000  -1.000   Economy  private    True\n",
       "4     4    16  6080  12.383  19.250   Economy  private    True"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "num_3 float64\n",
      "num_4 float64\n",
      "cat_5 object\n",
      "cat_6 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# poverty_A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.drivendata.org/competitions/50/worldbank-poverty-prediction/data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37560, 44)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>iid</th>\n",
       "      <th>HeUgMnzF</th>\n",
       "      <th>CaukPfUC</th>\n",
       "      <th>MzEtIdUF</th>\n",
       "      <th>gtnNTNam</th>\n",
       "      <th>SWoXNmPc</th>\n",
       "      <th>eXbOkwhI</th>\n",
       "      <th>OdXpbPGJ</th>\n",
       "      <th>XONDGWjH</th>\n",
       "      <th>KsFoQcUV</th>\n",
       "      <th>qYRZCuJD</th>\n",
       "      <th>FPQrjGnS</th>\n",
       "      <th>hOamrctW</th>\n",
       "      <th>XacGrSou</th>\n",
       "      <th>UsmeXdIS</th>\n",
       "      <th>igHwZsYz</th>\n",
       "      <th>cxWuAOZv</th>\n",
       "      <th>AQpdiRUz</th>\n",
       "      <th>AoLwmlEH</th>\n",
       "      <th>nLUXHpZr</th>\n",
       "      <th>CRLlSiFu</th>\n",
       "      <th>jYpOAjPW</th>\n",
       "      <th>NAxEQZVi</th>\n",
       "      <th>QvgxCmCV</th>\n",
       "      <th>kvMGuSDN</th>\n",
       "      <th>AjYsrafY</th>\n",
       "      <th>GNaMafZC</th>\n",
       "      <th>zTLPJYTw</th>\n",
       "      <th>QKzxTGus</th>\n",
       "      <th>SGeOiUlZ</th>\n",
       "      <th>MUrHEJeh</th>\n",
       "      <th>XBldkztv</th>\n",
       "      <th>tbgZsPXD</th>\n",
       "      <th>ukWqmeSS</th>\n",
       "      <th>qqVibbSA</th>\n",
       "      <th>MgCoFhXK</th>\n",
       "      <th>rFpoTXAq</th>\n",
       "      <th>RXcLsVAQ</th>\n",
       "      <th>rQWIpTiG</th>\n",
       "      <th>XizJGmbu</th>\n",
       "      <th>xqUooaNJ</th>\n",
       "      <th>poor</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80389</td>\n",
       "      <td>1</td>\n",
       "      <td>XJsPz</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>UFoKR</td>\n",
       "      <td>SSvEP</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>oArAw</td>\n",
       "      <td>kpkiH</td>\n",
       "      <td>fohru</td>\n",
       "      <td>scxJu</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>YEngm</td>\n",
       "      <td>pdgUV</td>\n",
       "      <td>YwljV</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>SowpV</td>\n",
       "      <td>XNPgB</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>dpMMl</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>hCKQi</td>\n",
       "      <td>vtkRP</td>\n",
       "      <td>kVYrO</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>VneGw</td>\n",
       "      <td>WNISg</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>XQevi</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>181</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>xUYIC</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>dSJoN</td>\n",
       "      <td>True</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80389</td>\n",
       "      <td>2</td>\n",
       "      <td>XJsPz</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>axSTs</td>\n",
       "      <td>CXizI</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>ccbZA</td>\n",
       "      <td>HgfUG</td>\n",
       "      <td>fohru</td>\n",
       "      <td>scxJu</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>EaHvf</td>\n",
       "      <td>pdgUV</td>\n",
       "      <td>YwljV</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>SowpV</td>\n",
       "      <td>XNPgB</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>SjaWF</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>hCKQi</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>OLBHI</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>VneGw</td>\n",
       "      <td>EAWFH</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>141</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>xUYIC</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>True</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>80389</td>\n",
       "      <td>3</td>\n",
       "      <td>TRFeI</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>axSTs</td>\n",
       "      <td>CXizI</td>\n",
       "      <td>NDnCs</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>fOUHD</td>\n",
       "      <td>HgfUG</td>\n",
       "      <td>oEref</td>\n",
       "      <td>bJTYb</td>\n",
       "      <td>kNZZn</td>\n",
       "      <td>zfTDU</td>\n",
       "      <td>olfwp</td>\n",
       "      <td>flBEG</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>QjTos</td>\n",
       "      <td>vSaJn</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>ndArQ</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>AyuSE</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>OLBHI</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>zncPX</td>\n",
       "      <td>EAWFH</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>41</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>gCSRj</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>rkLqZ</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>True</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80389</td>\n",
       "      <td>4</td>\n",
       "      <td>XJsPz</td>\n",
       "      <td>yAyAe</td>\n",
       "      <td>FRcdT</td>\n",
       "      <td>CXizI</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fOUHD</td>\n",
       "      <td>HgfUG</td>\n",
       "      <td>fohru</td>\n",
       "      <td>bJTYb</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>VzUws</td>\n",
       "      <td>olfwp</td>\n",
       "      <td>flBEG</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>nUKzL</td>\n",
       "      <td>vSaJn</td>\n",
       "      <td>vvXmD</td>\n",
       "      <td>ndArQ</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>AyuSE</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>OLBHI</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>rXEFU</td>\n",
       "      <td>aKoLM</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>mEGPl</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>16</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>jVHyH</td>\n",
       "      <td>GtHel</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>True</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>39883</td>\n",
       "      <td>1</td>\n",
       "      <td>XJsPz</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>UFoKR</td>\n",
       "      <td>HIvIU</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>oArAw</td>\n",
       "      <td>kpkiH</td>\n",
       "      <td>fohru</td>\n",
       "      <td>scxJu</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>HHynv</td>\n",
       "      <td>IZbuU</td>\n",
       "      <td>YwljV</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>SowpV</td>\n",
       "      <td>OeQKE</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>SjaWF</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>hCKQi</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>sqnlK</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>VneGw</td>\n",
       "      <td>EAWFH</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>381</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>xUYIC</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>UaIsy</td>\n",
       "      <td>False</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  iid HeUgMnzF CaukPfUC MzEtIdUF gtnNTNam SWoXNmPc eXbOkwhI  OdXpbPGJ  \\\n",
       "0  80389    1    XJsPz    mOlYV    UFoKR    SSvEP    onRNG    YXCNt       4.0   \n",
       "1  80389    2    XJsPz    mOlYV    axSTs    CXizI    onRNG    YXCNt       4.0   \n",
       "2  80389    3    TRFeI    mOlYV    axSTs    CXizI    NDnCs    YXCNt       4.0   \n",
       "3  80389    4    XJsPz    yAyAe    FRcdT    CXizI    onRNG    YXCNt       NaN   \n",
       "4  39883    1    XJsPz    mOlYV    UFoKR    HIvIU    onRNG    YXCNt       4.0   \n",
       "\n",
       "  XONDGWjH KsFoQcUV qYRZCuJD FPQrjGnS hOamrctW XacGrSou UsmeXdIS igHwZsYz  \\\n",
       "0    oArAw    kpkiH    fohru    scxJu    rrHdI    YEngm    pdgUV    YwljV   \n",
       "1    ccbZA    HgfUG    fohru    scxJu    rrHdI    EaHvf    pdgUV    YwljV   \n",
       "2    fOUHD    HgfUG    oEref    bJTYb    kNZZn    zfTDU    olfwp    flBEG   \n",
       "3    fOUHD    HgfUG    fohru    bJTYb    rrHdI    VzUws    olfwp    flBEG   \n",
       "4    oArAw    kpkiH    fohru    scxJu    rrHdI    HHynv    IZbuU    YwljV   \n",
       "\n",
       "  cxWuAOZv AQpdiRUz AoLwmlEH nLUXHpZr CRLlSiFu jYpOAjPW NAxEQZVi QvgxCmCV  \\\n",
       "0    QkRds    SowpV    XNPgB    CneHb    dpMMl    KOjYm    GIApU    hCKQi   \n",
       "1    QkRds    SowpV    XNPgB    CneHb    SjaWF    KOjYm    GIApU    hCKQi   \n",
       "2    QkRds    QjTos    vSaJn    CneHb    ndArQ    KOjYm    GIApU    AyuSE   \n",
       "3    QkRds    nUKzL    vSaJn    vvXmD    ndArQ    KOjYm    GIApU    AyuSE   \n",
       "4    QkRds    SowpV    OeQKE    CneHb    SjaWF    KOjYm    GIApU    hCKQi   \n",
       "\n",
       "  kvMGuSDN AjYsrafY GNaMafZC zTLPJYTw QKzxTGus SGeOiUlZ MUrHEJeh XBldkztv  \\\n",
       "0    vtkRP    kVYrO    sitaC    VneGw    WNISg    OlSuJ    qmOVd    XQevi   \n",
       "1    Qydia    OLBHI    sitaC    VneGw    EAWFH    OlSuJ    qmOVd    tbsMf   \n",
       "2    Qydia    OLBHI    sitaC    zncPX    EAWFH    OlSuJ    qmOVd    tbsMf   \n",
       "3    Qydia    OLBHI    sitaC    rXEFU    aKoLM    OlSuJ    mEGPl    tbsMf   \n",
       "4    Qydia    sqnlK    sitaC    VneGw    EAWFH    OlSuJ    qmOVd    tbsMf   \n",
       "\n",
       "  tbgZsPXD  ukWqmeSS qqVibbSA MgCoFhXK rFpoTXAq RXcLsVAQ rQWIpTiG XizJGmbu  \\\n",
       "0    yOwsR       181    QQdHS    uEstx    Hikoa    zQvdC    xUYIC    juMSt   \n",
       "1    yOwsR       141    QQdHS    uEstx    Hikoa    zQvdC    xUYIC    juMSt   \n",
       "2    yOwsR        41    QQdHS    gCSRj    Hikoa    zQvdC    rkLqZ    juMSt   \n",
       "3    yOwsR        16    QQdHS    uEstx    Hikoa    zQvdC    jVHyH    GtHel   \n",
       "4    yOwsR       381    QQdHS    uEstx    Hikoa    zQvdC    xUYIC    juMSt   \n",
       "\n",
       "  xqUooaNJ   poor country  \n",
       "0    dSJoN   True       A  \n",
       "1    JTCKs   True       A  \n",
       "2    JTCKs   True       A  \n",
       "3    JTCKs   True       A  \n",
       "4    UaIsy  False       A  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"poverty_A\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}A_indiv_train.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"id\", \"iid\", \"country\"],\n",
    "    \"cat_cols\": [col for col in data.columns if col not in [\"id\", \"iid\", \"poor\", \"ukWqmeSS\", \"OdXpbPGJ\"]],\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"poor\"] == 1\n",
    "data.drop([\"poor\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  38\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>cat_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>num_6</th>\n",
       "      <th>cat_7</th>\n",
       "      <th>cat_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>cat_10</th>\n",
       "      <th>cat_11</th>\n",
       "      <th>cat_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>cat_14</th>\n",
       "      <th>cat_15</th>\n",
       "      <th>cat_16</th>\n",
       "      <th>cat_17</th>\n",
       "      <th>cat_18</th>\n",
       "      <th>cat_19</th>\n",
       "      <th>cat_20</th>\n",
       "      <th>cat_21</th>\n",
       "      <th>cat_22</th>\n",
       "      <th>cat_23</th>\n",
       "      <th>cat_24</th>\n",
       "      <th>cat_25</th>\n",
       "      <th>cat_26</th>\n",
       "      <th>cat_27</th>\n",
       "      <th>cat_28</th>\n",
       "      <th>cat_29</th>\n",
       "      <th>cat_30</th>\n",
       "      <th>cat_31</th>\n",
       "      <th>num_32</th>\n",
       "      <th>cat_33</th>\n",
       "      <th>cat_34</th>\n",
       "      <th>cat_35</th>\n",
       "      <th>cat_36</th>\n",
       "      <th>cat_37</th>\n",
       "      <th>cat_38</th>\n",
       "      <th>cat_39</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XJsPz</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>UFoKR</td>\n",
       "      <td>SSvEP</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>oArAw</td>\n",
       "      <td>kpkiH</td>\n",
       "      <td>fohru</td>\n",
       "      <td>scxJu</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>YEngm</td>\n",
       "      <td>pdgUV</td>\n",
       "      <td>YwljV</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>SowpV</td>\n",
       "      <td>XNPgB</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>dpMMl</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>hCKQi</td>\n",
       "      <td>vtkRP</td>\n",
       "      <td>kVYrO</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>VneGw</td>\n",
       "      <td>WNISg</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>XQevi</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>181</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>xUYIC</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>dSJoN</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>XJsPz</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>axSTs</td>\n",
       "      <td>CXizI</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>ccbZA</td>\n",
       "      <td>HgfUG</td>\n",
       "      <td>fohru</td>\n",
       "      <td>scxJu</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>EaHvf</td>\n",
       "      <td>pdgUV</td>\n",
       "      <td>YwljV</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>SowpV</td>\n",
       "      <td>XNPgB</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>SjaWF</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>hCKQi</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>OLBHI</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>VneGw</td>\n",
       "      <td>EAWFH</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>141</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>xUYIC</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TRFeI</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>axSTs</td>\n",
       "      <td>CXizI</td>\n",
       "      <td>NDnCs</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>fOUHD</td>\n",
       "      <td>HgfUG</td>\n",
       "      <td>oEref</td>\n",
       "      <td>bJTYb</td>\n",
       "      <td>kNZZn</td>\n",
       "      <td>zfTDU</td>\n",
       "      <td>olfwp</td>\n",
       "      <td>flBEG</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>QjTos</td>\n",
       "      <td>vSaJn</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>ndArQ</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>AyuSE</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>OLBHI</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>zncPX</td>\n",
       "      <td>EAWFH</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>41</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>gCSRj</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>rkLqZ</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>XJsPz</td>\n",
       "      <td>yAyAe</td>\n",
       "      <td>FRcdT</td>\n",
       "      <td>CXizI</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fOUHD</td>\n",
       "      <td>HgfUG</td>\n",
       "      <td>fohru</td>\n",
       "      <td>bJTYb</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>VzUws</td>\n",
       "      <td>olfwp</td>\n",
       "      <td>flBEG</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>nUKzL</td>\n",
       "      <td>vSaJn</td>\n",
       "      <td>vvXmD</td>\n",
       "      <td>ndArQ</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>AyuSE</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>OLBHI</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>rXEFU</td>\n",
       "      <td>aKoLM</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>mEGPl</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>16</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>jVHyH</td>\n",
       "      <td>GtHel</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>XJsPz</td>\n",
       "      <td>mOlYV</td>\n",
       "      <td>UFoKR</td>\n",
       "      <td>HIvIU</td>\n",
       "      <td>onRNG</td>\n",
       "      <td>YXCNt</td>\n",
       "      <td>4.0</td>\n",
       "      <td>oArAw</td>\n",
       "      <td>kpkiH</td>\n",
       "      <td>fohru</td>\n",
       "      <td>scxJu</td>\n",
       "      <td>rrHdI</td>\n",
       "      <td>HHynv</td>\n",
       "      <td>IZbuU</td>\n",
       "      <td>YwljV</td>\n",
       "      <td>QkRds</td>\n",
       "      <td>SowpV</td>\n",
       "      <td>OeQKE</td>\n",
       "      <td>CneHb</td>\n",
       "      <td>SjaWF</td>\n",
       "      <td>KOjYm</td>\n",
       "      <td>GIApU</td>\n",
       "      <td>hCKQi</td>\n",
       "      <td>Qydia</td>\n",
       "      <td>sqnlK</td>\n",
       "      <td>sitaC</td>\n",
       "      <td>VneGw</td>\n",
       "      <td>EAWFH</td>\n",
       "      <td>OlSuJ</td>\n",
       "      <td>qmOVd</td>\n",
       "      <td>tbsMf</td>\n",
       "      <td>yOwsR</td>\n",
       "      <td>381</td>\n",
       "      <td>QQdHS</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>Hikoa</td>\n",
       "      <td>zQvdC</td>\n",
       "      <td>xUYIC</td>\n",
       "      <td>juMSt</td>\n",
       "      <td>UaIsy</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat_0  cat_1  cat_2  cat_3  cat_4  cat_5  num_6  cat_7  cat_8  cat_9  \\\n",
       "0  XJsPz  mOlYV  UFoKR  SSvEP  onRNG  YXCNt    4.0  oArAw  kpkiH  fohru   \n",
       "1  XJsPz  mOlYV  axSTs  CXizI  onRNG  YXCNt    4.0  ccbZA  HgfUG  fohru   \n",
       "2  TRFeI  mOlYV  axSTs  CXizI  NDnCs  YXCNt    4.0  fOUHD  HgfUG  oEref   \n",
       "3  XJsPz  yAyAe  FRcdT  CXizI  onRNG  YXCNt    NaN  fOUHD  HgfUG  fohru   \n",
       "4  XJsPz  mOlYV  UFoKR  HIvIU  onRNG  YXCNt    4.0  oArAw  kpkiH  fohru   \n",
       "\n",
       "  cat_10 cat_11 cat_12 cat_13 cat_14 cat_15 cat_16 cat_17 cat_18 cat_19  \\\n",
       "0  scxJu  rrHdI  YEngm  pdgUV  YwljV  QkRds  SowpV  XNPgB  CneHb  dpMMl   \n",
       "1  scxJu  rrHdI  EaHvf  pdgUV  YwljV  QkRds  SowpV  XNPgB  CneHb  SjaWF   \n",
       "2  bJTYb  kNZZn  zfTDU  olfwp  flBEG  QkRds  QjTos  vSaJn  CneHb  ndArQ   \n",
       "3  bJTYb  rrHdI  VzUws  olfwp  flBEG  QkRds  nUKzL  vSaJn  vvXmD  ndArQ   \n",
       "4  scxJu  rrHdI  HHynv  IZbuU  YwljV  QkRds  SowpV  OeQKE  CneHb  SjaWF   \n",
       "\n",
       "  cat_20 cat_21 cat_22 cat_23 cat_24 cat_25 cat_26 cat_27 cat_28 cat_29  \\\n",
       "0  KOjYm  GIApU  hCKQi  vtkRP  kVYrO  sitaC  VneGw  WNISg  OlSuJ  qmOVd   \n",
       "1  KOjYm  GIApU  hCKQi  Qydia  OLBHI  sitaC  VneGw  EAWFH  OlSuJ  qmOVd   \n",
       "2  KOjYm  GIApU  AyuSE  Qydia  OLBHI  sitaC  zncPX  EAWFH  OlSuJ  qmOVd   \n",
       "3  KOjYm  GIApU  AyuSE  Qydia  OLBHI  sitaC  rXEFU  aKoLM  OlSuJ  mEGPl   \n",
       "4  KOjYm  GIApU  hCKQi  Qydia  sqnlK  sitaC  VneGw  EAWFH  OlSuJ  qmOVd   \n",
       "\n",
       "  cat_30 cat_31  num_32 cat_33 cat_34 cat_35 cat_36 cat_37 cat_38 cat_39  \\\n",
       "0  XQevi  yOwsR     181  QQdHS  uEstx  Hikoa  zQvdC  xUYIC  juMSt  dSJoN   \n",
       "1  tbsMf  yOwsR     141  QQdHS  uEstx  Hikoa  zQvdC  xUYIC  juMSt  JTCKs   \n",
       "2  tbsMf  yOwsR      41  QQdHS  gCSRj  Hikoa  zQvdC  rkLqZ  juMSt  JTCKs   \n",
       "3  tbsMf  yOwsR      16  QQdHS  uEstx  Hikoa  zQvdC  jVHyH  GtHel  JTCKs   \n",
       "4  tbsMf  yOwsR     381  QQdHS  uEstx  Hikoa  zQvdC  xUYIC  juMSt  UaIsy   \n",
       "\n",
       "   target  \n",
       "0    True  \n",
       "1    True  \n",
       "2    True  \n",
       "3    True  \n",
       "4   False  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "cat_3 object\n",
      "cat_4 object\n",
      "cat_5 object\n",
      "num_6 float64\n",
      "cat_7 object\n",
      "cat_8 object\n",
      "cat_9 object\n",
      "cat_10 object\n",
      "cat_11 object\n",
      "cat_12 object\n",
      "cat_13 object\n",
      "cat_14 object\n",
      "cat_15 object\n",
      "cat_16 object\n",
      "cat_17 object\n",
      "cat_18 object\n",
      "cat_19 object\n",
      "cat_20 object\n",
      "cat_21 object\n",
      "cat_22 object\n",
      "cat_23 object\n",
      "cat_24 object\n",
      "cat_25 object\n",
      "cat_26 object\n",
      "cat_27 object\n",
      "cat_28 object\n",
      "cat_29 object\n",
      "cat_30 object\n",
      "cat_31 object\n",
      "num_32 int64\n",
      "cat_33 object\n",
      "cat_34 object\n",
      "cat_35 object\n",
      "cat_36 object\n",
      "cat_37 object\n",
      "cat_38 object\n",
      "cat_39 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# poverty_B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20252, 227)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>iid</th>\n",
       "      <th>MmKGUOrD</th>\n",
       "      <th>splgBZfe</th>\n",
       "      <th>JTCCBpjX</th>\n",
       "      <th>akGaJaYF</th>\n",
       "      <th>eYoMkEyN</th>\n",
       "      <th>bmhTgzYu</th>\n",
       "      <th>BoxViLPz</th>\n",
       "      <th>GjxOmHgp</th>\n",
       "      <th>qlLzyqpP</th>\n",
       "      <th>mYMBWfIH</th>\n",
       "      <th>SZucUQmM</th>\n",
       "      <th>EfqFGRSF</th>\n",
       "      <th>plDSQDzp</th>\n",
       "      <th>jyZvBZlo</th>\n",
       "      <th>gyxBUEPe</th>\n",
       "      <th>unRAgFtX</th>\n",
       "      <th>lTdxcpBA</th>\n",
       "      <th>yoijDqAm</th>\n",
       "      <th>LkhgHzsx</th>\n",
       "      <th>TJGiunYp</th>\n",
       "      <th>WmKLEUcd</th>\n",
       "      <th>poor</th>\n",
       "      <th>UUYXAsfo</th>\n",
       "      <th>IPBrAaYt</th>\n",
       "      <th>hCgDHGEc</th>\n",
       "      <th>GFXSuQvB</th>\n",
       "      <th>UPtsFglH</th>\n",
       "      <th>HZEsXrsF</th>\n",
       "      <th>kgVoFHRj</th>\n",
       "      <th>pqbNHKuK</th>\n",
       "      <th>gEqzZWXR</th>\n",
       "      <th>UkXhNUcd</th>\n",
       "      <th>hggXZmdr</th>\n",
       "      <th>uyhBLPFL</th>\n",
       "      <th>DYgxQeEi</th>\n",
       "      <th>jfsTwowc</th>\n",
       "      <th>MGfpfHam</th>\n",
       "      <th>hwJpBxAl</th>\n",
       "      <th>BXZfamTt</th>\n",
       "      <th>jkutpUMt</th>\n",
       "      <th>lFkBekuJ</th>\n",
       "      <th>aTEyAdSs</th>\n",
       "      <th>TdfmGIAn</th>\n",
       "      <th>kaVnoUbJ</th>\n",
       "      <th>bngqQTHD</th>\n",
       "      <th>cRQWeYKZ</th>\n",
       "      <th>esHWAAyG</th>\n",
       "      <th>ErqgHOEp</th>\n",
       "      <th>Ftzawcsw</th>\n",
       "      <th>SlqMdbUr</th>\n",
       "      <th>jyLFRgZs</th>\n",
       "      <th>TsredPtf</th>\n",
       "      <th>pFOEMbYu</th>\n",
       "      <th>nOoTxqCQ</th>\n",
       "      <th>xohFWnds</th>\n",
       "      <th>JYKPCImt</th>\n",
       "      <th>wEtnhPQs</th>\n",
       "      <th>LIoTrAzw</th>\n",
       "      <th>IoJgECCi</th>\n",
       "      <th>YwcPaCNf</th>\n",
       "      <th>DtcKwIEv</th>\n",
       "      <th>PgXpVWEU</th>\n",
       "      <th>ENQEzjVj</th>\n",
       "      <th>RFLvRzHv</th>\n",
       "      <th>xhqvDVIs</th>\n",
       "      <th>YhbMdUgd</th>\n",
       "      <th>cPHgLSMk</th>\n",
       "      <th>jBZoYONA</th>\n",
       "      <th>ETgxnJOM</th>\n",
       "      <th>HHbZiXXp</th>\n",
       "      <th>DsHJnzeZ</th>\n",
       "      <th>gKsBCLMY</th>\n",
       "      <th>PwxicsdL</th>\n",
       "      <th>UZLkbGnz</th>\n",
       "      <th>xNhKVNgq</th>\n",
       "      <th>nIgKFReH</th>\n",
       "      <th>FgnMveDS</th>\n",
       "      <th>NdBZKehA</th>\n",
       "      <th>mlyFwTHy</th>\n",
       "      <th>KjMhhJdX</th>\n",
       "      <th>zXonuRgn</th>\n",
       "      <th>MWVlKQoo</th>\n",
       "      <th>mQezyEOr</th>\n",
       "      <th>nPlbGBjS</th>\n",
       "      <th>nmrQVORl</th>\n",
       "      <th>KGtdytLE</th>\n",
       "      <th>DELHTNkd</th>\n",
       "      <th>rZfVzfDA</th>\n",
       "      <th>ZHOSEFIu</th>\n",
       "      <th>ogUWmSXZ</th>\n",
       "      <th>VWMwzteg</th>\n",
       "      <th>HaaXLOfY</th>\n",
       "      <th>NCyUzkfp</th>\n",
       "      <th>nAjgFRel</th>\n",
       "      <th>XZUEzGLM</th>\n",
       "      <th>OFkIQOzS</th>\n",
       "      <th>BqXRUQgi</th>\n",
       "      <th>TZDgOhYY</th>\n",
       "      <th>ywnHMkvq</th>\n",
       "      <th>eKkwQtwX</th>\n",
       "      <th>fwRLKHlN</th>\n",
       "      <th>jUwlkffQ</th>\n",
       "      <th>DAeoUuuN</th>\n",
       "      <th>gsofZtJJ</th>\n",
       "      <th>HQPtUibh</th>\n",
       "      <th>aqYWloSo</th>\n",
       "      <th>sWElQwuC</th>\n",
       "      <th>jzBRbsEG</th>\n",
       "      <th>RUTkmJrl</th>\n",
       "      <th>tQzcGkNE</th>\n",
       "      <th>VszNwfYx</th>\n",
       "      <th>hnwVYJyV</th>\n",
       "      <th>RsMBXisy</th>\n",
       "      <th>MiGIkxpV</th>\n",
       "      <th>xbOnnZEV</th>\n",
       "      <th>cGaHwCJA</th>\n",
       "      <th>dfchpzYE</th>\n",
       "      <th>fwBNLzWK</th>\n",
       "      <th>lLoRmWKX</th>\n",
       "      <th>GhzvKcjl</th>\n",
       "      <th>EpWsJmrF</th>\n",
       "      <th>wpoJgbis</th>\n",
       "      <th>KyTRKtbW</th>\n",
       "      <th>CLTXEwmz</th>\n",
       "      <th>xlJGUBXM</th>\n",
       "      <th>zuvYUDSL</th>\n",
       "      <th>ozvFfhhM</th>\n",
       "      <th>ljaNZowA</th>\n",
       "      <th>sbjoLEKh</th>\n",
       "      <th>pyvZUOyi</th>\n",
       "      <th>FldbnriM</th>\n",
       "      <th>FyXQULwx</th>\n",
       "      <th>WqEZQuJP</th>\n",
       "      <th>UUYYcars</th>\n",
       "      <th>xMysunyA</th>\n",
       "      <th>FOIfEcny</th>\n",
       "      <th>BhPvRtpj</th>\n",
       "      <th>JniMXoSr</th>\n",
       "      <th>AFSIVjPG</th>\n",
       "      <th>lnMlvpes</th>\n",
       "      <th>HrXGKbwr</th>\n",
       "      <th>NUefhhHz</th>\n",
       "      <th>UaUnOdpg</th>\n",
       "      <th>QIQcRKCg</th>\n",
       "      <th>dnmwvCng</th>\n",
       "      <th>yNHqwYaJ</th>\n",
       "      <th>rWlPNhUy</th>\n",
       "      <th>QcuPNLAv</th>\n",
       "      <th>nkxrhykC</th>\n",
       "      <th>zTEEwTzG</th>\n",
       "      <th>NMsLrKWd</th>\n",
       "      <th>iHCKmQSy</th>\n",
       "      <th>DSttkpSI</th>\n",
       "      <th>UPswDJlm</th>\n",
       "      <th>OhKTEjVy</th>\n",
       "      <th>jRXAvmTH</th>\n",
       "      <th>FHoVOXib</th>\n",
       "      <th>sIiSADFG</th>\n",
       "      <th>OATvAQNG</th>\n",
       "      <th>uDmhgsaQ</th>\n",
       "      <th>teSzyTms</th>\n",
       "      <th>hdDTwJhQ</th>\n",
       "      <th>FsUFnmgF</th>\n",
       "      <th>uDVGRBVU</th>\n",
       "      <th>WCMouTCp</th>\n",
       "      <th>VKUGzaxx</th>\n",
       "      <th>tkFVflRJ</th>\n",
       "      <th>qhffLQpJ</th>\n",
       "      <th>dFaxStnz</th>\n",
       "      <th>EzGVhUlW</th>\n",
       "      <th>pDeMFniB</th>\n",
       "      <th>AxEjAWSE</th>\n",
       "      <th>MhvfpnqH</th>\n",
       "      <th>jdYXWZgD</th>\n",
       "      <th>AJgudnHB</th>\n",
       "      <th>iZhWxnWa</th>\n",
       "      <th>LLZtRxnK</th>\n",
       "      <th>CtIAiLwz</th>\n",
       "      <th>NPNqQdIn</th>\n",
       "      <th>qzOZZsuS</th>\n",
       "      <th>fyfDnyQk</th>\n",
       "      <th>vquBDhol</th>\n",
       "      <th>MMWrfMBi</th>\n",
       "      <th>GshWgURK</th>\n",
       "      <th>LSpCPhcU</th>\n",
       "      <th>pHYVhLJD</th>\n",
       "      <th>NocVJZYG</th>\n",
       "      <th>zhbhQewN</th>\n",
       "      <th>wJthinfa</th>\n",
       "      <th>qxVAoxyX</th>\n",
       "      <th>sWADnjxA</th>\n",
       "      <th>zgJdRdgg</th>\n",
       "      <th>rxwugNti</th>\n",
       "      <th>XIHNcHXI</th>\n",
       "      <th>tugScLvF</th>\n",
       "      <th>xTZOLcSX</th>\n",
       "      <th>nxAFXxLQ</th>\n",
       "      <th>eWNUvdjK</th>\n",
       "      <th>mAeaImix</th>\n",
       "      <th>yxQmKfsS</th>\n",
       "      <th>PyTDmsQl</th>\n",
       "      <th>CnQHDRVZ</th>\n",
       "      <th>VxXEkbdS</th>\n",
       "      <th>vVyGhGoB</th>\n",
       "      <th>qRplnnEM</th>\n",
       "      <th>SCLDdPrm</th>\n",
       "      <th>HcutWGCs</th>\n",
       "      <th>GlkLFREL</th>\n",
       "      <th>NAZxDkiV</th>\n",
       "      <th>yjAfWuLb</th>\n",
       "      <th>HZqPmvkr</th>\n",
       "      <th>GlktDEEv</th>\n",
       "      <th>IIvMcTrH</th>\n",
       "      <th>JbDBFPPd</th>\n",
       "      <th>clCRDApV</th>\n",
       "      <th>aYGuxgyP</th>\n",
       "      <th>NZYkmhkD</th>\n",
       "      <th>fxWioPPP</th>\n",
       "      <th>ulQCDoYe</th>\n",
       "      <th>tzYvQeOb</th>\n",
       "      <th>DWmTWcUm</th>\n",
       "      <th>PxgyaWYq</th>\n",
       "      <th>NfpXxGQk</th>\n",
       "      <th>cavdrXpj</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62801</td>\n",
       "      <td>1</td>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>LRmij</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>ExcCa</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>tEehU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>CJciR</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>WuYiW</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>XYMAP</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>jdddH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>fWeeW</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>cBaJI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-20057.0</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>RljiF</td>\n",
       "      <td>JqFXv</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>uujhU</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>zDRYd</td>\n",
       "      <td>yAfaw</td>\n",
       "      <td>IoMyQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lOoVM</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>DHzXF</td>\n",
       "      <td>Aontx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pygde</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-109</td>\n",
       "      <td>scpMR</td>\n",
       "      <td>gLhRL</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>CiPSf</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>PaSty</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>cCsfg</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fvRSg</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>LwaMz</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>xxPXE</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>yUuwa</td>\n",
       "      <td>IOBmx</td>\n",
       "      <td>UcqME</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NWLcI</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kBfAd</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-2006.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>ETNhF</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>wzOxM</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>vxEOa</td>\n",
       "      <td>1</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>GGuOF</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>aLYmL</td>\n",
       "      <td>eztBC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>49.0</td>\n",
       "      <td>rrRFz</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LpWKt</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>jUoJv</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>tjJhO</td>\n",
       "      <td>xinaM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>nHeNd</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>haUyq</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>-4</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>ahACm</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>bywyW</td>\n",
       "      <td>KhlzK</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>Bovxn</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>26.0</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>JCGsD</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>VnOFM</td>\n",
       "      <td>-7827.0</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62801</td>\n",
       "      <td>2</td>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>lfpaq</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>EYUhv</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>tEehU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>WuYiW</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>uBqJD</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>jdddH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>RljiF</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>zDRYd</td>\n",
       "      <td>yAfaw</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lOoVM</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>Aontx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-9</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>CiPSf</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>XQuSp</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fvRSg</td>\n",
       "      <td>908.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>LwaMz</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>GwnPj</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ColQA</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kBfAd</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-1983.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>kVFfF</td>\n",
       "      <td>1</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>bcpJn</td>\n",
       "      <td>eztBC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LpWKt</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>xinaM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>nHeNd</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>-4</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>ahACm</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>bywyW</td>\n",
       "      <td>KhlzK</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>Bovxn</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>JCGsD</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>VnOFM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62801</td>\n",
       "      <td>3</td>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>cRkfb</td>\n",
       "      <td>LRmij</td>\n",
       "      <td>-68.0</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>mhxNR</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>EYUhv</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>ZIcaB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>kxGOb</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>sTSWL</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>jdddH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>KhKtT</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>fKLvO</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>LaZkH</td>\n",
       "      <td>smyLf</td>\n",
       "      <td>zSdpY</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>onbCV</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>KkNYn</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>hzjkK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-9</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>PysZH</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>SjPYj</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>EBoZt</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>gjpGX</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fvRSg</td>\n",
       "      <td>1040.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-4.0</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>wSDUU</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>hwbar</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>2.0</td>\n",
       "      <td>CcJnP</td>\n",
       "      <td>JrCwM</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kBfAd</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ropJW</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>kVFfF</td>\n",
       "      <td>1</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>noMvY</td>\n",
       "      <td>eztBC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tLaqd</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>vAVVy</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>yVVfU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>nHeNd</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>qjuXN</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>-4</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>LRMxq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>NMtVW</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>EyqjN</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>IwXNs</td>\n",
       "      <td>qmOxG</td>\n",
       "      <td>aDlJD</td>\n",
       "      <td>kIJMX</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>hdaYV</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>-82</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>VnOFM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20689</td>\n",
       "      <td>1</td>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>LRmij</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>qNDlo</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>tEehU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>WuYiW</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>XYMAP</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>wQNFL</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>RljiF</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>zDRYd</td>\n",
       "      <td>yAfaw</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lOoVM</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>Aontx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-119</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>CiPSf</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>PaSty</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>UaNRT</td>\n",
       "      <td>998.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>LwaMz</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>xxPXE</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rcvDK</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kIHlc</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-1979.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>muyFb</td>\n",
       "      <td>5</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>bcpJn</td>\n",
       "      <td>Bjenx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LpWKt</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>Ujfiw</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>xinaM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-15833.0</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>Sypvt</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>3</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>ahACm</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>bywyW</td>\n",
       "      <td>KhlzK</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>Bovxn</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>JCGsD</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>-6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>ppEcI</td>\n",
       "      <td>-7867.0</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20689</td>\n",
       "      <td>2</td>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>lfpaq</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>EYUhv</td>\n",
       "      <td>QfwOP</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>ZIcaB</td>\n",
       "      <td>-32.0</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>kxGOb</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>sTSWL</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>GVHhH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>jItWj</td>\n",
       "      <td>KhKtT</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>YNWtv</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>zgniP</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>YcIvR</td>\n",
       "      <td>oejpw</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>8.0</td>\n",
       "      <td>JHnUf</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>IOvtf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-9</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>einSo</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>SjPYj</td>\n",
       "      <td>TDgoQ</td>\n",
       "      <td>PaSty</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>mwvLo</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fkdNn</td>\n",
       "      <td>968.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>lVZOx</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>-24.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pIPMN</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>wSDUU</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>GwnPj</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rcvDK</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kIHlc</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-1974.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>kVFfF</td>\n",
       "      <td>5</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>sZics</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>bcpJn</td>\n",
       "      <td>Bjenx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>oRzdr</td>\n",
       "      <td>bDMtf</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>wcjbH</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>yVVfU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-15833.0</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>4.0</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>Sypvt</td>\n",
       "      <td>Ucdwk</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>3</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>EyqjN</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>IwXNs</td>\n",
       "      <td>qmOxG</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>kIJMX</td>\n",
       "      <td>ASpbn</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>hdaYV</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>ppEcI</td>\n",
       "      <td>-7987.0</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  iid MmKGUOrD splgBZfe JTCCBpjX akGaJaYF eYoMkEyN bmhTgzYu  BoxViLPz  \\\n",
       "0  62801    1    ZZKZW    CLRvF    QEcpz    wmLgk    VsLed    LRmij       NaN   \n",
       "1  62801    2    ZZKZW    CLRvF    QEcpz    wmLgk    VsLed    lfpaq       NaN   \n",
       "2  62801    3    ZZKZW    CLRvF    QEcpz    wmLgk    cRkfb    LRmij     -68.0   \n",
       "3  20689    1    ZZKZW    CLRvF    QEcpz    wmLgk    VsLed    LRmij       NaN   \n",
       "4  20689    2    ZZKZW    CLRvF    QEcpz    wmLgk    VsLed    lfpaq       NaN   \n",
       "\n",
       "  GjxOmHgp  qlLzyqpP mYMBWfIH SZucUQmM EfqFGRSF plDSQDzp jyZvBZlo gyxBUEPe  \\\n",
       "0    rLPoG       NaN    wBmmA    SYvDi    ExcCa    fzxDF    nTjeS    tEehU   \n",
       "1    rLPoG       NaN    wBmmA    SYvDi    EYUhv    fzxDF    nTjeS    tEehU   \n",
       "2    rLPoG       NaN    mhxNR    SYvDi    EYUhv    fzxDF    nTjeS    ZIcaB   \n",
       "3    rLPoG       NaN    wBmmA    SYvDi    qNDlo    fzxDF    nTjeS    tEehU   \n",
       "4    rLPoG      -9.0    wBmmA    SYvDi    EYUhv    QfwOP    nTjeS    ZIcaB   \n",
       "\n",
       "   unRAgFtX lTdxcpBA yoijDqAm LkhgHzsx  TJGiunYp  WmKLEUcd   poor UUYXAsfo  \\\n",
       "0       NaN    AwAZH    CJciR    gouHj       NaN       NaN  False    WuYiW   \n",
       "1       NaN    AwAZH    pVzHd    gouHj       NaN       NaN  False    WuYiW   \n",
       "2       NaN    AwAZH    pVzHd    gouHj       0.0       NaN  False    kxGOb   \n",
       "3       NaN    AwAZH    pVzHd    gouHj       NaN       NaN   True    WuYiW   \n",
       "4     -32.0    AwAZH    pVzHd    gouHj       NaN       NaN   True    kxGOb   \n",
       "\n",
       "  IPBrAaYt hCgDHGEc GFXSuQvB UPtsFglH HZEsXrsF kgVoFHRj pqbNHKuK gEqzZWXR  \\\n",
       "0    DMMRj    XYMAP    BAepu    ZKHtO    jdddH    MwltS    fWeeW    mUfCv   \n",
       "1    DMMRj    uBqJD    BAepu    ZKHtO    jdddH    MwltS    mJIJq    mUfCv   \n",
       "2    DMMRj    sTSWL    BAepu    ZKHtO    jdddH    MwltS    mJIJq    mUfCv   \n",
       "3    DMMRj    XYMAP    BAepu    ZKHtO    wQNFL    MwltS    mJIJq    mUfCv   \n",
       "4    DMMRj    sTSWL    BAepu    ZKHtO    GVHhH    MwltS    mJIJq    mUfCv   \n",
       "\n",
       "  UkXhNUcd hggXZmdr uyhBLPFL  DYgxQeEi  jfsTwowc  MGfpfHam hwJpBxAl BXZfamTt  \\\n",
       "0    QhCVe    vteNx    cBaJI       NaN       NaN  -20057.0    jpGgs    OMAba   \n",
       "1    QhCVe    vteNx    lczKW       NaN       NaN       NaN    jpGgs    OMAba   \n",
       "2    QhCVe    vteNx    lczKW       NaN       NaN       NaN    jpGgs    OMAba   \n",
       "3    QhCVe    vteNx    lczKW       NaN       NaN       NaN    jpGgs    OMAba   \n",
       "4    QhCVe    vteNx    lczKW       NaN       NaN       NaN    jpGgs    OMAba   \n",
       "\n",
       "  jkutpUMt lFkBekuJ aTEyAdSs TdfmGIAn kaVnoUbJ bngqQTHD cRQWeYKZ  esHWAAyG  \\\n",
       "0    ZwKYC    RljiF    JqFXv    cPQsP    qXssi    zzQiQ    vGLhB      -1.0   \n",
       "1    ZwKYC    RljiF    bXLdG    cPQsP    qXssi    zzQiQ    vGLhB      -1.0   \n",
       "2    ZwKYC    KhKtT    bXLdG    cPQsP    qXssi    zzQiQ    vGLhB      -1.0   \n",
       "3    ZwKYC    RljiF    bXLdG    cPQsP    qXssi    zzQiQ    vGLhB      -8.0   \n",
       "4    jItWj    KhKtT    bXLdG    cPQsP    qXssi    YNWtv    vGLhB      -8.0   \n",
       "\n",
       "  ErqgHOEp Ftzawcsw SlqMdbUr jyLFRgZs TsredPtf pFOEMbYu nOoTxqCQ xohFWnds  \\\n",
       "0    lGbPx    wnWvh    jnMFm    SCNcV    uujhU    IUOva    pyjch    sbRVc   \n",
       "1    lGbPx    wnWvh    jnMFm    SCNcV    JQNZD    IUOva    pyjch    sbRVc   \n",
       "2    lGbPx    wnWvh    jnMFm    SCNcV    JQNZD    IUOva    fKLvO    sbRVc   \n",
       "3    lGbPx    wnWvh    jnMFm    SCNcV    JQNZD    IUOva    pyjch    sbRVc   \n",
       "4    lGbPx    zgniP    jnMFm    SCNcV    JQNZD    IUOva    pyjch    sbRVc   \n",
       "\n",
       "  JYKPCImt wEtnhPQs LIoTrAzw IoJgECCi YwcPaCNf  DtcKwIEv PgXpVWEU ENQEzjVj  \\\n",
       "0    xSJVZ    VprmC    zDRYd    yAfaw    IoMyQ       NaN    lOoVM    AJXyE   \n",
       "1    xSJVZ    VprmC    zDRYd    yAfaw    SflVy       NaN    lOoVM    AJXyE   \n",
       "2    xSJVZ    LaZkH    smyLf    zSdpY    SflVy       NaN    onbCV    AJXyE   \n",
       "3    xSJVZ    VprmC    zDRYd    yAfaw    SflVy       NaN    lOoVM    AJXyE   \n",
       "4    xSJVZ    VprmC    YcIvR    oejpw    SflVy       8.0    JHnUf    AJXyE   \n",
       "\n",
       "  RFLvRzHv xhqvDVIs YhbMdUgd cPHgLSMk jBZoYONA  ETgxnJOM HHbZiXXp DsHJnzeZ  \\\n",
       "0    gcgvz    aIbya    VMwUL    DHzXF    Aontx       NaN    pygde    xdvtE   \n",
       "1    gcgvz    aIbya    VMwUL    QCFuV    Aontx       NaN    LvUAW    xdvtE   \n",
       "2    gcgvz    aIbya    KkNYn    QCFuV    hzjkK       NaN    LvUAW    xdvtE   \n",
       "3    gcgvz    aIbya    VMwUL    QCFuV    Aontx       NaN    LvUAW    xdvtE   \n",
       "4    gcgvz    aIbya    VMwUL    QCFuV    IOvtf       NaN    LvUAW    xdvtE   \n",
       "\n",
       "   gKsBCLMY PwxicsdL UZLkbGnz xNhKVNgq nIgKFReH FgnMveDS NdBZKehA mlyFwTHy  \\\n",
       "0      -109    scpMR    gLhRL    nRABU    aHxXb    YCDxr    GsGPK    zMlZf   \n",
       "1        -9    YEKGi    DDjIC    nRABU    aHxXb    YCDxr    GsGPK    zMlZf   \n",
       "2        -9    YEKGi    DDjIC    nRABU    aHxXb    YCDxr    GsGPK    zMlZf   \n",
       "3      -119    YEKGi    DDjIC    nRABU    aHxXb    YCDxr    GsGPK    zMlZf   \n",
       "4        -9    YEKGi    DDjIC    nRABU    einSo    YCDxr    GsGPK    zMlZf   \n",
       "\n",
       "  KjMhhJdX zXonuRgn MWVlKQoo mQezyEOr nPlbGBjS nmrQVORl KGtdytLE DELHTNkd  \\\n",
       "0    oJJFE    uGmbE    NVWEr    fHGmP    CiPSf    bZaYr    PaSty    ExaxN   \n",
       "1    oJJFE    uGmbE    NVWEr    fHGmP    CiPSf    bZaYr    XQuSp    ExaxN   \n",
       "2    oJJFE    PysZH    NVWEr    fHGmP    SjPYj    bZaYr    EBoZt    ExaxN   \n",
       "3    oJJFE    uGmbE    NVWEr    fHGmP    CiPSf    bZaYr    PaSty    ExaxN   \n",
       "4    oJJFE    uGmbE    NVWEr    fHGmP    SjPYj    TDgoQ    PaSty    ExaxN   \n",
       "\n",
       "  rZfVzfDA ZHOSEFIu ogUWmSXZ VWMwzteg HaaXLOfY NCyUzkfp nAjgFRel XZUEzGLM  \\\n",
       "0    sItvx    wrELJ    IUoqV    gxFBa    ENXfH    aMDvF    cCsfg    jmbmU   \n",
       "1    sItvx    wrELJ    IUoqV    gxFBa    ENXfH    aMDvF    WomgD    jmbmU   \n",
       "2    sItvx    wrELJ    IUoqV    gxFBa    ENXfH    gjpGX    WomgD    jmbmU   \n",
       "3    sItvx    wrELJ    IUoqV    gxFBa    ENXfH    aMDvF    WomgD    jmbmU   \n",
       "4    sItvx    wrELJ    mwvLo    gxFBa    ENXfH    aMDvF    WomgD    jmbmU   \n",
       "\n",
       "  OFkIQOzS BqXRUQgi  TZDgOhYY ywnHMkvq eKkwQtwX fwRLKHlN jUwlkffQ DAeoUuuN  \\\n",
       "0    EFgSK    fvRSg    1052.0    dVQbr    vijmq    ICjTy    ujKUp    kOlhN   \n",
       "1    EFgSK    fvRSg     908.0    dVQbr    vijmq    ICjTy    ujKUp    kOlhN   \n",
       "2    EFgSK    fvRSg    1040.0    dVQbr    vijmq    ICjTy    ujKUp    kOlhN   \n",
       "3    EFgSK    UaNRT     998.0    dVQbr    vijmq    ICjTy    ujKUp    kOlhN   \n",
       "4    EFgSK    fkdNn     968.0    dVQbr    vijmq    ICjTy    ujKUp    kOlhN   \n",
       "\n",
       "  gsofZtJJ HQPtUibh aqYWloSo  sWElQwuC  jzBRbsEG RUTkmJrl tQzcGkNE VszNwfYx  \\\n",
       "0    zSWWI    UUiGC    MRHGy       NaN       NaN    Urxue    NAvSQ    LwaMz   \n",
       "1    zSWWI    UUiGC    MRHGy       NaN       NaN    Urxue    NAvSQ    LwaMz   \n",
       "2    zSWWI    UUiGC    MRHGy       NaN      -4.0    Urxue    NAvSQ    wSDUU   \n",
       "3    zSWWI    UUiGC    MRHGy       NaN       NaN    Urxue    NAvSQ    LwaMz   \n",
       "4    lVZOx    UUiGC    MRHGy     -24.0       NaN    pIPMN    NAvSQ    wSDUU   \n",
       "\n",
       "  hnwVYJyV RsMBXisy MiGIkxpV xbOnnZEV cGaHwCJA dfchpzYE fwBNLzWK lLoRmWKX  \\\n",
       "0    Baacf    NgOVA    xxPXE    IfRPA    nMWJh    NSfTc    yUuwa    IOBmx   \n",
       "1    Baacf    NgOVA    GwnPj    IfRPA    nMWJh    NSfTc    FzIHG    ijEHl   \n",
       "2    Baacf    NgOVA    hwbar    IfRPA    nMWJh    NSfTc    FzIHG    ijEHl   \n",
       "3    Baacf    NgOVA    xxPXE    IfRPA    nMWJh    NSfTc    FzIHG    ijEHl   \n",
       "4    Baacf    NgOVA    GwnPj    IfRPA    nMWJh    NSfTc    FzIHG    ijEHl   \n",
       "\n",
       "  GhzvKcjl EpWsJmrF wpoJgbis KyTRKtbW  CLTXEwmz xlJGUBXM zuvYUDSL ozvFfhhM  \\\n",
       "0    UcqME    KToyu    FxHQQ    ewPOf       NaN    NWLcI    MWWYS    GIMJt   \n",
       "1    sqGjf    KToyu    FxHQQ    ewPOf       NaN    ColQA    MWWYS    GIMJt   \n",
       "2    sqGjf    KToyu    FxHQQ    ewPOf       2.0    CcJnP    JrCwM    GIMJt   \n",
       "3    sqGjf    KToyu    FxHQQ    ewPOf       NaN    rcvDK    MWWYS    GIMJt   \n",
       "4    sqGjf    KToyu    FxHQQ    ewPOf       NaN    rcvDK    MWWYS    GIMJt   \n",
       "\n",
       "  ljaNZowA sbjoLEKh pyvZUOyi FldbnriM FyXQULwx  WqEZQuJP UUYYcars xMysunyA  \\\n",
       "0    dCjbC    lLRPM    CgQye    kBfAd    OBaph   -2006.0    iKuWQ    ssoMZ   \n",
       "1    dCjbC    lLRPM    CgQye    kBfAd    OBaph   -1983.0    iKuWQ    ssoMZ   \n",
       "2    dCjbC    lLRPM    CgQye    kBfAd    OBaph       NaN    ropJW    ssoMZ   \n",
       "3    dCjbC    lLRPM    CgQye    kIHlc    OBaph   -1979.0    iKuWQ    ssoMZ   \n",
       "4    dCjbC    lLRPM    CgQye    kIHlc    OBaph   -1974.0    iKuWQ    ssoMZ   \n",
       "\n",
       "  FOIfEcny BhPvRtpj JniMXoSr AFSIVjPG lnMlvpes HrXGKbwr NUefhhHz UaUnOdpg  \\\n",
       "0    ETNhF    ugfFd    mPuLc    zxuaL    wzOxM    Wsdvj    iNYVL    TbWBG   \n",
       "1    WRMpA    ugfFd    mPuLc    zxuaL    MkimP    Wsdvj    iNYVL    TbWBG   \n",
       "2    WRMpA    ugfFd    mPuLc    zxuaL    MkimP    Wsdvj    iNYVL    TbWBG   \n",
       "3    WRMpA    ugfFd    mPuLc    zxuaL    MkimP    Wsdvj    iNYVL    TbWBG   \n",
       "4    WRMpA    ugfFd    mPuLc    zxuaL    MkimP    Wsdvj    iNYVL    TbWBG   \n",
       "\n",
       "  QIQcRKCg  dnmwvCng yNHqwYaJ rWlPNhUy QcuPNLAv nkxrhykC zTEEwTzG NMsLrKWd  \\\n",
       "0    vxEOa         1    uadNh    GGuOF    ZmwnX    xzhZC    VloRD    qASvW   \n",
       "1    kVFfF         1    uadNh    kbAmh    ZmwnX    xzhZC    VloRD    qASvW   \n",
       "2    kVFfF         1    uadNh    kbAmh    ZmwnX    xzhZC    VloRD    qASvW   \n",
       "3    muyFb         5    uadNh    kbAmh    ZmwnX    xzhZC    VloRD    qASvW   \n",
       "4    kVFfF         5    uadNh    kbAmh    ZmwnX    xzhZC    VloRD    qASvW   \n",
       "\n",
       "  iHCKmQSy  DSttkpSI UPswDJlm OhKTEjVy jRXAvmTH FHoVOXib  sIiSADFG OATvAQNG  \\\n",
       "0    GriJs       NaN    eKCJh    puFAh    aLYmL    eztBC       NaN    dcLfg   \n",
       "1    GriJs       NaN    eKCJh    puFAh    bcpJn    eztBC       NaN    dcLfg   \n",
       "2    GriJs      -3.0    eKCJh    puFAh    noMvY    eztBC       NaN    dcLfg   \n",
       "3    GriJs       NaN    eKCJh    puFAh    bcpJn    Bjenx       NaN    dcLfg   \n",
       "4    GriJs       NaN    sZics    puFAh    bcpJn    Bjenx       NaN    dcLfg   \n",
       "\n",
       "   uDmhgsaQ teSzyTms  hdDTwJhQ FsUFnmgF uDVGRBVU WCMouTCp VKUGzaxx tkFVflRJ  \\\n",
       "0      49.0    rrRFz       NaN    LpWKt    utTVH    UVpbm    jUoJv    BplVZ   \n",
       "1       NaN    yhGmw       NaN    LpWKt    utTVH    UVpbm    JqRWC    BplVZ   \n",
       "2       NaN    yhGmw       NaN    tLaqd    utTVH    UVpbm    JqRWC    vAVVy   \n",
       "3       NaN    yhGmw       NaN    LpWKt    utTVH    Ujfiw    JqRWC    BplVZ   \n",
       "4       NaN    yhGmw       NaN    oRzdr    bDMtf    UVpbm    JqRWC    BplVZ   \n",
       "\n",
       "  qhffLQpJ dFaxStnz EzGVhUlW pDeMFniB AxEjAWSE MhvfpnqH jdYXWZgD  AJgudnHB  \\\n",
       "0    YGwec    MGxdE    cbuDg    JMirk    rOmBS    tjJhO    xinaM       NaN   \n",
       "1    YGwec    MGxdE    cbuDg    JMirk    rOmBS    JAnza    xinaM       NaN   \n",
       "2    YGwec    MGxdE    cbuDg    JMirk    rOmBS    JAnza    yVVfU       NaN   \n",
       "3    YGwec    MGxdE    cbuDg    JMirk    rOmBS    JAnza    xinaM       NaN   \n",
       "4    wcjbH    MGxdE    cbuDg    JMirk    rOmBS    JAnza    yVVfU       NaN   \n",
       "\n",
       "   iZhWxnWa LLZtRxnK CtIAiLwz NPNqQdIn qzOZZsuS  fyfDnyQk vquBDhol MMWrfMBi  \\\n",
       "0       NaN    EHjSq    WGkAx    YopdI    yfAST       NaN    elRKB    nHeNd   \n",
       "1       NaN    EHjSq    WGkAx    YopdI    yfAST       NaN    elRKB    nHeNd   \n",
       "2       NaN    EHjSq    WGkAx    YopdI    yfAST       NaN    elRKB    nHeNd   \n",
       "3  -15833.0    EHjSq    WGkAx    YopdI    yfAST       NaN    elRKB    Sypvt   \n",
       "4  -15833.0    EHjSq    WGkAx    YopdI    yfAST       4.0    elRKB    Sypvt   \n",
       "\n",
       "  GshWgURK LSpCPhcU pHYVhLJD NocVJZYG zhbhQewN  wJthinfa qxVAoxyX sWADnjxA  \\\n",
       "0    ILNCl    haUyq    bTxAJ    ZujmJ    qVMHa        -4    QvEVs    GPQFq   \n",
       "1    ILNCl    ojvZG    bTxAJ    ZujmJ    qVMHa        -4    QvEVs    GPQFq   \n",
       "2    ILNCl    ojvZG    bTxAJ    qjuXN    qVMHa        -4    QvEVs    LRMxq   \n",
       "3    ILNCl    ojvZG    bTxAJ    ZujmJ    qVMHa         3    QvEVs    GPQFq   \n",
       "4    Ucdwk    ojvZG    bTxAJ    ZujmJ    qVMHa         3    QvEVs    GPQFq   \n",
       "\n",
       "  zgJdRdgg rxwugNti XIHNcHXI tugScLvF xTZOLcSX  nxAFXxLQ eWNUvdjK  mAeaImix  \\\n",
       "0    dCpjP    IqJgG    HyDNL    kJcMb    BJIIK       NaN    DTzrG      -8.0   \n",
       "1    dCpjP    IqJgG    HyDNL    kJcMb    BJIIK       NaN    DTzrG      -8.0   \n",
       "2    dCpjP    IqJgG    HyDNL    kJcMb    BJIIK       NaN    DTzrG      -8.0   \n",
       "3    dCpjP    IqJgG    HyDNL    kJcMb    BJIIK       NaN    DTzrG      -8.0   \n",
       "4    dCpjP    IqJgG    HyDNL    kJcMb    BJIIK       NaN    DTzrG      -8.0   \n",
       "\n",
       "  yxQmKfsS PyTDmsQl CnQHDRVZ VxXEkbdS vVyGhGoB qRplnnEM SCLDdPrm HcutWGCs  \\\n",
       "0    bfgeg    wIdgm    lUoRY    ahACm    sOBnN    bywyW    KhlzK    TdcoU   \n",
       "1    bfgeg    wIdgm    lUoRY    ahACm    sOBnN    bywyW    KhlzK    TdcoU   \n",
       "2    bfgeg    NMtVW    lUoRY    EyqjN    sOBnN    IwXNs    qmOxG    aDlJD   \n",
       "3    bfgeg    wIdgm    lUoRY    ahACm    sOBnN    bywyW    KhlzK    TdcoU   \n",
       "4    bfgeg    wIdgm    lUoRY    EyqjN    sOBnN    IwXNs    qmOxG    TdcoU   \n",
       "\n",
       "  GlkLFREL NAZxDkiV yjAfWuLb  HZqPmvkr GlktDEEv IIvMcTrH JbDBFPPd clCRDApV  \\\n",
       "0    Bovxn    HzgoY    hEagh      26.0    JehJJ    JCGsD    VBmxq    yWVRJ   \n",
       "1    Bovxn    HzgoY    hEagh       NaN    JehJJ    JCGsD    VBmxq    yWVRJ   \n",
       "2    kIJMX    HzgoY    hEagh       NaN    JehJJ    hdaYV    VBmxq    yWVRJ   \n",
       "3    Bovxn    HzgoY    hEagh       NaN    JehJJ    JCGsD    VBmxq    yWVRJ   \n",
       "4    kIJMX    ASpbn    hEagh       NaN    JehJJ    hdaYV    VBmxq    yWVRJ   \n",
       "\n",
       "  aYGuxgyP NZYkmhkD fxWioPPP  ulQCDoYe  tzYvQeOb DWmTWcUm PxgyaWYq  NfpXxGQk  \\\n",
       "0    rlTrR    uCOQO    UYIFp         9       NaN    MAFfK    VnOFM   -7827.0   \n",
       "1    rlTrR    uCOQO    UYIFp        29       NaN    MAFfK    VnOFM       NaN   \n",
       "2    rlTrR    uCOQO    UYIFp       -82       NaN    MAFfK    VnOFM       NaN   \n",
       "3    rlTrR    uCOQO    UYIFp        -6       NaN    MAFfK    ppEcI   -7867.0   \n",
       "4    rlTrR    uCOQO    UYIFp        19       NaN    MAFfK    ppEcI   -7987.0   \n",
       "\n",
       "  cavdrXpj country  \n",
       "0    uJXdA       B  \n",
       "1    uJXdA       B  \n",
       "2    uJXdA       B  \n",
       "3    uJXdA       B  \n",
       "4    uJXdA       B  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"poverty_B\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}B_indiv_train.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols = [\n",
    "    'BoxViLPz', 'qlLzyqpP', 'unRAgFtX', 'TJGiunYp', 'WmKLEUcd',\n",
    "    'DYgxQeEi', 'jfsTwowc', 'MGfpfHam', 'esHWAAyG', 'DtcKwIEv',\n",
    "    'ETgxnJOM', 'gKsBCLMY', 'TZDgOhYY', 'sWElQwuC', 'jzBRbsEG', \n",
    "    'CLTXEwmz', 'WqEZQuJP', 'dnmwvCng', 'DSttkpSI', 'sIiSADFG', \n",
    "    'uDmhgsaQ', 'hdDTwJhQ', 'AJgudnHB', 'iZhWxnWa', 'fyfDnyQk', \n",
    "    'wJthinfa', 'nxAFXxLQ', 'mAeaImix', 'HZqPmvkr', 'ulQCDoYe', \n",
    "    'tzYvQeOb', 'NfpXxGQk'\n",
    "]\n",
    "\n",
    "cols_dict = {\n",
    "    \"id_cols\": [\"id\", \"iid\", \"country\"],\n",
    "    \"cat_cols\": [col for col in data.columns if col not in num_cols],\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"poor\"] == 1\n",
    "data.drop([\"poor\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  191\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>cat_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>num_6</th>\n",
       "      <th>cat_7</th>\n",
       "      <th>num_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>cat_10</th>\n",
       "      <th>cat_11</th>\n",
       "      <th>cat_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>cat_14</th>\n",
       "      <th>num_15</th>\n",
       "      <th>cat_16</th>\n",
       "      <th>cat_17</th>\n",
       "      <th>cat_18</th>\n",
       "      <th>num_19</th>\n",
       "      <th>num_20</th>\n",
       "      <th>cat_21</th>\n",
       "      <th>cat_22</th>\n",
       "      <th>cat_23</th>\n",
       "      <th>cat_24</th>\n",
       "      <th>cat_25</th>\n",
       "      <th>cat_26</th>\n",
       "      <th>cat_27</th>\n",
       "      <th>cat_28</th>\n",
       "      <th>cat_29</th>\n",
       "      <th>cat_30</th>\n",
       "      <th>cat_31</th>\n",
       "      <th>cat_32</th>\n",
       "      <th>num_33</th>\n",
       "      <th>num_34</th>\n",
       "      <th>num_35</th>\n",
       "      <th>cat_36</th>\n",
       "      <th>cat_37</th>\n",
       "      <th>cat_38</th>\n",
       "      <th>cat_39</th>\n",
       "      <th>cat_40</th>\n",
       "      <th>cat_41</th>\n",
       "      <th>cat_42</th>\n",
       "      <th>cat_43</th>\n",
       "      <th>cat_44</th>\n",
       "      <th>num_45</th>\n",
       "      <th>cat_46</th>\n",
       "      <th>cat_47</th>\n",
       "      <th>cat_48</th>\n",
       "      <th>cat_49</th>\n",
       "      <th>cat_50</th>\n",
       "      <th>cat_51</th>\n",
       "      <th>cat_52</th>\n",
       "      <th>cat_53</th>\n",
       "      <th>cat_54</th>\n",
       "      <th>cat_55</th>\n",
       "      <th>cat_56</th>\n",
       "      <th>cat_57</th>\n",
       "      <th>cat_58</th>\n",
       "      <th>num_59</th>\n",
       "      <th>cat_60</th>\n",
       "      <th>cat_61</th>\n",
       "      <th>cat_62</th>\n",
       "      <th>cat_63</th>\n",
       "      <th>cat_64</th>\n",
       "      <th>cat_65</th>\n",
       "      <th>cat_66</th>\n",
       "      <th>num_67</th>\n",
       "      <th>cat_68</th>\n",
       "      <th>cat_69</th>\n",
       "      <th>num_70</th>\n",
       "      <th>cat_71</th>\n",
       "      <th>cat_72</th>\n",
       "      <th>cat_73</th>\n",
       "      <th>cat_74</th>\n",
       "      <th>cat_75</th>\n",
       "      <th>cat_76</th>\n",
       "      <th>cat_77</th>\n",
       "      <th>cat_78</th>\n",
       "      <th>cat_79</th>\n",
       "      <th>cat_80</th>\n",
       "      <th>cat_81</th>\n",
       "      <th>cat_82</th>\n",
       "      <th>cat_83</th>\n",
       "      <th>cat_84</th>\n",
       "      <th>cat_85</th>\n",
       "      <th>cat_86</th>\n",
       "      <th>cat_87</th>\n",
       "      <th>cat_88</th>\n",
       "      <th>cat_89</th>\n",
       "      <th>cat_90</th>\n",
       "      <th>cat_91</th>\n",
       "      <th>cat_92</th>\n",
       "      <th>cat_93</th>\n",
       "      <th>cat_94</th>\n",
       "      <th>cat_95</th>\n",
       "      <th>num_96</th>\n",
       "      <th>cat_97</th>\n",
       "      <th>cat_98</th>\n",
       "      <th>cat_99</th>\n",
       "      <th>cat_100</th>\n",
       "      <th>cat_101</th>\n",
       "      <th>cat_102</th>\n",
       "      <th>cat_103</th>\n",
       "      <th>cat_104</th>\n",
       "      <th>num_105</th>\n",
       "      <th>num_106</th>\n",
       "      <th>cat_107</th>\n",
       "      <th>cat_108</th>\n",
       "      <th>cat_109</th>\n",
       "      <th>cat_110</th>\n",
       "      <th>cat_111</th>\n",
       "      <th>cat_112</th>\n",
       "      <th>cat_113</th>\n",
       "      <th>cat_114</th>\n",
       "      <th>cat_115</th>\n",
       "      <th>cat_116</th>\n",
       "      <th>cat_117</th>\n",
       "      <th>cat_118</th>\n",
       "      <th>cat_119</th>\n",
       "      <th>cat_120</th>\n",
       "      <th>cat_121</th>\n",
       "      <th>num_122</th>\n",
       "      <th>cat_123</th>\n",
       "      <th>cat_124</th>\n",
       "      <th>cat_125</th>\n",
       "      <th>cat_126</th>\n",
       "      <th>cat_127</th>\n",
       "      <th>cat_128</th>\n",
       "      <th>cat_129</th>\n",
       "      <th>cat_130</th>\n",
       "      <th>num_131</th>\n",
       "      <th>cat_132</th>\n",
       "      <th>cat_133</th>\n",
       "      <th>cat_134</th>\n",
       "      <th>cat_135</th>\n",
       "      <th>cat_136</th>\n",
       "      <th>cat_137</th>\n",
       "      <th>cat_138</th>\n",
       "      <th>cat_139</th>\n",
       "      <th>cat_140</th>\n",
       "      <th>cat_141</th>\n",
       "      <th>cat_142</th>\n",
       "      <th>num_143</th>\n",
       "      <th>cat_144</th>\n",
       "      <th>cat_145</th>\n",
       "      <th>cat_146</th>\n",
       "      <th>cat_147</th>\n",
       "      <th>cat_148</th>\n",
       "      <th>cat_149</th>\n",
       "      <th>cat_150</th>\n",
       "      <th>num_151</th>\n",
       "      <th>cat_152</th>\n",
       "      <th>cat_153</th>\n",
       "      <th>cat_154</th>\n",
       "      <th>cat_155</th>\n",
       "      <th>num_156</th>\n",
       "      <th>cat_157</th>\n",
       "      <th>num_158</th>\n",
       "      <th>cat_159</th>\n",
       "      <th>num_160</th>\n",
       "      <th>cat_161</th>\n",
       "      <th>cat_162</th>\n",
       "      <th>cat_163</th>\n",
       "      <th>cat_164</th>\n",
       "      <th>cat_165</th>\n",
       "      <th>cat_166</th>\n",
       "      <th>cat_167</th>\n",
       "      <th>cat_168</th>\n",
       "      <th>cat_169</th>\n",
       "      <th>cat_170</th>\n",
       "      <th>cat_171</th>\n",
       "      <th>cat_172</th>\n",
       "      <th>num_173</th>\n",
       "      <th>num_174</th>\n",
       "      <th>cat_175</th>\n",
       "      <th>cat_176</th>\n",
       "      <th>cat_177</th>\n",
       "      <th>cat_178</th>\n",
       "      <th>num_179</th>\n",
       "      <th>cat_180</th>\n",
       "      <th>cat_181</th>\n",
       "      <th>cat_182</th>\n",
       "      <th>cat_183</th>\n",
       "      <th>cat_184</th>\n",
       "      <th>cat_185</th>\n",
       "      <th>cat_186</th>\n",
       "      <th>num_187</th>\n",
       "      <th>cat_188</th>\n",
       "      <th>cat_189</th>\n",
       "      <th>cat_190</th>\n",
       "      <th>cat_191</th>\n",
       "      <th>cat_192</th>\n",
       "      <th>cat_193</th>\n",
       "      <th>cat_194</th>\n",
       "      <th>num_195</th>\n",
       "      <th>cat_196</th>\n",
       "      <th>num_197</th>\n",
       "      <th>cat_198</th>\n",
       "      <th>cat_199</th>\n",
       "      <th>cat_200</th>\n",
       "      <th>cat_201</th>\n",
       "      <th>cat_202</th>\n",
       "      <th>cat_203</th>\n",
       "      <th>cat_204</th>\n",
       "      <th>cat_205</th>\n",
       "      <th>cat_206</th>\n",
       "      <th>cat_207</th>\n",
       "      <th>cat_208</th>\n",
       "      <th>num_209</th>\n",
       "      <th>cat_210</th>\n",
       "      <th>cat_211</th>\n",
       "      <th>cat_212</th>\n",
       "      <th>cat_213</th>\n",
       "      <th>cat_214</th>\n",
       "      <th>cat_215</th>\n",
       "      <th>cat_216</th>\n",
       "      <th>num_217</th>\n",
       "      <th>num_218</th>\n",
       "      <th>cat_219</th>\n",
       "      <th>cat_220</th>\n",
       "      <th>num_221</th>\n",
       "      <th>cat_222</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>LRmij</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>ExcCa</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>tEehU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>CJciR</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WuYiW</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>XYMAP</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>jdddH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>fWeeW</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>cBaJI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-20057.0</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>RljiF</td>\n",
       "      <td>JqFXv</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>uujhU</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>zDRYd</td>\n",
       "      <td>yAfaw</td>\n",
       "      <td>IoMyQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lOoVM</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>DHzXF</td>\n",
       "      <td>Aontx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pygde</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-109</td>\n",
       "      <td>scpMR</td>\n",
       "      <td>gLhRL</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>CiPSf</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>PaSty</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>cCsfg</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fvRSg</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>LwaMz</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>xxPXE</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>yUuwa</td>\n",
       "      <td>IOBmx</td>\n",
       "      <td>UcqME</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NWLcI</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kBfAd</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-2006.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>ETNhF</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>wzOxM</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>vxEOa</td>\n",
       "      <td>1</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>GGuOF</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>aLYmL</td>\n",
       "      <td>eztBC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>49.0</td>\n",
       "      <td>rrRFz</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LpWKt</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>jUoJv</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>tjJhO</td>\n",
       "      <td>xinaM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>nHeNd</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>haUyq</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>-4</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>ahACm</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>bywyW</td>\n",
       "      <td>KhlzK</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>Bovxn</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>26.0</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>JCGsD</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>VnOFM</td>\n",
       "      <td>-7827.0</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>lfpaq</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>EYUhv</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>tEehU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WuYiW</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>uBqJD</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>jdddH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>RljiF</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>zDRYd</td>\n",
       "      <td>yAfaw</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lOoVM</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>Aontx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-9</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>CiPSf</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>XQuSp</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fvRSg</td>\n",
       "      <td>908.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>LwaMz</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>GwnPj</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ColQA</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kBfAd</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-1983.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>kVFfF</td>\n",
       "      <td>1</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>bcpJn</td>\n",
       "      <td>eztBC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LpWKt</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>xinaM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>nHeNd</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>-4</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>ahACm</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>bywyW</td>\n",
       "      <td>KhlzK</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>Bovxn</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>JCGsD</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>VnOFM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>cRkfb</td>\n",
       "      <td>LRmij</td>\n",
       "      <td>-68.0</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>mhxNR</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>EYUhv</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>ZIcaB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kxGOb</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>sTSWL</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>jdddH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>KhKtT</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>fKLvO</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>LaZkH</td>\n",
       "      <td>smyLf</td>\n",
       "      <td>zSdpY</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>onbCV</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>KkNYn</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>hzjkK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-9</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>PysZH</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>SjPYj</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>EBoZt</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>gjpGX</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fvRSg</td>\n",
       "      <td>1040.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-4.0</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>wSDUU</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>hwbar</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>2.0</td>\n",
       "      <td>CcJnP</td>\n",
       "      <td>JrCwM</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kBfAd</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ropJW</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>kVFfF</td>\n",
       "      <td>1</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>noMvY</td>\n",
       "      <td>eztBC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tLaqd</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>vAVVy</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>yVVfU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>nHeNd</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>qjuXN</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>-4</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>LRMxq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>NMtVW</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>EyqjN</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>IwXNs</td>\n",
       "      <td>qmOxG</td>\n",
       "      <td>aDlJD</td>\n",
       "      <td>kIJMX</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>hdaYV</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>-82</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>VnOFM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>LRmij</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>qNDlo</td>\n",
       "      <td>fzxDF</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>tEehU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WuYiW</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>XYMAP</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>wQNFL</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>ZwKYC</td>\n",
       "      <td>RljiF</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>zzQiQ</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>wnWvh</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>zDRYd</td>\n",
       "      <td>yAfaw</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lOoVM</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>Aontx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-119</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>aHxXb</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>CiPSf</td>\n",
       "      <td>bZaYr</td>\n",
       "      <td>PaSty</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>IUoqV</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>UaNRT</td>\n",
       "      <td>998.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>zSWWI</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Urxue</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>LwaMz</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>xxPXE</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rcvDK</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kIHlc</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-1979.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>muyFb</td>\n",
       "      <td>5</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eKCJh</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>bcpJn</td>\n",
       "      <td>Bjenx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LpWKt</td>\n",
       "      <td>utTVH</td>\n",
       "      <td>Ujfiw</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>YGwec</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>xinaM</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-15833.0</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>Sypvt</td>\n",
       "      <td>ILNCl</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>3</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>ahACm</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>bywyW</td>\n",
       "      <td>KhlzK</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>Bovxn</td>\n",
       "      <td>HzgoY</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>JCGsD</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>-6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>ppEcI</td>\n",
       "      <td>-7867.0</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ZZKZW</td>\n",
       "      <td>CLRvF</td>\n",
       "      <td>QEcpz</td>\n",
       "      <td>wmLgk</td>\n",
       "      <td>VsLed</td>\n",
       "      <td>lfpaq</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rLPoG</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>wBmmA</td>\n",
       "      <td>SYvDi</td>\n",
       "      <td>EYUhv</td>\n",
       "      <td>QfwOP</td>\n",
       "      <td>nTjeS</td>\n",
       "      <td>ZIcaB</td>\n",
       "      <td>-32.0</td>\n",
       "      <td>AwAZH</td>\n",
       "      <td>pVzHd</td>\n",
       "      <td>gouHj</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kxGOb</td>\n",
       "      <td>DMMRj</td>\n",
       "      <td>sTSWL</td>\n",
       "      <td>BAepu</td>\n",
       "      <td>ZKHtO</td>\n",
       "      <td>GVHhH</td>\n",
       "      <td>MwltS</td>\n",
       "      <td>mJIJq</td>\n",
       "      <td>mUfCv</td>\n",
       "      <td>QhCVe</td>\n",
       "      <td>vteNx</td>\n",
       "      <td>lczKW</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jpGgs</td>\n",
       "      <td>OMAba</td>\n",
       "      <td>jItWj</td>\n",
       "      <td>KhKtT</td>\n",
       "      <td>bXLdG</td>\n",
       "      <td>cPQsP</td>\n",
       "      <td>qXssi</td>\n",
       "      <td>YNWtv</td>\n",
       "      <td>vGLhB</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>lGbPx</td>\n",
       "      <td>zgniP</td>\n",
       "      <td>jnMFm</td>\n",
       "      <td>SCNcV</td>\n",
       "      <td>JQNZD</td>\n",
       "      <td>IUOva</td>\n",
       "      <td>pyjch</td>\n",
       "      <td>sbRVc</td>\n",
       "      <td>xSJVZ</td>\n",
       "      <td>VprmC</td>\n",
       "      <td>YcIvR</td>\n",
       "      <td>oejpw</td>\n",
       "      <td>SflVy</td>\n",
       "      <td>8.0</td>\n",
       "      <td>JHnUf</td>\n",
       "      <td>AJXyE</td>\n",
       "      <td>gcgvz</td>\n",
       "      <td>aIbya</td>\n",
       "      <td>VMwUL</td>\n",
       "      <td>QCFuV</td>\n",
       "      <td>IOvtf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LvUAW</td>\n",
       "      <td>xdvtE</td>\n",
       "      <td>-9</td>\n",
       "      <td>YEKGi</td>\n",
       "      <td>DDjIC</td>\n",
       "      <td>nRABU</td>\n",
       "      <td>einSo</td>\n",
       "      <td>YCDxr</td>\n",
       "      <td>GsGPK</td>\n",
       "      <td>zMlZf</td>\n",
       "      <td>oJJFE</td>\n",
       "      <td>uGmbE</td>\n",
       "      <td>NVWEr</td>\n",
       "      <td>fHGmP</td>\n",
       "      <td>SjPYj</td>\n",
       "      <td>TDgoQ</td>\n",
       "      <td>PaSty</td>\n",
       "      <td>ExaxN</td>\n",
       "      <td>sItvx</td>\n",
       "      <td>wrELJ</td>\n",
       "      <td>mwvLo</td>\n",
       "      <td>gxFBa</td>\n",
       "      <td>ENXfH</td>\n",
       "      <td>aMDvF</td>\n",
       "      <td>WomgD</td>\n",
       "      <td>jmbmU</td>\n",
       "      <td>EFgSK</td>\n",
       "      <td>fkdNn</td>\n",
       "      <td>968.0</td>\n",
       "      <td>dVQbr</td>\n",
       "      <td>vijmq</td>\n",
       "      <td>ICjTy</td>\n",
       "      <td>ujKUp</td>\n",
       "      <td>kOlhN</td>\n",
       "      <td>lVZOx</td>\n",
       "      <td>UUiGC</td>\n",
       "      <td>MRHGy</td>\n",
       "      <td>-24.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pIPMN</td>\n",
       "      <td>NAvSQ</td>\n",
       "      <td>wSDUU</td>\n",
       "      <td>Baacf</td>\n",
       "      <td>NgOVA</td>\n",
       "      <td>GwnPj</td>\n",
       "      <td>IfRPA</td>\n",
       "      <td>nMWJh</td>\n",
       "      <td>NSfTc</td>\n",
       "      <td>FzIHG</td>\n",
       "      <td>ijEHl</td>\n",
       "      <td>sqGjf</td>\n",
       "      <td>KToyu</td>\n",
       "      <td>FxHQQ</td>\n",
       "      <td>ewPOf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>rcvDK</td>\n",
       "      <td>MWWYS</td>\n",
       "      <td>GIMJt</td>\n",
       "      <td>dCjbC</td>\n",
       "      <td>lLRPM</td>\n",
       "      <td>CgQye</td>\n",
       "      <td>kIHlc</td>\n",
       "      <td>OBaph</td>\n",
       "      <td>-1974.0</td>\n",
       "      <td>iKuWQ</td>\n",
       "      <td>ssoMZ</td>\n",
       "      <td>WRMpA</td>\n",
       "      <td>ugfFd</td>\n",
       "      <td>mPuLc</td>\n",
       "      <td>zxuaL</td>\n",
       "      <td>MkimP</td>\n",
       "      <td>Wsdvj</td>\n",
       "      <td>iNYVL</td>\n",
       "      <td>TbWBG</td>\n",
       "      <td>kVFfF</td>\n",
       "      <td>5</td>\n",
       "      <td>uadNh</td>\n",
       "      <td>kbAmh</td>\n",
       "      <td>ZmwnX</td>\n",
       "      <td>xzhZC</td>\n",
       "      <td>VloRD</td>\n",
       "      <td>qASvW</td>\n",
       "      <td>GriJs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>sZics</td>\n",
       "      <td>puFAh</td>\n",
       "      <td>bcpJn</td>\n",
       "      <td>Bjenx</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dcLfg</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yhGmw</td>\n",
       "      <td>NaN</td>\n",
       "      <td>oRzdr</td>\n",
       "      <td>bDMtf</td>\n",
       "      <td>UVpbm</td>\n",
       "      <td>JqRWC</td>\n",
       "      <td>BplVZ</td>\n",
       "      <td>wcjbH</td>\n",
       "      <td>MGxdE</td>\n",
       "      <td>cbuDg</td>\n",
       "      <td>JMirk</td>\n",
       "      <td>rOmBS</td>\n",
       "      <td>JAnza</td>\n",
       "      <td>yVVfU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-15833.0</td>\n",
       "      <td>EHjSq</td>\n",
       "      <td>WGkAx</td>\n",
       "      <td>YopdI</td>\n",
       "      <td>yfAST</td>\n",
       "      <td>4.0</td>\n",
       "      <td>elRKB</td>\n",
       "      <td>Sypvt</td>\n",
       "      <td>Ucdwk</td>\n",
       "      <td>ojvZG</td>\n",
       "      <td>bTxAJ</td>\n",
       "      <td>ZujmJ</td>\n",
       "      <td>qVMHa</td>\n",
       "      <td>3</td>\n",
       "      <td>QvEVs</td>\n",
       "      <td>GPQFq</td>\n",
       "      <td>dCpjP</td>\n",
       "      <td>IqJgG</td>\n",
       "      <td>HyDNL</td>\n",
       "      <td>kJcMb</td>\n",
       "      <td>BJIIK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DTzrG</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>bfgeg</td>\n",
       "      <td>wIdgm</td>\n",
       "      <td>lUoRY</td>\n",
       "      <td>EyqjN</td>\n",
       "      <td>sOBnN</td>\n",
       "      <td>IwXNs</td>\n",
       "      <td>qmOxG</td>\n",
       "      <td>TdcoU</td>\n",
       "      <td>kIJMX</td>\n",
       "      <td>ASpbn</td>\n",
       "      <td>hEagh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JehJJ</td>\n",
       "      <td>hdaYV</td>\n",
       "      <td>VBmxq</td>\n",
       "      <td>yWVRJ</td>\n",
       "      <td>rlTrR</td>\n",
       "      <td>uCOQO</td>\n",
       "      <td>UYIFp</td>\n",
       "      <td>19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MAFfK</td>\n",
       "      <td>ppEcI</td>\n",
       "      <td>-7987.0</td>\n",
       "      <td>uJXdA</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat_0  cat_1  cat_2  cat_3  cat_4  cat_5  num_6  cat_7  num_8  cat_9  \\\n",
       "0  ZZKZW  CLRvF  QEcpz  wmLgk  VsLed  LRmij    NaN  rLPoG    NaN  wBmmA   \n",
       "1  ZZKZW  CLRvF  QEcpz  wmLgk  VsLed  lfpaq    NaN  rLPoG    NaN  wBmmA   \n",
       "2  ZZKZW  CLRvF  QEcpz  wmLgk  cRkfb  LRmij  -68.0  rLPoG    NaN  mhxNR   \n",
       "3  ZZKZW  CLRvF  QEcpz  wmLgk  VsLed  LRmij    NaN  rLPoG    NaN  wBmmA   \n",
       "4  ZZKZW  CLRvF  QEcpz  wmLgk  VsLed  lfpaq    NaN  rLPoG   -9.0  wBmmA   \n",
       "\n",
       "  cat_10 cat_11 cat_12 cat_13 cat_14  num_15 cat_16 cat_17 cat_18  num_19  \\\n",
       "0  SYvDi  ExcCa  fzxDF  nTjeS  tEehU     NaN  AwAZH  CJciR  gouHj     NaN   \n",
       "1  SYvDi  EYUhv  fzxDF  nTjeS  tEehU     NaN  AwAZH  pVzHd  gouHj     NaN   \n",
       "2  SYvDi  EYUhv  fzxDF  nTjeS  ZIcaB     NaN  AwAZH  pVzHd  gouHj     0.0   \n",
       "3  SYvDi  qNDlo  fzxDF  nTjeS  tEehU     NaN  AwAZH  pVzHd  gouHj     NaN   \n",
       "4  SYvDi  EYUhv  QfwOP  nTjeS  ZIcaB   -32.0  AwAZH  pVzHd  gouHj     NaN   \n",
       "\n",
       "   num_20 cat_21 cat_22 cat_23 cat_24 cat_25 cat_26 cat_27 cat_28 cat_29  \\\n",
       "0     NaN  WuYiW  DMMRj  XYMAP  BAepu  ZKHtO  jdddH  MwltS  fWeeW  mUfCv   \n",
       "1     NaN  WuYiW  DMMRj  uBqJD  BAepu  ZKHtO  jdddH  MwltS  mJIJq  mUfCv   \n",
       "2     NaN  kxGOb  DMMRj  sTSWL  BAepu  ZKHtO  jdddH  MwltS  mJIJq  mUfCv   \n",
       "3     NaN  WuYiW  DMMRj  XYMAP  BAepu  ZKHtO  wQNFL  MwltS  mJIJq  mUfCv   \n",
       "4     NaN  kxGOb  DMMRj  sTSWL  BAepu  ZKHtO  GVHhH  MwltS  mJIJq  mUfCv   \n",
       "\n",
       "  cat_30 cat_31 cat_32  num_33  num_34   num_35 cat_36 cat_37 cat_38 cat_39  \\\n",
       "0  QhCVe  vteNx  cBaJI     NaN     NaN -20057.0  jpGgs  OMAba  ZwKYC  RljiF   \n",
       "1  QhCVe  vteNx  lczKW     NaN     NaN      NaN  jpGgs  OMAba  ZwKYC  RljiF   \n",
       "2  QhCVe  vteNx  lczKW     NaN     NaN      NaN  jpGgs  OMAba  ZwKYC  KhKtT   \n",
       "3  QhCVe  vteNx  lczKW     NaN     NaN      NaN  jpGgs  OMAba  ZwKYC  RljiF   \n",
       "4  QhCVe  vteNx  lczKW     NaN     NaN      NaN  jpGgs  OMAba  jItWj  KhKtT   \n",
       "\n",
       "  cat_40 cat_41 cat_42 cat_43 cat_44  num_45 cat_46 cat_47 cat_48 cat_49  \\\n",
       "0  JqFXv  cPQsP  qXssi  zzQiQ  vGLhB    -1.0  lGbPx  wnWvh  jnMFm  SCNcV   \n",
       "1  bXLdG  cPQsP  qXssi  zzQiQ  vGLhB    -1.0  lGbPx  wnWvh  jnMFm  SCNcV   \n",
       "2  bXLdG  cPQsP  qXssi  zzQiQ  vGLhB    -1.0  lGbPx  wnWvh  jnMFm  SCNcV   \n",
       "3  bXLdG  cPQsP  qXssi  zzQiQ  vGLhB    -8.0  lGbPx  wnWvh  jnMFm  SCNcV   \n",
       "4  bXLdG  cPQsP  qXssi  YNWtv  vGLhB    -8.0  lGbPx  zgniP  jnMFm  SCNcV   \n",
       "\n",
       "  cat_50 cat_51 cat_52 cat_53 cat_54 cat_55 cat_56 cat_57 cat_58  num_59  \\\n",
       "0  uujhU  IUOva  pyjch  sbRVc  xSJVZ  VprmC  zDRYd  yAfaw  IoMyQ     NaN   \n",
       "1  JQNZD  IUOva  pyjch  sbRVc  xSJVZ  VprmC  zDRYd  yAfaw  SflVy     NaN   \n",
       "2  JQNZD  IUOva  fKLvO  sbRVc  xSJVZ  LaZkH  smyLf  zSdpY  SflVy     NaN   \n",
       "3  JQNZD  IUOva  pyjch  sbRVc  xSJVZ  VprmC  zDRYd  yAfaw  SflVy     NaN   \n",
       "4  JQNZD  IUOva  pyjch  sbRVc  xSJVZ  VprmC  YcIvR  oejpw  SflVy     8.0   \n",
       "\n",
       "  cat_60 cat_61 cat_62 cat_63 cat_64 cat_65 cat_66  num_67 cat_68 cat_69  \\\n",
       "0  lOoVM  AJXyE  gcgvz  aIbya  VMwUL  DHzXF  Aontx     NaN  pygde  xdvtE   \n",
       "1  lOoVM  AJXyE  gcgvz  aIbya  VMwUL  QCFuV  Aontx     NaN  LvUAW  xdvtE   \n",
       "2  onbCV  AJXyE  gcgvz  aIbya  KkNYn  QCFuV  hzjkK     NaN  LvUAW  xdvtE   \n",
       "3  lOoVM  AJXyE  gcgvz  aIbya  VMwUL  QCFuV  Aontx     NaN  LvUAW  xdvtE   \n",
       "4  JHnUf  AJXyE  gcgvz  aIbya  VMwUL  QCFuV  IOvtf     NaN  LvUAW  xdvtE   \n",
       "\n",
       "   num_70 cat_71 cat_72 cat_73 cat_74 cat_75 cat_76 cat_77 cat_78 cat_79  \\\n",
       "0    -109  scpMR  gLhRL  nRABU  aHxXb  YCDxr  GsGPK  zMlZf  oJJFE  uGmbE   \n",
       "1      -9  YEKGi  DDjIC  nRABU  aHxXb  YCDxr  GsGPK  zMlZf  oJJFE  uGmbE   \n",
       "2      -9  YEKGi  DDjIC  nRABU  aHxXb  YCDxr  GsGPK  zMlZf  oJJFE  PysZH   \n",
       "3    -119  YEKGi  DDjIC  nRABU  aHxXb  YCDxr  GsGPK  zMlZf  oJJFE  uGmbE   \n",
       "4      -9  YEKGi  DDjIC  nRABU  einSo  YCDxr  GsGPK  zMlZf  oJJFE  uGmbE   \n",
       "\n",
       "  cat_80 cat_81 cat_82 cat_83 cat_84 cat_85 cat_86 cat_87 cat_88 cat_89  \\\n",
       "0  NVWEr  fHGmP  CiPSf  bZaYr  PaSty  ExaxN  sItvx  wrELJ  IUoqV  gxFBa   \n",
       "1  NVWEr  fHGmP  CiPSf  bZaYr  XQuSp  ExaxN  sItvx  wrELJ  IUoqV  gxFBa   \n",
       "2  NVWEr  fHGmP  SjPYj  bZaYr  EBoZt  ExaxN  sItvx  wrELJ  IUoqV  gxFBa   \n",
       "3  NVWEr  fHGmP  CiPSf  bZaYr  PaSty  ExaxN  sItvx  wrELJ  IUoqV  gxFBa   \n",
       "4  NVWEr  fHGmP  SjPYj  TDgoQ  PaSty  ExaxN  sItvx  wrELJ  mwvLo  gxFBa   \n",
       "\n",
       "  cat_90 cat_91 cat_92 cat_93 cat_94 cat_95  num_96 cat_97 cat_98 cat_99  \\\n",
       "0  ENXfH  aMDvF  cCsfg  jmbmU  EFgSK  fvRSg  1052.0  dVQbr  vijmq  ICjTy   \n",
       "1  ENXfH  aMDvF  WomgD  jmbmU  EFgSK  fvRSg   908.0  dVQbr  vijmq  ICjTy   \n",
       "2  ENXfH  gjpGX  WomgD  jmbmU  EFgSK  fvRSg  1040.0  dVQbr  vijmq  ICjTy   \n",
       "3  ENXfH  aMDvF  WomgD  jmbmU  EFgSK  UaNRT   998.0  dVQbr  vijmq  ICjTy   \n",
       "4  ENXfH  aMDvF  WomgD  jmbmU  EFgSK  fkdNn   968.0  dVQbr  vijmq  ICjTy   \n",
       "\n",
       "  cat_100 cat_101 cat_102 cat_103 cat_104  num_105  num_106 cat_107 cat_108  \\\n",
       "0   ujKUp   kOlhN   zSWWI   UUiGC   MRHGy      NaN      NaN   Urxue   NAvSQ   \n",
       "1   ujKUp   kOlhN   zSWWI   UUiGC   MRHGy      NaN      NaN   Urxue   NAvSQ   \n",
       "2   ujKUp   kOlhN   zSWWI   UUiGC   MRHGy      NaN     -4.0   Urxue   NAvSQ   \n",
       "3   ujKUp   kOlhN   zSWWI   UUiGC   MRHGy      NaN      NaN   Urxue   NAvSQ   \n",
       "4   ujKUp   kOlhN   lVZOx   UUiGC   MRHGy    -24.0      NaN   pIPMN   NAvSQ   \n",
       "\n",
       "  cat_109 cat_110 cat_111 cat_112 cat_113 cat_114 cat_115 cat_116 cat_117  \\\n",
       "0   LwaMz   Baacf   NgOVA   xxPXE   IfRPA   nMWJh   NSfTc   yUuwa   IOBmx   \n",
       "1   LwaMz   Baacf   NgOVA   GwnPj   IfRPA   nMWJh   NSfTc   FzIHG   ijEHl   \n",
       "2   wSDUU   Baacf   NgOVA   hwbar   IfRPA   nMWJh   NSfTc   FzIHG   ijEHl   \n",
       "3   LwaMz   Baacf   NgOVA   xxPXE   IfRPA   nMWJh   NSfTc   FzIHG   ijEHl   \n",
       "4   wSDUU   Baacf   NgOVA   GwnPj   IfRPA   nMWJh   NSfTc   FzIHG   ijEHl   \n",
       "\n",
       "  cat_118 cat_119 cat_120 cat_121  num_122 cat_123 cat_124 cat_125 cat_126  \\\n",
       "0   UcqME   KToyu   FxHQQ   ewPOf      NaN   NWLcI   MWWYS   GIMJt   dCjbC   \n",
       "1   sqGjf   KToyu   FxHQQ   ewPOf      NaN   ColQA   MWWYS   GIMJt   dCjbC   \n",
       "2   sqGjf   KToyu   FxHQQ   ewPOf      2.0   CcJnP   JrCwM   GIMJt   dCjbC   \n",
       "3   sqGjf   KToyu   FxHQQ   ewPOf      NaN   rcvDK   MWWYS   GIMJt   dCjbC   \n",
       "4   sqGjf   KToyu   FxHQQ   ewPOf      NaN   rcvDK   MWWYS   GIMJt   dCjbC   \n",
       "\n",
       "  cat_127 cat_128 cat_129 cat_130  num_131 cat_132 cat_133 cat_134 cat_135  \\\n",
       "0   lLRPM   CgQye   kBfAd   OBaph  -2006.0   iKuWQ   ssoMZ   ETNhF   ugfFd   \n",
       "1   lLRPM   CgQye   kBfAd   OBaph  -1983.0   iKuWQ   ssoMZ   WRMpA   ugfFd   \n",
       "2   lLRPM   CgQye   kBfAd   OBaph      NaN   ropJW   ssoMZ   WRMpA   ugfFd   \n",
       "3   lLRPM   CgQye   kIHlc   OBaph  -1979.0   iKuWQ   ssoMZ   WRMpA   ugfFd   \n",
       "4   lLRPM   CgQye   kIHlc   OBaph  -1974.0   iKuWQ   ssoMZ   WRMpA   ugfFd   \n",
       "\n",
       "  cat_136 cat_137 cat_138 cat_139 cat_140 cat_141 cat_142  num_143 cat_144  \\\n",
       "0   mPuLc   zxuaL   wzOxM   Wsdvj   iNYVL   TbWBG   vxEOa        1   uadNh   \n",
       "1   mPuLc   zxuaL   MkimP   Wsdvj   iNYVL   TbWBG   kVFfF        1   uadNh   \n",
       "2   mPuLc   zxuaL   MkimP   Wsdvj   iNYVL   TbWBG   kVFfF        1   uadNh   \n",
       "3   mPuLc   zxuaL   MkimP   Wsdvj   iNYVL   TbWBG   muyFb        5   uadNh   \n",
       "4   mPuLc   zxuaL   MkimP   Wsdvj   iNYVL   TbWBG   kVFfF        5   uadNh   \n",
       "\n",
       "  cat_145 cat_146 cat_147 cat_148 cat_149 cat_150  num_151 cat_152 cat_153  \\\n",
       "0   GGuOF   ZmwnX   xzhZC   VloRD   qASvW   GriJs      NaN   eKCJh   puFAh   \n",
       "1   kbAmh   ZmwnX   xzhZC   VloRD   qASvW   GriJs      NaN   eKCJh   puFAh   \n",
       "2   kbAmh   ZmwnX   xzhZC   VloRD   qASvW   GriJs     -3.0   eKCJh   puFAh   \n",
       "3   kbAmh   ZmwnX   xzhZC   VloRD   qASvW   GriJs      NaN   eKCJh   puFAh   \n",
       "4   kbAmh   ZmwnX   xzhZC   VloRD   qASvW   GriJs      NaN   sZics   puFAh   \n",
       "\n",
       "  cat_154 cat_155  num_156 cat_157  num_158 cat_159  num_160 cat_161 cat_162  \\\n",
       "0   aLYmL   eztBC      NaN   dcLfg     49.0   rrRFz      NaN   LpWKt   utTVH   \n",
       "1   bcpJn   eztBC      NaN   dcLfg      NaN   yhGmw      NaN   LpWKt   utTVH   \n",
       "2   noMvY   eztBC      NaN   dcLfg      NaN   yhGmw      NaN   tLaqd   utTVH   \n",
       "3   bcpJn   Bjenx      NaN   dcLfg      NaN   yhGmw      NaN   LpWKt   utTVH   \n",
       "4   bcpJn   Bjenx      NaN   dcLfg      NaN   yhGmw      NaN   oRzdr   bDMtf   \n",
       "\n",
       "  cat_163 cat_164 cat_165 cat_166 cat_167 cat_168 cat_169 cat_170 cat_171  \\\n",
       "0   UVpbm   jUoJv   BplVZ   YGwec   MGxdE   cbuDg   JMirk   rOmBS   tjJhO   \n",
       "1   UVpbm   JqRWC   BplVZ   YGwec   MGxdE   cbuDg   JMirk   rOmBS   JAnza   \n",
       "2   UVpbm   JqRWC   vAVVy   YGwec   MGxdE   cbuDg   JMirk   rOmBS   JAnza   \n",
       "3   Ujfiw   JqRWC   BplVZ   YGwec   MGxdE   cbuDg   JMirk   rOmBS   JAnza   \n",
       "4   UVpbm   JqRWC   BplVZ   wcjbH   MGxdE   cbuDg   JMirk   rOmBS   JAnza   \n",
       "\n",
       "  cat_172  num_173  num_174 cat_175 cat_176 cat_177 cat_178  num_179 cat_180  \\\n",
       "0   xinaM      NaN      NaN   EHjSq   WGkAx   YopdI   yfAST      NaN   elRKB   \n",
       "1   xinaM      NaN      NaN   EHjSq   WGkAx   YopdI   yfAST      NaN   elRKB   \n",
       "2   yVVfU      NaN      NaN   EHjSq   WGkAx   YopdI   yfAST      NaN   elRKB   \n",
       "3   xinaM      NaN -15833.0   EHjSq   WGkAx   YopdI   yfAST      NaN   elRKB   \n",
       "4   yVVfU      NaN -15833.0   EHjSq   WGkAx   YopdI   yfAST      4.0   elRKB   \n",
       "\n",
       "  cat_181 cat_182 cat_183 cat_184 cat_185 cat_186  num_187 cat_188 cat_189  \\\n",
       "0   nHeNd   ILNCl   haUyq   bTxAJ   ZujmJ   qVMHa       -4   QvEVs   GPQFq   \n",
       "1   nHeNd   ILNCl   ojvZG   bTxAJ   ZujmJ   qVMHa       -4   QvEVs   GPQFq   \n",
       "2   nHeNd   ILNCl   ojvZG   bTxAJ   qjuXN   qVMHa       -4   QvEVs   LRMxq   \n",
       "3   Sypvt   ILNCl   ojvZG   bTxAJ   ZujmJ   qVMHa        3   QvEVs   GPQFq   \n",
       "4   Sypvt   Ucdwk   ojvZG   bTxAJ   ZujmJ   qVMHa        3   QvEVs   GPQFq   \n",
       "\n",
       "  cat_190 cat_191 cat_192 cat_193 cat_194  num_195 cat_196  num_197 cat_198  \\\n",
       "0   dCpjP   IqJgG   HyDNL   kJcMb   BJIIK      NaN   DTzrG     -8.0   bfgeg   \n",
       "1   dCpjP   IqJgG   HyDNL   kJcMb   BJIIK      NaN   DTzrG     -8.0   bfgeg   \n",
       "2   dCpjP   IqJgG   HyDNL   kJcMb   BJIIK      NaN   DTzrG     -8.0   bfgeg   \n",
       "3   dCpjP   IqJgG   HyDNL   kJcMb   BJIIK      NaN   DTzrG     -8.0   bfgeg   \n",
       "4   dCpjP   IqJgG   HyDNL   kJcMb   BJIIK      NaN   DTzrG     -8.0   bfgeg   \n",
       "\n",
       "  cat_199 cat_200 cat_201 cat_202 cat_203 cat_204 cat_205 cat_206 cat_207  \\\n",
       "0   wIdgm   lUoRY   ahACm   sOBnN   bywyW   KhlzK   TdcoU   Bovxn   HzgoY   \n",
       "1   wIdgm   lUoRY   ahACm   sOBnN   bywyW   KhlzK   TdcoU   Bovxn   HzgoY   \n",
       "2   NMtVW   lUoRY   EyqjN   sOBnN   IwXNs   qmOxG   aDlJD   kIJMX   HzgoY   \n",
       "3   wIdgm   lUoRY   ahACm   sOBnN   bywyW   KhlzK   TdcoU   Bovxn   HzgoY   \n",
       "4   wIdgm   lUoRY   EyqjN   sOBnN   IwXNs   qmOxG   TdcoU   kIJMX   ASpbn   \n",
       "\n",
       "  cat_208  num_209 cat_210 cat_211 cat_212 cat_213 cat_214 cat_215 cat_216  \\\n",
       "0   hEagh     26.0   JehJJ   JCGsD   VBmxq   yWVRJ   rlTrR   uCOQO   UYIFp   \n",
       "1   hEagh      NaN   JehJJ   JCGsD   VBmxq   yWVRJ   rlTrR   uCOQO   UYIFp   \n",
       "2   hEagh      NaN   JehJJ   hdaYV   VBmxq   yWVRJ   rlTrR   uCOQO   UYIFp   \n",
       "3   hEagh      NaN   JehJJ   JCGsD   VBmxq   yWVRJ   rlTrR   uCOQO   UYIFp   \n",
       "4   hEagh      NaN   JehJJ   hdaYV   VBmxq   yWVRJ   rlTrR   uCOQO   UYIFp   \n",
       "\n",
       "   num_217  num_218 cat_219 cat_220  num_221 cat_222  target  \n",
       "0        9      NaN   MAFfK   VnOFM  -7827.0   uJXdA   False  \n",
       "1       29      NaN   MAFfK   VnOFM      NaN   uJXdA   False  \n",
       "2      -82      NaN   MAFfK   VnOFM      NaN   uJXdA   False  \n",
       "3       -6      NaN   MAFfK   ppEcI  -7867.0   uJXdA    True  \n",
       "4       19      NaN   MAFfK   ppEcI  -7987.0   uJXdA    True  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "cat_3 object\n",
      "cat_4 object\n",
      "cat_5 object\n",
      "num_6 float64\n",
      "cat_7 object\n",
      "num_8 float64\n",
      "cat_9 object\n",
      "cat_10 object\n",
      "cat_11 object\n",
      "cat_12 object\n",
      "cat_13 object\n",
      "cat_14 object\n",
      "num_15 float64\n",
      "cat_16 object\n",
      "cat_17 object\n",
      "cat_18 object\n",
      "num_19 float64\n",
      "num_20 float64\n",
      "cat_21 object\n",
      "cat_22 object\n",
      "cat_23 object\n",
      "cat_24 object\n",
      "cat_25 object\n",
      "cat_26 object\n",
      "cat_27 object\n",
      "cat_28 object\n",
      "cat_29 object\n",
      "cat_30 object\n",
      "cat_31 object\n",
      "cat_32 object\n",
      "num_33 float64\n",
      "num_34 float64\n",
      "num_35 float64\n",
      "cat_36 object\n",
      "cat_37 object\n",
      "cat_38 object\n",
      "cat_39 object\n",
      "cat_40 object\n",
      "cat_41 object\n",
      "cat_42 object\n",
      "cat_43 object\n",
      "cat_44 object\n",
      "num_45 float64\n",
      "cat_46 object\n",
      "cat_47 object\n",
      "cat_48 object\n",
      "cat_49 object\n",
      "cat_50 object\n",
      "cat_51 object\n",
      "cat_52 object\n",
      "cat_53 object\n",
      "cat_54 object\n",
      "cat_55 object\n",
      "cat_56 object\n",
      "cat_57 object\n",
      "cat_58 object\n",
      "num_59 float64\n",
      "cat_60 object\n",
      "cat_61 object\n",
      "cat_62 object\n",
      "cat_63 object\n",
      "cat_64 object\n",
      "cat_65 object\n",
      "cat_66 object\n",
      "num_67 float64\n",
      "cat_68 object\n",
      "cat_69 object\n",
      "num_70 int64\n",
      "cat_71 object\n",
      "cat_72 object\n",
      "cat_73 object\n",
      "cat_74 object\n",
      "cat_75 object\n",
      "cat_76 object\n",
      "cat_77 object\n",
      "cat_78 object\n",
      "cat_79 object\n",
      "cat_80 object\n",
      "cat_81 object\n",
      "cat_82 object\n",
      "cat_83 object\n",
      "cat_84 object\n",
      "cat_85 object\n",
      "cat_86 object\n",
      "cat_87 object\n",
      "cat_88 object\n",
      "cat_89 object\n",
      "cat_90 object\n",
      "cat_91 object\n",
      "cat_92 object\n",
      "cat_93 object\n",
      "cat_94 object\n",
      "cat_95 object\n",
      "num_96 float64\n",
      "cat_97 object\n",
      "cat_98 object\n",
      "cat_99 object\n",
      "cat_100 object\n",
      "cat_101 object\n",
      "cat_102 object\n",
      "cat_103 object\n",
      "cat_104 object\n",
      "num_105 float64\n",
      "num_106 float64\n",
      "cat_107 object\n",
      "cat_108 object\n",
      "cat_109 object\n",
      "cat_110 object\n",
      "cat_111 object\n",
      "cat_112 object\n",
      "cat_113 object\n",
      "cat_114 object\n",
      "cat_115 object\n",
      "cat_116 object\n",
      "cat_117 object\n",
      "cat_118 object\n",
      "cat_119 object\n",
      "cat_120 object\n",
      "cat_121 object\n",
      "num_122 float64\n",
      "cat_123 object\n",
      "cat_124 object\n",
      "cat_125 object\n",
      "cat_126 object\n",
      "cat_127 object\n",
      "cat_128 object\n",
      "cat_129 object\n",
      "cat_130 object\n",
      "num_131 float64\n",
      "cat_132 object\n",
      "cat_133 object\n",
      "cat_134 object\n",
      "cat_135 object\n",
      "cat_136 object\n",
      "cat_137 object\n",
      "cat_138 object\n",
      "cat_139 object\n",
      "cat_140 object\n",
      "cat_141 object\n",
      "cat_142 object\n",
      "num_143 int64\n",
      "cat_144 object\n",
      "cat_145 object\n",
      "cat_146 object\n",
      "cat_147 object\n",
      "cat_148 object\n",
      "cat_149 object\n",
      "cat_150 object\n",
      "num_151 float64\n",
      "cat_152 object\n",
      "cat_153 object\n",
      "cat_154 object\n",
      "cat_155 object\n",
      "num_156 float64\n",
      "cat_157 object\n",
      "num_158 float64\n",
      "cat_159 object\n",
      "num_160 float64\n",
      "cat_161 object\n",
      "cat_162 object\n",
      "cat_163 object\n",
      "cat_164 object\n",
      "cat_165 object\n",
      "cat_166 object\n",
      "cat_167 object\n",
      "cat_168 object\n",
      "cat_169 object\n",
      "cat_170 object\n",
      "cat_171 object\n",
      "cat_172 object\n",
      "num_173 float64\n",
      "num_174 float64\n",
      "cat_175 object\n",
      "cat_176 object\n",
      "cat_177 object\n",
      "cat_178 object\n",
      "num_179 float64\n",
      "cat_180 object\n",
      "cat_181 object\n",
      "cat_182 object\n",
      "cat_183 object\n",
      "cat_184 object\n",
      "cat_185 object\n",
      "cat_186 object\n",
      "num_187 int64\n",
      "cat_188 object\n",
      "cat_189 object\n",
      "cat_190 object\n",
      "cat_191 object\n",
      "cat_192 object\n",
      "cat_193 object\n",
      "cat_194 object\n",
      "num_195 float64\n",
      "cat_196 object\n",
      "num_197 float64\n",
      "cat_198 object\n",
      "cat_199 object\n",
      "cat_200 object\n",
      "cat_201 object\n",
      "cat_202 object\n",
      "cat_203 object\n",
      "cat_204 object\n",
      "cat_205 object\n",
      "cat_206 object\n",
      "cat_207 object\n",
      "cat_208 object\n",
      "num_209 float64\n",
      "cat_210 object\n",
      "cat_211 object\n",
      "cat_212 object\n",
      "cat_213 object\n",
      "cat_214 object\n",
      "cat_215 object\n",
      "cat_216 object\n",
      "num_217 int64\n",
      "num_218 float64\n",
      "cat_219 object\n",
      "cat_220 object\n",
      "num_221 float64\n",
      "cat_222 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# poverty_C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(29913, 44)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>iid</th>\n",
       "      <th>OoqEwyJF</th>\n",
       "      <th>cJPCnaAs</th>\n",
       "      <th>vIUDQhEE</th>\n",
       "      <th>cRHfBOQd</th>\n",
       "      <th>dAmhsMgC</th>\n",
       "      <th>skzSFBmO</th>\n",
       "      <th>IsqnlKAj</th>\n",
       "      <th>AOSWkWKB</th>\n",
       "      <th>IThGOCux</th>\n",
       "      <th>gBTGbFhG</th>\n",
       "      <th>XKQWlRjk</th>\n",
       "      <th>CggtnNTN</th>\n",
       "      <th>TfEUOijE</th>\n",
       "      <th>tOfmAJyI</th>\n",
       "      <th>vWNISgEA</th>\n",
       "      <th>FRcdTUFo</th>\n",
       "      <th>wXJsPzSl</th>\n",
       "      <th>bsMfXBld</th>\n",
       "      <th>EaHvfzfT</th>\n",
       "      <th>XDnwpOpR</th>\n",
       "      <th>uOlSuJDG</th>\n",
       "      <th>CjuMStGt</th>\n",
       "      <th>ymHYOZZv</th>\n",
       "      <th>eXdISXQe</th>\n",
       "      <th>WqmeSStq</th>\n",
       "      <th>XKyOwsRR</th>\n",
       "      <th>XDzQvdCw</th>\n",
       "      <th>CgAkQtOd</th>\n",
       "      <th>UTyHNliH</th>\n",
       "      <th>poor</th>\n",
       "      <th>gxCmCVUk</th>\n",
       "      <th>qGqYslGF</th>\n",
       "      <th>dArQTUaf</th>\n",
       "      <th>NAxEQZVi</th>\n",
       "      <th>ShCKQiAy</th>\n",
       "      <th>rkLqZrQW</th>\n",
       "      <th>VGJlUgVG</th>\n",
       "      <th>kMVbipfP</th>\n",
       "      <th>sCTSWhXf</th>\n",
       "      <th>rVneGwzn</th>\n",
       "      <th>uVFOfrpa</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30639</td>\n",
       "      <td>1</td>\n",
       "      <td>cEcbt</td>\n",
       "      <td>QhmHR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>sgjYl</td>\n",
       "      <td>IpTiG</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>GpnOQ</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>PVIia</td>\n",
       "      <td>amOeQ</td>\n",
       "      <td>122</td>\n",
       "      <td>RxYsa</td>\n",
       "      <td>ucqiX</td>\n",
       "      <td>mlNXN</td>\n",
       "      <td>9</td>\n",
       "      <td>efxMi</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>9.0</td>\n",
       "      <td>xHsSQ</td>\n",
       "      <td>DUVzU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>GHDuu</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>-3</td>\n",
       "      <td>PBUwt</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>hHSXB</td>\n",
       "      <td>False</td>\n",
       "      <td>uSErG</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iRUzd</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>INYbJ</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>VlcEt</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>xgpHA</td>\n",
       "      <td>DnIbO</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30639</td>\n",
       "      <td>2</td>\n",
       "      <td>cEcbt</td>\n",
       "      <td>QhmHR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>sgjYl</td>\n",
       "      <td>IpTiG</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>UaIsy</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>MyWVa</td>\n",
       "      <td>amOeQ</td>\n",
       "      <td>68</td>\n",
       "      <td>RxYsa</td>\n",
       "      <td>AzTqj</td>\n",
       "      <td>mlNXN</td>\n",
       "      <td>9</td>\n",
       "      <td>toWkd</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>9.0</td>\n",
       "      <td>xHsSQ</td>\n",
       "      <td>DUVzU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>GHDuu</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>DGWjH</td>\n",
       "      <td>-3</td>\n",
       "      <td>PBUwt</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>hHSXB</td>\n",
       "      <td>False</td>\n",
       "      <td>uSErG</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>VYpjn</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>TYbsc</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>VlcEt</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>xgpHA</td>\n",
       "      <td>DnIbO</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30639</td>\n",
       "      <td>3</td>\n",
       "      <td>cEcbt</td>\n",
       "      <td>cMeuH</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>sgjYl</td>\n",
       "      <td>IpTiG</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>ehUOC</td>\n",
       "      <td>KEvSa</td>\n",
       "      <td>1</td>\n",
       "      <td>hAGot</td>\n",
       "      <td>ISjaW</td>\n",
       "      <td>SSvEP</td>\n",
       "      <td>9</td>\n",
       "      <td>NwYCR</td>\n",
       "      <td>EFUMy</td>\n",
       "      <td>9.0</td>\n",
       "      <td>xHsSQ</td>\n",
       "      <td>DUVzU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>GHDuu</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>-3</td>\n",
       "      <td>PBUwt</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>FbTRU</td>\n",
       "      <td>False</td>\n",
       "      <td>wNFGx</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iQpoG</td>\n",
       "      <td>GkrMH</td>\n",
       "      <td>xJurw</td>\n",
       "      <td>pbPGJ</td>\n",
       "      <td>YYwlj</td>\n",
       "      <td>rPkFE</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>ldKFc</td>\n",
       "      <td>kXobL</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30639</td>\n",
       "      <td>9</td>\n",
       "      <td>cEcbt</td>\n",
       "      <td>uSuzR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>GpRit</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>UrHEJ</td>\n",
       "      <td>JnveI</td>\n",
       "      <td>1</td>\n",
       "      <td>hAGot</td>\n",
       "      <td>ISjaW</td>\n",
       "      <td>SSvEP</td>\n",
       "      <td>104</td>\n",
       "      <td>NwYCR</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>-19.2</td>\n",
       "      <td>yFSGe</td>\n",
       "      <td>nvyhU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>Engma</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>249</td>\n",
       "      <td>bgZsP</td>\n",
       "      <td>-53.8699</td>\n",
       "      <td>XFrNG</td>\n",
       "      <td>False</td>\n",
       "      <td>wNFGx</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iQpoG</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>iuiyo</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>YYwlj</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>QGHnL</td>\n",
       "      <td>xRxWC</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30639</td>\n",
       "      <td>10</td>\n",
       "      <td>cEcbt</td>\n",
       "      <td>uSuzR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>GpRit</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>UrHEJ</td>\n",
       "      <td>JnveI</td>\n",
       "      <td>1</td>\n",
       "      <td>hAGot</td>\n",
       "      <td>ISjaW</td>\n",
       "      <td>mlNXN</td>\n",
       "      <td>113</td>\n",
       "      <td>NwYCR</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>-27.0</td>\n",
       "      <td>yFSGe</td>\n",
       "      <td>nvyhU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>Engma</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>237</td>\n",
       "      <td>bgZsP</td>\n",
       "      <td>-56.9260</td>\n",
       "      <td>XFrNG</td>\n",
       "      <td>False</td>\n",
       "      <td>wNFGx</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iQpoG</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>iuiyo</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>YYwlj</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>QGHnL</td>\n",
       "      <td>xRxWC</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  iid OoqEwyJF cJPCnaAs vIUDQhEE cRHfBOQd dAmhsMgC skzSFBmO IsqnlKAj  \\\n",
       "0  30639    1    cEcbt    QhmHR    lJvCX    sgjYl    IpTiG    WFHaK    GpnOQ   \n",
       "1  30639    2    cEcbt    QhmHR    lJvCX    sgjYl    IpTiG    WFHaK    UaIsy   \n",
       "2  30639    3    cEcbt    cMeuH    lJvCX    sgjYl    IpTiG    WFHaK    JTCKs   \n",
       "3  30639    9    cEcbt    uSuzR    lJvCX    GpRit    uEstx    WFHaK    JTCKs   \n",
       "4  30639   10    cEcbt    uSuzR    lJvCX    GpRit    uEstx    WFHaK    JTCKs   \n",
       "\n",
       "  AOSWkWKB IThGOCux gBTGbFhG  XKQWlRjk CggtnNTN TfEUOijE tOfmAJyI  vWNISgEA  \\\n",
       "0    nLEWH    PVIia    amOeQ       122    RxYsa    ucqiX    mlNXN         9   \n",
       "1    nLEWH    MyWVa    amOeQ        68    RxYsa    AzTqj    mlNXN         9   \n",
       "2    nLEWH    ehUOC    KEvSa         1    hAGot    ISjaW    SSvEP         9   \n",
       "3    nLEWH    UrHEJ    JnveI         1    hAGot    ISjaW    SSvEP       104   \n",
       "4    nLEWH    UrHEJ    JnveI         1    hAGot    ISjaW    mlNXN       113   \n",
       "\n",
       "  FRcdTUFo wXJsPzSl  bsMfXBld EaHvfzfT XDnwpOpR uOlSuJDG CjuMStGt ymHYOZZv  \\\n",
       "0    efxMi    cPXrX       9.0    xHsSQ    DUVzU    HelFU    lYVyA    GHDuu   \n",
       "1    toWkd    cPXrX       9.0    xHsSQ    DUVzU    HelFU    lYVyA    GHDuu   \n",
       "2    NwYCR    EFUMy       9.0    xHsSQ    DUVzU    HelFU    lYVyA    GHDuu   \n",
       "3    NwYCR    cPXrX     -19.2    yFSGe    nvyhU    HelFU    lYVyA    Engma   \n",
       "4    NwYCR    cPXrX     -27.0    yFSGe    nvyhU    HelFU    lYVyA    Engma   \n",
       "\n",
       "  eXdISXQe WqmeSStq  XKyOwsRR XDzQvdCw  CgAkQtOd UTyHNliH   poor gxCmCVUk  \\\n",
       "0    dxzZA    XvXON        -3    PBUwt   -7.0000    hHSXB  False    uSErG   \n",
       "1    dxzZA    DGWjH        -3    PBUwt   -7.0000    hHSXB  False    uSErG   \n",
       "2    dxzZA    XvXON        -3    PBUwt   -7.0000    FbTRU  False    wNFGx   \n",
       "3    dxzZA    XvXON       249    bgZsP  -53.8699    XFrNG  False    wNFGx   \n",
       "4    dxzZA    XvXON       237    bgZsP  -56.9260    XFrNG  False    wNFGx   \n",
       "\n",
       "  qGqYslGF dArQTUaf NAxEQZVi ShCKQiAy rkLqZrQW VGJlUgVG kMVbipfP sCTSWhXf  \\\n",
       "0    SDNoA    iRUzd    Rihyc    INYbJ    SoOdX    VlcEt    zzxBZ    yQhuJ   \n",
       "1    SDNoA    VYpjn    Rihyc    TYbsc    SoOdX    VlcEt    zzxBZ    yQhuJ   \n",
       "2    SDNoA    iQpoG    GkrMH    xJurw    pbPGJ    YYwlj    rPkFE    yQhuJ   \n",
       "3    SDNoA    iQpoG    Rihyc    iuiyo    SoOdX    YYwlj    zzxBZ    yQhuJ   \n",
       "4    SDNoA    iQpoG    Rihyc    iuiyo    SoOdX    YYwlj    zzxBZ    yQhuJ   \n",
       "\n",
       "  rVneGwzn uVFOfrpa country  \n",
       "0    xgpHA    DnIbO       C  \n",
       "1    xgpHA    DnIbO       C  \n",
       "2    ldKFc    kXobL       C  \n",
       "3    QGHnL    xRxWC       C  \n",
       "4    QGHnL    xRxWC       C  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"poverty_C\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}C_indiv_train.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"id\", \"iid\", \"country\"],\n",
    "    \"cat_cols\": [col for col in data.columns if col not in [\"id\", \"iid\", \"poor\", \"XKQWlRjk\", \"vWNISgEA\", \"bsMfXBld\", \n",
    "                                                            \"XKyOwsRR\", \"CgAkQtOd\"]],\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"poor\"] == 1\n",
    "data.drop([\"poor\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  35\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>cat_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>cat_6</th>\n",
       "      <th>cat_7</th>\n",
       "      <th>cat_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>num_10</th>\n",
       "      <th>cat_11</th>\n",
       "      <th>cat_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>num_14</th>\n",
       "      <th>cat_15</th>\n",
       "      <th>cat_16</th>\n",
       "      <th>num_17</th>\n",
       "      <th>cat_18</th>\n",
       "      <th>cat_19</th>\n",
       "      <th>cat_20</th>\n",
       "      <th>cat_21</th>\n",
       "      <th>cat_22</th>\n",
       "      <th>cat_23</th>\n",
       "      <th>cat_24</th>\n",
       "      <th>num_25</th>\n",
       "      <th>cat_26</th>\n",
       "      <th>num_27</th>\n",
       "      <th>cat_28</th>\n",
       "      <th>cat_29</th>\n",
       "      <th>cat_30</th>\n",
       "      <th>cat_31</th>\n",
       "      <th>cat_32</th>\n",
       "      <th>cat_33</th>\n",
       "      <th>cat_34</th>\n",
       "      <th>cat_35</th>\n",
       "      <th>cat_36</th>\n",
       "      <th>cat_37</th>\n",
       "      <th>cat_38</th>\n",
       "      <th>cat_39</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cEcbt</td>\n",
       "      <td>QhmHR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>sgjYl</td>\n",
       "      <td>IpTiG</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>GpnOQ</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>PVIia</td>\n",
       "      <td>amOeQ</td>\n",
       "      <td>122</td>\n",
       "      <td>RxYsa</td>\n",
       "      <td>ucqiX</td>\n",
       "      <td>mlNXN</td>\n",
       "      <td>9</td>\n",
       "      <td>efxMi</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>9.0</td>\n",
       "      <td>xHsSQ</td>\n",
       "      <td>DUVzU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>GHDuu</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>-3</td>\n",
       "      <td>PBUwt</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>hHSXB</td>\n",
       "      <td>uSErG</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iRUzd</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>INYbJ</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>VlcEt</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>xgpHA</td>\n",
       "      <td>DnIbO</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cEcbt</td>\n",
       "      <td>QhmHR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>sgjYl</td>\n",
       "      <td>IpTiG</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>UaIsy</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>MyWVa</td>\n",
       "      <td>amOeQ</td>\n",
       "      <td>68</td>\n",
       "      <td>RxYsa</td>\n",
       "      <td>AzTqj</td>\n",
       "      <td>mlNXN</td>\n",
       "      <td>9</td>\n",
       "      <td>toWkd</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>9.0</td>\n",
       "      <td>xHsSQ</td>\n",
       "      <td>DUVzU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>GHDuu</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>DGWjH</td>\n",
       "      <td>-3</td>\n",
       "      <td>PBUwt</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>hHSXB</td>\n",
       "      <td>uSErG</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>VYpjn</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>TYbsc</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>VlcEt</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>xgpHA</td>\n",
       "      <td>DnIbO</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cEcbt</td>\n",
       "      <td>cMeuH</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>sgjYl</td>\n",
       "      <td>IpTiG</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>ehUOC</td>\n",
       "      <td>KEvSa</td>\n",
       "      <td>1</td>\n",
       "      <td>hAGot</td>\n",
       "      <td>ISjaW</td>\n",
       "      <td>SSvEP</td>\n",
       "      <td>9</td>\n",
       "      <td>NwYCR</td>\n",
       "      <td>EFUMy</td>\n",
       "      <td>9.0</td>\n",
       "      <td>xHsSQ</td>\n",
       "      <td>DUVzU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>GHDuu</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>-3</td>\n",
       "      <td>PBUwt</td>\n",
       "      <td>-7.0000</td>\n",
       "      <td>FbTRU</td>\n",
       "      <td>wNFGx</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iQpoG</td>\n",
       "      <td>GkrMH</td>\n",
       "      <td>xJurw</td>\n",
       "      <td>pbPGJ</td>\n",
       "      <td>YYwlj</td>\n",
       "      <td>rPkFE</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>ldKFc</td>\n",
       "      <td>kXobL</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cEcbt</td>\n",
       "      <td>uSuzR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>GpRit</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>UrHEJ</td>\n",
       "      <td>JnveI</td>\n",
       "      <td>1</td>\n",
       "      <td>hAGot</td>\n",
       "      <td>ISjaW</td>\n",
       "      <td>SSvEP</td>\n",
       "      <td>104</td>\n",
       "      <td>NwYCR</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>-19.2</td>\n",
       "      <td>yFSGe</td>\n",
       "      <td>nvyhU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>Engma</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>249</td>\n",
       "      <td>bgZsP</td>\n",
       "      <td>-53.8699</td>\n",
       "      <td>XFrNG</td>\n",
       "      <td>wNFGx</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iQpoG</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>iuiyo</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>YYwlj</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>QGHnL</td>\n",
       "      <td>xRxWC</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cEcbt</td>\n",
       "      <td>uSuzR</td>\n",
       "      <td>lJvCX</td>\n",
       "      <td>GpRit</td>\n",
       "      <td>uEstx</td>\n",
       "      <td>WFHaK</td>\n",
       "      <td>JTCKs</td>\n",
       "      <td>nLEWH</td>\n",
       "      <td>UrHEJ</td>\n",
       "      <td>JnveI</td>\n",
       "      <td>1</td>\n",
       "      <td>hAGot</td>\n",
       "      <td>ISjaW</td>\n",
       "      <td>mlNXN</td>\n",
       "      <td>113</td>\n",
       "      <td>NwYCR</td>\n",
       "      <td>cPXrX</td>\n",
       "      <td>-27.0</td>\n",
       "      <td>yFSGe</td>\n",
       "      <td>nvyhU</td>\n",
       "      <td>HelFU</td>\n",
       "      <td>lYVyA</td>\n",
       "      <td>Engma</td>\n",
       "      <td>dxzZA</td>\n",
       "      <td>XvXON</td>\n",
       "      <td>237</td>\n",
       "      <td>bgZsP</td>\n",
       "      <td>-56.9260</td>\n",
       "      <td>XFrNG</td>\n",
       "      <td>wNFGx</td>\n",
       "      <td>SDNoA</td>\n",
       "      <td>iQpoG</td>\n",
       "      <td>Rihyc</td>\n",
       "      <td>iuiyo</td>\n",
       "      <td>SoOdX</td>\n",
       "      <td>YYwlj</td>\n",
       "      <td>zzxBZ</td>\n",
       "      <td>yQhuJ</td>\n",
       "      <td>QGHnL</td>\n",
       "      <td>xRxWC</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat_0  cat_1  cat_2  cat_3  cat_4  cat_5  cat_6  cat_7  cat_8  cat_9  \\\n",
       "0  cEcbt  QhmHR  lJvCX  sgjYl  IpTiG  WFHaK  GpnOQ  nLEWH  PVIia  amOeQ   \n",
       "1  cEcbt  QhmHR  lJvCX  sgjYl  IpTiG  WFHaK  UaIsy  nLEWH  MyWVa  amOeQ   \n",
       "2  cEcbt  cMeuH  lJvCX  sgjYl  IpTiG  WFHaK  JTCKs  nLEWH  ehUOC  KEvSa   \n",
       "3  cEcbt  uSuzR  lJvCX  GpRit  uEstx  WFHaK  JTCKs  nLEWH  UrHEJ  JnveI   \n",
       "4  cEcbt  uSuzR  lJvCX  GpRit  uEstx  WFHaK  JTCKs  nLEWH  UrHEJ  JnveI   \n",
       "\n",
       "   num_10 cat_11 cat_12 cat_13  num_14 cat_15 cat_16  num_17 cat_18 cat_19  \\\n",
       "0     122  RxYsa  ucqiX  mlNXN       9  efxMi  cPXrX     9.0  xHsSQ  DUVzU   \n",
       "1      68  RxYsa  AzTqj  mlNXN       9  toWkd  cPXrX     9.0  xHsSQ  DUVzU   \n",
       "2       1  hAGot  ISjaW  SSvEP       9  NwYCR  EFUMy     9.0  xHsSQ  DUVzU   \n",
       "3       1  hAGot  ISjaW  SSvEP     104  NwYCR  cPXrX   -19.2  yFSGe  nvyhU   \n",
       "4       1  hAGot  ISjaW  mlNXN     113  NwYCR  cPXrX   -27.0  yFSGe  nvyhU   \n",
       "\n",
       "  cat_20 cat_21 cat_22 cat_23 cat_24  num_25 cat_26   num_27 cat_28 cat_29  \\\n",
       "0  HelFU  lYVyA  GHDuu  dxzZA  XvXON      -3  PBUwt  -7.0000  hHSXB  uSErG   \n",
       "1  HelFU  lYVyA  GHDuu  dxzZA  DGWjH      -3  PBUwt  -7.0000  hHSXB  uSErG   \n",
       "2  HelFU  lYVyA  GHDuu  dxzZA  XvXON      -3  PBUwt  -7.0000  FbTRU  wNFGx   \n",
       "3  HelFU  lYVyA  Engma  dxzZA  XvXON     249  bgZsP -53.8699  XFrNG  wNFGx   \n",
       "4  HelFU  lYVyA  Engma  dxzZA  XvXON     237  bgZsP -56.9260  XFrNG  wNFGx   \n",
       "\n",
       "  cat_30 cat_31 cat_32 cat_33 cat_34 cat_35 cat_36 cat_37 cat_38 cat_39  \\\n",
       "0  SDNoA  iRUzd  Rihyc  INYbJ  SoOdX  VlcEt  zzxBZ  yQhuJ  xgpHA  DnIbO   \n",
       "1  SDNoA  VYpjn  Rihyc  TYbsc  SoOdX  VlcEt  zzxBZ  yQhuJ  xgpHA  DnIbO   \n",
       "2  SDNoA  iQpoG  GkrMH  xJurw  pbPGJ  YYwlj  rPkFE  yQhuJ  ldKFc  kXobL   \n",
       "3  SDNoA  iQpoG  Rihyc  iuiyo  SoOdX  YYwlj  zzxBZ  yQhuJ  QGHnL  xRxWC   \n",
       "4  SDNoA  iQpoG  Rihyc  iuiyo  SoOdX  YYwlj  zzxBZ  yQhuJ  QGHnL  xRxWC   \n",
       "\n",
       "   target  \n",
       "0   False  \n",
       "1   False  \n",
       "2   False  \n",
       "3   False  \n",
       "4   False  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "cat_3 object\n",
      "cat_4 object\n",
      "cat_5 object\n",
      "cat_6 object\n",
      "cat_7 object\n",
      "cat_8 object\n",
      "cat_9 object\n",
      "num_10 int64\n",
      "cat_11 object\n",
      "cat_12 object\n",
      "cat_13 object\n",
      "num_14 int64\n",
      "cat_15 object\n",
      "cat_16 object\n",
      "num_17 float64\n",
      "cat_18 object\n",
      "cat_19 object\n",
      "cat_20 object\n",
      "cat_21 object\n",
      "cat_22 object\n",
      "cat_23 object\n",
      "cat_24 object\n",
      "num_25 int64\n",
      "cat_26 object\n",
      "num_27 float64\n",
      "cat_28 object\n",
      "cat_29 object\n",
      "cat_30 object\n",
      "cat_31 object\n",
      "cat_32 object\n",
      "cat_33 object\n",
      "cat_34 object\n",
      "cat_35 object\n",
      "cat_36 object\n",
      "cat_37 object\n",
      "cat_38 object\n",
      "cat_39 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# credit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/c/home-credit-default-risk/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(307511, 122)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>NAME_TYPE_SUITE</th>\n",
       "      <th>NAME_INCOME_TYPE</th>\n",
       "      <th>NAME_EDUCATION_TYPE</th>\n",
       "      <th>NAME_FAMILY_STATUS</th>\n",
       "      <th>NAME_HOUSING_TYPE</th>\n",
       "      <th>REGION_POPULATION_RELATIVE</th>\n",
       "      <th>DAYS_BIRTH</th>\n",
       "      <th>DAYS_EMPLOYED</th>\n",
       "      <th>DAYS_REGISTRATION</th>\n",
       "      <th>DAYS_ID_PUBLISH</th>\n",
       "      <th>OWN_CAR_AGE</th>\n",
       "      <th>FLAG_MOBIL</th>\n",
       "      <th>FLAG_EMP_PHONE</th>\n",
       "      <th>FLAG_WORK_PHONE</th>\n",
       "      <th>FLAG_CONT_MOBILE</th>\n",
       "      <th>FLAG_PHONE</th>\n",
       "      <th>FLAG_EMAIL</th>\n",
       "      <th>OCCUPATION_TYPE</th>\n",
       "      <th>CNT_FAM_MEMBERS</th>\n",
       "      <th>REGION_RATING_CLIENT</th>\n",
       "      <th>REGION_RATING_CLIENT_W_CITY</th>\n",
       "      <th>WEEKDAY_APPR_PROCESS_START</th>\n",
       "      <th>HOUR_APPR_PROCESS_START</th>\n",
       "      <th>REG_REGION_NOT_LIVE_REGION</th>\n",
       "      <th>REG_REGION_NOT_WORK_REGION</th>\n",
       "      <th>LIVE_REGION_NOT_WORK_REGION</th>\n",
       "      <th>REG_CITY_NOT_LIVE_CITY</th>\n",
       "      <th>REG_CITY_NOT_WORK_CITY</th>\n",
       "      <th>LIVE_CITY_NOT_WORK_CITY</th>\n",
       "      <th>ORGANIZATION_TYPE</th>\n",
       "      <th>EXT_SOURCE_1</th>\n",
       "      <th>EXT_SOURCE_2</th>\n",
       "      <th>EXT_SOURCE_3</th>\n",
       "      <th>APARTMENTS_AVG</th>\n",
       "      <th>BASEMENTAREA_AVG</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_AVG</th>\n",
       "      <th>YEARS_BUILD_AVG</th>\n",
       "      <th>COMMONAREA_AVG</th>\n",
       "      <th>ELEVATORS_AVG</th>\n",
       "      <th>ENTRANCES_AVG</th>\n",
       "      <th>FLOORSMAX_AVG</th>\n",
       "      <th>FLOORSMIN_AVG</th>\n",
       "      <th>LANDAREA_AVG</th>\n",
       "      <th>LIVINGAPARTMENTS_AVG</th>\n",
       "      <th>LIVINGAREA_AVG</th>\n",
       "      <th>NONLIVINGAPARTMENTS_AVG</th>\n",
       "      <th>NONLIVINGAREA_AVG</th>\n",
       "      <th>APARTMENTS_MODE</th>\n",
       "      <th>BASEMENTAREA_MODE</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_MODE</th>\n",
       "      <th>YEARS_BUILD_MODE</th>\n",
       "      <th>COMMONAREA_MODE</th>\n",
       "      <th>ELEVATORS_MODE</th>\n",
       "      <th>ENTRANCES_MODE</th>\n",
       "      <th>FLOORSMAX_MODE</th>\n",
       "      <th>FLOORSMIN_MODE</th>\n",
       "      <th>LANDAREA_MODE</th>\n",
       "      <th>LIVINGAPARTMENTS_MODE</th>\n",
       "      <th>LIVINGAREA_MODE</th>\n",
       "      <th>NONLIVINGAPARTMENTS_MODE</th>\n",
       "      <th>NONLIVINGAREA_MODE</th>\n",
       "      <th>APARTMENTS_MEDI</th>\n",
       "      <th>BASEMENTAREA_MEDI</th>\n",
       "      <th>YEARS_BEGINEXPLUATATION_MEDI</th>\n",
       "      <th>YEARS_BUILD_MEDI</th>\n",
       "      <th>COMMONAREA_MEDI</th>\n",
       "      <th>ELEVATORS_MEDI</th>\n",
       "      <th>ENTRANCES_MEDI</th>\n",
       "      <th>FLOORSMAX_MEDI</th>\n",
       "      <th>FLOORSMIN_MEDI</th>\n",
       "      <th>LANDAREA_MEDI</th>\n",
       "      <th>LIVINGAPARTMENTS_MEDI</th>\n",
       "      <th>LIVINGAREA_MEDI</th>\n",
       "      <th>NONLIVINGAPARTMENTS_MEDI</th>\n",
       "      <th>NONLIVINGAREA_MEDI</th>\n",
       "      <th>FONDKAPREMONT_MODE</th>\n",
       "      <th>HOUSETYPE_MODE</th>\n",
       "      <th>TOTALAREA_MODE</th>\n",
       "      <th>WALLSMATERIAL_MODE</th>\n",
       "      <th>EMERGENCYSTATE_MODE</th>\n",
       "      <th>OBS_30_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DEF_30_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>OBS_60_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DEF_60_CNT_SOCIAL_CIRCLE</th>\n",
       "      <th>DAYS_LAST_PHONE_CHANGE</th>\n",
       "      <th>FLAG_DOCUMENT_2</th>\n",
       "      <th>FLAG_DOCUMENT_3</th>\n",
       "      <th>FLAG_DOCUMENT_4</th>\n",
       "      <th>FLAG_DOCUMENT_5</th>\n",
       "      <th>FLAG_DOCUMENT_6</th>\n",
       "      <th>FLAG_DOCUMENT_7</th>\n",
       "      <th>FLAG_DOCUMENT_8</th>\n",
       "      <th>FLAG_DOCUMENT_9</th>\n",
       "      <th>FLAG_DOCUMENT_10</th>\n",
       "      <th>FLAG_DOCUMENT_11</th>\n",
       "      <th>FLAG_DOCUMENT_12</th>\n",
       "      <th>FLAG_DOCUMENT_13</th>\n",
       "      <th>FLAG_DOCUMENT_14</th>\n",
       "      <th>FLAG_DOCUMENT_15</th>\n",
       "      <th>FLAG_DOCUMENT_16</th>\n",
       "      <th>FLAG_DOCUMENT_17</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.018801</td>\n",
       "      <td>-9461</td>\n",
       "      <td>-637</td>\n",
       "      <td>-3648.0</td>\n",
       "      <td>-2120</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>0.083037</td>\n",
       "      <td>0.262949</td>\n",
       "      <td>0.139376</td>\n",
       "      <td>0.0247</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6192</td>\n",
       "      <td>0.0143</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.0202</td>\n",
       "      <td>0.0190</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0252</td>\n",
       "      <td>0.0383</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6341</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0377</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0.0198</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0250</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6243</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0375</td>\n",
       "      <td>0.0205</td>\n",
       "      <td>0.0193</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.0149</td>\n",
       "      <td>Stone, brick</td>\n",
       "      <td>No</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-1134.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>Family</td>\n",
       "      <td>State servant</td>\n",
       "      <td>Higher education</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>-16765</td>\n",
       "      <td>-1188</td>\n",
       "      <td>-1186.0</td>\n",
       "      <td>-291</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>MONDAY</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>School</td>\n",
       "      <td>0.311267</td>\n",
       "      <td>0.622246</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0959</td>\n",
       "      <td>0.0529</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.7960</td>\n",
       "      <td>0.0605</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0130</td>\n",
       "      <td>0.0773</td>\n",
       "      <td>0.0549</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.0098</td>\n",
       "      <td>0.0924</td>\n",
       "      <td>0.0538</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.8040</td>\n",
       "      <td>0.0497</td>\n",
       "      <td>0.0806</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0128</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.0554</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0968</td>\n",
       "      <td>0.0529</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.7987</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.0787</td>\n",
       "      <td>0.0558</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.01</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>Block</td>\n",
       "      <td>No</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-828.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.010032</td>\n",
       "      <td>-19046</td>\n",
       "      <td>-225</td>\n",
       "      <td>-4260.0</td>\n",
       "      <td>-2531</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>MONDAY</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Government</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.555912</td>\n",
       "      <td>0.729567</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-815.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Civil marriage</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.008019</td>\n",
       "      <td>-19005</td>\n",
       "      <td>-3039</td>\n",
       "      <td>-9833.0</td>\n",
       "      <td>-2437</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-617.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.028663</td>\n",
       "      <td>-19932</td>\n",
       "      <td>-3038</td>\n",
       "      <td>-4311.0</td>\n",
       "      <td>-3458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>THURSDAY</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Religion</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1106.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002       1         Cash loans           M            N   \n",
       "1      100003       0         Cash loans           F            N   \n",
       "2      100004       0    Revolving loans           M            Y   \n",
       "3      100006       0         Cash loans           F            N   \n",
       "4      100007       0         Cash loans           M            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "2               Y             0           67500.0    135000.0       6750.0   \n",
       "3               Y             0          135000.0    312682.5      29686.5   \n",
       "4               Y             0          121500.0    513000.0      21865.5   \n",
       "\n",
       "   AMT_GOODS_PRICE NAME_TYPE_SUITE NAME_INCOME_TYPE  \\\n",
       "0         351000.0   Unaccompanied          Working   \n",
       "1        1129500.0          Family    State servant   \n",
       "2         135000.0   Unaccompanied          Working   \n",
       "3         297000.0   Unaccompanied          Working   \n",
       "4         513000.0   Unaccompanied          Working   \n",
       "\n",
       "             NAME_EDUCATION_TYPE    NAME_FAMILY_STATUS  NAME_HOUSING_TYPE  \\\n",
       "0  Secondary / secondary special  Single / not married  House / apartment   \n",
       "1               Higher education               Married  House / apartment   \n",
       "2  Secondary / secondary special  Single / not married  House / apartment   \n",
       "3  Secondary / secondary special        Civil marriage  House / apartment   \n",
       "4  Secondary / secondary special  Single / not married  House / apartment   \n",
       "\n",
       "   REGION_POPULATION_RELATIVE  DAYS_BIRTH  DAYS_EMPLOYED  DAYS_REGISTRATION  \\\n",
       "0                    0.018801       -9461           -637            -3648.0   \n",
       "1                    0.003541      -16765          -1188            -1186.0   \n",
       "2                    0.010032      -19046           -225            -4260.0   \n",
       "3                    0.008019      -19005          -3039            -9833.0   \n",
       "4                    0.028663      -19932          -3038            -4311.0   \n",
       "\n",
       "   DAYS_ID_PUBLISH  OWN_CAR_AGE  FLAG_MOBIL  FLAG_EMP_PHONE  FLAG_WORK_PHONE  \\\n",
       "0            -2120          NaN           1               1                0   \n",
       "1             -291          NaN           1               1                0   \n",
       "2            -2531         26.0           1               1                1   \n",
       "3            -2437          NaN           1               1                0   \n",
       "4            -3458          NaN           1               1                0   \n",
       "\n",
       "   FLAG_CONT_MOBILE  FLAG_PHONE  FLAG_EMAIL OCCUPATION_TYPE  CNT_FAM_MEMBERS  \\\n",
       "0                 1           1           0        Laborers              1.0   \n",
       "1                 1           1           0      Core staff              2.0   \n",
       "2                 1           1           0        Laborers              1.0   \n",
       "3                 1           0           0        Laborers              2.0   \n",
       "4                 1           0           0      Core staff              1.0   \n",
       "\n",
       "   REGION_RATING_CLIENT  REGION_RATING_CLIENT_W_CITY  \\\n",
       "0                     2                            2   \n",
       "1                     1                            1   \n",
       "2                     2                            2   \n",
       "3                     2                            2   \n",
       "4                     2                            2   \n",
       "\n",
       "  WEEKDAY_APPR_PROCESS_START  HOUR_APPR_PROCESS_START  \\\n",
       "0                  WEDNESDAY                       10   \n",
       "1                     MONDAY                       11   \n",
       "2                     MONDAY                        9   \n",
       "3                  WEDNESDAY                       17   \n",
       "4                   THURSDAY                       11   \n",
       "\n",
       "   REG_REGION_NOT_LIVE_REGION  REG_REGION_NOT_WORK_REGION  \\\n",
       "0                           0                           0   \n",
       "1                           0                           0   \n",
       "2                           0                           0   \n",
       "3                           0                           0   \n",
       "4                           0                           0   \n",
       "\n",
       "   LIVE_REGION_NOT_WORK_REGION  REG_CITY_NOT_LIVE_CITY  \\\n",
       "0                            0                       0   \n",
       "1                            0                       0   \n",
       "2                            0                       0   \n",
       "3                            0                       0   \n",
       "4                            0                       0   \n",
       "\n",
       "   REG_CITY_NOT_WORK_CITY  LIVE_CITY_NOT_WORK_CITY       ORGANIZATION_TYPE  \\\n",
       "0                       0                        0  Business Entity Type 3   \n",
       "1                       0                        0                  School   \n",
       "2                       0                        0              Government   \n",
       "3                       0                        0  Business Entity Type 3   \n",
       "4                       1                        1                Religion   \n",
       "\n",
       "   EXT_SOURCE_1  EXT_SOURCE_2  EXT_SOURCE_3  APARTMENTS_AVG  BASEMENTAREA_AVG  \\\n",
       "0      0.083037      0.262949      0.139376          0.0247            0.0369   \n",
       "1      0.311267      0.622246           NaN          0.0959            0.0529   \n",
       "2           NaN      0.555912      0.729567             NaN               NaN   \n",
       "3           NaN      0.650442           NaN             NaN               NaN   \n",
       "4           NaN      0.322738           NaN             NaN               NaN   \n",
       "\n",
       "   YEARS_BEGINEXPLUATATION_AVG  YEARS_BUILD_AVG  COMMONAREA_AVG  \\\n",
       "0                       0.9722           0.6192          0.0143   \n",
       "1                       0.9851           0.7960          0.0605   \n",
       "2                          NaN              NaN             NaN   \n",
       "3                          NaN              NaN             NaN   \n",
       "4                          NaN              NaN             NaN   \n",
       "\n",
       "   ELEVATORS_AVG  ENTRANCES_AVG  FLOORSMAX_AVG  FLOORSMIN_AVG  LANDAREA_AVG  \\\n",
       "0           0.00         0.0690         0.0833         0.1250        0.0369   \n",
       "1           0.08         0.0345         0.2917         0.3333        0.0130   \n",
       "2            NaN            NaN            NaN            NaN           NaN   \n",
       "3            NaN            NaN            NaN            NaN           NaN   \n",
       "4            NaN            NaN            NaN            NaN           NaN   \n",
       "\n",
       "   LIVINGAPARTMENTS_AVG  LIVINGAREA_AVG  NONLIVINGAPARTMENTS_AVG  \\\n",
       "0                0.0202          0.0190                   0.0000   \n",
       "1                0.0773          0.0549                   0.0039   \n",
       "2                   NaN             NaN                      NaN   \n",
       "3                   NaN             NaN                      NaN   \n",
       "4                   NaN             NaN                      NaN   \n",
       "\n",
       "   NONLIVINGAREA_AVG  APARTMENTS_MODE  BASEMENTAREA_MODE  \\\n",
       "0             0.0000           0.0252             0.0383   \n",
       "1             0.0098           0.0924             0.0538   \n",
       "2                NaN              NaN                NaN   \n",
       "3                NaN              NaN                NaN   \n",
       "4                NaN              NaN                NaN   \n",
       "\n",
       "   YEARS_BEGINEXPLUATATION_MODE  YEARS_BUILD_MODE  COMMONAREA_MODE  \\\n",
       "0                        0.9722            0.6341           0.0144   \n",
       "1                        0.9851            0.8040           0.0497   \n",
       "2                           NaN               NaN              NaN   \n",
       "3                           NaN               NaN              NaN   \n",
       "4                           NaN               NaN              NaN   \n",
       "\n",
       "   ELEVATORS_MODE  ENTRANCES_MODE  FLOORSMAX_MODE  FLOORSMIN_MODE  \\\n",
       "0          0.0000          0.0690          0.0833          0.1250   \n",
       "1          0.0806          0.0345          0.2917          0.3333   \n",
       "2             NaN             NaN             NaN             NaN   \n",
       "3             NaN             NaN             NaN             NaN   \n",
       "4             NaN             NaN             NaN             NaN   \n",
       "\n",
       "   LANDAREA_MODE  LIVINGAPARTMENTS_MODE  LIVINGAREA_MODE  \\\n",
       "0         0.0377                  0.022           0.0198   \n",
       "1         0.0128                  0.079           0.0554   \n",
       "2            NaN                    NaN              NaN   \n",
       "3            NaN                    NaN              NaN   \n",
       "4            NaN                    NaN              NaN   \n",
       "\n",
       "   NONLIVINGAPARTMENTS_MODE  NONLIVINGAREA_MODE  APARTMENTS_MEDI  \\\n",
       "0                       0.0                 0.0           0.0250   \n",
       "1                       0.0                 0.0           0.0968   \n",
       "2                       NaN                 NaN              NaN   \n",
       "3                       NaN                 NaN              NaN   \n",
       "4                       NaN                 NaN              NaN   \n",
       "\n",
       "   BASEMENTAREA_MEDI  YEARS_BEGINEXPLUATATION_MEDI  YEARS_BUILD_MEDI  \\\n",
       "0             0.0369                        0.9722            0.6243   \n",
       "1             0.0529                        0.9851            0.7987   \n",
       "2                NaN                           NaN               NaN   \n",
       "3                NaN                           NaN               NaN   \n",
       "4                NaN                           NaN               NaN   \n",
       "\n",
       "   COMMONAREA_MEDI  ELEVATORS_MEDI  ENTRANCES_MEDI  FLOORSMAX_MEDI  \\\n",
       "0           0.0144            0.00          0.0690          0.0833   \n",
       "1           0.0608            0.08          0.0345          0.2917   \n",
       "2              NaN             NaN             NaN             NaN   \n",
       "3              NaN             NaN             NaN             NaN   \n",
       "4              NaN             NaN             NaN             NaN   \n",
       "\n",
       "   FLOORSMIN_MEDI  LANDAREA_MEDI  LIVINGAPARTMENTS_MEDI  LIVINGAREA_MEDI  \\\n",
       "0          0.1250         0.0375                 0.0205           0.0193   \n",
       "1          0.3333         0.0132                 0.0787           0.0558   \n",
       "2             NaN            NaN                    NaN              NaN   \n",
       "3             NaN            NaN                    NaN              NaN   \n",
       "4             NaN            NaN                    NaN              NaN   \n",
       "\n",
       "   NONLIVINGAPARTMENTS_MEDI  NONLIVINGAREA_MEDI FONDKAPREMONT_MODE  \\\n",
       "0                    0.0000                0.00   reg oper account   \n",
       "1                    0.0039                0.01   reg oper account   \n",
       "2                       NaN                 NaN                NaN   \n",
       "3                       NaN                 NaN                NaN   \n",
       "4                       NaN                 NaN                NaN   \n",
       "\n",
       "   HOUSETYPE_MODE  TOTALAREA_MODE WALLSMATERIAL_MODE EMERGENCYSTATE_MODE  \\\n",
       "0  block of flats          0.0149       Stone, brick                  No   \n",
       "1  block of flats          0.0714              Block                  No   \n",
       "2             NaN             NaN                NaN                 NaN   \n",
       "3             NaN             NaN                NaN                 NaN   \n",
       "4             NaN             NaN                NaN                 NaN   \n",
       "\n",
       "   OBS_30_CNT_SOCIAL_CIRCLE  DEF_30_CNT_SOCIAL_CIRCLE  \\\n",
       "0                       2.0                       2.0   \n",
       "1                       1.0                       0.0   \n",
       "2                       0.0                       0.0   \n",
       "3                       2.0                       0.0   \n",
       "4                       0.0                       0.0   \n",
       "\n",
       "   OBS_60_CNT_SOCIAL_CIRCLE  DEF_60_CNT_SOCIAL_CIRCLE  DAYS_LAST_PHONE_CHANGE  \\\n",
       "0                       2.0                       2.0                 -1134.0   \n",
       "1                       1.0                       0.0                  -828.0   \n",
       "2                       0.0                       0.0                  -815.0   \n",
       "3                       2.0                       0.0                  -617.0   \n",
       "4                       0.0                       0.0                 -1106.0   \n",
       "\n",
       "   FLAG_DOCUMENT_2  FLAG_DOCUMENT_3  FLAG_DOCUMENT_4  FLAG_DOCUMENT_5  \\\n",
       "0                0                1                0                0   \n",
       "1                0                1                0                0   \n",
       "2                0                0                0                0   \n",
       "3                0                1                0                0   \n",
       "4                0                0                0                0   \n",
       "\n",
       "   FLAG_DOCUMENT_6  FLAG_DOCUMENT_7  FLAG_DOCUMENT_8  FLAG_DOCUMENT_9  \\\n",
       "0                0                0                0                0   \n",
       "1                0                0                0                0   \n",
       "2                0                0                0                0   \n",
       "3                0                0                0                0   \n",
       "4                0                0                1                0   \n",
       "\n",
       "   FLAG_DOCUMENT_10  FLAG_DOCUMENT_11  FLAG_DOCUMENT_12  FLAG_DOCUMENT_13  \\\n",
       "0                 0                 0                 0                 0   \n",
       "1                 0                 0                 0                 0   \n",
       "2                 0                 0                 0                 0   \n",
       "3                 0                 0                 0                 0   \n",
       "4                 0                 0                 0                 0   \n",
       "\n",
       "   FLAG_DOCUMENT_14  FLAG_DOCUMENT_15  FLAG_DOCUMENT_16  FLAG_DOCUMENT_17  \\\n",
       "0                 0                 0                 0                 0   \n",
       "1                 0                 0                 0                 0   \n",
       "2                 0                 0                 0                 0   \n",
       "3                 0                 0                 0                 0   \n",
       "4                 0                 0                 0                 0   \n",
       "\n",
       "   FLAG_DOCUMENT_18  FLAG_DOCUMENT_19  FLAG_DOCUMENT_20  FLAG_DOCUMENT_21  \\\n",
       "0                 0                 0                 0                 0   \n",
       "1                 0                 0                 0                 0   \n",
       "2                 0                 0                 0                 0   \n",
       "3                 0                 0                 0                 0   \n",
       "4                 0                 0                 0                 0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_HOUR  AMT_REQ_CREDIT_BUREAU_DAY  \\\n",
       "0                         0.0                        0.0   \n",
       "1                         0.0                        0.0   \n",
       "2                         0.0                        0.0   \n",
       "3                         NaN                        NaN   \n",
       "4                         0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  \\\n",
       "0                         0.0                        0.0   \n",
       "1                         0.0                        0.0   \n",
       "2                         0.0                        0.0   \n",
       "3                         NaN                        NaN   \n",
       "4                         0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                        0.0                         1.0  \n",
       "1                        0.0                         0.0  \n",
       "2                        0.0                         0.0  \n",
       "3                        NaN                         NaN  \n",
       "4                        0.0                         0.0  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"credit\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}application_train.csv.zip\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"SK_ID_CURR\"],\n",
    "    \"cat_cols\": ['NAME_CONTRACT_TYPE', 'CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY',\n",
    "                 \"REGION_RATING_CLIENT\", \"REGION_RATING_CLIENT_W_CITY\", \"NAME_TYPE_SUITE\",\n",
    "                 'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE',\n",
    "                 'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE',\n",
    "                 'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE',\n",
    "                 'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE'],\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"TARGET\"] == 1\n",
    "data.drop([\"TARGET\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  18\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>num_4</th>\n",
       "      <th>num_5</th>\n",
       "      <th>num_6</th>\n",
       "      <th>num_7</th>\n",
       "      <th>num_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>cat_10</th>\n",
       "      <th>cat_11</th>\n",
       "      <th>cat_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>num_14</th>\n",
       "      <th>num_15</th>\n",
       "      <th>num_16</th>\n",
       "      <th>num_17</th>\n",
       "      <th>num_18</th>\n",
       "      <th>num_19</th>\n",
       "      <th>num_20</th>\n",
       "      <th>num_21</th>\n",
       "      <th>num_22</th>\n",
       "      <th>num_23</th>\n",
       "      <th>num_24</th>\n",
       "      <th>num_25</th>\n",
       "      <th>cat_26</th>\n",
       "      <th>num_27</th>\n",
       "      <th>cat_28</th>\n",
       "      <th>cat_29</th>\n",
       "      <th>cat_30</th>\n",
       "      <th>num_31</th>\n",
       "      <th>num_32</th>\n",
       "      <th>num_33</th>\n",
       "      <th>num_34</th>\n",
       "      <th>num_35</th>\n",
       "      <th>num_36</th>\n",
       "      <th>num_37</th>\n",
       "      <th>cat_38</th>\n",
       "      <th>num_39</th>\n",
       "      <th>num_40</th>\n",
       "      <th>num_41</th>\n",
       "      <th>num_42</th>\n",
       "      <th>num_43</th>\n",
       "      <th>num_44</th>\n",
       "      <th>num_45</th>\n",
       "      <th>num_46</th>\n",
       "      <th>num_47</th>\n",
       "      <th>num_48</th>\n",
       "      <th>num_49</th>\n",
       "      <th>num_50</th>\n",
       "      <th>num_51</th>\n",
       "      <th>num_52</th>\n",
       "      <th>num_53</th>\n",
       "      <th>num_54</th>\n",
       "      <th>num_55</th>\n",
       "      <th>num_56</th>\n",
       "      <th>num_57</th>\n",
       "      <th>num_58</th>\n",
       "      <th>num_59</th>\n",
       "      <th>num_60</th>\n",
       "      <th>num_61</th>\n",
       "      <th>num_62</th>\n",
       "      <th>num_63</th>\n",
       "      <th>num_64</th>\n",
       "      <th>num_65</th>\n",
       "      <th>num_66</th>\n",
       "      <th>num_67</th>\n",
       "      <th>num_68</th>\n",
       "      <th>num_69</th>\n",
       "      <th>num_70</th>\n",
       "      <th>num_71</th>\n",
       "      <th>num_72</th>\n",
       "      <th>num_73</th>\n",
       "      <th>num_74</th>\n",
       "      <th>num_75</th>\n",
       "      <th>num_76</th>\n",
       "      <th>num_77</th>\n",
       "      <th>num_78</th>\n",
       "      <th>num_79</th>\n",
       "      <th>num_80</th>\n",
       "      <th>num_81</th>\n",
       "      <th>num_82</th>\n",
       "      <th>num_83</th>\n",
       "      <th>cat_84</th>\n",
       "      <th>cat_85</th>\n",
       "      <th>num_86</th>\n",
       "      <th>cat_87</th>\n",
       "      <th>cat_88</th>\n",
       "      <th>num_89</th>\n",
       "      <th>num_90</th>\n",
       "      <th>num_91</th>\n",
       "      <th>num_92</th>\n",
       "      <th>num_93</th>\n",
       "      <th>num_94</th>\n",
       "      <th>num_95</th>\n",
       "      <th>num_96</th>\n",
       "      <th>num_97</th>\n",
       "      <th>num_98</th>\n",
       "      <th>num_99</th>\n",
       "      <th>num_100</th>\n",
       "      <th>num_101</th>\n",
       "      <th>num_102</th>\n",
       "      <th>num_103</th>\n",
       "      <th>num_104</th>\n",
       "      <th>num_105</th>\n",
       "      <th>num_106</th>\n",
       "      <th>num_107</th>\n",
       "      <th>num_108</th>\n",
       "      <th>num_109</th>\n",
       "      <th>num_110</th>\n",
       "      <th>num_111</th>\n",
       "      <th>num_112</th>\n",
       "      <th>num_113</th>\n",
       "      <th>num_114</th>\n",
       "      <th>num_115</th>\n",
       "      <th>num_116</th>\n",
       "      <th>num_117</th>\n",
       "      <th>num_118</th>\n",
       "      <th>num_119</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.018801</td>\n",
       "      <td>-9461</td>\n",
       "      <td>-637</td>\n",
       "      <td>-3648.0</td>\n",
       "      <td>-2120</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>0.083037</td>\n",
       "      <td>0.262949</td>\n",
       "      <td>0.139376</td>\n",
       "      <td>0.0247</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6192</td>\n",
       "      <td>0.0143</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.0202</td>\n",
       "      <td>0.0190</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0252</td>\n",
       "      <td>0.0383</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6341</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0377</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0.0198</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0250</td>\n",
       "      <td>0.0369</td>\n",
       "      <td>0.9722</td>\n",
       "      <td>0.6243</td>\n",
       "      <td>0.0144</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0690</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.0375</td>\n",
       "      <td>0.0205</td>\n",
       "      <td>0.0193</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.0149</td>\n",
       "      <td>Stone, brick</td>\n",
       "      <td>No</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-1134.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>Family</td>\n",
       "      <td>State servant</td>\n",
       "      <td>Higher education</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>-16765</td>\n",
       "      <td>-1188</td>\n",
       "      <td>-1186.0</td>\n",
       "      <td>-291</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>MONDAY</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>School</td>\n",
       "      <td>0.311267</td>\n",
       "      <td>0.622246</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0959</td>\n",
       "      <td>0.0529</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.7960</td>\n",
       "      <td>0.0605</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0130</td>\n",
       "      <td>0.0773</td>\n",
       "      <td>0.0549</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.0098</td>\n",
       "      <td>0.0924</td>\n",
       "      <td>0.0538</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.8040</td>\n",
       "      <td>0.0497</td>\n",
       "      <td>0.0806</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0128</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.0554</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0968</td>\n",
       "      <td>0.0529</td>\n",
       "      <td>0.9851</td>\n",
       "      <td>0.7987</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.0345</td>\n",
       "      <td>0.2917</td>\n",
       "      <td>0.3333</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.0787</td>\n",
       "      <td>0.0558</td>\n",
       "      <td>0.0039</td>\n",
       "      <td>0.01</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>Block</td>\n",
       "      <td>No</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-828.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.010032</td>\n",
       "      <td>-19046</td>\n",
       "      <td>-225</td>\n",
       "      <td>-4260.0</td>\n",
       "      <td>-2531</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>MONDAY</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Government</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.555912</td>\n",
       "      <td>0.729567</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-815.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Civil marriage</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.008019</td>\n",
       "      <td>-19005</td>\n",
       "      <td>-3039</td>\n",
       "      <td>-9833.0</td>\n",
       "      <td>-2437</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-617.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Working</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>0.028663</td>\n",
       "      <td>-19932</td>\n",
       "      <td>-3038</td>\n",
       "      <td>-4311.0</td>\n",
       "      <td>-3458</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>THURSDAY</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Religion</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.322738</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1106.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             cat_0 cat_1 cat_2 cat_3  num_4     num_5      num_6    num_7  \\\n",
       "0       Cash loans     M     N     Y      0  202500.0   406597.5  24700.5   \n",
       "1       Cash loans     F     N     N      0  270000.0  1293502.5  35698.5   \n",
       "2  Revolving loans     M     Y     Y      0   67500.0   135000.0   6750.0   \n",
       "3       Cash loans     F     N     Y      0  135000.0   312682.5  29686.5   \n",
       "4       Cash loans     M     N     Y      0  121500.0   513000.0  21865.5   \n",
       "\n",
       "       num_8          cat_9         cat_10                         cat_11  \\\n",
       "0   351000.0  Unaccompanied        Working  Secondary / secondary special   \n",
       "1  1129500.0         Family  State servant               Higher education   \n",
       "2   135000.0  Unaccompanied        Working  Secondary / secondary special   \n",
       "3   297000.0  Unaccompanied        Working  Secondary / secondary special   \n",
       "4   513000.0  Unaccompanied        Working  Secondary / secondary special   \n",
       "\n",
       "                 cat_12             cat_13    num_14  num_15  num_16  num_17  \\\n",
       "0  Single / not married  House / apartment  0.018801   -9461    -637 -3648.0   \n",
       "1               Married  House / apartment  0.003541  -16765   -1188 -1186.0   \n",
       "2  Single / not married  House / apartment  0.010032  -19046    -225 -4260.0   \n",
       "3        Civil marriage  House / apartment  0.008019  -19005   -3039 -9833.0   \n",
       "4  Single / not married  House / apartment  0.028663  -19932   -3038 -4311.0   \n",
       "\n",
       "   num_18  num_19  num_20  num_21  num_22  num_23  num_24  num_25      cat_26  \\\n",
       "0   -2120     NaN       1       1       0       1       1       0    Laborers   \n",
       "1    -291     NaN       1       1       0       1       1       0  Core staff   \n",
       "2   -2531    26.0       1       1       1       1       1       0    Laborers   \n",
       "3   -2437     NaN       1       1       0       1       0       0    Laborers   \n",
       "4   -3458     NaN       1       1       0       1       0       0  Core staff   \n",
       "\n",
       "   num_27 cat_28 cat_29     cat_30  num_31  num_32  num_33  num_34  num_35  \\\n",
       "0     1.0      2      2  WEDNESDAY      10       0       0       0       0   \n",
       "1     2.0      1      1     MONDAY      11       0       0       0       0   \n",
       "2     1.0      2      2     MONDAY       9       0       0       0       0   \n",
       "3     2.0      2      2  WEDNESDAY      17       0       0       0       0   \n",
       "4     1.0      2      2   THURSDAY      11       0       0       0       0   \n",
       "\n",
       "   num_36  num_37                  cat_38    num_39    num_40    num_41  \\\n",
       "0       0       0  Business Entity Type 3  0.083037  0.262949  0.139376   \n",
       "1       0       0                  School  0.311267  0.622246       NaN   \n",
       "2       0       0              Government       NaN  0.555912  0.729567   \n",
       "3       0       0  Business Entity Type 3       NaN  0.650442       NaN   \n",
       "4       1       1                Religion       NaN  0.322738       NaN   \n",
       "\n",
       "   num_42  num_43  num_44  num_45  num_46  num_47  num_48  num_49  num_50  \\\n",
       "0  0.0247  0.0369  0.9722  0.6192  0.0143    0.00  0.0690  0.0833  0.1250   \n",
       "1  0.0959  0.0529  0.9851  0.7960  0.0605    0.08  0.0345  0.2917  0.3333   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_51  num_52  num_53  num_54  num_55  num_56  num_57  num_58  num_59  \\\n",
       "0  0.0369  0.0202  0.0190  0.0000  0.0000  0.0252  0.0383  0.9722  0.6341   \n",
       "1  0.0130  0.0773  0.0549  0.0039  0.0098  0.0924  0.0538  0.9851  0.8040   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_60  num_61  num_62  num_63  num_64  num_65  num_66  num_67  num_68  \\\n",
       "0  0.0144  0.0000  0.0690  0.0833  0.1250  0.0377   0.022  0.0198     0.0   \n",
       "1  0.0497  0.0806  0.0345  0.2917  0.3333  0.0128   0.079  0.0554     0.0   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_69  num_70  num_71  num_72  num_73  num_74  num_75  num_76  num_77  \\\n",
       "0     0.0  0.0250  0.0369  0.9722  0.6243  0.0144    0.00  0.0690  0.0833   \n",
       "1     0.0  0.0968  0.0529  0.9851  0.7987  0.0608    0.08  0.0345  0.2917   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_78  num_79  num_80  num_81  num_82  num_83            cat_84  \\\n",
       "0  0.1250  0.0375  0.0205  0.0193  0.0000    0.00  reg oper account   \n",
       "1  0.3333  0.0132  0.0787  0.0558  0.0039    0.01  reg oper account   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN  new_category_nan   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN  new_category_nan   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN  new_category_nan   \n",
       "\n",
       "             cat_85  num_86            cat_87            cat_88  num_89  \\\n",
       "0    block of flats  0.0149      Stone, brick                No     2.0   \n",
       "1    block of flats  0.0714             Block                No     1.0   \n",
       "2  new_category_nan     NaN  new_category_nan  new_category_nan     0.0   \n",
       "3  new_category_nan     NaN  new_category_nan  new_category_nan     2.0   \n",
       "4  new_category_nan     NaN  new_category_nan  new_category_nan     0.0   \n",
       "\n",
       "   num_90  num_91  num_92  num_93  num_94  num_95  num_96  num_97  num_98  \\\n",
       "0     2.0     2.0     2.0 -1134.0       0       1       0       0       0   \n",
       "1     0.0     1.0     0.0  -828.0       0       1       0       0       0   \n",
       "2     0.0     0.0     0.0  -815.0       0       0       0       0       0   \n",
       "3     0.0     2.0     0.0  -617.0       0       1       0       0       0   \n",
       "4     0.0     0.0     0.0 -1106.0       0       0       0       0       0   \n",
       "\n",
       "   num_99  num_100  num_101  num_102  num_103  num_104  num_105  num_106  \\\n",
       "0       0        0        0        0        0        0        0        0   \n",
       "1       0        0        0        0        0        0        0        0   \n",
       "2       0        0        0        0        0        0        0        0   \n",
       "3       0        0        0        0        0        0        0        0   \n",
       "4       0        1        0        0        0        0        0        0   \n",
       "\n",
       "   num_107  num_108  num_109  num_110  num_111  num_112  num_113  num_114  \\\n",
       "0        0        0        0        0        0        0        0      0.0   \n",
       "1        0        0        0        0        0        0        0      0.0   \n",
       "2        0        0        0        0        0        0        0      0.0   \n",
       "3        0        0        0        0        0        0        0      NaN   \n",
       "4        0        0        0        0        0        0        0      0.0   \n",
       "\n",
       "   num_115  num_116  num_117  num_118  num_119  target  \n",
       "0      0.0      0.0      0.0      0.0      1.0    True  \n",
       "1      0.0      0.0      0.0      0.0      0.0   False  \n",
       "2      0.0      0.0      0.0      0.0      0.0   False  \n",
       "3      NaN      NaN      NaN      NaN      NaN   False  \n",
       "4      0.0      0.0      0.0      0.0      0.0   False  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "cat_3 object\n",
      "num_4 int64\n",
      "num_5 float64\n",
      "num_6 float64\n",
      "num_7 float64\n",
      "num_8 float64\n",
      "cat_9 object\n",
      "cat_10 object\n",
      "cat_11 object\n",
      "cat_12 object\n",
      "cat_13 object\n",
      "num_14 float64\n",
      "num_15 int64\n",
      "num_16 int64\n",
      "num_17 float64\n",
      "num_18 int64\n",
      "num_19 float64\n",
      "num_20 int64\n",
      "num_21 int64\n",
      "num_22 int64\n",
      "num_23 int64\n",
      "num_24 int64\n",
      "num_25 int64\n",
      "cat_26 object\n",
      "num_27 float64\n",
      "cat_28 object\n",
      "cat_29 object\n",
      "cat_30 object\n",
      "num_31 int64\n",
      "num_32 int64\n",
      "num_33 int64\n",
      "num_34 int64\n",
      "num_35 int64\n",
      "num_36 int64\n",
      "num_37 int64\n",
      "cat_38 object\n",
      "num_39 float64\n",
      "num_40 float64\n",
      "num_41 float64\n",
      "num_42 float64\n",
      "num_43 float64\n",
      "num_44 float64\n",
      "num_45 float64\n",
      "num_46 float64\n",
      "num_47 float64\n",
      "num_48 float64\n",
      "num_49 float64\n",
      "num_50 float64\n",
      "num_51 float64\n",
      "num_52 float64\n",
      "num_53 float64\n",
      "num_54 float64\n",
      "num_55 float64\n",
      "num_56 float64\n",
      "num_57 float64\n",
      "num_58 float64\n",
      "num_59 float64\n",
      "num_60 float64\n",
      "num_61 float64\n",
      "num_62 float64\n",
      "num_63 float64\n",
      "num_64 float64\n",
      "num_65 float64\n",
      "num_66 float64\n",
      "num_67 float64\n",
      "num_68 float64\n",
      "num_69 float64\n",
      "num_70 float64\n",
      "num_71 float64\n",
      "num_72 float64\n",
      "num_73 float64\n",
      "num_74 float64\n",
      "num_75 float64\n",
      "num_76 float64\n",
      "num_77 float64\n",
      "num_78 float64\n",
      "num_79 float64\n",
      "num_80 float64\n",
      "num_81 float64\n",
      "num_82 float64\n",
      "num_83 float64\n",
      "cat_84 object\n",
      "cat_85 object\n",
      "num_86 float64\n",
      "cat_87 object\n",
      "cat_88 object\n",
      "num_89 float64\n",
      "num_90 float64\n",
      "num_91 float64\n",
      "num_92 float64\n",
      "num_93 float64\n",
      "num_94 int64\n",
      "num_95 int64\n",
      "num_96 int64\n",
      "num_97 int64\n",
      "num_98 int64\n",
      "num_99 int64\n",
      "num_100 int64\n",
      "num_101 int64\n",
      "num_102 int64\n",
      "num_103 int64\n",
      "num_104 int64\n",
      "num_105 int64\n",
      "num_106 int64\n",
      "num_107 int64\n",
      "num_108 int64\n",
      "num_109 int64\n",
      "num_110 int64\n",
      "num_111 int64\n",
      "num_112 int64\n",
      "num_113 int64\n",
      "num_114 float64\n",
      "num_115 float64\n",
      "num_116 float64\n",
      "num_117 float64\n",
      "num_118 float64\n",
      "num_119 float64\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# adult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/wenruliu/adult-income-dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48842, 15)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>educational-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>gender</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>Private</td>\n",
       "      <td>226802</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>89814</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Farming-fishing</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>336951</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44</td>\n",
       "      <td>Private</td>\n",
       "      <td>160323</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>7688</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>?</td>\n",
       "      <td>103497</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>?</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  workclass  fnlwgt     education  educational-num      marital-status  \\\n",
       "0   25    Private  226802          11th                7       Never-married   \n",
       "1   38    Private   89814       HS-grad                9  Married-civ-spouse   \n",
       "2   28  Local-gov  336951    Assoc-acdm               12  Married-civ-spouse   \n",
       "3   44    Private  160323  Some-college               10  Married-civ-spouse   \n",
       "4   18          ?  103497  Some-college               10       Never-married   \n",
       "\n",
       "          occupation relationship   race  gender  capital-gain  capital-loss  \\\n",
       "0  Machine-op-inspct    Own-child  Black    Male             0             0   \n",
       "1    Farming-fishing      Husband  White    Male             0             0   \n",
       "2    Protective-serv      Husband  White    Male             0             0   \n",
       "3  Machine-op-inspct      Husband  Black    Male          7688             0   \n",
       "4                  ?    Own-child  White  Female             0             0   \n",
       "\n",
       "   hours-per-week native-country income  \n",
       "0              40  United-States  <=50K  \n",
       "1              50  United-States  <=50K  \n",
       "2              40  United-States   >50K  \n",
       "3              40  United-States   >50K  \n",
       "4              30  United-States  <=50K  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"adult\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}adult.csv.zip\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [],\n",
    "    \"cat_cols\": [\"workclass\", \"education\", \"marital-status\", \"occupation\", \"relationship\", \"race\", \"gender\", \"native-country\"],\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"income\"] == \">50K\"\n",
    "data.drop([\"income\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>num_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>num_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>cat_6</th>\n",
       "      <th>cat_7</th>\n",
       "      <th>cat_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>num_10</th>\n",
       "      <th>num_11</th>\n",
       "      <th>num_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>Private</td>\n",
       "      <td>226802</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>89814</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Farming-fishing</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>United-States</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>336951</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44</td>\n",
       "      <td>Private</td>\n",
       "      <td>160323</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>7688</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>?</td>\n",
       "      <td>103497</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>?</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>United-States</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   num_0      cat_1   num_2         cat_3  num_4               cat_5  \\\n",
       "0     25    Private  226802          11th      7       Never-married   \n",
       "1     38    Private   89814       HS-grad      9  Married-civ-spouse   \n",
       "2     28  Local-gov  336951    Assoc-acdm     12  Married-civ-spouse   \n",
       "3     44    Private  160323  Some-college     10  Married-civ-spouse   \n",
       "4     18          ?  103497  Some-college     10       Never-married   \n",
       "\n",
       "               cat_6      cat_7  cat_8   cat_9  num_10  num_11  num_12  \\\n",
       "0  Machine-op-inspct  Own-child  Black    Male       0       0      40   \n",
       "1    Farming-fishing    Husband  White    Male       0       0      50   \n",
       "2    Protective-serv    Husband  White    Male       0       0      40   \n",
       "3  Machine-op-inspct    Husband  Black    Male    7688       0      40   \n",
       "4                  ?  Own-child  White  Female       0       0      30   \n",
       "\n",
       "          cat_13  target  \n",
       "0  United-States   False  \n",
       "1  United-States   False  \n",
       "2  United-States    True  \n",
       "3  United-States    True  \n",
       "4  United-States   False  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_0 int64\n",
      "cat_1 object\n",
      "num_2 int64\n",
      "cat_3 object\n",
      "num_4 int64\n",
      "cat_5 object\n",
      "cat_6 object\n",
      "cat_7 object\n",
      "cat_8 object\n",
      "cat_9 object\n",
      "num_10 int64\n",
      "num_11 int64\n",
      "num_12 int64\n",
      "cat_13 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# employee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/c/amazon-employee-access-challenge/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32769, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACTION</th>\n",
       "      <th>RESOURCE</th>\n",
       "      <th>MGR_ID</th>\n",
       "      <th>ROLE_ROLLUP_1</th>\n",
       "      <th>ROLE_ROLLUP_2</th>\n",
       "      <th>ROLE_DEPTNAME</th>\n",
       "      <th>ROLE_TITLE</th>\n",
       "      <th>ROLE_FAMILY_DESC</th>\n",
       "      <th>ROLE_FAMILY</th>\n",
       "      <th>ROLE_CODE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>39353</td>\n",
       "      <td>85475</td>\n",
       "      <td>117961</td>\n",
       "      <td>118300</td>\n",
       "      <td>123472</td>\n",
       "      <td>117905</td>\n",
       "      <td>117906</td>\n",
       "      <td>290919</td>\n",
       "      <td>117908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>17183</td>\n",
       "      <td>1540</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>123125</td>\n",
       "      <td>118536</td>\n",
       "      <td>118536</td>\n",
       "      <td>308574</td>\n",
       "      <td>118539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>36724</td>\n",
       "      <td>14457</td>\n",
       "      <td>118219</td>\n",
       "      <td>118220</td>\n",
       "      <td>117884</td>\n",
       "      <td>117879</td>\n",
       "      <td>267952</td>\n",
       "      <td>19721</td>\n",
       "      <td>117880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>36135</td>\n",
       "      <td>5396</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>119993</td>\n",
       "      <td>118321</td>\n",
       "      <td>240983</td>\n",
       "      <td>290919</td>\n",
       "      <td>118322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>42680</td>\n",
       "      <td>5905</td>\n",
       "      <td>117929</td>\n",
       "      <td>117930</td>\n",
       "      <td>119569</td>\n",
       "      <td>119323</td>\n",
       "      <td>123932</td>\n",
       "      <td>19793</td>\n",
       "      <td>119325</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ACTION  RESOURCE  MGR_ID  ROLE_ROLLUP_1  ROLE_ROLLUP_2  ROLE_DEPTNAME  \\\n",
       "0       1     39353   85475         117961         118300         123472   \n",
       "1       1     17183    1540         117961         118343         123125   \n",
       "2       1     36724   14457         118219         118220         117884   \n",
       "3       1     36135    5396         117961         118343         119993   \n",
       "4       1     42680    5905         117929         117930         119569   \n",
       "\n",
       "   ROLE_TITLE  ROLE_FAMILY_DESC  ROLE_FAMILY  ROLE_CODE  \n",
       "0      117905            117906       290919     117908  \n",
       "1      118536            118536       308574     118539  \n",
       "2      117879            267952        19721     117880  \n",
       "3      118321            240983       290919     118322  \n",
       "4      119323            123932        19793     119325  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"employee\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}train.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [],\n",
    "    \"cat_cols\": data.columns,\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"ACTION\"] == 1\n",
    "data.drop([\"ACTION\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  9\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>cat_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>cat_6</th>\n",
       "      <th>cat_7</th>\n",
       "      <th>cat_8</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39353</td>\n",
       "      <td>85475</td>\n",
       "      <td>117961</td>\n",
       "      <td>118300</td>\n",
       "      <td>123472</td>\n",
       "      <td>117905</td>\n",
       "      <td>117906</td>\n",
       "      <td>290919</td>\n",
       "      <td>117908</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17183</td>\n",
       "      <td>1540</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>123125</td>\n",
       "      <td>118536</td>\n",
       "      <td>118536</td>\n",
       "      <td>308574</td>\n",
       "      <td>118539</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36724</td>\n",
       "      <td>14457</td>\n",
       "      <td>118219</td>\n",
       "      <td>118220</td>\n",
       "      <td>117884</td>\n",
       "      <td>117879</td>\n",
       "      <td>267952</td>\n",
       "      <td>19721</td>\n",
       "      <td>117880</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36135</td>\n",
       "      <td>5396</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>119993</td>\n",
       "      <td>118321</td>\n",
       "      <td>240983</td>\n",
       "      <td>290919</td>\n",
       "      <td>118322</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>42680</td>\n",
       "      <td>5905</td>\n",
       "      <td>117929</td>\n",
       "      <td>117930</td>\n",
       "      <td>119569</td>\n",
       "      <td>119323</td>\n",
       "      <td>123932</td>\n",
       "      <td>19793</td>\n",
       "      <td>119325</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat_0  cat_1   cat_2   cat_3   cat_4   cat_5   cat_6   cat_7   cat_8  \\\n",
       "0  39353  85475  117961  118300  123472  117905  117906  290919  117908   \n",
       "1  17183   1540  117961  118343  123125  118536  118536  308574  118539   \n",
       "2  36724  14457  118219  118220  117884  117879  267952   19721  117880   \n",
       "3  36135   5396  117961  118343  119993  118321  240983  290919  118322   \n",
       "4  42680   5905  117929  117930  119569  119323  123932   19793  119325   \n",
       "\n",
       "   target  \n",
       "0    True  \n",
       "1    True  \n",
       "2    True  \n",
       "3    True  \n",
       "4    True  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "cat_3 object\n",
      "cat_4 object\n",
      "cat_5 object\n",
      "cat_6 object\n",
      "cat_7 object\n",
      "cat_8 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# kick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/c/DontGetKicked/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(72983, 34)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RefId</th>\n",
       "      <th>IsBadBuy</th>\n",
       "      <th>PurchDate</th>\n",
       "      <th>Auction</th>\n",
       "      <th>VehYear</th>\n",
       "      <th>VehicleAge</th>\n",
       "      <th>Make</th>\n",
       "      <th>Model</th>\n",
       "      <th>Trim</th>\n",
       "      <th>SubModel</th>\n",
       "      <th>Color</th>\n",
       "      <th>Transmission</th>\n",
       "      <th>WheelTypeID</th>\n",
       "      <th>WheelType</th>\n",
       "      <th>VehOdo</th>\n",
       "      <th>Nationality</th>\n",
       "      <th>Size</th>\n",
       "      <th>TopThreeAmericanName</th>\n",
       "      <th>MMRAcquisitionAuctionAveragePrice</th>\n",
       "      <th>MMRAcquisitionAuctionCleanPrice</th>\n",
       "      <th>MMRAcquisitionRetailAveragePrice</th>\n",
       "      <th>MMRAcquisitonRetailCleanPrice</th>\n",
       "      <th>MMRCurrentAuctionAveragePrice</th>\n",
       "      <th>MMRCurrentAuctionCleanPrice</th>\n",
       "      <th>MMRCurrentRetailAveragePrice</th>\n",
       "      <th>MMRCurrentRetailCleanPrice</th>\n",
       "      <th>PRIMEUNIT</th>\n",
       "      <th>AUCGUART</th>\n",
       "      <th>BYRNO</th>\n",
       "      <th>VNZIP1</th>\n",
       "      <th>VNST</th>\n",
       "      <th>VehBCost</th>\n",
       "      <th>IsOnlineSale</th>\n",
       "      <th>WarrantyCost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12/7/2009</td>\n",
       "      <td>ADESA</td>\n",
       "      <td>2006</td>\n",
       "      <td>3</td>\n",
       "      <td>MAZDA</td>\n",
       "      <td>MAZDA3</td>\n",
       "      <td>i</td>\n",
       "      <td>4D SEDAN I</td>\n",
       "      <td>RED</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Alloy</td>\n",
       "      <td>89046</td>\n",
       "      <td>OTHER ASIAN</td>\n",
       "      <td>MEDIUM</td>\n",
       "      <td>OTHER</td>\n",
       "      <td>8155.0</td>\n",
       "      <td>9829.0</td>\n",
       "      <td>11636.0</td>\n",
       "      <td>13600.0</td>\n",
       "      <td>7451.0</td>\n",
       "      <td>8552.0</td>\n",
       "      <td>11597.0</td>\n",
       "      <td>12409.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21973</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>7100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12/7/2009</td>\n",
       "      <td>ADESA</td>\n",
       "      <td>2004</td>\n",
       "      <td>5</td>\n",
       "      <td>DODGE</td>\n",
       "      <td>1500 RAM PICKUP 2WD</td>\n",
       "      <td>ST</td>\n",
       "      <td>QUAD CAB 4.7L SLT</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Alloy</td>\n",
       "      <td>93593</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>LARGE TRUCK</td>\n",
       "      <td>CHRYSLER</td>\n",
       "      <td>6854.0</td>\n",
       "      <td>8383.0</td>\n",
       "      <td>10897.0</td>\n",
       "      <td>12572.0</td>\n",
       "      <td>7456.0</td>\n",
       "      <td>9222.0</td>\n",
       "      <td>11374.0</td>\n",
       "      <td>12791.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>7600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>12/7/2009</td>\n",
       "      <td>ADESA</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "      <td>DODGE</td>\n",
       "      <td>STRATUS V6</td>\n",
       "      <td>SXT</td>\n",
       "      <td>4D SEDAN SXT FFV</td>\n",
       "      <td>MAROON</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Covers</td>\n",
       "      <td>73807</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>MEDIUM</td>\n",
       "      <td>CHRYSLER</td>\n",
       "      <td>3202.0</td>\n",
       "      <td>4760.0</td>\n",
       "      <td>6943.0</td>\n",
       "      <td>8457.0</td>\n",
       "      <td>4035.0</td>\n",
       "      <td>5557.0</td>\n",
       "      <td>7146.0</td>\n",
       "      <td>8702.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>4900.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>12/7/2009</td>\n",
       "      <td>ADESA</td>\n",
       "      <td>2004</td>\n",
       "      <td>5</td>\n",
       "      <td>DODGE</td>\n",
       "      <td>NEON</td>\n",
       "      <td>SXT</td>\n",
       "      <td>4D SEDAN</td>\n",
       "      <td>SILVER</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Alloy</td>\n",
       "      <td>65617</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>COMPACT</td>\n",
       "      <td>CHRYSLER</td>\n",
       "      <td>1893.0</td>\n",
       "      <td>2675.0</td>\n",
       "      <td>4658.0</td>\n",
       "      <td>5690.0</td>\n",
       "      <td>1844.0</td>\n",
       "      <td>2646.0</td>\n",
       "      <td>4375.0</td>\n",
       "      <td>5518.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>4100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>12/7/2009</td>\n",
       "      <td>ADESA</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "      <td>FORD</td>\n",
       "      <td>FOCUS</td>\n",
       "      <td>ZX3</td>\n",
       "      <td>2D COUPE ZX3</td>\n",
       "      <td>SILVER</td>\n",
       "      <td>MANUAL</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Covers</td>\n",
       "      <td>69367</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>COMPACT</td>\n",
       "      <td>FORD</td>\n",
       "      <td>3913.0</td>\n",
       "      <td>5054.0</td>\n",
       "      <td>7723.0</td>\n",
       "      <td>8707.0</td>\n",
       "      <td>3247.0</td>\n",
       "      <td>4384.0</td>\n",
       "      <td>6739.0</td>\n",
       "      <td>7911.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   RefId  IsBadBuy  PurchDate Auction  VehYear  VehicleAge   Make  \\\n",
       "0      1         0  12/7/2009   ADESA     2006           3  MAZDA   \n",
       "1      2         0  12/7/2009   ADESA     2004           5  DODGE   \n",
       "2      3         0  12/7/2009   ADESA     2005           4  DODGE   \n",
       "3      4         0  12/7/2009   ADESA     2004           5  DODGE   \n",
       "4      5         0  12/7/2009   ADESA     2005           4   FORD   \n",
       "\n",
       "                 Model Trim           SubModel   Color Transmission  \\\n",
       "0               MAZDA3    i         4D SEDAN I     RED         AUTO   \n",
       "1  1500 RAM PICKUP 2WD   ST  QUAD CAB 4.7L SLT   WHITE         AUTO   \n",
       "2           STRATUS V6  SXT   4D SEDAN SXT FFV  MAROON         AUTO   \n",
       "3                 NEON  SXT           4D SEDAN  SILVER         AUTO   \n",
       "4                FOCUS  ZX3       2D COUPE ZX3  SILVER       MANUAL   \n",
       "\n",
       "   WheelTypeID WheelType  VehOdo  Nationality         Size  \\\n",
       "0          1.0     Alloy   89046  OTHER ASIAN       MEDIUM   \n",
       "1          1.0     Alloy   93593     AMERICAN  LARGE TRUCK   \n",
       "2          2.0    Covers   73807     AMERICAN       MEDIUM   \n",
       "3          1.0     Alloy   65617     AMERICAN      COMPACT   \n",
       "4          2.0    Covers   69367     AMERICAN      COMPACT   \n",
       "\n",
       "  TopThreeAmericanName  MMRAcquisitionAuctionAveragePrice  \\\n",
       "0                OTHER                             8155.0   \n",
       "1             CHRYSLER                             6854.0   \n",
       "2             CHRYSLER                             3202.0   \n",
       "3             CHRYSLER                             1893.0   \n",
       "4                 FORD                             3913.0   \n",
       "\n",
       "   MMRAcquisitionAuctionCleanPrice  MMRAcquisitionRetailAveragePrice  \\\n",
       "0                           9829.0                           11636.0   \n",
       "1                           8383.0                           10897.0   \n",
       "2                           4760.0                            6943.0   \n",
       "3                           2675.0                            4658.0   \n",
       "4                           5054.0                            7723.0   \n",
       "\n",
       "   MMRAcquisitonRetailCleanPrice  MMRCurrentAuctionAveragePrice  \\\n",
       "0                        13600.0                         7451.0   \n",
       "1                        12572.0                         7456.0   \n",
       "2                         8457.0                         4035.0   \n",
       "3                         5690.0                         1844.0   \n",
       "4                         8707.0                         3247.0   \n",
       "\n",
       "   MMRCurrentAuctionCleanPrice  MMRCurrentRetailAveragePrice  \\\n",
       "0                       8552.0                       11597.0   \n",
       "1                       9222.0                       11374.0   \n",
       "2                       5557.0                        7146.0   \n",
       "3                       2646.0                        4375.0   \n",
       "4                       4384.0                        6739.0   \n",
       "\n",
       "   MMRCurrentRetailCleanPrice PRIMEUNIT AUCGUART  BYRNO  VNZIP1 VNST  \\\n",
       "0                     12409.0       NaN      NaN  21973   33619   FL   \n",
       "1                     12791.0       NaN      NaN  19638   33619   FL   \n",
       "2                      8702.0       NaN      NaN  19638   33619   FL   \n",
       "3                      5518.0       NaN      NaN  19638   33619   FL   \n",
       "4                      7911.0       NaN      NaN  19638   33619   FL   \n",
       "\n",
       "   VehBCost  IsOnlineSale  WarrantyCost  \n",
       "0    7100.0             0          1113  \n",
       "1    7600.0             0          1053  \n",
       "2    4900.0             0          1389  \n",
       "3    4100.0             0           630  \n",
       "4    4000.0             0          1020  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"kick\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}training.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"RefId\"],\n",
    "    \"cat_cols\": [\"Auction\", \"Make\", \"Model\", \"Trim\", \"SubModel\", \"Color\", \"Transmission\", \n",
    "                 \"WheelTypeID\", \"WheelType\", \"VehOdo\", \"Nationality\", \"Size\", \"TopThreeAmericanName\", \n",
    "                 \"PRIMEUNIT\", \"AUCGUART\", \"BYRNO\", \"VNZIP1\", \"VNST\", \"IsOnlineSale\"],\n",
    "    \"time_cols\": [\"PurchDate\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"IsBadBuy\"] == 1\n",
    "data.drop([\"IsBadBuy\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  19\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>num_1</th>\n",
       "      <th>num_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>cat_4</th>\n",
       "      <th>cat_5</th>\n",
       "      <th>cat_6</th>\n",
       "      <th>cat_7</th>\n",
       "      <th>cat_8</th>\n",
       "      <th>cat_9</th>\n",
       "      <th>cat_10</th>\n",
       "      <th>cat_11</th>\n",
       "      <th>cat_12</th>\n",
       "      <th>cat_13</th>\n",
       "      <th>cat_14</th>\n",
       "      <th>num_15</th>\n",
       "      <th>num_16</th>\n",
       "      <th>num_17</th>\n",
       "      <th>num_18</th>\n",
       "      <th>num_19</th>\n",
       "      <th>num_20</th>\n",
       "      <th>num_21</th>\n",
       "      <th>num_22</th>\n",
       "      <th>cat_23</th>\n",
       "      <th>cat_24</th>\n",
       "      <th>cat_25</th>\n",
       "      <th>cat_26</th>\n",
       "      <th>cat_27</th>\n",
       "      <th>num_28</th>\n",
       "      <th>cat_29</th>\n",
       "      <th>num_30</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ADESA</td>\n",
       "      <td>2006</td>\n",
       "      <td>3</td>\n",
       "      <td>MAZDA</td>\n",
       "      <td>MAZDA3</td>\n",
       "      <td>i</td>\n",
       "      <td>4D SEDAN I</td>\n",
       "      <td>RED</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Alloy</td>\n",
       "      <td>89046</td>\n",
       "      <td>OTHER ASIAN</td>\n",
       "      <td>MEDIUM</td>\n",
       "      <td>OTHER</td>\n",
       "      <td>8155.0</td>\n",
       "      <td>9829.0</td>\n",
       "      <td>11636.0</td>\n",
       "      <td>13600.0</td>\n",
       "      <td>7451.0</td>\n",
       "      <td>8552.0</td>\n",
       "      <td>11597.0</td>\n",
       "      <td>12409.0</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>21973</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>7100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1113</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADESA</td>\n",
       "      <td>2004</td>\n",
       "      <td>5</td>\n",
       "      <td>DODGE</td>\n",
       "      <td>1500 RAM PICKUP 2WD</td>\n",
       "      <td>ST</td>\n",
       "      <td>QUAD CAB 4.7L SLT</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Alloy</td>\n",
       "      <td>93593</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>LARGE TRUCK</td>\n",
       "      <td>CHRYSLER</td>\n",
       "      <td>6854.0</td>\n",
       "      <td>8383.0</td>\n",
       "      <td>10897.0</td>\n",
       "      <td>12572.0</td>\n",
       "      <td>7456.0</td>\n",
       "      <td>9222.0</td>\n",
       "      <td>11374.0</td>\n",
       "      <td>12791.0</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>7600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1053</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ADESA</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "      <td>DODGE</td>\n",
       "      <td>STRATUS V6</td>\n",
       "      <td>SXT</td>\n",
       "      <td>4D SEDAN SXT FFV</td>\n",
       "      <td>MAROON</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Covers</td>\n",
       "      <td>73807</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>MEDIUM</td>\n",
       "      <td>CHRYSLER</td>\n",
       "      <td>3202.0</td>\n",
       "      <td>4760.0</td>\n",
       "      <td>6943.0</td>\n",
       "      <td>8457.0</td>\n",
       "      <td>4035.0</td>\n",
       "      <td>5557.0</td>\n",
       "      <td>7146.0</td>\n",
       "      <td>8702.0</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>4900.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1389</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ADESA</td>\n",
       "      <td>2004</td>\n",
       "      <td>5</td>\n",
       "      <td>DODGE</td>\n",
       "      <td>NEON</td>\n",
       "      <td>SXT</td>\n",
       "      <td>4D SEDAN</td>\n",
       "      <td>SILVER</td>\n",
       "      <td>AUTO</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Alloy</td>\n",
       "      <td>65617</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>COMPACT</td>\n",
       "      <td>CHRYSLER</td>\n",
       "      <td>1893.0</td>\n",
       "      <td>2675.0</td>\n",
       "      <td>4658.0</td>\n",
       "      <td>5690.0</td>\n",
       "      <td>1844.0</td>\n",
       "      <td>2646.0</td>\n",
       "      <td>4375.0</td>\n",
       "      <td>5518.0</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>4100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>630</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ADESA</td>\n",
       "      <td>2005</td>\n",
       "      <td>4</td>\n",
       "      <td>FORD</td>\n",
       "      <td>FOCUS</td>\n",
       "      <td>ZX3</td>\n",
       "      <td>2D COUPE ZX3</td>\n",
       "      <td>SILVER</td>\n",
       "      <td>MANUAL</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Covers</td>\n",
       "      <td>69367</td>\n",
       "      <td>AMERICAN</td>\n",
       "      <td>COMPACT</td>\n",
       "      <td>FORD</td>\n",
       "      <td>3913.0</td>\n",
       "      <td>5054.0</td>\n",
       "      <td>7723.0</td>\n",
       "      <td>8707.0</td>\n",
       "      <td>3247.0</td>\n",
       "      <td>4384.0</td>\n",
       "      <td>6739.0</td>\n",
       "      <td>7911.0</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>19638</td>\n",
       "      <td>33619</td>\n",
       "      <td>FL</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1020</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat_0  num_1  num_2  cat_3                cat_4 cat_5              cat_6  \\\n",
       "0  ADESA   2006      3  MAZDA               MAZDA3     i         4D SEDAN I   \n",
       "1  ADESA   2004      5  DODGE  1500 RAM PICKUP 2WD    ST  QUAD CAB 4.7L SLT   \n",
       "2  ADESA   2005      4  DODGE           STRATUS V6   SXT   4D SEDAN SXT FFV   \n",
       "3  ADESA   2004      5  DODGE                 NEON   SXT           4D SEDAN   \n",
       "4  ADESA   2005      4   FORD                FOCUS   ZX3       2D COUPE ZX3   \n",
       "\n",
       "    cat_7   cat_8 cat_9  cat_10 cat_11       cat_12       cat_13    cat_14  \\\n",
       "0     RED    AUTO   1.0   Alloy  89046  OTHER ASIAN       MEDIUM     OTHER   \n",
       "1   WHITE    AUTO   1.0   Alloy  93593     AMERICAN  LARGE TRUCK  CHRYSLER   \n",
       "2  MAROON    AUTO   2.0  Covers  73807     AMERICAN       MEDIUM  CHRYSLER   \n",
       "3  SILVER    AUTO   1.0   Alloy  65617     AMERICAN      COMPACT  CHRYSLER   \n",
       "4  SILVER  MANUAL   2.0  Covers  69367     AMERICAN      COMPACT      FORD   \n",
       "\n",
       "   num_15  num_16   num_17   num_18  num_19  num_20   num_21   num_22  \\\n",
       "0  8155.0  9829.0  11636.0  13600.0  7451.0  8552.0  11597.0  12409.0   \n",
       "1  6854.0  8383.0  10897.0  12572.0  7456.0  9222.0  11374.0  12791.0   \n",
       "2  3202.0  4760.0   6943.0   8457.0  4035.0  5557.0   7146.0   8702.0   \n",
       "3  1893.0  2675.0   4658.0   5690.0  1844.0  2646.0   4375.0   5518.0   \n",
       "4  3913.0  5054.0   7723.0   8707.0  3247.0  4384.0   6739.0   7911.0   \n",
       "\n",
       "             cat_23            cat_24 cat_25 cat_26 cat_27  num_28 cat_29  \\\n",
       "0  new_category_nan  new_category_nan  21973  33619     FL  7100.0      0   \n",
       "1  new_category_nan  new_category_nan  19638  33619     FL  7600.0      0   \n",
       "2  new_category_nan  new_category_nan  19638  33619     FL  4900.0      0   \n",
       "3  new_category_nan  new_category_nan  19638  33619     FL  4100.0      0   \n",
       "4  new_category_nan  new_category_nan  19638  33619     FL  4000.0      0   \n",
       "\n",
       "   num_30  target  \n",
       "0    1113   False  \n",
       "1    1053   False  \n",
       "2    1389   False  \n",
       "3     630   False  \n",
       "4    1020   False  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "num_1 int64\n",
      "num_2 int64\n",
      "cat_3 object\n",
      "cat_4 object\n",
      "cat_5 object\n",
      "cat_6 object\n",
      "cat_7 object\n",
      "cat_8 object\n",
      "cat_9 object\n",
      "cat_10 object\n",
      "cat_11 object\n",
      "cat_12 object\n",
      "cat_13 object\n",
      "cat_14 object\n",
      "num_15 float64\n",
      "num_16 float64\n",
      "num_17 float64\n",
      "num_18 float64\n",
      "num_19 float64\n",
      "num_20 float64\n",
      "num_21 float64\n",
      "num_22 float64\n",
      "cat_23 object\n",
      "cat_24 object\n",
      "cat_25 object\n",
      "cat_26 object\n",
      "cat_27 object\n",
      "num_28 float64\n",
      "cat_29 object\n",
      "num_30 int64\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# promotion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://datahack.analyticsvidhya.com/contest/wns-analytics-hackathon-2018/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54808, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>employee_id</th>\n",
       "      <th>department</th>\n",
       "      <th>region</th>\n",
       "      <th>education</th>\n",
       "      <th>gender</th>\n",
       "      <th>recruitment_channel</th>\n",
       "      <th>no_of_trainings</th>\n",
       "      <th>age</th>\n",
       "      <th>previous_year_rating</th>\n",
       "      <th>length_of_service</th>\n",
       "      <th>KPIs_met &gt;80%</th>\n",
       "      <th>awards_won?</th>\n",
       "      <th>avg_training_score</th>\n",
       "      <th>is_promoted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65438</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_7</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65141</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_22</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7513</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_19</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2542</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_23</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48945</td>\n",
       "      <td>Technology</td>\n",
       "      <td>region_26</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   employee_id         department     region         education gender  \\\n",
       "0        65438  Sales & Marketing   region_7  Master's & above      f   \n",
       "1        65141         Operations  region_22        Bachelor's      m   \n",
       "2         7513  Sales & Marketing  region_19        Bachelor's      m   \n",
       "3         2542  Sales & Marketing  region_23        Bachelor's      m   \n",
       "4        48945         Technology  region_26        Bachelor's      m   \n",
       "\n",
       "  recruitment_channel  no_of_trainings  age  previous_year_rating  \\\n",
       "0            sourcing                1   35                   5.0   \n",
       "1               other                1   30                   5.0   \n",
       "2            sourcing                1   34                   3.0   \n",
       "3               other                2   39                   1.0   \n",
       "4               other                1   45                   3.0   \n",
       "\n",
       "   length_of_service  KPIs_met >80%  awards_won?  avg_training_score  \\\n",
       "0                  8              1            0                  49   \n",
       "1                  4              0            0                  60   \n",
       "2                  7              0            0                  50   \n",
       "3                 10              0            0                  50   \n",
       "4                  2              0            0                  73   \n",
       "\n",
       "   is_promoted  \n",
       "0            0  \n",
       "1            0  \n",
       "2            0  \n",
       "3            0  \n",
       "4            0  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"promotion\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}train_LZdllcl.csv\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [\"employee_id\"],\n",
    "    \"cat_cols\": [\"department\", \"region\", \"education\", \"gender\", \"recruitment_channel\"],\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = data[\"is_promoted\"] == 1\n",
    "data.drop([\"is_promoted\"] + cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>cat_4</th>\n",
       "      <th>num_5</th>\n",
       "      <th>num_6</th>\n",
       "      <th>num_7</th>\n",
       "      <th>num_8</th>\n",
       "      <th>num_9</th>\n",
       "      <th>num_10</th>\n",
       "      <th>num_11</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_7</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Operations</td>\n",
       "      <td>region_22</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_19</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_23</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Technology</td>\n",
       "      <td>region_26</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               cat_0      cat_1             cat_2 cat_3     cat_4  num_5  \\\n",
       "0  Sales & Marketing   region_7  Master's & above     f  sourcing      1   \n",
       "1         Operations  region_22        Bachelor's     m     other      1   \n",
       "2  Sales & Marketing  region_19        Bachelor's     m  sourcing      1   \n",
       "3  Sales & Marketing  region_23        Bachelor's     m     other      2   \n",
       "4         Technology  region_26        Bachelor's     m     other      1   \n",
       "\n",
       "   num_6  num_7  num_8  num_9  num_10  num_11  target  \n",
       "0     35    5.0      8      1       0      49   False  \n",
       "1     30    5.0      4      0       0      60   False  \n",
       "2     34    3.0      7      0       0      50   False  \n",
       "3     39    1.0     10      0       0      50   False  \n",
       "4     45    3.0      2      0       0      73   False  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat_0 object\n",
      "cat_1 object\n",
      "cat_2 object\n",
      "cat_3 object\n",
      "cat_4 object\n",
      "num_5 int64\n",
      "num_6 int64\n",
      "num_7 float64\n",
      "num_8 int64\n",
      "num_9 int64\n",
      "num_10 int64\n",
      "num_11 int64\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# kdd_upselling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kdd.org/kdd-cup/view/kdd-cup-2009/Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 230)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Var1</th>\n",
       "      <th>Var2</th>\n",
       "      <th>Var3</th>\n",
       "      <th>Var4</th>\n",
       "      <th>Var5</th>\n",
       "      <th>Var6</th>\n",
       "      <th>Var7</th>\n",
       "      <th>Var8</th>\n",
       "      <th>Var9</th>\n",
       "      <th>Var10</th>\n",
       "      <th>Var11</th>\n",
       "      <th>Var12</th>\n",
       "      <th>Var13</th>\n",
       "      <th>Var14</th>\n",
       "      <th>Var15</th>\n",
       "      <th>Var16</th>\n",
       "      <th>Var17</th>\n",
       "      <th>Var18</th>\n",
       "      <th>Var19</th>\n",
       "      <th>Var20</th>\n",
       "      <th>Var21</th>\n",
       "      <th>Var22</th>\n",
       "      <th>Var23</th>\n",
       "      <th>Var24</th>\n",
       "      <th>Var25</th>\n",
       "      <th>Var26</th>\n",
       "      <th>Var27</th>\n",
       "      <th>Var28</th>\n",
       "      <th>Var29</th>\n",
       "      <th>Var30</th>\n",
       "      <th>Var31</th>\n",
       "      <th>Var32</th>\n",
       "      <th>Var33</th>\n",
       "      <th>Var34</th>\n",
       "      <th>Var35</th>\n",
       "      <th>Var36</th>\n",
       "      <th>Var37</th>\n",
       "      <th>Var38</th>\n",
       "      <th>Var39</th>\n",
       "      <th>Var40</th>\n",
       "      <th>Var41</th>\n",
       "      <th>Var42</th>\n",
       "      <th>Var43</th>\n",
       "      <th>Var44</th>\n",
       "      <th>Var45</th>\n",
       "      <th>Var46</th>\n",
       "      <th>Var47</th>\n",
       "      <th>Var48</th>\n",
       "      <th>Var49</th>\n",
       "      <th>Var50</th>\n",
       "      <th>Var51</th>\n",
       "      <th>Var52</th>\n",
       "      <th>Var53</th>\n",
       "      <th>Var54</th>\n",
       "      <th>Var55</th>\n",
       "      <th>Var56</th>\n",
       "      <th>Var57</th>\n",
       "      <th>Var58</th>\n",
       "      <th>Var59</th>\n",
       "      <th>Var60</th>\n",
       "      <th>Var61</th>\n",
       "      <th>Var62</th>\n",
       "      <th>Var63</th>\n",
       "      <th>Var64</th>\n",
       "      <th>Var65</th>\n",
       "      <th>Var66</th>\n",
       "      <th>Var67</th>\n",
       "      <th>Var68</th>\n",
       "      <th>Var69</th>\n",
       "      <th>Var70</th>\n",
       "      <th>Var71</th>\n",
       "      <th>Var72</th>\n",
       "      <th>Var73</th>\n",
       "      <th>Var74</th>\n",
       "      <th>Var75</th>\n",
       "      <th>Var76</th>\n",
       "      <th>Var77</th>\n",
       "      <th>Var78</th>\n",
       "      <th>Var79</th>\n",
       "      <th>Var80</th>\n",
       "      <th>Var81</th>\n",
       "      <th>Var82</th>\n",
       "      <th>Var83</th>\n",
       "      <th>Var84</th>\n",
       "      <th>Var85</th>\n",
       "      <th>Var86</th>\n",
       "      <th>Var87</th>\n",
       "      <th>Var88</th>\n",
       "      <th>Var89</th>\n",
       "      <th>Var90</th>\n",
       "      <th>Var91</th>\n",
       "      <th>Var92</th>\n",
       "      <th>Var93</th>\n",
       "      <th>Var94</th>\n",
       "      <th>Var95</th>\n",
       "      <th>Var96</th>\n",
       "      <th>Var97</th>\n",
       "      <th>Var98</th>\n",
       "      <th>Var99</th>\n",
       "      <th>Var100</th>\n",
       "      <th>Var101</th>\n",
       "      <th>Var102</th>\n",
       "      <th>Var103</th>\n",
       "      <th>Var104</th>\n",
       "      <th>Var105</th>\n",
       "      <th>Var106</th>\n",
       "      <th>Var107</th>\n",
       "      <th>Var108</th>\n",
       "      <th>Var109</th>\n",
       "      <th>Var110</th>\n",
       "      <th>Var111</th>\n",
       "      <th>Var112</th>\n",
       "      <th>Var113</th>\n",
       "      <th>Var114</th>\n",
       "      <th>Var115</th>\n",
       "      <th>Var116</th>\n",
       "      <th>Var117</th>\n",
       "      <th>Var118</th>\n",
       "      <th>Var119</th>\n",
       "      <th>Var120</th>\n",
       "      <th>Var121</th>\n",
       "      <th>Var122</th>\n",
       "      <th>Var123</th>\n",
       "      <th>Var124</th>\n",
       "      <th>Var125</th>\n",
       "      <th>Var126</th>\n",
       "      <th>Var127</th>\n",
       "      <th>Var128</th>\n",
       "      <th>Var129</th>\n",
       "      <th>Var130</th>\n",
       "      <th>Var131</th>\n",
       "      <th>Var132</th>\n",
       "      <th>Var133</th>\n",
       "      <th>Var134</th>\n",
       "      <th>Var135</th>\n",
       "      <th>Var136</th>\n",
       "      <th>Var137</th>\n",
       "      <th>Var138</th>\n",
       "      <th>Var139</th>\n",
       "      <th>Var140</th>\n",
       "      <th>Var141</th>\n",
       "      <th>Var142</th>\n",
       "      <th>Var143</th>\n",
       "      <th>Var144</th>\n",
       "      <th>Var145</th>\n",
       "      <th>Var146</th>\n",
       "      <th>Var147</th>\n",
       "      <th>Var148</th>\n",
       "      <th>Var149</th>\n",
       "      <th>Var150</th>\n",
       "      <th>Var151</th>\n",
       "      <th>Var152</th>\n",
       "      <th>Var153</th>\n",
       "      <th>Var154</th>\n",
       "      <th>Var155</th>\n",
       "      <th>Var156</th>\n",
       "      <th>Var157</th>\n",
       "      <th>Var158</th>\n",
       "      <th>Var159</th>\n",
       "      <th>Var160</th>\n",
       "      <th>Var161</th>\n",
       "      <th>Var162</th>\n",
       "      <th>Var163</th>\n",
       "      <th>Var164</th>\n",
       "      <th>Var165</th>\n",
       "      <th>Var166</th>\n",
       "      <th>Var167</th>\n",
       "      <th>Var168</th>\n",
       "      <th>Var169</th>\n",
       "      <th>Var170</th>\n",
       "      <th>Var171</th>\n",
       "      <th>Var172</th>\n",
       "      <th>Var173</th>\n",
       "      <th>Var174</th>\n",
       "      <th>Var175</th>\n",
       "      <th>Var176</th>\n",
       "      <th>Var177</th>\n",
       "      <th>Var178</th>\n",
       "      <th>Var179</th>\n",
       "      <th>Var180</th>\n",
       "      <th>Var181</th>\n",
       "      <th>Var182</th>\n",
       "      <th>Var183</th>\n",
       "      <th>Var184</th>\n",
       "      <th>Var185</th>\n",
       "      <th>Var186</th>\n",
       "      <th>Var187</th>\n",
       "      <th>Var188</th>\n",
       "      <th>Var189</th>\n",
       "      <th>Var190</th>\n",
       "      <th>Var191</th>\n",
       "      <th>Var192</th>\n",
       "      <th>Var193</th>\n",
       "      <th>Var194</th>\n",
       "      <th>Var195</th>\n",
       "      <th>Var196</th>\n",
       "      <th>Var197</th>\n",
       "      <th>Var198</th>\n",
       "      <th>Var199</th>\n",
       "      <th>Var200</th>\n",
       "      <th>Var201</th>\n",
       "      <th>Var202</th>\n",
       "      <th>Var203</th>\n",
       "      <th>Var204</th>\n",
       "      <th>Var205</th>\n",
       "      <th>Var206</th>\n",
       "      <th>Var207</th>\n",
       "      <th>Var208</th>\n",
       "      <th>Var209</th>\n",
       "      <th>Var210</th>\n",
       "      <th>Var211</th>\n",
       "      <th>Var212</th>\n",
       "      <th>Var213</th>\n",
       "      <th>Var214</th>\n",
       "      <th>Var215</th>\n",
       "      <th>Var216</th>\n",
       "      <th>Var217</th>\n",
       "      <th>Var218</th>\n",
       "      <th>Var219</th>\n",
       "      <th>Var220</th>\n",
       "      <th>Var221</th>\n",
       "      <th>Var222</th>\n",
       "      <th>Var223</th>\n",
       "      <th>Var224</th>\n",
       "      <th>Var225</th>\n",
       "      <th>Var226</th>\n",
       "      <th>Var227</th>\n",
       "      <th>Var228</th>\n",
       "      <th>Var229</th>\n",
       "      <th>Var230</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1526.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>184.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>464.0</td>\n",
       "      <td>580.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>166.56</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3570.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.076907</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36</td>\n",
       "      <td>35.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1350864.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7333.11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>104.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>168.0</td>\n",
       "      <td>117625.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1175.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>720.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1212385.0</td>\n",
       "      <td>69134.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>185.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>397579.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1812252.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>142.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38418.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>462.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>bZkvyxLkBI</td>\n",
       "      <td>RO12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>lK27</td>\n",
       "      <td>ka_ns41</td>\n",
       "      <td>nQUveAzAF7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>dXGu</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>FbIm</td>\n",
       "      <td>VpdQ</td>\n",
       "      <td>haYg</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>XfqtO3UdzaXh_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>XTbPUYD</td>\n",
       "      <td>sH5Z</td>\n",
       "      <td>cJvF</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>1YVfGrO</td>\n",
       "      <td>oslk</td>\n",
       "      <td>fXVEsaq</td>\n",
       "      <td>jySVZNlOJy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>xb3V</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>168.0</td>\n",
       "      <td>210.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>353.52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4764966.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.408032</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>26</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2872928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>151098.90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58158.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>-356411.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>590.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4136430.0</td>\n",
       "      <td>357038.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>278334.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10439160.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>238572.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CEat0G8rTN</td>\n",
       "      <td>RO12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>2Ix5</td>\n",
       "      <td>qEdASpP</td>\n",
       "      <td>y2LIM01bE1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>lg1t</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>k13i</td>\n",
       "      <td>sJzTlal</td>\n",
       "      <td>zm5i</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>NhsEn4L</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kZJyVg2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>0AJo2f2</td>\n",
       "      <td>oslk</td>\n",
       "      <td>2Kb5FSF</td>\n",
       "      <td>LM8l689qOp</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fKCe</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5236.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>904.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1212.0</td>\n",
       "      <td>1515.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>26.0</td>\n",
       "      <td>816.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>220.08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5883894.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.599658</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>130</td>\n",
       "      <td>518.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1675776.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16211.58</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>312.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>336.0</td>\n",
       "      <td>405104.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3230.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>114.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5967.0</td>\n",
       "      <td>-28.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3478905.0</td>\n",
       "      <td>248932.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>800.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>320565.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9826360.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>206.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>434946.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>eOQt0GoOh3</td>\n",
       "      <td>AERks4l</td>\n",
       "      <td>SEuy</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>ffXs</td>\n",
       "      <td>NldASpP</td>\n",
       "      <td>y4g9XoZ</td>\n",
       "      <td>vynJTq9</td>\n",
       "      <td>smXZ</td>\n",
       "      <td>4bTR</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>MGOA</td>\n",
       "      <td>VpdQ</td>\n",
       "      <td>haYg</td>\n",
       "      <td>DHn_WUyBhW_whjA88g9bvA64_</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>UbxQ8lZ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TTGHfSv</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pMWAe2U</td>\n",
       "      <td>bHR7</td>\n",
       "      <td>UYBR</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>JFM1BiF</td>\n",
       "      <td>Al6ZaUT</td>\n",
       "      <td>NKv4yOc</td>\n",
       "      <td>jySVZNlOJy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kG3k</td>\n",
       "      <td>Qu4f</td>\n",
       "      <td>02N6s8f</td>\n",
       "      <td>ib5G6X1eUxUn6</td>\n",
       "      <td>am7c</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.988250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-275703.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-14.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jg69tYsGvO</td>\n",
       "      <td>RO12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>ssAy</td>\n",
       "      <td>_ybO0dd</td>\n",
       "      <td>4hMlgkf58mhwh</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>W8mQ</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>YULl</td>\n",
       "      <td>VpdQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>Mtgm</td>\n",
       "      <td>NhsEn4L</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kq0dQfu</td>\n",
       "      <td>eKej</td>\n",
       "      <td>UYBR</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>L91KIiz</td>\n",
       "      <td>oslk</td>\n",
       "      <td>CE7uk3u</td>\n",
       "      <td>LM8l689qOp</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FSa2</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1029.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3216.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.552446</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>82</td>\n",
       "      <td>224.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>784448.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37423.50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>89754.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56.0</td>\n",
       "      <td>10714.84</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>215.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15111.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>150650.0</td>\n",
       "      <td>66046.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3255.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>267162.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>644836.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IXSgUHShse</td>\n",
       "      <td>RO12</td>\n",
       "      <td>SEuy</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>uNkU</td>\n",
       "      <td>EKR938I</td>\n",
       "      <td>ThrHXVS</td>\n",
       "      <td>0v21jmy</td>\n",
       "      <td>smXZ</td>\n",
       "      <td>xklU</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>RVjC</td>\n",
       "      <td>sJzTlal</td>\n",
       "      <td>6JmL</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>XfqtO3UdzaXh_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SJs3duv</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11p4mKe</td>\n",
       "      <td>H3p7</td>\n",
       "      <td>UYBR</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>OrnLfvc</td>\n",
       "      <td>oslk</td>\n",
       "      <td>1J2cvxe</td>\n",
       "      <td>LM8l689qOp</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kG3k</td>\n",
       "      <td>FSa2</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>mj86</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Var1  Var2  Var3  Var4  Var5    Var6  Var7  Var8  Var9  Var10  Var11  \\\n",
       "0   NaN   NaN   NaN   NaN   NaN  1526.0   7.0   NaN   NaN    NaN    NaN   \n",
       "1   NaN   NaN   NaN   NaN   NaN   525.0   0.0   NaN   NaN    NaN    NaN   \n",
       "2   NaN   NaN   NaN   NaN   NaN  5236.0   7.0   NaN   NaN    NaN    NaN   \n",
       "3   NaN   NaN   NaN   NaN   NaN     NaN   0.0   NaN   NaN    NaN    NaN   \n",
       "4   NaN   NaN   NaN   NaN   NaN  1029.0   7.0   NaN   NaN    NaN    NaN   \n",
       "\n",
       "   Var12   Var13  Var14  Var15  Var16  Var17  Var18  Var19  Var20   Var21  \\\n",
       "0    NaN   184.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN   464.0   \n",
       "1    NaN     0.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN   168.0   \n",
       "2    NaN   904.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN  1212.0   \n",
       "3    NaN     0.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN     NaN   \n",
       "4    NaN  3216.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN    64.0   \n",
       "\n",
       "    Var22  Var23  Var24  Var25  Var26  Var27   Var28  Var29  Var30  Var31  \\\n",
       "0   580.0    NaN   14.0  128.0    NaN    NaN  166.56    NaN    NaN    NaN   \n",
       "1   210.0    NaN    2.0   24.0    NaN    NaN  353.52    NaN    NaN    NaN   \n",
       "2  1515.0    NaN   26.0  816.0    NaN    NaN  220.08    NaN    NaN    NaN   \n",
       "3     0.0    NaN    NaN    0.0    NaN    NaN   22.08    NaN    NaN    NaN   \n",
       "4    80.0    NaN    4.0   64.0    NaN    NaN  200.00    NaN    NaN    NaN   \n",
       "\n",
       "   Var32  Var33  Var34  Var35  Var36  Var37      Var38  Var39  Var40  Var41  \\\n",
       "0    NaN    NaN    NaN    0.0    NaN    NaN     3570.0    NaN    NaN    NaN   \n",
       "1    NaN    NaN    NaN    0.0    NaN    NaN  4764966.0    NaN    NaN    NaN   \n",
       "2    NaN    NaN    NaN    0.0    NaN    NaN  5883894.0    NaN    NaN    NaN   \n",
       "3    NaN    NaN    NaN    0.0    NaN    NaN        0.0    NaN    NaN    NaN   \n",
       "4    NaN    NaN    NaN    0.0    NaN    NaN        0.0    NaN    NaN    NaN   \n",
       "\n",
       "   Var42  Var43  Var44  Var45  Var46  Var47  Var48  Var49  Var50  Var51  \\\n",
       "0    NaN    NaN    0.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "1    NaN    NaN    0.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "2    NaN    NaN    0.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "3    NaN    NaN    0.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "4    NaN    NaN    0.0    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "\n",
       "   Var52  Var53  Var54  Var55  Var56     Var57  Var58  Var59  Var60  Var61  \\\n",
       "0    NaN    NaN    NaN    NaN    NaN  4.076907    NaN    NaN    NaN    NaN   \n",
       "1    NaN    NaN    NaN    NaN    NaN  5.408032    NaN    NaN    NaN    NaN   \n",
       "2    NaN    NaN    NaN    NaN    NaN  6.599658    NaN    NaN    NaN    NaN   \n",
       "3    NaN    NaN    NaN    NaN    NaN  1.988250    NaN    NaN    NaN    NaN   \n",
       "4    NaN    NaN    NaN    NaN    NaN  4.552446    NaN    NaN    NaN    NaN   \n",
       "\n",
       "   Var62  Var63  Var64  Var65  Var66  Var67  Var68  Var69  Var70  Var71  \\\n",
       "0    NaN    NaN    NaN    9.0    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "1    NaN    NaN    NaN    9.0    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "2    NaN    NaN    NaN    9.0    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "3    NaN    NaN    NaN    9.0    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "4    NaN    NaN    NaN   18.0    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "\n",
       "   Var72  Var73  Var74  Var75      Var76  Var77  Var78  Var79  Var80  \\\n",
       "0    NaN     36   35.0    NaN  1350864.0    NaN    0.0    NaN    NaN   \n",
       "1    3.0     26    0.0    NaN  2872928.0    NaN    3.0    NaN    NaN   \n",
       "2    NaN    130  518.0    NaN  1675776.0    NaN    0.0    NaN    NaN   \n",
       "3    NaN     12    0.0    NaN        0.0    NaN    0.0    NaN    NaN   \n",
       "4    3.0     82  224.0    NaN   784448.0    NaN    0.0    NaN    NaN   \n",
       "\n",
       "       Var81  Var82  Var83  Var84  Var85  Var86  Var87  Var88  Var89  Var90  \\\n",
       "0    7333.11    NaN    5.0    NaN   12.0    NaN    NaN    NaN    NaN    NaN   \n",
       "1  151098.90    NaN   25.0    NaN    2.0    NaN    NaN    NaN    NaN    NaN   \n",
       "2   16211.58    NaN   40.0    NaN   58.0    NaN    NaN    NaN    NaN    NaN   \n",
       "3        NaN    NaN    0.0    NaN    0.0    NaN    NaN    NaN    NaN    NaN   \n",
       "4   37423.50    NaN    0.0    NaN    0.0    NaN    NaN    NaN    NaN    NaN   \n",
       "\n",
       "   Var91  Var92  Var93    Var94  Var95  Var96  Var97  Var98  Var99  Var100  \\\n",
       "0    NaN    NaN    NaN      NaN    NaN    NaN    NaN    NaN    NaN     NaN   \n",
       "1    NaN    NaN    NaN  58158.0    NaN    NaN    NaN    NaN    NaN     NaN   \n",
       "2    NaN    NaN    NaN      NaN    NaN    NaN    NaN    NaN    NaN     NaN   \n",
       "3    NaN    NaN    NaN      NaN    NaN    NaN    NaN    NaN    NaN     NaN   \n",
       "4    NaN    NaN    NaN  89754.0    NaN    NaN    NaN    NaN    NaN     NaN   \n",
       "\n",
       "   Var101  Var102  Var103  Var104  Var105  Var106  Var107  Var108  Var109  \\\n",
       "0     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   104.0   \n",
       "1     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN    40.0   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   312.0   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN    32.0   \n",
       "\n",
       "   Var110  Var111  Var112     Var113  Var114  Var115  Var116  Var117  Var118  \\\n",
       "0     NaN     NaN   168.0  117625.60     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN     NaN    40.0 -356411.60     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN     NaN   336.0  405104.00     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     0.0 -275703.60     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN    56.0   10714.84     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   Var119  Var120  Var121  Var122  Var123  Var124   Var125  Var126  Var127  \\\n",
       "0  1175.0     NaN     NaN     NaN     6.0     NaN    720.0     8.0     NaN   \n",
       "1   590.0     NaN     NaN     NaN    72.0     NaN      0.0     NaN     NaN   \n",
       "2  3230.0     NaN     NaN     NaN   114.0     NaN   5967.0   -28.0     NaN   \n",
       "3     NaN     NaN     NaN     NaN     0.0     NaN      0.0   -14.0     NaN   \n",
       "4   215.0     NaN     NaN     NaN     0.0     NaN  15111.0    58.0     NaN   \n",
       "\n",
       "   Var128  Var129  Var130  Var131  Var132     Var133    Var134  Var135  \\\n",
       "0     NaN     NaN     NaN     NaN     0.0  1212385.0   69134.0     NaN   \n",
       "1     NaN     NaN     NaN     NaN     8.0  4136430.0  357038.0     NaN   \n",
       "2     NaN     NaN     NaN     NaN     0.0  3478905.0  248932.0     NaN   \n",
       "3     NaN     NaN     NaN     NaN     0.0        0.0       0.0     NaN   \n",
       "4     NaN     NaN     NaN     NaN     0.0   150650.0   66046.0     NaN   \n",
       "\n",
       "   Var136  Var137  Var138  Var139  Var140  Var141  Var142  Var143  Var144  \\\n",
       "0     NaN     NaN     NaN     NaN   185.0     NaN     NaN     0.0     9.0   \n",
       "1     NaN     NaN     NaN     NaN     0.0     NaN     NaN     0.0     9.0   \n",
       "2     NaN     NaN     NaN     NaN   800.0     NaN     NaN     0.0    36.0   \n",
       "3     NaN     NaN     NaN     NaN     0.0     NaN     NaN     0.0     NaN   \n",
       "4     NaN     NaN     NaN     NaN  3255.0     NaN     NaN     0.0     9.0   \n",
       "\n",
       "   Var145  Var146  Var147  Var148    Var149  Var150  Var151  Var152  \\\n",
       "0     NaN     NaN     NaN     NaN  397579.0     NaN     NaN     NaN   \n",
       "1     NaN     NaN     NaN     NaN  278334.0     NaN     NaN     NaN   \n",
       "2     NaN     NaN     NaN     NaN  320565.0     NaN     NaN     NaN   \n",
       "3     NaN     NaN     NaN     NaN       NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN  267162.0     NaN     NaN     NaN   \n",
       "\n",
       "       Var153  Var154  Var155  Var156  Var157  Var158  Var159  Var160  Var161  \\\n",
       "0   1812252.0     NaN     NaN     NaN     NaN     NaN     NaN   142.0     NaN   \n",
       "1  10439160.0     NaN     NaN     NaN     NaN     NaN     NaN    32.0     NaN   \n",
       "2   9826360.0     NaN     NaN     NaN     NaN     NaN     NaN   206.0     NaN   \n",
       "3         0.0     NaN     NaN     NaN     NaN     NaN     NaN     0.0     NaN   \n",
       "4    644836.0     NaN     NaN     NaN     NaN     NaN     NaN     2.0     NaN   \n",
       "\n",
       "   Var162    Var163  Var164  Var165  Var166  Var167  Var168  Var169  Var170  \\\n",
       "0     NaN   38418.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN  238572.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN  434946.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN       0.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN       0.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   Var171  Var172  Var173  Var174  Var175  Var176  Var177  Var178  Var179  \\\n",
       "0     NaN     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   Var180  Var181  Var182  Var183  Var184  Var185  Var186  Var187  Var188  \\\n",
       "0     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   Var189  Var190 Var191      Var192   Var193 Var194 Var195 Var196 Var197  \\\n",
       "0   462.0     NaN    NaN  bZkvyxLkBI     RO12    NaN   taul   1K8T   lK27   \n",
       "1     NaN     NaN    NaN  CEat0G8rTN     RO12    NaN   taul   1K8T   2Ix5   \n",
       "2     NaN     NaN    NaN  eOQt0GoOh3  AERks4l   SEuy   taul   1K8T   ffXs   \n",
       "3     NaN     NaN    NaN  jg69tYsGvO     RO12    NaN   taul   1K8T   ssAy   \n",
       "4     NaN     NaN    NaN  IXSgUHShse     RO12   SEuy   taul   1K8T   uNkU   \n",
       "\n",
       "    Var198         Var199   Var200 Var201 Var202 Var203 Var204   Var205  \\\n",
       "0  ka_ns41     nQUveAzAF7      NaN    NaN   dXGu   9_Y1   FbIm     VpdQ   \n",
       "1  qEdASpP     y2LIM01bE1      NaN    NaN   lg1t   9_Y1   k13i  sJzTlal   \n",
       "2  NldASpP        y4g9XoZ  vynJTq9   smXZ   4bTR   9_Y1   MGOA     VpdQ   \n",
       "3  _ybO0dd  4hMlgkf58mhwh      NaN    NaN   W8mQ   9_Y1   YULl     VpdQ   \n",
       "4  EKR938I        ThrHXVS  0v21jmy   smXZ   xklU   9_Y1   RVjC  sJzTlal   \n",
       "\n",
       "  Var206                     Var207 Var208  Var209 Var210 Var211  \\\n",
       "0   haYg                 me75fM6ugJ   kIsH     NaN   uKAI   L84s   \n",
       "1   zm5i                 me75fM6ugJ   kIsH     NaN   uKAI   L84s   \n",
       "2   haYg  DHn_WUyBhW_whjA88g9bvA64_   kIsH     NaN   uKAI   L84s   \n",
       "3    NaN                 me75fM6ugJ   kIsH     NaN   uKAI   Mtgm   \n",
       "4   6JmL                 me75fM6ugJ   kIsH     NaN   uKAI   L84s   \n",
       "\n",
       "          Var212 Var213   Var214 Var215   Var216 Var217 Var218 Var219  \\\n",
       "0  XfqtO3UdzaXh_    NaN      NaN    NaN  XTbPUYD   sH5Z   cJvF   FzaX   \n",
       "1        NhsEn4L    NaN      NaN    NaN  kZJyVg2    NaN    NaN   FzaX   \n",
       "2        UbxQ8lZ    NaN  TTGHfSv    NaN  pMWAe2U   bHR7   UYBR   FzaX   \n",
       "3        NhsEn4L    NaN      NaN    NaN  kq0dQfu   eKej   UYBR   FzaX   \n",
       "4  XfqtO3UdzaXh_    NaN  SJs3duv    NaN  11p4mKe   H3p7   UYBR   FzaX   \n",
       "\n",
       "    Var220   Var221   Var222      Var223 Var224 Var225 Var226   Var227  \\\n",
       "0  1YVfGrO     oslk  fXVEsaq  jySVZNlOJy    NaN    NaN   xb3V     RAYp   \n",
       "1  0AJo2f2     oslk  2Kb5FSF  LM8l689qOp    NaN    NaN   fKCe     RAYp   \n",
       "2  JFM1BiF  Al6ZaUT  NKv4yOc  jySVZNlOJy    NaN   kG3k   Qu4f  02N6s8f   \n",
       "3  L91KIiz     oslk  CE7uk3u  LM8l689qOp    NaN    NaN   FSa2     RAYp   \n",
       "4  OrnLfvc     oslk  1J2cvxe  LM8l689qOp    NaN   kG3k   FSa2     RAYp   \n",
       "\n",
       "          Var228 Var229  Var230  \n",
       "0  F2FyR07IdsN7I    NaN     NaN  \n",
       "1  F2FyR07IdsN7I    NaN     NaN  \n",
       "2  ib5G6X1eUxUn6   am7c     NaN  \n",
       "3  F2FyR07IdsN7I    NaN     NaN  \n",
       "4  F2FyR07IdsN7I   mj86     NaN  "
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_name = \"kdd_upselling\"\n",
    "data_pth = f\"./data/{dataset_name}/\"\n",
    "data = pd.read_csv(f\"{data_pth}orange_small_train.data\", sep=\"\\t\")\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = pd.read_csv(f\"{data_pth}orange_small_train_upselling.labels.txt\", header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_dict = {\n",
    "    \"id_cols\": [],\n",
    "    \"cat_cols\": data.columns[-40:],\n",
    "    \"time_cols\": []\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"target\"] = targets.values == 1\n",
    "data.drop(cols_dict[\"time_cols\"] + cols_dict[\"id_cols\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tot cat columns :  40\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_0</th>\n",
       "      <th>num_1</th>\n",
       "      <th>num_2</th>\n",
       "      <th>num_3</th>\n",
       "      <th>num_4</th>\n",
       "      <th>num_5</th>\n",
       "      <th>num_6</th>\n",
       "      <th>num_7</th>\n",
       "      <th>num_8</th>\n",
       "      <th>num_9</th>\n",
       "      <th>num_10</th>\n",
       "      <th>num_11</th>\n",
       "      <th>num_12</th>\n",
       "      <th>num_13</th>\n",
       "      <th>num_14</th>\n",
       "      <th>num_15</th>\n",
       "      <th>num_16</th>\n",
       "      <th>num_17</th>\n",
       "      <th>num_18</th>\n",
       "      <th>num_19</th>\n",
       "      <th>num_20</th>\n",
       "      <th>num_21</th>\n",
       "      <th>num_22</th>\n",
       "      <th>num_23</th>\n",
       "      <th>num_24</th>\n",
       "      <th>num_25</th>\n",
       "      <th>num_26</th>\n",
       "      <th>num_27</th>\n",
       "      <th>num_28</th>\n",
       "      <th>num_29</th>\n",
       "      <th>num_30</th>\n",
       "      <th>num_31</th>\n",
       "      <th>num_32</th>\n",
       "      <th>num_33</th>\n",
       "      <th>num_34</th>\n",
       "      <th>num_35</th>\n",
       "      <th>num_36</th>\n",
       "      <th>num_37</th>\n",
       "      <th>num_38</th>\n",
       "      <th>num_39</th>\n",
       "      <th>num_40</th>\n",
       "      <th>num_41</th>\n",
       "      <th>num_42</th>\n",
       "      <th>num_43</th>\n",
       "      <th>num_44</th>\n",
       "      <th>num_45</th>\n",
       "      <th>num_46</th>\n",
       "      <th>num_47</th>\n",
       "      <th>num_48</th>\n",
       "      <th>num_49</th>\n",
       "      <th>num_50</th>\n",
       "      <th>num_51</th>\n",
       "      <th>num_52</th>\n",
       "      <th>num_53</th>\n",
       "      <th>num_54</th>\n",
       "      <th>num_55</th>\n",
       "      <th>num_56</th>\n",
       "      <th>num_57</th>\n",
       "      <th>num_58</th>\n",
       "      <th>num_59</th>\n",
       "      <th>num_60</th>\n",
       "      <th>num_61</th>\n",
       "      <th>num_62</th>\n",
       "      <th>num_63</th>\n",
       "      <th>num_64</th>\n",
       "      <th>num_65</th>\n",
       "      <th>num_66</th>\n",
       "      <th>num_67</th>\n",
       "      <th>num_68</th>\n",
       "      <th>num_69</th>\n",
       "      <th>num_70</th>\n",
       "      <th>num_71</th>\n",
       "      <th>num_72</th>\n",
       "      <th>num_73</th>\n",
       "      <th>num_74</th>\n",
       "      <th>num_75</th>\n",
       "      <th>num_76</th>\n",
       "      <th>num_77</th>\n",
       "      <th>num_78</th>\n",
       "      <th>num_79</th>\n",
       "      <th>num_80</th>\n",
       "      <th>num_81</th>\n",
       "      <th>num_82</th>\n",
       "      <th>num_83</th>\n",
       "      <th>num_84</th>\n",
       "      <th>num_85</th>\n",
       "      <th>num_86</th>\n",
       "      <th>num_87</th>\n",
       "      <th>num_88</th>\n",
       "      <th>num_89</th>\n",
       "      <th>num_90</th>\n",
       "      <th>num_91</th>\n",
       "      <th>num_92</th>\n",
       "      <th>num_93</th>\n",
       "      <th>num_94</th>\n",
       "      <th>num_95</th>\n",
       "      <th>num_96</th>\n",
       "      <th>num_97</th>\n",
       "      <th>num_98</th>\n",
       "      <th>num_99</th>\n",
       "      <th>num_100</th>\n",
       "      <th>num_101</th>\n",
       "      <th>num_102</th>\n",
       "      <th>num_103</th>\n",
       "      <th>num_104</th>\n",
       "      <th>num_105</th>\n",
       "      <th>num_106</th>\n",
       "      <th>num_107</th>\n",
       "      <th>num_108</th>\n",
       "      <th>num_109</th>\n",
       "      <th>num_110</th>\n",
       "      <th>num_111</th>\n",
       "      <th>num_112</th>\n",
       "      <th>num_113</th>\n",
       "      <th>num_114</th>\n",
       "      <th>num_115</th>\n",
       "      <th>num_116</th>\n",
       "      <th>num_117</th>\n",
       "      <th>num_118</th>\n",
       "      <th>num_119</th>\n",
       "      <th>num_120</th>\n",
       "      <th>num_121</th>\n",
       "      <th>num_122</th>\n",
       "      <th>num_123</th>\n",
       "      <th>num_124</th>\n",
       "      <th>num_125</th>\n",
       "      <th>num_126</th>\n",
       "      <th>num_127</th>\n",
       "      <th>num_128</th>\n",
       "      <th>num_129</th>\n",
       "      <th>num_130</th>\n",
       "      <th>num_131</th>\n",
       "      <th>num_132</th>\n",
       "      <th>num_133</th>\n",
       "      <th>num_134</th>\n",
       "      <th>num_135</th>\n",
       "      <th>num_136</th>\n",
       "      <th>num_137</th>\n",
       "      <th>num_138</th>\n",
       "      <th>num_139</th>\n",
       "      <th>num_140</th>\n",
       "      <th>num_141</th>\n",
       "      <th>num_142</th>\n",
       "      <th>num_143</th>\n",
       "      <th>num_144</th>\n",
       "      <th>num_145</th>\n",
       "      <th>num_146</th>\n",
       "      <th>num_147</th>\n",
       "      <th>num_148</th>\n",
       "      <th>num_149</th>\n",
       "      <th>num_150</th>\n",
       "      <th>num_151</th>\n",
       "      <th>num_152</th>\n",
       "      <th>num_153</th>\n",
       "      <th>num_154</th>\n",
       "      <th>num_155</th>\n",
       "      <th>num_156</th>\n",
       "      <th>num_157</th>\n",
       "      <th>num_158</th>\n",
       "      <th>num_159</th>\n",
       "      <th>num_160</th>\n",
       "      <th>num_161</th>\n",
       "      <th>num_162</th>\n",
       "      <th>num_163</th>\n",
       "      <th>num_164</th>\n",
       "      <th>num_165</th>\n",
       "      <th>num_166</th>\n",
       "      <th>num_167</th>\n",
       "      <th>num_168</th>\n",
       "      <th>num_169</th>\n",
       "      <th>num_170</th>\n",
       "      <th>num_171</th>\n",
       "      <th>num_172</th>\n",
       "      <th>num_173</th>\n",
       "      <th>num_174</th>\n",
       "      <th>num_175</th>\n",
       "      <th>num_176</th>\n",
       "      <th>num_177</th>\n",
       "      <th>num_178</th>\n",
       "      <th>num_179</th>\n",
       "      <th>num_180</th>\n",
       "      <th>num_181</th>\n",
       "      <th>num_182</th>\n",
       "      <th>num_183</th>\n",
       "      <th>num_184</th>\n",
       "      <th>num_185</th>\n",
       "      <th>num_186</th>\n",
       "      <th>num_187</th>\n",
       "      <th>num_188</th>\n",
       "      <th>num_189</th>\n",
       "      <th>cat_190</th>\n",
       "      <th>cat_191</th>\n",
       "      <th>cat_192</th>\n",
       "      <th>cat_193</th>\n",
       "      <th>cat_194</th>\n",
       "      <th>cat_195</th>\n",
       "      <th>cat_196</th>\n",
       "      <th>cat_197</th>\n",
       "      <th>cat_198</th>\n",
       "      <th>cat_199</th>\n",
       "      <th>cat_200</th>\n",
       "      <th>cat_201</th>\n",
       "      <th>cat_202</th>\n",
       "      <th>cat_203</th>\n",
       "      <th>cat_204</th>\n",
       "      <th>cat_205</th>\n",
       "      <th>cat_206</th>\n",
       "      <th>cat_207</th>\n",
       "      <th>cat_208</th>\n",
       "      <th>cat_209</th>\n",
       "      <th>cat_210</th>\n",
       "      <th>cat_211</th>\n",
       "      <th>cat_212</th>\n",
       "      <th>cat_213</th>\n",
       "      <th>cat_214</th>\n",
       "      <th>cat_215</th>\n",
       "      <th>cat_216</th>\n",
       "      <th>cat_217</th>\n",
       "      <th>cat_218</th>\n",
       "      <th>cat_219</th>\n",
       "      <th>cat_220</th>\n",
       "      <th>cat_221</th>\n",
       "      <th>cat_222</th>\n",
       "      <th>cat_223</th>\n",
       "      <th>cat_224</th>\n",
       "      <th>cat_225</th>\n",
       "      <th>cat_226</th>\n",
       "      <th>cat_227</th>\n",
       "      <th>cat_228</th>\n",
       "      <th>cat_229</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1526.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>184.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>464.0</td>\n",
       "      <td>580.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>166.56</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3570.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.076907</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36</td>\n",
       "      <td>35.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1350864.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7333.11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>104.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>168.0</td>\n",
       "      <td>117625.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1175.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>720.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1212385.0</td>\n",
       "      <td>69134.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>185.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>397579.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1812252.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>142.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38418.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>462.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>bZkvyxLkBI</td>\n",
       "      <td>RO12</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>lK27</td>\n",
       "      <td>ka_ns41</td>\n",
       "      <td>nQUveAzAF7</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>dXGu</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>FbIm</td>\n",
       "      <td>VpdQ</td>\n",
       "      <td>haYg</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>XfqtO3UdzaXh_</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>XTbPUYD</td>\n",
       "      <td>sH5Z</td>\n",
       "      <td>cJvF</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>1YVfGrO</td>\n",
       "      <td>oslk</td>\n",
       "      <td>fXVEsaq</td>\n",
       "      <td>jySVZNlOJy</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>xb3V</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>168.0</td>\n",
       "      <td>210.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>353.52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4764966.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.408032</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>26</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2872928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>151098.90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58158.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>-356411.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>590.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4136430.0</td>\n",
       "      <td>357038.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>278334.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10439160.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>238572.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>CEat0G8rTN</td>\n",
       "      <td>RO12</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>2Ix5</td>\n",
       "      <td>qEdASpP</td>\n",
       "      <td>y2LIM01bE1</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>lg1t</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>k13i</td>\n",
       "      <td>sJzTlal</td>\n",
       "      <td>zm5i</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>NhsEn4L</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>kZJyVg2</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>0AJo2f2</td>\n",
       "      <td>oslk</td>\n",
       "      <td>2Kb5FSF</td>\n",
       "      <td>LM8l689qOp</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>fKCe</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5236.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>904.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1212.0</td>\n",
       "      <td>1515.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>26.0</td>\n",
       "      <td>816.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>220.08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5883894.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.599658</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>130</td>\n",
       "      <td>518.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1675776.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16211.58</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>312.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>336.0</td>\n",
       "      <td>405104.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3230.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>114.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5967.0</td>\n",
       "      <td>-28.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3478905.0</td>\n",
       "      <td>248932.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>800.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>320565.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9826360.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>206.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>434946.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>eOQt0GoOh3</td>\n",
       "      <td>AERks4l</td>\n",
       "      <td>SEuy</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>ffXs</td>\n",
       "      <td>NldASpP</td>\n",
       "      <td>y4g9XoZ</td>\n",
       "      <td>vynJTq9</td>\n",
       "      <td>smXZ</td>\n",
       "      <td>4bTR</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>MGOA</td>\n",
       "      <td>VpdQ</td>\n",
       "      <td>haYg</td>\n",
       "      <td>DHn_WUyBhW_whjA88g9bvA64_</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>UbxQ8lZ</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>TTGHfSv</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>pMWAe2U</td>\n",
       "      <td>bHR7</td>\n",
       "      <td>UYBR</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>JFM1BiF</td>\n",
       "      <td>Al6ZaUT</td>\n",
       "      <td>NKv4yOc</td>\n",
       "      <td>jySVZNlOJy</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>kG3k</td>\n",
       "      <td>Qu4f</td>\n",
       "      <td>02N6s8f</td>\n",
       "      <td>ib5G6X1eUxUn6</td>\n",
       "      <td>am7c</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.988250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-275703.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-14.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>jg69tYsGvO</td>\n",
       "      <td>RO12</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>ssAy</td>\n",
       "      <td>_ybO0dd</td>\n",
       "      <td>4hMlgkf58mhwh</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>W8mQ</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>YULl</td>\n",
       "      <td>VpdQ</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>Mtgm</td>\n",
       "      <td>NhsEn4L</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>kq0dQfu</td>\n",
       "      <td>eKej</td>\n",
       "      <td>UYBR</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>L91KIiz</td>\n",
       "      <td>oslk</td>\n",
       "      <td>CE7uk3u</td>\n",
       "      <td>LM8l689qOp</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>FSa2</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1029.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3216.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.552446</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>82</td>\n",
       "      <td>224.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>784448.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37423.50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>89754.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56.0</td>\n",
       "      <td>10714.84</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>215.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15111.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>150650.0</td>\n",
       "      <td>66046.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3255.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>267162.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>644836.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>IXSgUHShse</td>\n",
       "      <td>RO12</td>\n",
       "      <td>SEuy</td>\n",
       "      <td>taul</td>\n",
       "      <td>1K8T</td>\n",
       "      <td>uNkU</td>\n",
       "      <td>EKR938I</td>\n",
       "      <td>ThrHXVS</td>\n",
       "      <td>0v21jmy</td>\n",
       "      <td>smXZ</td>\n",
       "      <td>xklU</td>\n",
       "      <td>9_Y1</td>\n",
       "      <td>RVjC</td>\n",
       "      <td>sJzTlal</td>\n",
       "      <td>6JmL</td>\n",
       "      <td>me75fM6ugJ</td>\n",
       "      <td>kIsH</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>uKAI</td>\n",
       "      <td>L84s</td>\n",
       "      <td>XfqtO3UdzaXh_</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>SJs3duv</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>11p4mKe</td>\n",
       "      <td>H3p7</td>\n",
       "      <td>UYBR</td>\n",
       "      <td>FzaX</td>\n",
       "      <td>OrnLfvc</td>\n",
       "      <td>oslk</td>\n",
       "      <td>1J2cvxe</td>\n",
       "      <td>LM8l689qOp</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>kG3k</td>\n",
       "      <td>FSa2</td>\n",
       "      <td>RAYp</td>\n",
       "      <td>F2FyR07IdsN7I</td>\n",
       "      <td>mj86</td>\n",
       "      <td>new_category_nan</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   num_0  num_1  num_2  num_3  num_4   num_5  num_6  num_7  num_8  num_9  \\\n",
       "0    NaN    NaN    NaN    NaN    NaN  1526.0    7.0    NaN    NaN    NaN   \n",
       "1    NaN    NaN    NaN    NaN    NaN   525.0    0.0    NaN    NaN    NaN   \n",
       "2    NaN    NaN    NaN    NaN    NaN  5236.0    7.0    NaN    NaN    NaN   \n",
       "3    NaN    NaN    NaN    NaN    NaN     NaN    0.0    NaN    NaN    NaN   \n",
       "4    NaN    NaN    NaN    NaN    NaN  1029.0    7.0    NaN    NaN    NaN   \n",
       "\n",
       "   num_10  num_11  num_12  num_13  num_14  num_15  num_16  num_17  num_18  \\\n",
       "0     NaN     NaN   184.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN     NaN   904.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     0.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN  3216.0     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_19  num_20  num_21  num_22  num_23  num_24  num_25  num_26  num_27  \\\n",
       "0     NaN   464.0   580.0     NaN    14.0   128.0     NaN     NaN  166.56   \n",
       "1     NaN   168.0   210.0     NaN     2.0    24.0     NaN     NaN  353.52   \n",
       "2     NaN  1212.0  1515.0     NaN    26.0   816.0     NaN     NaN  220.08   \n",
       "3     NaN     NaN     0.0     NaN     NaN     0.0     NaN     NaN   22.08   \n",
       "4     NaN    64.0    80.0     NaN     4.0    64.0     NaN     NaN  200.00   \n",
       "\n",
       "   num_28  num_29  num_30  num_31  num_32  num_33  num_34  num_35  num_36  \\\n",
       "0     NaN     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "1     NaN     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "\n",
       "      num_37  num_38  num_39  num_40  num_41  num_42  num_43  num_44  num_45  \\\n",
       "0     3570.0     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "1  4764966.0     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "2  5883894.0     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "3        0.0     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "4        0.0     NaN     NaN     NaN     NaN     NaN     0.0     NaN     NaN   \n",
       "\n",
       "   num_46  num_47  num_48  num_49  num_50  num_51  num_52  num_53  num_54  \\\n",
       "0     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_55    num_56  num_57  num_58  num_59  num_60  num_61  num_62  num_63  \\\n",
       "0     NaN  4.076907     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN  5.408032     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN  6.599658     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN  1.988250     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN  4.552446     NaN     NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_64  num_65  num_66  num_67  num_68  num_69  num_70  num_71  num_72  \\\n",
       "0     9.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN      36   \n",
       "1     9.0     NaN     NaN     NaN     NaN     NaN     NaN     3.0      26   \n",
       "2     9.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN     130   \n",
       "3     9.0     NaN     NaN     NaN     NaN     NaN     NaN     NaN      12   \n",
       "4    18.0     NaN     NaN     NaN     NaN     NaN     NaN     3.0      82   \n",
       "\n",
       "   num_73  num_74     num_75  num_76  num_77  num_78  num_79     num_80  \\\n",
       "0    35.0     NaN  1350864.0     NaN     0.0     NaN     NaN    7333.11   \n",
       "1     0.0     NaN  2872928.0     NaN     3.0     NaN     NaN  151098.90   \n",
       "2   518.0     NaN  1675776.0     NaN     0.0     NaN     NaN   16211.58   \n",
       "3     0.0     NaN        0.0     NaN     0.0     NaN     NaN        NaN   \n",
       "4   224.0     NaN   784448.0     NaN     0.0     NaN     NaN   37423.50   \n",
       "\n",
       "   num_81  num_82  num_83  num_84  num_85  num_86  num_87  num_88  num_89  \\\n",
       "0     NaN     5.0     NaN    12.0     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN    25.0     NaN     2.0     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN    40.0     NaN    58.0     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     0.0     NaN     0.0     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     0.0     NaN     0.0     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_90  num_91  num_92   num_93  num_94  num_95  num_96  num_97  num_98  \\\n",
       "0     NaN     NaN     NaN      NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "1     NaN     NaN     NaN  58158.0     NaN     NaN     NaN     NaN     NaN   \n",
       "2     NaN     NaN     NaN      NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "3     NaN     NaN     NaN      NaN     NaN     NaN     NaN     NaN     NaN   \n",
       "4     NaN     NaN     NaN  89754.0     NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   num_99  num_100  num_101  num_102  num_103  num_104  num_105  num_106  \\\n",
       "0     NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "1     NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "2     NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "3     NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "4     NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "\n",
       "   num_107  num_108  num_109  num_110  num_111    num_112  num_113  num_114  \\\n",
       "0      NaN    104.0      NaN      NaN    168.0  117625.60      NaN      NaN   \n",
       "1      NaN     40.0      NaN      NaN     40.0 -356411.60      NaN      NaN   \n",
       "2      NaN    312.0      NaN      NaN    336.0  405104.00      NaN      NaN   \n",
       "3      NaN      NaN      NaN      NaN      0.0 -275703.60      NaN      NaN   \n",
       "4      NaN     32.0      NaN      NaN     56.0   10714.84      NaN      NaN   \n",
       "\n",
       "   num_115  num_116  num_117  num_118  num_119  num_120  num_121  num_122  \\\n",
       "0      NaN      NaN      NaN   1175.0      NaN      NaN      NaN      6.0   \n",
       "1      NaN      NaN      NaN    590.0      NaN      NaN      NaN     72.0   \n",
       "2      NaN      NaN      NaN   3230.0      NaN      NaN      NaN    114.0   \n",
       "3      NaN      NaN      NaN      NaN      NaN      NaN      NaN      0.0   \n",
       "4      NaN      NaN      NaN    215.0      NaN      NaN      NaN      0.0   \n",
       "\n",
       "   num_123  num_124  num_125  num_126  num_127  num_128  num_129  num_130  \\\n",
       "0      NaN    720.0      8.0      NaN      NaN      NaN      NaN      NaN   \n",
       "1      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "2      NaN   5967.0    -28.0      NaN      NaN      NaN      NaN      NaN   \n",
       "3      NaN      0.0    -14.0      NaN      NaN      NaN      NaN      NaN   \n",
       "4      NaN  15111.0     58.0      NaN      NaN      NaN      NaN      NaN   \n",
       "\n",
       "   num_131    num_132   num_133  num_134  num_135  num_136  num_137  num_138  \\\n",
       "0      0.0  1212385.0   69134.0      NaN      NaN      NaN      NaN      NaN   \n",
       "1      8.0  4136430.0  357038.0      NaN      NaN      NaN      NaN      NaN   \n",
       "2      0.0  3478905.0  248932.0      NaN      NaN      NaN      NaN      NaN   \n",
       "3      0.0        0.0       0.0      NaN      NaN      NaN      NaN      NaN   \n",
       "4      0.0   150650.0   66046.0      NaN      NaN      NaN      NaN      NaN   \n",
       "\n",
       "   num_139  num_140  num_141  num_142  num_143  num_144  num_145  num_146  \\\n",
       "0    185.0      NaN      NaN      0.0      9.0      NaN      NaN      NaN   \n",
       "1      0.0      NaN      NaN      0.0      9.0      NaN      NaN      NaN   \n",
       "2    800.0      NaN      NaN      0.0     36.0      NaN      NaN      NaN   \n",
       "3      0.0      NaN      NaN      0.0      NaN      NaN      NaN      NaN   \n",
       "4   3255.0      NaN      NaN      0.0      9.0      NaN      NaN      NaN   \n",
       "\n",
       "   num_147   num_148  num_149  num_150  num_151     num_152  num_153  num_154  \\\n",
       "0      NaN  397579.0      NaN      NaN      NaN   1812252.0      NaN      NaN   \n",
       "1      NaN  278334.0      NaN      NaN      NaN  10439160.0      NaN      NaN   \n",
       "2      NaN  320565.0      NaN      NaN      NaN   9826360.0      NaN      NaN   \n",
       "3      NaN       NaN      NaN      NaN      NaN         0.0      NaN      NaN   \n",
       "4      NaN  267162.0      NaN      NaN      NaN    644836.0      NaN      NaN   \n",
       "\n",
       "   num_155  num_156  num_157  num_158  num_159  num_160  num_161   num_162  \\\n",
       "0      NaN      NaN      NaN      NaN    142.0      NaN      NaN   38418.0   \n",
       "1      NaN      NaN      NaN      NaN     32.0      NaN      NaN  238572.0   \n",
       "2      NaN      NaN      NaN      NaN    206.0      NaN      NaN  434946.0   \n",
       "3      NaN      NaN      NaN      NaN      0.0      NaN      NaN       0.0   \n",
       "4      NaN      NaN      NaN      NaN      2.0      NaN      NaN       0.0   \n",
       "\n",
       "   num_163  num_164  num_165  num_166  num_167  num_168  num_169  num_170  \\\n",
       "0      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "1      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "2      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "3      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "4      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "\n",
       "   num_171  num_172  num_173  num_174  num_175  num_176  num_177  num_178  \\\n",
       "0      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "1      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "2      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "3      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "4      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "\n",
       "   num_179  num_180  num_181  num_182  num_183  num_184  num_185  num_186  \\\n",
       "0      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "1      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "2      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "3      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "4      NaN      0.0      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "\n",
       "   num_187  num_188  num_189           cat_190     cat_191  cat_192  \\\n",
       "0      NaN    462.0      NaN  new_category_nan  bZkvyxLkBI     RO12   \n",
       "1      NaN      NaN      NaN  new_category_nan  CEat0G8rTN     RO12   \n",
       "2      NaN      NaN      NaN  new_category_nan  eOQt0GoOh3  AERks4l   \n",
       "3      NaN      NaN      NaN  new_category_nan  jg69tYsGvO     RO12   \n",
       "4      NaN      NaN      NaN  new_category_nan  IXSgUHShse     RO12   \n",
       "\n",
       "            cat_193 cat_194 cat_195 cat_196  cat_197        cat_198  \\\n",
       "0  new_category_nan    taul    1K8T    lK27  ka_ns41     nQUveAzAF7   \n",
       "1  new_category_nan    taul    1K8T    2Ix5  qEdASpP     y2LIM01bE1   \n",
       "2              SEuy    taul    1K8T    ffXs  NldASpP        y4g9XoZ   \n",
       "3  new_category_nan    taul    1K8T    ssAy  _ybO0dd  4hMlgkf58mhwh   \n",
       "4              SEuy    taul    1K8T    uNkU  EKR938I        ThrHXVS   \n",
       "\n",
       "            cat_199           cat_200 cat_201 cat_202 cat_203  cat_204  \\\n",
       "0  new_category_nan  new_category_nan    dXGu    9_Y1    FbIm     VpdQ   \n",
       "1  new_category_nan  new_category_nan    lg1t    9_Y1    k13i  sJzTlal   \n",
       "2           vynJTq9              smXZ    4bTR    9_Y1    MGOA     VpdQ   \n",
       "3  new_category_nan  new_category_nan    W8mQ    9_Y1    YULl     VpdQ   \n",
       "4           0v21jmy              smXZ    xklU    9_Y1    RVjC  sJzTlal   \n",
       "\n",
       "            cat_205                    cat_206 cat_207           cat_208  \\\n",
       "0              haYg                 me75fM6ugJ    kIsH  new_category_nan   \n",
       "1              zm5i                 me75fM6ugJ    kIsH  new_category_nan   \n",
       "2              haYg  DHn_WUyBhW_whjA88g9bvA64_    kIsH  new_category_nan   \n",
       "3  new_category_nan                 me75fM6ugJ    kIsH  new_category_nan   \n",
       "4              6JmL                 me75fM6ugJ    kIsH  new_category_nan   \n",
       "\n",
       "  cat_209 cat_210        cat_211           cat_212           cat_213  \\\n",
       "0    uKAI    L84s  XfqtO3UdzaXh_  new_category_nan  new_category_nan   \n",
       "1    uKAI    L84s        NhsEn4L  new_category_nan  new_category_nan   \n",
       "2    uKAI    L84s        UbxQ8lZ  new_category_nan           TTGHfSv   \n",
       "3    uKAI    Mtgm        NhsEn4L  new_category_nan  new_category_nan   \n",
       "4    uKAI    L84s  XfqtO3UdzaXh_  new_category_nan           SJs3duv   \n",
       "\n",
       "            cat_214  cat_215           cat_216           cat_217 cat_218  \\\n",
       "0  new_category_nan  XTbPUYD              sH5Z              cJvF    FzaX   \n",
       "1  new_category_nan  kZJyVg2  new_category_nan  new_category_nan    FzaX   \n",
       "2  new_category_nan  pMWAe2U              bHR7              UYBR    FzaX   \n",
       "3  new_category_nan  kq0dQfu              eKej              UYBR    FzaX   \n",
       "4  new_category_nan  11p4mKe              H3p7              UYBR    FzaX   \n",
       "\n",
       "   cat_219  cat_220  cat_221     cat_222           cat_223           cat_224  \\\n",
       "0  1YVfGrO     oslk  fXVEsaq  jySVZNlOJy  new_category_nan  new_category_nan   \n",
       "1  0AJo2f2     oslk  2Kb5FSF  LM8l689qOp  new_category_nan  new_category_nan   \n",
       "2  JFM1BiF  Al6ZaUT  NKv4yOc  jySVZNlOJy  new_category_nan              kG3k   \n",
       "3  L91KIiz     oslk  CE7uk3u  LM8l689qOp  new_category_nan  new_category_nan   \n",
       "4  OrnLfvc     oslk  1J2cvxe  LM8l689qOp  new_category_nan              kG3k   \n",
       "\n",
       "  cat_225  cat_226        cat_227           cat_228           cat_229  target  \n",
       "0    xb3V     RAYp  F2FyR07IdsN7I  new_category_nan  new_category_nan   False  \n",
       "1    fKCe     RAYp  F2FyR07IdsN7I  new_category_nan  new_category_nan   False  \n",
       "2    Qu4f  02N6s8f  ib5G6X1eUxUn6              am7c  new_category_nan   False  \n",
       "3    FSa2     RAYp  F2FyR07IdsN7I  new_category_nan  new_category_nan   False  \n",
       "4    FSa2     RAYp  F2FyR07IdsN7I              mj86  new_category_nan   False  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = process_cols(data, cols_dict)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_0 float64\n",
      "num_1 float64\n",
      "num_2 float64\n",
      "num_3 float64\n",
      "num_4 float64\n",
      "num_5 float64\n",
      "num_6 float64\n",
      "num_7 float64\n",
      "num_8 float64\n",
      "num_9 float64\n",
      "num_10 float64\n",
      "num_11 float64\n",
      "num_12 float64\n",
      "num_13 float64\n",
      "num_14 float64\n",
      "num_15 float64\n",
      "num_16 float64\n",
      "num_17 float64\n",
      "num_18 float64\n",
      "num_19 float64\n",
      "num_20 float64\n",
      "num_21 float64\n",
      "num_22 float64\n",
      "num_23 float64\n",
      "num_24 float64\n",
      "num_25 float64\n",
      "num_26 float64\n",
      "num_27 float64\n",
      "num_28 float64\n",
      "num_29 float64\n",
      "num_30 float64\n",
      "num_31 float64\n",
      "num_32 float64\n",
      "num_33 float64\n",
      "num_34 float64\n",
      "num_35 float64\n",
      "num_36 float64\n",
      "num_37 float64\n",
      "num_38 float64\n",
      "num_39 float64\n",
      "num_40 float64\n",
      "num_41 float64\n",
      "num_42 float64\n",
      "num_43 float64\n",
      "num_44 float64\n",
      "num_45 float64\n",
      "num_46 float64\n",
      "num_47 float64\n",
      "num_48 float64\n",
      "num_49 float64\n",
      "num_50 float64\n",
      "num_51 float64\n",
      "num_52 float64\n",
      "num_53 float64\n",
      "num_54 float64\n",
      "num_55 float64\n",
      "num_56 float64\n",
      "num_57 float64\n",
      "num_58 float64\n",
      "num_59 float64\n",
      "num_60 float64\n",
      "num_61 float64\n",
      "num_62 float64\n",
      "num_63 float64\n",
      "num_64 float64\n",
      "num_65 float64\n",
      "num_66 float64\n",
      "num_67 float64\n",
      "num_68 float64\n",
      "num_69 float64\n",
      "num_70 float64\n",
      "num_71 float64\n",
      "num_72 int64\n",
      "num_73 float64\n",
      "num_74 float64\n",
      "num_75 float64\n",
      "num_76 float64\n",
      "num_77 float64\n",
      "num_78 float64\n",
      "num_79 float64\n",
      "num_80 float64\n",
      "num_81 float64\n",
      "num_82 float64\n",
      "num_83 float64\n",
      "num_84 float64\n",
      "num_85 float64\n",
      "num_86 float64\n",
      "num_87 float64\n",
      "num_88 float64\n",
      "num_89 float64\n",
      "num_90 float64\n",
      "num_91 float64\n",
      "num_92 float64\n",
      "num_93 float64\n",
      "num_94 float64\n",
      "num_95 float64\n",
      "num_96 float64\n",
      "num_97 float64\n",
      "num_98 float64\n",
      "num_99 float64\n",
      "num_100 float64\n",
      "num_101 float64\n",
      "num_102 float64\n",
      "num_103 float64\n",
      "num_104 float64\n",
      "num_105 float64\n",
      "num_106 float64\n",
      "num_107 float64\n",
      "num_108 float64\n",
      "num_109 float64\n",
      "num_110 float64\n",
      "num_111 float64\n",
      "num_112 float64\n",
      "num_113 float64\n",
      "num_114 float64\n",
      "num_115 float64\n",
      "num_116 float64\n",
      "num_117 float64\n",
      "num_118 float64\n",
      "num_119 float64\n",
      "num_120 float64\n",
      "num_121 float64\n",
      "num_122 float64\n",
      "num_123 float64\n",
      "num_124 float64\n",
      "num_125 float64\n",
      "num_126 float64\n",
      "num_127 float64\n",
      "num_128 float64\n",
      "num_129 float64\n",
      "num_130 float64\n",
      "num_131 float64\n",
      "num_132 float64\n",
      "num_133 float64\n",
      "num_134 float64\n",
      "num_135 float64\n",
      "num_136 float64\n",
      "num_137 float64\n",
      "num_138 float64\n",
      "num_139 float64\n",
      "num_140 float64\n",
      "num_141 float64\n",
      "num_142 float64\n",
      "num_143 float64\n",
      "num_144 float64\n",
      "num_145 float64\n",
      "num_146 float64\n",
      "num_147 float64\n",
      "num_148 float64\n",
      "num_149 float64\n",
      "num_150 float64\n",
      "num_151 float64\n",
      "num_152 float64\n",
      "num_153 float64\n",
      "num_154 float64\n",
      "num_155 float64\n",
      "num_156 float64\n",
      "num_157 float64\n",
      "num_158 float64\n",
      "num_159 float64\n",
      "num_160 float64\n",
      "num_161 float64\n",
      "num_162 float64\n",
      "num_163 float64\n",
      "num_164 float64\n",
      "num_165 float64\n",
      "num_166 float64\n",
      "num_167 float64\n",
      "num_168 float64\n",
      "num_169 float64\n",
      "num_170 float64\n",
      "num_171 float64\n",
      "num_172 float64\n",
      "num_173 float64\n",
      "num_174 float64\n",
      "num_175 float64\n",
      "num_176 float64\n",
      "num_177 float64\n",
      "num_178 float64\n",
      "num_179 float64\n",
      "num_180 float64\n",
      "num_181 float64\n",
      "num_182 float64\n",
      "num_183 float64\n",
      "num_184 float64\n",
      "num_185 float64\n",
      "num_186 float64\n",
      "num_187 float64\n",
      "num_188 float64\n",
      "num_189 float64\n",
      "cat_190 object\n",
      "cat_191 object\n",
      "cat_192 object\n",
      "cat_193 object\n",
      "cat_194 object\n",
      "cat_195 object\n",
      "cat_196 object\n",
      "cat_197 object\n",
      "cat_198 object\n",
      "cat_199 object\n",
      "cat_200 object\n",
      "cat_201 object\n",
      "cat_202 object\n",
      "cat_203 object\n",
      "cat_204 object\n",
      "cat_205 object\n",
      "cat_206 object\n",
      "cat_207 object\n",
      "cat_208 object\n",
      "cat_209 object\n",
      "cat_210 object\n",
      "cat_211 object\n",
      "cat_212 object\n",
      "cat_213 object\n",
      "cat_214 object\n",
      "cat_215 object\n",
      "cat_216 object\n",
      "cat_217 object\n",
      "cat_218 object\n",
      "cat_219 object\n",
      "cat_220 object\n",
      "cat_221 object\n",
      "cat_222 object\n",
      "cat_223 object\n",
      "cat_224 object\n",
      "cat_225 object\n",
      "cat_226 object\n",
      "cat_227 object\n",
      "cat_228 object\n",
      "cat_229 object\n",
      "target bool\n"
     ]
    }
   ],
   "source": [
    "for col in data.columns:\n",
    "    print(col, data[col].dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(f\"{data_pth}{dataset_name}.gz\", compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}