metric-proto.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "from os import path\n",
    "import pickle\n",
    "import itertools\n",
    "from functools import partial\n",
    "import pprint\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "\n",
    "import intra\n",
    "import util\n",
    "from util import md, highlight_max_min\n",
    "from experiment import new_exp, get_loader, run_once, run_permutation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = get_loader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ver = 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Small"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 2,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 10,\n",
      "    'nvec': 1000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GELS|GV|LSI|LSG|MC|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"471\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 1000 10\n",
      "gv: ave 59.38, min 58.47, max 59.77\n",
      "lsi: ave 58.06, min 57.67, max 58.51\n",
      "lsg: 35.14\n",
      "rn: 24.57\n",
      "ge10: 59.74\n",
      "gels10: 59.74\n",
      "ge30: 59.19\n",
      "gels30: 59.87\n",
      "ge50: 60.08\n",
      "gels50: 60.08\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:00:07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ge70: 60.72\n",
      "gels70: 60.72\n",
      "ge95: 60.60\n",
      "gels95: 60.60\n",
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 10,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 10,\n",
      "    'nvec': 1000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GELS|GV|LSI|LSG|MC|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"473\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 1000 10\n",
      "gv: ave 109.24, min 108.11, max 109.80\n",
      "lsi: ave 109.89, min 109.68, max 110.19\n",
      "lsg: 88.84\n",
      "rn: 83.03\n",
      "ge10: 110.47\n",
      "gels10: 111.49\n",
      "ge30: 110.53\n",
      "gels30: 111.04\n",
      "ge50: 109.76\n",
      "gels50: 110.97\n",
      "ge70: 110.45\n",
      "gels70: 111.25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ge95: 110.15\n",
      "gels95: 110.81\n",
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 2,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 100,\n",
      "    'nvec': 1000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GELS|GV|LSI|LSG|MC|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"475\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 1000 10\n",
      "Warning: less selections than specified 909/1000\n",
      "Warning: less selections than specified 909/1000\n",
      "Warning: less selections than specified 908/1000\n",
      "Warning: less selections than specified 911/1000\n",
      "Warning: less selections than specified 913/1000\n",
      "Warning: less selections than specified 899/1000\n",
      "Warning: less selections than specified 908/1000\n",
      "Warning: less selections than specified 909/1000\n",
      "Warning: less selections than specified 917/1000\n",
      "Warning: less selections than specified 908/1000\n",
      "gv: ave 2806.80, min 2764.05, max 2860.26\n",
      "Warning: less selections than specified 998/1000\n",
      "Warning: less selections than specified 998/1000\n",
      "Warning: less selections than specified 977/1000\n",
      "Warning: less selections than specified 977/1000\n",
      "Warning: less selections than specified 997/1000\n",
      "Warning: less selections than specified 997/1000\n",
      "Warning: less selections than specified 974/1000\n",
      "Warning: less selections than specified 974/1000\n",
      "Warning: less selections than specified 987/1000\n",
      "Warning: less selections than specified 987/1000\n",
      "Warning: less selections than specified 979/1000\n",
      "Warning: less selections than specified 979/1000\n",
      "Warning: less selections than specified 977/1000\n",
      "Warning: less selections than specified 977/1000\n",
      "Warning: less selections than specified 964/1000\n",
      "Warning: less selections than specified 964/1000\n",
      "Warning: less selections than specified 961/1000\n",
      "Warning: less selections than specified 961/1000\n",
      "lsi: ave 3235.12, min 3034.81, max 3464.11\n",
      "lsg: 3464.11\n",
      "rn: 3464.11\n",
      "Warning: less selections than specified 976/1000\n",
      "ge10: 3314.76\n",
      "Warning: less selections than specified 976/1000\n",
      "gels10: 3314.76\n",
      "Warning: less selections than specified 982/1000\n",
      "ge30: 3366.04\n",
      "Warning: less selections than specified 982/1000\n",
      "gels30: 3366.04\n",
      "Warning: less selections than specified 986/1000\n",
      "ge50: 3391.64\n",
      "Warning: less selections than specified 986/1000\n",
      "gels50: 3391.64\n",
      "Warning: less selections than specified 986/1000\n",
      "ge70: 3387.80\n",
      "Warning: less selections than specified 986/1000\n",
      "gels70: 3387.80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:00:21\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: less selections than specified 980/1000\n",
      "ge95: 3372.06\n",
      "Warning: less selections than specified 980/1000\n",
      "gels95: 3372.06\n",
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 10,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 100,\n",
      "    'nvec': 1000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GELS|GV|LSI|LSG|MC|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"476\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 1000 10\n",
      "Warning: less selections than specified 992/1000\n",
      "Warning: less selections than specified 992/1000\n",
      "Warning: less selections than specified 992/1000\n",
      "Warning: less selections than specified 992/1000\n",
      "Warning: less selections than specified 993/1000\n",
      "Warning: less selections than specified 992/1000\n",
      "Warning: less selections than specified 992/1000\n",
      "Warning: less selections than specified 992/1000\n",
      "Warning: less selections than specified 993/1000\n",
      "Warning: less selections than specified 992/1000\n",
      "gv: ave 8444.17, min 8438.63, max 8453.31\n",
      "Warning: less selections than specified 998/1000\n",
      "Warning: less selections than specified 998/1000\n",
      "Warning: less selections than specified 998/1000\n",
      "Warning: less selections than specified 998/1000\n",
      "Warning: less selections than specified 999/1000\n",
      "Warning: less selections than specified 999/1000\n",
      "Warning: less selections than specified 997/1000\n",
      "Warning: less selections than specified 997/1000\n",
      "Warning: less selections than specified 999/1000\n",
      "Warning: less selections than specified 999/1000\n",
      "Warning: less selections than specified 999/1000\n",
      "Warning: less selections than specified 999/1000\n",
      "lsi: ave 8548.16, min 8516.51, max 8565.17\n",
      "lsg: 8565.17\n",
      "rn: 8565.17\n",
      "Warning: less selections than specified 998/1000\n",
      "ge10: 8536.65\n",
      "Warning: less selections than specified 998/1000\n",
      "gels10: 8536.65\n",
      "Warning: less selections than specified 994/1000\n",
      "ge30: 8477.24\n",
      "Warning: less selections than specified 994/1000\n",
      "gels30: 8477.24\n",
      "Warning: less selections than specified 994/1000\n",
      "ge50: 8473.93\n",
      "Warning: less selections than specified 994/1000\n",
      "gels50: 8473.93\n",
      "Warning: less selections than specified 998/1000\n",
      "ge70: 8534.07\n",
      "Warning: less selections than specified 998/1000\n",
      "gels70: 8534.07\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:00:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: less selections than specified 992/1000\n",
      "ge95: 8451.89\n",
      "Warning: less selections than specified 992/1000\n",
      "gels95: 8452.17\n"
     ]
    }
   ],
   "source": [
    "ex = new_exp()\n",
    "    \n",
    "# r = ex.run()\n",
    "for nvec, ncls, noise, nsel, dim, alphas in itertools.product([1000], \n",
    "                                                        [10], \n",
    "                                                        [0.2], \n",
    "                                                        #[0.5], \n",
    "                                                        [10,100],\n",
    "                                                        [2,10],\n",
    "                                                        [[0.1, 0.3, 0.5, 0.7, 0.95]]):\n",
    "    conf = {'which': 'proto', \n",
    "        'whichalgo': 'GE|GELS|GV|LSI|LSG|MC|RN',\n",
    "        'dataset': 'proto-v{}'.format(ver), \n",
    "        'nvec': nvec, \n",
    "        'ncls': ncls, \n",
    "        'noise': noise, \n",
    "        'nsel': nsel, \n",
    "        'alphas': alphas,\n",
    "        'dim': dim}\n",
    "    pp.pprint(conf)\n",
    "    r = ex.run(config_updates=conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.find_latest().to_dict()\n",
    "\n",
    "query = {\"$and\": [\n",
    "    {\"status\": 'COMPLETED'},\n",
    "    {\"config.dataset\": 'proto-v{}'.format(ver)},\n",
    "    {\"config.nvec\": 1000},\n",
    "    {\"config.noise\": 0.2},\n",
    "        ]}\n",
    "qs = loader.find(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col6 {\n",
       "            color:  green;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col15 {\n",
       "            color:  red;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col4 {\n",
       "            color:  green;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col15 {\n",
       "            color:  red;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col8 {\n",
       "            color:  red;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col13 {\n",
       "            color:  green;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col14 {\n",
       "            color:  green;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col15 {\n",
       "            color:  green;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col8 {\n",
       "            color:  red;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col13 {\n",
       "            color:  green;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col14 {\n",
       "            color:  green;\n",
       "        }    #T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col15 {\n",
       "            color:  green;\n",
       "        }</style><table id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >nvec</th>        <th class=\"col_heading level0 col1\" >nsel</th>        <th class=\"col_heading level0 col2\" >dim</th>        <th class=\"col_heading level0 col3\" >ge10_mean</th>        <th class=\"col_heading level0 col4\" >ge30_mean</th>        <th class=\"col_heading level0 col5\" >ge50_mean</th>        <th class=\"col_heading level0 col6\" >ge70_mean</th>        <th class=\"col_heading level0 col7\" >ge95_mean</th>        <th class=\"col_heading level0 col8\" >gv_min_mean</th>        <th class=\"col_heading level0 col9\" >gv_ave_mean</th>        <th class=\"col_heading level0 col10\" >gv_max_mean</th>        <th class=\"col_heading level0 col11\" >lsi_min_mean</th>        <th class=\"col_heading level0 col12\" >lsi_ave_mean</th>        <th class=\"col_heading level0 col13\" >lsi_max_mean</th>        <th class=\"col_heading level0 col14\" >lsg_mean</th>        <th class=\"col_heading level0 col15\" >rn_mean</th>    </tr>    <tr>        <th class=\"index_name level0\" >exp_id</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590level0_row0\" class=\"row_heading level0 row0\" >471</th>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col0\" class=\"data row0 col0\" >1000</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col1\" class=\"data row0 col1\" >10</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col2\" class=\"data row0 col2\" >2</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col3\" class=\"data row0 col3\" >59.7423</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col4\" class=\"data row0 col4\" >59.1905</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col5\" class=\"data row0 col5\" >60.0804</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col6\" class=\"data row0 col6\" >60.7223</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col7\" class=\"data row0 col7\" >60.6018</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col8\" class=\"data row0 col8\" >58.4698</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col9\" class=\"data row0 col9\" >59.3775</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col10\" class=\"data row0 col10\" >59.767</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col11\" class=\"data row0 col11\" >57.6659</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col12\" class=\"data row0 col12\" >58.0591</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col13\" class=\"data row0 col13\" >58.513</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col14\" class=\"data row0 col14\" >35.1376</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row0_col15\" class=\"data row0 col15\" >24.5678</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590level0_row1\" class=\"row_heading level0 row1\" >473</th>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col0\" class=\"data row1 col0\" >1000</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col1\" class=\"data row1 col1\" >10</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col2\" class=\"data row1 col2\" >10</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col3\" class=\"data row1 col3\" >110.474</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col4\" class=\"data row1 col4\" >110.534</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col5\" class=\"data row1 col5\" >109.762</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col6\" class=\"data row1 col6\" >110.446</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col7\" class=\"data row1 col7\" >110.152</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col8\" class=\"data row1 col8\" >108.113</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col9\" class=\"data row1 col9\" >109.235</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col10\" class=\"data row1 col10\" >109.804</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col11\" class=\"data row1 col11\" >109.677</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col12\" class=\"data row1 col12\" >109.889</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col13\" class=\"data row1 col13\" >110.188</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col14\" class=\"data row1 col14\" >88.8406</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row1_col15\" class=\"data row1 col15\" >83.0338</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590level0_row2\" class=\"row_heading level0 row2\" >475</th>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col0\" class=\"data row2 col0\" >1000</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col1\" class=\"data row2 col1\" >100</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col2\" class=\"data row2 col2\" >2</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col3\" class=\"data row2 col3\" >3314.76</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col4\" class=\"data row2 col4\" >3366.04</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col5\" class=\"data row2 col5\" >3391.64</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col6\" class=\"data row2 col6\" >3387.8</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col7\" class=\"data row2 col7\" >3372.06</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col8\" class=\"data row2 col8\" >2764.05</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col9\" class=\"data row2 col9\" >2806.8</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col10\" class=\"data row2 col10\" >2860.26</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col11\" class=\"data row2 col11\" >3034.81</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col12\" class=\"data row2 col12\" >3235.12</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col13\" class=\"data row2 col13\" >3464.11</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col14\" class=\"data row2 col14\" >3464.11</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row2_col15\" class=\"data row2 col15\" >3464.11</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590level0_row3\" class=\"row_heading level0 row3\" >476</th>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col0\" class=\"data row3 col0\" >1000</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col1\" class=\"data row3 col1\" >100</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col2\" class=\"data row3 col2\" >10</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col3\" class=\"data row3 col3\" >8536.65</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col4\" class=\"data row3 col4\" >8477.24</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col5\" class=\"data row3 col5\" >8473.93</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col6\" class=\"data row3 col6\" >8534.07</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col7\" class=\"data row3 col7\" >8451.89</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col8\" class=\"data row3 col8\" >8438.63</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col9\" class=\"data row3 col9\" >8444.17</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col10\" class=\"data row3 col10\" >8453.31</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col11\" class=\"data row3 col11\" >8516.51</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col12\" class=\"data row3 col12\" >8548.16</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col13\" class=\"data row3 col13\" >8565.17</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col14\" class=\"data row3 col14\" >8565.17</td>\n",
       "                        <td id=\"T_bd4bcf12_e37d_11e9_ac97_5dc85b020590row3_col15\" class=\"data row3 col15\" >8565.17</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f9344137e50>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# full, alpha = 0.95\n",
    "f = np.mean\n",
    "df = qs.project(on=[\"config.nvec\", \"config.nsel\", \"config.dim\",\n",
    "               {\"metrics.ge10\": f},\n",
    "               {\"metrics.ge30\": f},\n",
    "               {\"metrics.ge50\": f},\n",
    "               {\"metrics.ge70\": f},\n",
    "               {\"metrics.ge95\": f},\n",
    "#                {\"metrics.gels10\": f},\n",
    "#                {\"metrics.gels30\": f},\n",
    "#                {\"metrics.gels50\": f},\n",
    "#                {\"metrics.gels70\": f},\n",
    "#                {\"metrics.gels95\": f},\n",
    "               {\"metrics.gv_min\": f},\n",
    "               {\"metrics.gv_ave\": f},\n",
    "               {\"metrics.gv_max\": f},\n",
    "               {\"metrics.lsi_min\": f},\n",
    "               {\"metrics.lsi_ave\": f},\n",
    "               {\"metrics.lsi_max\": f},\n",
    "               {\"metrics.lsg\": f},\n",
    "               {\"metrics.rn\": f},\n",
    "              ])\n",
    "h = partial(highlight_max_min, start=4)\n",
    "df.style.apply(h, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llrrrrrrrrrrrrrr}\n",
      "\\toprule\n",
      "{} & Dataset &     N &    P &   D &  GEmin &  GEavg &  GEmax &  GVmin &  GVavg &  GVmax &  LSImin &  LSIavg &  LSImax &    LSG &     RN \\\\\n",
      "exp\\_id &         &       &      &     &        &        &        &        &        &        &         &         &         &        &        \\\\\n",
      "\\midrule\n",
      "471    &   proto &  1000 &   10 &   2 &  0.975 &  0.989 &  1.000 &  0.963 &  0.978 &  0.984 &   0.950 &   0.956 &   0.964 &  0.579 &  0.405 \\\\\n",
      "473    &   proto &  1000 &   10 &  10 &  0.993 &  0.998 &  1.000 &  0.978 &  0.988 &  0.993 &   0.992 &   0.994 &   0.997 &  0.804 &  0.751 \\\\\n",
      "475    &   proto &  1000 &  100 &   2 &  0.957 &  0.972 &  0.979 &  0.798 &  0.810 &  0.826 &   0.876 &   0.934 &   1.000 &  1.000 &  1.000 \\\\\n",
      "476    &   proto &  1000 &  100 &  10 &  0.987 &  0.992 &  0.997 &  0.985 &  0.986 &  0.987 &   0.994 &   0.998 &   1.000 &  1.000 &  1.000 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = util.aggregate_tbl(df, names=['ge'])\n",
    "df = util.relative_tbl(df, beg=3)\n",
    "df.columns = list(map(lambda x: md[x] if x in md else x, df.columns))\n",
    "df.insert(0, column='Dataset', value=['proto']*len(df))\n",
    "print(df.round(3).to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 2,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 10,\n",
      "    'nvec': 100000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GV|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"477\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 100000 10\n",
      "gv: ave 94.52, min 94.33, max 94.72\n",
      "rn: 16.23\n",
      "ge10: 94.17\n",
      "ge30: 94.21\n",
      "ge50: 95.52\n",
      "ge70: 95.64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:00:14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ge95: 95.27\n",
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 10,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 10,\n",
      "    'nvec': 100000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GV|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"478\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 100000 10\n",
      "gv: ave 143.27, min 142.56, max 144.23\n",
      "rn: 79.55\n",
      "ge10: 143.23\n",
      "ge30: 143.83\n",
      "ge50: 144.01\n",
      "ge70: 144.78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:00:11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ge95: 143.98\n",
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 2,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 100,\n",
      "    'nvec': 100000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GV|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"479\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 100000 10\n",
      "gv: ave 8466.75, min 8458.09, max 8470.95\n",
      "rn: 1912.05\n",
      "ge10: 8490.28\n",
      "ge30: 8488.66\n",
      "ge50: 8491.71\n",
      "ge70: 8493.10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:05:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ge95: 8496.39\n",
      "{   'alphas': [0.1, 0.3, 0.5, 0.7, 0.95],\n",
      "    'dataset': 'proto-v8',\n",
      "    'dim': 10,\n",
      "    'ncls': 10,\n",
      "    'noise': 0.2,\n",
      "    'nsel': 100,\n",
      "    'nvec': 100000,\n",
      "    'which': 'proto',\n",
      "    'whichalgo': 'GE|GV|RN'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "/u/67/zhangg1/unix/.local/lib/python3.7/site-packages/sacred/experiment.py:566: UserWarning: Subclassing `CommandLineOption` is deprecated. Please use the `sacred.cli_option` decorator and pass the function to the Experiment constructor.\n",
      "  \"Subclassing `CommandLineOption` is deprecated. Please \"\n",
      "INFO - jupyter_ex - Running command 'my_main'\n",
      "INFO - jupyter_ex - Started run with ID \"484\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvec, ncls: 100000 10\n",
      "gv: ave 13963.37, min 13956.79, max 13969.30\n",
      "rn: 8398.08\n",
      "ge10: 13989.23\n",
      "ge30: 13986.26\n",
      "ge50: 13987.39\n",
      "ge70: 13998.01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - jupyter_ex - Completed after 0:03:40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ge95: 13988.24\n"
     ]
    }
   ],
   "source": [
    "ex = new_exp()\n",
    "    \n",
    "# r = ex.run()\n",
    "for nvec, ncls, noise, nsel, dim, alphas in itertools.product([100000], \n",
    "                                                        [10], \n",
    "                                                        [0.2], \n",
    "                                                        [10,100],\n",
    "                                                        [2,10],\n",
    "                                                        [[0.1, 0.3, 0.5, 0.7, 0.95]]):\n",
    "    conf = {'which': 'proto', \n",
    "        'whichalgo': 'GE|GV|RN',\n",
    "        'dataset': 'proto-v{}'.format(ver), \n",
    "        'nvec': nvec, \n",
    "        'ncls': ncls, \n",
    "        'noise': noise, \n",
    "        'nsel': nsel, \n",
    "        'alphas': alphas,\n",
    "        'dim': dim}\n",
    "    pp.pprint(conf)\n",
    "    r = ex.run(config_updates=conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader.find_latest().to_dict()\n",
    "\n",
    "query = {\"$and\": [\n",
    "    {\"status\": 'COMPLETED'},\n",
    "    {\"config.dataset\": 'proto-v{}'.format(ver)},\n",
    "    {\"config.nvec\": 100000},\n",
    "        ]}\n",
    "qs = loader.find(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col6 {\n",
       "            color:  green;\n",
       "        }    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col11 {\n",
       "            color:  red;\n",
       "        }    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col6 {\n",
       "            color:  green;\n",
       "        }    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col11 {\n",
       "            color:  red;\n",
       "        }    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col7 {\n",
       "            color:  green;\n",
       "        }    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col11 {\n",
       "            color:  red;\n",
       "        }    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col6 {\n",
       "            color:  green;\n",
       "        }    #T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col11 {\n",
       "            color:  red;\n",
       "        }</style><table id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >nvec</th>        <th class=\"col_heading level0 col1\" >nsel</th>        <th class=\"col_heading level0 col2\" >dim</th>        <th class=\"col_heading level0 col3\" >ge10_mean</th>        <th class=\"col_heading level0 col4\" >ge30_mean</th>        <th class=\"col_heading level0 col5\" >ge50_mean</th>        <th class=\"col_heading level0 col6\" >ge70_mean</th>        <th class=\"col_heading level0 col7\" >ge95_mean</th>        <th class=\"col_heading level0 col8\" >gv_min_mean</th>        <th class=\"col_heading level0 col9\" >gv_ave_mean</th>        <th class=\"col_heading level0 col10\" >gv_max_mean</th>        <th class=\"col_heading level0 col11\" >rn_mean</th>    </tr>    <tr>        <th class=\"index_name level0\" >exp_id</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590level0_row0\" class=\"row_heading level0 row0\" >477</th>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col0\" class=\"data row0 col0\" >100000</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col1\" class=\"data row0 col1\" >10</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col2\" class=\"data row0 col2\" >2</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col3\" class=\"data row0 col3\" >94.1714</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col4\" class=\"data row0 col4\" >94.2062</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col5\" class=\"data row0 col5\" >95.5165</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col6\" class=\"data row0 col6\" >95.6396</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col7\" class=\"data row0 col7\" >95.2704</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col8\" class=\"data row0 col8\" >94.3325</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col9\" class=\"data row0 col9\" >94.5179</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col10\" class=\"data row0 col10\" >94.7185</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row0_col11\" class=\"data row0 col11\" >16.2302</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590level0_row1\" class=\"row_heading level0 row1\" >478</th>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col0\" class=\"data row1 col0\" >100000</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col1\" class=\"data row1 col1\" >10</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col2\" class=\"data row1 col2\" >10</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col3\" class=\"data row1 col3\" >143.235</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col4\" class=\"data row1 col4\" >143.826</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col5\" class=\"data row1 col5\" >144.01</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col6\" class=\"data row1 col6\" >144.782</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col7\" class=\"data row1 col7\" >143.979</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col8\" class=\"data row1 col8\" >142.557</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col9\" class=\"data row1 col9\" >143.266</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col10\" class=\"data row1 col10\" >144.232</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row1_col11\" class=\"data row1 col11\" >79.5495</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590level0_row2\" class=\"row_heading level0 row2\" >479</th>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col0\" class=\"data row2 col0\" >100000</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col1\" class=\"data row2 col1\" >100</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col2\" class=\"data row2 col2\" >2</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col3\" class=\"data row2 col3\" >8490.28</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col4\" class=\"data row2 col4\" >8488.66</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col5\" class=\"data row2 col5\" >8491.71</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col6\" class=\"data row2 col6\" >8493.1</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col7\" class=\"data row2 col7\" >8496.39</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col8\" class=\"data row2 col8\" >8458.09</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col9\" class=\"data row2 col9\" >8466.75</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col10\" class=\"data row2 col10\" >8470.95</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row2_col11\" class=\"data row2 col11\" >1912.05</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590level0_row3\" class=\"row_heading level0 row3\" >484</th>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col0\" class=\"data row3 col0\" >100000</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col1\" class=\"data row3 col1\" >100</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col2\" class=\"data row3 col2\" >10</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col3\" class=\"data row3 col3\" >13989.2</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col4\" class=\"data row3 col4\" >13986.3</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col5\" class=\"data row3 col5\" >13987.4</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col6\" class=\"data row3 col6\" >13998</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col7\" class=\"data row3 col7\" >13988.2</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col8\" class=\"data row3 col8\" >13956.8</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col9\" class=\"data row3 col9\" >13963.4</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col10\" class=\"data row3 col10\" >13969.3</td>\n",
       "                        <td id=\"T_ff1a7dbc_e37d_11e9_ac97_5dc85b020590row3_col11\" class=\"data row3 col11\" >8398.08</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f93700e8810>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# full, alpha = 0.95\n",
    "f = np.mean\n",
    "df = qs.project(on=[\"config.nvec\", \"config.nsel\", \"config.dim\", \n",
    "               {\"metrics.ge10\": f},\n",
    "               {\"metrics.ge30\": f},\n",
    "               {\"metrics.ge50\": f},\n",
    "               {\"metrics.ge70\": f},\n",
    "               {\"metrics.ge95\": f},\n",
    "               {\"metrics.gv_min\": f},\n",
    "               {\"metrics.gv_ave\": f},\n",
    "               {\"metrics.gv_max\": f},\n",
    "               {\"metrics.rn\": f},\n",
    "              ])\n",
    "h = partial(highlight_max_min, start=3)\n",
    "df.style.apply(h, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llrrrrrrrrrr}\n",
      "\\toprule\n",
      "{} & Dataset &       N &    P &   D &  GEmin &  GEavg &  GEmax &  GVmin &  GVavg &  GVmax &     RN \\\\\n",
      "exp\\_id &         &         &      &     &        &        &        &        &        &        &        \\\\\n",
      "\\midrule\n",
      "477    &   proto &  100000 &   10 &   2 &  0.985 &  0.993 &    1.0 &  0.986 &  0.988 &  0.990 &  0.170 \\\\\n",
      "478    &   proto &  100000 &   10 &  10 &  0.989 &  0.994 &    1.0 &  0.985 &  0.990 &  0.996 &  0.549 \\\\\n",
      "479    &   proto &  100000 &  100 &   2 &  0.999 &  0.999 &    1.0 &  0.995 &  0.997 &  0.997 &  0.225 \\\\\n",
      "484    &   proto &  100000 &  100 &  10 &  0.999 &  0.999 &    1.0 &  0.997 &  0.998 &  0.998 &  0.600 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = util.aggregate_tbl(df, names=['ge'])\n",
    "df = util.relative_tbl(df, beg=3)\n",
    "df.columns = list(map(lambda x: md[x] if x in md else x, df.columns))\n",
    "df.insert(0, column='Dataset', value=['proto']*len(df))\n",
    "print(df.round(3).to_latex())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}