misc_save_survival_metadata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Creates metadata file of images used for model training from the parsed and filtered dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# auto reload modules\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from utils.helpers import get_areds_data_dir\n",
    "from utils.create_survival_data import get_survival_data\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parsed and filtered metadata file from AREDS, relative to the data root as specified in configs/dirs.yml\n",
    "FILTERED_METADATA_CSV = \"data_processed/metadata/image_level_metadata_filtered.csv\"\n",
    "\n",
    "visit_numbers = None\n",
    "remove_odd_visits = False\n",
    "event_grade = 9 # This is the zero-based AREDS severity score, i.e. 9 and above is late AMD\n",
    "keep_metadata = True\n",
    "data_root = Path(get_areds_data_dir(scratch=True))\n",
    "metadata_csv = data_root.joinpath(FILTERED_METADATA_CSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of parsed and filtered data:  133293\n"
     ]
    }
   ],
   "source": [
    "print(\"length of parsed and filtered data: \", len(pd.read_csv(metadata_csv)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decide on metadata file to create (i.e., normal (one view per eye) or with stereo pairs) <br>\n",
    "Uncomment the appropriate code block and run the notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard metadata. For all except the stereo pair model (Babenko et al.)\n",
    "filename = \"metadata_surv.csv\"\n",
    "keep_stereo_pairs = False\n",
    "delete_initial_converters = False\n",
    "shift_odd_visits = False\n",
    "keep_converted = False\n",
    "\n",
    "# # Stereo pair version for Babenko et al.\n",
    "# filename = \"metadata_surv_stereo.csv\"\n",
    "# keep_stereo_pairs = True\n",
    "# delete_initial_converters = False\n",
    "# shift_odd_visits = False\n",
    "# keep_converted = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_after_event = False if \"keep_converted\" in locals() and keep_converted else True\n",
    "\n",
    "df_surv = get_survival_data(\n",
    "        metadata_csv.__str__(),\n",
    "        event_grade=event_grade,\n",
    "        remove_odd_visits=remove_odd_visits,\n",
    "        visit_numbers=visit_numbers,\n",
    "        impute=True if keep_metadata else False,\n",
    "        absolute_durations=False,\n",
    "        drop_after_event=drop_after_event,\n",
    "        keep_stereo_pairs=keep_stereo_pairs,\n",
    "    )\n",
    "\n",
    "if not drop_after_event: # keeping converted\n",
    "    df_surv.loc[df_surv[\"duration\"] < 0, \"duration\"] = 0\n",
    "\n",
    "if delete_initial_converters:\n",
    "    df_surv = df_surv[\n",
    "        ~((df_surv[\"event\"] == 1) & (df_surv[\"duration\"] == 0))\n",
    "    ]\n",
    "\n",
    "if shift_odd_visits:\n",
    "    # add 1 to all odd durations -> Converted in the time until the next even visit\n",
    "    df_surv.loc[df_surv[\"duration\"] % 2 == 1, \"duration\"] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>event</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1001_left_RS</td>\n",
       "      <td>False</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1001_right_LS</td>\n",
       "      <td>False</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1001_left_LS</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1001_right_LS</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001_left_RS</td>\n",
       "      <td>False</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      patient_id  event  duration\n",
       "0   1001_left_RS  False        16\n",
       "1  1001_right_LS  False        16\n",
       "2   1001_left_LS  False        14\n",
       "3  1001_right_LS  False        14\n",
       "4   1001_left_RS  False        12"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_surv[[\"patient_id\", \"event\", \"duration\"]].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52544"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_surv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGdCAYAAAASUnlxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAu8klEQVR4nO3df3RU9Z3/8VcSkgkgSQgsmUwNMdUWUH4KEuIPFktI+KErSl2jaaVtCqsmriG7oHQhBtAiQZCfK8u2Sj1LXGRXqQKLmQUhKkOAaBZBpOhiscUJWxFGiCRDMt8//M49DOFXYMIl+Twf5+Qc5t7PfO77vrkTXtx7ZyYiEAgEBAAAYIBIuwsAAAC4Ugg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjtLO7ADs1Njbq0KFD6tSpkyIiIuwuBwAAXIRAIKBvvvlGLpdLkZHNO4djdPA5dOiQUlJS7C4DAABcgi+++ELXXntts55jdPDp1KmTpO8aFxcXF7Z5/X6/ysvLlZWVpejo6LDNi/Oj7/ag7/ag7/ag7/Y4s+8+n08pKSnWv+PNYXTwCV7eiouLC3vw6dChg+Li4nhhXEH03R703R703R703R7n6vul3KbCzc0AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxmhndwFtWe+St1XXEGF3GRft8+fG2F0CAAAtijM+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACM0ezgU1FRobvvvlsul0sRERFas2aNtc7v9+vJJ59Unz591LFjR7lcLj388MM6dOhQyBxHjhxRbm6u4uLilJCQoLy8PB0/fjxkzK5du3THHXcoNjZWKSkpKi0tbVLL6tWr1bNnT8XGxqpPnz5av359c3cHAAAYpNnB58SJE+rXr5+WLl3aZF1tba0++OADTZ8+XR988IFef/117du3T3/zN38TMi43N1d79uyR2+3W2rVrVVFRoYkTJ1rrfT6fsrKylJqaqqqqKs2dO1clJSVavny5NWbr1q168MEHlZeXpw8//FBjx47V2LFjtXv37ubuEgAAMESzP8dn1KhRGjVq1FnXxcfHy+12hyxbsmSJBg8erIMHD6p79+7au3evNmzYoB07dmjQoEGSpMWLF2v06NF6/vnn5XK5tHLlStXX1+ull15STEyMbrrpJlVXV2v+/PlWQFq4cKFGjhypyZMnS5JmzZolt9utJUuWaNmyZc3dLQAAYIAWv8fn2LFjioiIUEJCgiTJ4/EoISHBCj2SlJmZqcjISFVWVlpjhg4dqpiYGGtMdna29u3bp6+//toak5mZGbKt7OxseTyeFt4jAADQWrXoJzefPHlSTz75pB588EHFxcVJkrxer7p16xZaRLt2SkxMlNfrtcakpaWFjElKSrLWde7cWV6v11p2+pjgHGdTV1enuro667HP55P03b1Jfr//EveyqeBcjshA2Oa8EsLZAzsE62/t+9Ha0Hd70Hd70Hd7nNn3y+l/iwUfv9+vv/3bv1UgENCLL77YUptpltmzZ2vGjBlNlpeXl6tDhw5h396sQY1hn7MltZWbw8+83Iorg77bg77bg77bI9j32traS56jRYJPMPT88Y9/1KZNm6yzPZLkdDp1+PDhkPGnTp3SkSNH5HQ6rTE1NTUhY4KPLzQmuP5spk6dqqKiIuuxz+dTSkqKsrKyQmq8XH6/X263W9N3RqqusfV8V9fukmy7S7gswb6PGDFC0dHRdpdjDPpuD/puD/pujzP7HrxicynCHnyCoWf//v1655131KVLl5D1GRkZOnr0qKqqqjRw4EBJ0qZNm9TY2Kj09HRrzD/90z/J7/dbB5bb7VaPHj3UuXNna8zGjRtVWFhoze12u5WRkXHO2hwOhxwOR5Pl0dHRLXIA1zVGtKovKW0rL+KW+vvE+dF3e9B3e9B3ewT7fjm9b/bNzcePH1d1dbWqq6slSQcOHFB1dbUOHjwov9+vH//4x9q5c6dWrlyphoYGeb1eeb1e1dfXS5J69eqlkSNHasKECdq+fbvef/99FRQUKCcnRy6XS5L00EMPKSYmRnl5edqzZ49WrVqlhQsXhpyteeKJJ7RhwwbNmzdPn3zyiUpKSrRz504VFBRccjMAAEDb1uzgs3PnTg0YMEADBgyQJBUVFWnAgAEqLi7Wn//8Z7355pv605/+pP79+ys5Odn62bp1qzXHypUr1bNnTw0fPlyjR4/W7bffHvIZPfHx8SovL9eBAwc0cOBA/cM//IOKi4tDPuvn1ltvVVlZmZYvX65+/frpP/7jP7RmzRr17t37cvoBAADasGZf6ho2bJgCgXO/W+l864ISExNVVlZ23jF9+/bVu+++e94x999/v+6///4Lbg8AAEDiu7oAAIBBCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACM0ezgU1FRobvvvlsul0sRERFas2ZNyPpAIKDi4mIlJyerffv2yszM1P79+0PGHDlyRLm5uYqLi1NCQoLy8vJ0/PjxkDG7du3SHXfcodjYWKWkpKi0tLRJLatXr1bPnj0VGxurPn36aP369c3dHQAAYJBmB58TJ06oX79+Wrp06VnXl5aWatGiRVq2bJkqKyvVsWNHZWdn6+TJk9aY3Nxc7dmzR263W2vXrlVFRYUmTpxorff5fMrKylJqaqqqqqo0d+5clZSUaPny5daYrVu36sEHH1ReXp4+/PBDjR07VmPHjtXu3bubu0sAAMAQ7Zr7hFGjRmnUqFFnXRcIBLRgwQJNmzZN99xzjyTplVdeUVJSktasWaOcnBzt3btXGzZs0I4dOzRo0CBJ0uLFizV69Gg9//zzcrlcWrlyperr6/XSSy8pJiZGN910k6qrqzV//nwrIC1cuFAjR47U5MmTJUmzZs2S2+3WkiVLtGzZsktqBgAAaNuaHXzO58CBA/J6vcrMzLSWxcfHKz09XR6PRzk5OfJ4PEpISLBCjyRlZmYqMjJSlZWVuvfee+XxeDR06FDFxMRYY7KzszVnzhx9/fXX6ty5szwej4qKikK2n52d3eTS2+nq6upUV1dnPfb5fJIkv98vv99/ubtvCc7liAyEbc4rIZw9sEOw/ta+H60NfbcHfbcHfbfHmX2/nP6HNfh4vV5JUlJSUsjypKQka53X61W3bt1Ci2jXTomJiSFj0tLSmswRXNe5c2d5vd7zbudsZs+erRkzZjRZXl5erg4dOlzMLjbLrEGNYZ+zJbWVe6TcbrfdJRiJvtuDvtuDvtsj2Pfa2tpLniOswedqN3Xq1JCzRD6fTykpKcrKylJcXFzYtuP3++V2uzV9Z6TqGiPCNm9L212SbXcJlyXY9xEjRig6OtrucoxB3+1B3+1B3+1xZt+DV2wuRViDj9PplCTV1NQoOTnZWl5TU6P+/ftbYw4fPhzyvFOnTunIkSPW851Op2pqakLGBB9faExw/dk4HA45HI4my6Ojo1vkAK5rjFBdQ+sJPm3lRdxSf584P/puD/puD/puj2DfL6f3Yf0cn7S0NDmdTm3cuNFa5vP5VFlZqYyMDElSRkaGjh49qqqqKmvMpk2b1NjYqPT0dGtMRUVFyDU8t9utHj16qHPnztaY07cTHBPcDgAAwJmaHXyOHz+u6upqVVdXS/ruhubq6modPHhQERERKiws1DPPPKM333xTH330kR5++GG5XC6NHTtWktSrVy+NHDlSEyZM0Pbt2/X++++roKBAOTk5crlckqSHHnpIMTExysvL0549e7Rq1SotXLgw5DLVE088oQ0bNmjevHn65JNPVFJSop07d6qgoODyuwIAANqkZl/q2rlzp+68807rcTCMjB8/XitWrNCUKVN04sQJTZw4UUePHtXtt9+uDRs2KDY21nrOypUrVVBQoOHDhysyMlLjxo3TokWLrPXx8fEqLy9Xfn6+Bg4cqK5du6q4uDjks35uvfVWlZWVadq0afrVr36lH/zgB1qzZo169+59SY0AAABtX7ODz7BhwxQInPtt2hEREZo5c6Zmzpx5zjGJiYkqKys773b69u2rd99997xj7r//ft1///3nLxgAAOD/47u6AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADBGO7sLAICWct1T6+wuodk+f26M3SUAbRpnfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMcIefBoaGjR9+nSlpaWpffv2uv766zVr1iwFAgFrTCAQUHFxsZKTk9W+fXtlZmZq//79IfMcOXJEubm5iouLU0JCgvLy8nT8+PGQMbt27dIdd9yh2NhYpaSkqLS0NNy7AwAA2pCwB585c+boxRdf1JIlS7R3717NmTNHpaWlWrx4sTWmtLRUixYt0rJly1RZWamOHTsqOztbJ0+etMbk5uZqz549crvdWrt2rSoqKjRx4kRrvc/nU1ZWllJTU1VVVaW5c+eqpKREy5cvD/cuAQCANiLs39W1detW3XPPPRoz5rvvm7nuuuv06quvavv27ZK+O9uzYMECTZs2Tffcc48k6ZVXXlFSUpLWrFmjnJwc7d27Vxs2bNCOHTs0aNAgSdLixYs1evRoPf/883K5XFq5cqXq6+v10ksvKSYmRjfddJOqq6s1f/78kIAEAAAQFPbgc+utt2r58uX6wx/+oB/+8If6n//5H7333nuaP3++JOnAgQPyer3KzMy0nhMfH6/09HR5PB7l5OTI4/EoISHBCj2SlJmZqcjISFVWVuree++Vx+PR0KFDFRMTY43Jzs7WnDlz9PXXX6tz585Naqurq1NdXZ312OfzSZL8fr/8fn/YehCcyxEZuMDIq0s4e2CHYP2tfT9am6u5746o1vUalC6+j1dz39sy+m6PM/t+Of0Pe/B56qmn5PP51LNnT0VFRamhoUHPPvuscnNzJUler1eSlJSUFPK8pKQka53X61W3bt1CC23XTomJiSFj0tLSmswRXHe24DN79mzNmDGjyfLy8nJ16NDhUnb3vGYNagz7nC1p/fr1dpcQFm632+4SjHQ19r10sN0VNF9zX4dXY99NQN/tEex7bW3tJc8R9uDz2muvaeXKlSorK7MuPxUWFsrlcmn8+PHh3lyzTJ06VUVFRdZjn8+nlJQUZWVlKS4uLmzb8fv9crvdmr4zUnWNEWGbt6XtLsm2u4TLEuz7iBEjFB0dbXc5xria+9675G27S2i2i30dXs19b8vouz3O7Hvwis2lCHvwmTx5sp566inl5ORIkvr06aM//vGPmj17tsaPHy+n0ylJqqmpUXJysvW8mpoa9e/fX5LkdDp1+PDhkHlPnTqlI0eOWM93Op2qqakJGRN8HBxzJofDIYfD0WR5dHR0ixzAdY0RqmtoPcGnrbyIW+rvE+d3Nfa9Nb3+gprbw6ux7yag7/YI9v1yeh/2d3XV1tYqMjJ02qioKDU2fnfZJy0tTU6nUxs3brTW+3w+VVZWKiMjQ5KUkZGho0ePqqqqyhqzadMmNTY2Kj093RpTUVERcp3P7XarR48eZ73MBQAAEPbgc/fdd+vZZ5/VunXr9Pnnn+uNN97Q/Pnzde+990qSIiIiVFhYqGeeeUZvvvmmPvroIz388MNyuVwaO3asJKlXr14aOXKkJkyYoO3bt+v9999XQUGBcnJy5HK5JEkPPfSQYmJilJeXpz179mjVqlVauHBhyKUsAACA04X9UtfixYs1ffp0PfbYYzp8+LBcLpf+7u/+TsXFxdaYKVOm6MSJE5o4caKOHj2q22+/XRs2bFBsbKw1ZuXKlSooKNDw4cMVGRmpcePGadGiRdb6+Ph4lZeXKz8/XwMHDlTXrl1VXFzMW9kBAMA5hT34dOrUSQsWLNCCBQvOOSYiIkIzZ87UzJkzzzkmMTFRZWVl591W37599e67715qqQAAwDB8VxcAADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxWiT4/PnPf9ZPfvITdenSRe3bt1efPn20c+dOa30gEFBxcbGSk5PVvn17ZWZmav/+/SFzHDlyRLm5uYqLi1NCQoLy8vJ0/PjxkDG7du3SHXfcodjYWKWkpKi0tLQldgcAALQRYQ8+X3/9tW677TZFR0frv/7rv/Txxx9r3rx56ty5szWmtLRUixYt0rJly1RZWamOHTsqOztbJ0+etMbk5uZqz549crvdWrt2rSoqKjRx4kRrvc/nU1ZWllJTU1VVVaW5c+eqpKREy5cvD/cuAQCANqJduCecM2eOUlJS9PLLL1vL0tLSrD8HAgEtWLBA06ZN0z333CNJeuWVV5SUlKQ1a9YoJydHe/fu1YYNG7Rjxw4NGjRIkrR48WKNHj1azz//vFwul1auXKn6+nq99NJLiomJ0U033aTq6mrNnz8/JCABAAAEhT34vPnmm8rOztb999+vLVu26Hvf+54ee+wxTZgwQZJ04MABeb1eZWZmWs+Jj49Xenq6PB6PcnJy5PF4lJCQYIUeScrMzFRkZKQqKyt17733yuPxaOjQoYqJibHGZGdna86cOfr6669DzjAF1dXVqa6uznrs8/kkSX6/X36/P2w9CM7liAyEbc4rIZw9sEOw/ta+H63N1dx3R1Treg1KF9/Hq7nvbRl9t8eZfb+c/oc9+Pzv//6vXnzxRRUVFelXv/qVduzYob//+79XTEyMxo8fL6/XK0lKSkoKeV5SUpK1zuv1qlu3bqGFtmunxMTEkDGnn0k6fU6v13vW4DN79mzNmDGjyfLy8nJ16NDhEvf43GYNagz7nC1p/fr1dpcQFm632+4SjHQ19r10sN0VNF9zX4dXY99NQN/tEex7bW3tJc8R9uDT2NioQYMG6de//rUkacCAAdq9e7eWLVum8ePHh3tzzTJ16lQVFRVZj30+n1JSUpSVlaW4uLiwbcfv98vtdmv6zkjVNUaEbd6Wtrsk2+4SLkuw7yNGjFB0dLTd5Rjjau5775K37S6h2S72dXg1970to+/2OLPvwSs2lyLswSc5OVk33nhjyLJevXrpP//zPyVJTqdTklRTU6Pk5GRrTE1Njfr372+NOXz4cMgcp06d0pEjR6znO51O1dTUhIwJPg6OOZPD4ZDD4WiyPDo6ukUO4LrGCNU1tJ7g01ZexC3194nzuxr73ppef0HN7eHV2HcT0Hd7BPt+Ob0P+7u6brvtNu3bty9k2R/+8AelpqZK+u5GZ6fTqY0bN1rrfT6fKisrlZGRIUnKyMjQ0aNHVVVVZY3ZtGmTGhsblZ6ebo2pqKgIuc7ndrvVo0ePs17mAgAACHvwmTRpkrZt26Zf//rX+vTTT1VWVqbly5crPz9fkhQREaHCwkI988wzevPNN/XRRx/p4Ycflsvl0tixYyV9d4Zo5MiRmjBhgrZv3673339fBQUFysnJkcvlkiQ99NBDiomJUV5envbs2aNVq1Zp4cKFIZeyAAAAThf2S1233HKL3njjDU2dOlUzZ85UWlqaFixYoNzcXGvMlClTdOLECU2cOFFHjx7V7bffrg0bNig2NtYas3LlShUUFGj48OGKjIzUuHHjtGjRImt9fHy8ysvLlZ+fr4EDB6pr164qLi7mrewAAOCcwh58JOmuu+7SXXfddc71ERERmjlzpmbOnHnOMYmJiSorKzvvdvr27at33333kusEAABm4bu6AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjtMgHGAJXynVPrbP+7IgKqHTwd9/IfTV/OeXnz42xuwQAMBZnfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAY3N8Ny+o3CAAC0RZzxAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIzRzu4CALQO1z217qzLHVEBlQ6Wepe8rbqGiCtcFQA0D8EHuMLOFSAAAC2PS10AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxmjx4PPcc88pIiJChYWF1rKTJ08qPz9fXbp00TXXXKNx48appqYm5HkHDx7UmDFj1KFDB3Xr1k2TJ0/WqVOnQsZs3rxZN998sxwOh2644QatWLGipXcHAAC0Yi0afHbs2KF/+Zd/Ud++fUOWT5o0SW+99ZZWr16tLVu26NChQ7rvvvus9Q0NDRozZozq6+u1detW/e53v9OKFStUXFxsjTlw4IDGjBmjO++8U9XV1SosLNQvf/lLvf322y25SwAAoBVrseBz/Phx5ebm6l//9V/VuXNna/mxY8f029/+VvPnz9ePfvQjDRw4UC+//LK2bt2qbdu2SZLKy8v18ccf69/+7d/Uv39/jRo1SrNmzdLSpUtVX18vSVq2bJnS0tI0b9489erVSwUFBfrxj3+sF154oaV2CQAAtHLtWmri/Px8jRkzRpmZmXrmmWes5VVVVfL7/crMzLSW9ezZU927d5fH49GQIUPk8XjUp08fJSUlWWOys7P16KOPas+ePRowYIA8Hk/IHMExp19SO1NdXZ3q6uqsxz6fT5Lk9/vl9/svd5ctwbkckYGwzYkLC/abvl9Z9D28LvZ3UXBcOH934cLouz3O7Pvl9L9Fgs+///u/64MPPtCOHTuarPN6vYqJiVFCQkLI8qSkJHm9XmvM6aEnuD647nxjfD6fvv32W7Vv377JtmfPnq0ZM2Y0WV5eXq4OHTpc/A5epFmDGsM+Jy6MvtuDvofH+vXrmzXe7Xa3UCU4H/puj2Dfa2trL3mOsAefL774Qk888YTcbrdiY2PDPf1lmTp1qoqKiqzHPp9PKSkpysrKUlxcXNi24/f75Xa7NX1npOoaI8I2L87PERnQrEGN9P0Ko+/htbsk+6LGBX/PjBgxQtHR0S1cFYLouz3O7Hvwis2lCHvwqaqq0uHDh3XzzTdbyxoaGlRRUaElS5bo7bffVn19vY4ePRpy1qempkZOp1OS5HQ6tX379pB5g+/6On3Mme8Eq6mpUVxc3FnP9kiSw+GQw+Fosjw6OrpFDuC6xgjVNfAPwZVG3+1B38Ojub+LWur3F86Pvtsj2PfL6X3Yb24ePny4PvroI1VXV1s/gwYNUm5urvXn6Ohobdy40XrOvn37dPDgQWVkZEiSMjIy9NFHH+nw4cPWGLfbrbi4ON14443WmNPnCI4JzgEAAHCmsJ/x6dSpk3r37h2yrGPHjurSpYu1PC8vT0VFRUpMTFRcXJwef/xxZWRkaMiQIZKkrKws3XjjjfrpT3+q0tJSeb1eTZs2Tfn5+dYZm0ceeURLlizRlClT9Itf/EKbNm3Sa6+9pnXr1oV7lwDgirnuqYv7HeaICqh0sNS75G3bz7R9/twYW7cPNEeLvavrfF544QVFRkZq3LhxqqurU3Z2tv75n//ZWh8VFaW1a9fq0UcfVUZGhjp27Kjx48dr5syZ1pi0tDStW7dOkyZN0sKFC3XttdfqN7/5jbKzL+76OAAAMM8VCT6bN28OeRwbG6ulS5dq6dKl53xOamrqBd/dMGzYMH344YfhKBEAABiA7+oCAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMEY7uwsAALRu1z21zu4Smu3z58bYXQJswhkfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxwh58Zs+erVtuuUWdOnVSt27dNHbsWO3bty9kzMmTJ5Wfn68uXbrommuu0bhx41RTUxMy5uDBgxozZow6dOigbt26afLkyTp16lTImM2bN+vmm2+Ww+HQDTfcoBUrVoR7dwAAQBsS9uCzZcsW5efna9u2bXK73fL7/crKytKJEyesMZMmTdJbb72l1atXa8uWLTp06JDuu+8+a31DQ4PGjBmj+vp6bd26Vb/73e+0YsUKFRcXW2MOHDigMWPG6M4771R1dbUKCwv1y1/+Um+//Xa4dwkAALQRYf/k5g0bNoQ8XrFihbp166aqqioNHTpUx44d029/+1uVlZXpRz/6kSTp5ZdfVq9evbRt2zYNGTJE5eXl+vjjj/Xf//3fSkpKUv/+/TVr1iw9+eSTKikpUUxMjJYtW6a0tDTNmzdPktSrVy+99957euGFF5SdnR3u3QIAAG1Ai39lxbFjxyRJiYmJkqSqqir5/X5lZmZaY3r27Knu3bvL4/FoyJAh8ng86tOnj5KSkqwx2dnZevTRR7Vnzx4NGDBAHo8nZI7gmMLCwnPWUldXp7q6Ouuxz+eTJPn9fvn9/sve16DgXI7IQNjmxIUF+03fryz6bg/6fnku9Xd+8Hnh/DcDF3Zm3y+n/y0afBobG1VYWKjbbrtNvXv3liR5vV7FxMQoISEhZGxSUpK8Xq815vTQE1wfXHe+MT6fT99++63at2/fpJ7Zs2drxowZTZaXl5erQ4cOl7aT5zFrUGPY58SF0Xd70Hd70PdLs379+st6vtvtDlMlaI5g32tray95jhYNPvn5+dq9e7fee++9ltzMRZs6daqKioqsxz6fTykpKcrKylJcXFzYtuP3++V2uzV9Z6TqGiPCNi/OzxEZ0KxBjfT9CqPv9qDvl2d3yaXdEhH8/T5ixAhFR0eHuSqcy5l9D16xuRQtFnwKCgq0du1aVVRU6Nprr7WWO51O1dfX6+jRoyFnfWpqauR0Oq0x27dvD5kv+K6v08ec+U6wmpoaxcXFnfVsjyQ5HA45HI4my6Ojo1vkAK5rjFBdA7+QrjT6bg/6bg/6fmku93d+S/27gfML9v1yeh/24BMIBPT444/rjTfe0ObNm5WWlhayfuDAgYqOjtbGjRs1btw4SdK+fft08OBBZWRkSJIyMjL07LPP6vDhw+rWrZuk705vxcXF6cYbb7TGnHmq0u12W3NcDXY78hTdeNLuMi7adSfL7C4BAIAWFfbgk5+fr7KyMv3+979Xp06drHty4uPj1b59e8XHxysvL09FRUVKTExUXFycHn/8cWVkZGjIkCGSpKysLN1444366U9/qtLSUnm9Xk2bNk35+fnWGZtHHnlES5Ys0ZQpU/SLX/xCmzZt0muvvaZ169aFe5cAAEAbEfbP8XnxxRd17NgxDRs2TMnJydbPqlWrrDEvvPCC7rrrLo0bN05Dhw6V0+nU66+/bq2PiorS2rVrFRUVpYyMDP3kJz/Rww8/rJkzZ1pj0tLStG7dOrndbvXr10/z5s3Tb37zG97KDgAAzqlFLnVdSGxsrJYuXaqlS5eec0xqauoF77ofNmyYPvzww2bXCAAAzMR3dQEAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADBGO7sLAADgSrvuqXWX9DxHVEClg6XeJW+rriEizFWd3+fPjbmi22urOOMDAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjNHO7gJw9fg89iG7S7gs/shYrddy7XbkKbrxpN3lnNN1J8vsLgEAjEXwAdBmtcYwTzAGWhaXugAAgDE44wNcYa3xLITEmQgAbQPBB8BFOVdgay33VgGt3XVPrbO7hGb7/LkxdpfQBJe6AACAMQg+AADAGK3+UtfSpUs1d+5ceb1e9evXT4sXL9bgwYPtLgsALsnF3gN2NV1i5P4vtCat+ozPqlWrVFRUpKeffloffPCB+vXrp+zsbB0+fNju0gAAwFWoVZ/xmT9/viZMmKCf//znkqRly5Zp3bp1eumll/TUU0/ZXB0AmKE1vlORs1TmarXBp76+XlVVVZo6daq1LDIyUpmZmfJ4PGd9Tl1dnerq6qzHx44dkyQdOXJEfr8/bLX5/X7V1tbqq/oYRTc2hm1enJ8/Moa+24C+24O+X56qyJ9d0vP8EbF6p7ZUWyKmKDryyl5iTK9fekW3Fw5fffVVWOax/l396itFR0frm2++kSQFAoFmz9Vqg89f/vIXNTQ0KCkpKWR5UlKSPvnkk7M+Z/bs2ZoxY0aT5WlpaS1SI+zQ+v7n2TbQd3vQd3vY1fcHbNrupes6r2Xn/+abbxQfH9+s57Ta4HMppk6dqqKiIutxY2Ojjhw5oi5duigiIiJs2/H5fEpJSdEXX3yhuLi4sM2L86Pv9qDv9qDv9qDv9jiz74FAQN98841cLlez52q1wadr166KiopSTU1NyPKamho5nc6zPsfhcMjhcIQsS0hIaKkSFRcXxwvDBvTdHvTdHvTdHvTdHqf3vblneoJa7bu6YmJiNHDgQG3cuNFa1tjYqI0bNyojI8PGygAAwNWq1Z7xkaSioiKNHz9egwYN0uDBg7VgwQKdOHHCepcXAADA6Vp18HnggQf0f//3fyouLpbX61X//v21YcOGJjc8X2kOh0NPP/10k8tqaFn03R703R703R703R7h7HtE4FLeCwYAANAKtdp7fAAAAJqL4AMAAIxB8AEAAMYg+AAAAGMQfFrA0qVLdd111yk2Nlbp6enavn273SW1aSUlJYqIiAj56dmzp91ltTkVFRW6++675XK5FBERoTVr1oSsDwQCKi4uVnJystq3b6/MzEzt37/fnmLbkAv1/Wc/+1mT43/kyJH2FNtGzJ49W7fccos6deqkbt26aezYsdq3b1/ImJMnTyo/P19dunTRNddco3HjxjX5QF00z8X0fdiwYU2O90ceeaRZ2yH4hNmqVatUVFSkp59+Wh988IH69eun7OxsHT582O7S2rSbbrpJX375pfXz3nvv2V1Sm3PixAn169dPS5ee/YsSS0tLtWjRIi1btkyVlZXq2LGjsrOzdfLklf0ix7bmQn2XpJEjR4Yc/6+++uoVrLDt2bJli/Lz87Vt2za53W75/X5lZWXpxIkT1phJkybprbfe0urVq7VlyxYdOnRI9913n41Vt34X03dJmjBhQsjxXlpa2rwNBRBWgwcPDuTn51uPGxoaAi6XKzB79mwbq2rbnn766UC/fv3sLsMokgJvvPGG9bixsTHgdDoDc+fOtZYdPXo04HA4Aq+++qoNFbZNZ/Y9EAgExo8fH7jnnntsqccUhw8fDkgKbNmyJRAIfHdsR0dHB1avXm2N2bt3b0BSwOPx2FVmm3Nm3wOBQOCv//qvA0888cRlzcsZnzCqr69XVVWVMjMzrWWRkZHKzMyUx+OxsbK2b//+/XK5XPr+97+v3NxcHTx40O6SjHLgwAF5vd6QYz8+Pl7p6ekc+1fA5s2b1a1bN/Xo0UOPPvqovvrqK7tLalOOHTsmSUpMTJQkVVVVye/3hxzvPXv2VPfu3Tnew+jMvgetXLlSXbt2Ve/evTV16lTV1tY2a95W/cnNV5u//OUvamhoaPLJ0UlJSfrkk09sqqrtS09P14oVK9SjRw99+eWXmjFjhu644w7t3r1bnTp1srs8I3i9Xkk667EfXIeWMXLkSN13331KS0vTZ599pl/96lcaNWqUPB6PoqKi7C6v1WtsbFRhYaFuu+029e7dW9J3x3tMTEyTL7nmeA+fs/Vdkh566CGlpqbK5XJp165devLJJ7Vv3z69/vrrFz03wQet3qhRo6w/9+3bV+np6UpNTdVrr72mvLw8GysDWl5OTo715z59+qhv3766/vrrtXnzZg0fPtzGytqG/Px87d69m/sGr7Bz9X3ixInWn/v06aPk5GQNHz5cn332ma6//vqLmptLXWHUtWtXRUVFNbmzv6amRk6n06aqzJOQkKAf/vCH+vTTT+0uxRjB45tj337f//731bVrV47/MCgoKNDatWv1zjvv6Nprr7WWO51O1dfX6+jRoyHjOd7D41x9P5v09HRJatbxTvAJo5iYGA0cOFAbN260ljU2Nmrjxo3KyMiwsTKzHD9+XJ999pmSk5PtLsUYaWlpcjqdIce+z+dTZWUlx/4V9qc//UlfffUVx/9lCAQCKigo0BtvvKFNmzYpLS0tZP3AgQMVHR0dcrzv27dPBw8e5Hi/DBfq+9lUV1dLUrOOdy51hVlRUZHGjx+vQYMGafDgwVqwYIFOnDihn//853aX1mb94z/+o+6++26lpqbq0KFDevrppxUVFaUHH3zQ7tLalOPHj4f8r+rAgQOqrq5WYmKiunfvrsLCQj3zzDP6wQ9+oLS0NE2fPl0ul0tjx461r+g24Hx9T0xM1IwZMzRu3Dg5nU599tlnmjJlim644QZlZ2fbWHXrlp+fr7KyMv3+979Xp06drPt24uPj1b59e8XHxysvL09FRUVKTExUXFycHn/8cWVkZGjIkCE2V996Xajvn332mcrKyjR69Gh16dJFu3bt0qRJkzR06FD17dv34jd0We8Jw1ktXrw40L1790BMTExg8ODBgW3bttldUpv2wAMPBJKTkwMxMTGB733ve4EHHngg8Omnn9pdVpvzzjvvBCQ1+Rk/fnwgEPjuLe3Tp08PJCUlBRwOR2D48OGBffv22Vt0G3C+vtfW1gaysrICf/VXfxWIjo4OpKamBiZMmBDwer12l92qna3fkgIvv/yyNebbb78NPPbYY4HOnTsHOnToELj33nsDX375pX1FtwEX6vvBgwcDQ4cODSQmJgYcDkfghhtuCEyePDlw7NixZm0n4v9vDAAAoM3jHh8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjPH/AEnl4brtMbcSAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the distribution of the duration by converter status\n",
    "for event in df_surv[\"event\"].unique():\n",
    "    df_surv[df_surv[\"event\"] == event][\"duration\"].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    48636\n",
       "True      3908\n",
       "Name: event, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_surv[\"event\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shuffle the data but alternate between converters and non-converters. Good for testing runs with subset of data\n",
    "df_surv_converters = df_surv[df_surv[\"event\"] == True].sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "df_surv_converters.index = np.arange(1, len(df_surv_converters)*2, 2)\n",
    "\n",
    "df_surv_non_converters = df_surv[df_surv[\"event\"] == False].sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "df_surv_non_converters.index = np.arange(0, len(df_surv_non_converters)*2, 2)\n",
    "\n",
    "df_surv = pd.concat([df_surv_converters, df_surv_non_converters]).sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_surv.to_csv(f'data/{filename}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### (Optional) Load and check if IPCW can be computed: <br>Estimation of censoring distribution, needed as weight for censoring adjusted metrics from sksurv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data.cnn_surv_dataloader import get_dataset\n",
    "from utils.helpers import get_config\n",
    "from utils.cnn_survival_utils import check_ipcw_calc\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_surv = pd.read_csv(f'data/{filename}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = get_config(\"sparsebagnet_cox.yml\")\n",
    "c.metadata_csv = f'data/{filename}'\n",
    "c.cnn.gpu = 0\n",
    "c.cnn.test_run[\"enabled\"] = False\n",
    "c.cnn.train_set_fraction = 1.0\n",
    "c.cnn.val_set_fraction = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Len data (Number of images in train split): 31459\n",
      "Len data (Number of images in val split): 10496\n",
      "Len data (Number of images in test split): 10589\n"
     ]
    }
   ],
   "source": [
    "y_train_set = get_dataset(split=\"train\", c=c)\n",
    "y_val_set = get_dataset(split=\"val\", c=c)\n",
    "y_test_set = get_dataset(split=\"test\", c=c)\n",
    "\n",
    "y_survival = {\n",
    "    \"train\": y_train_set.get_e_t(),\n",
    "    \"val\": y_val_set.get_e_t(),\n",
    "    \"test\": y_test_set.get_e_t(),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_y_train = pd.DataFrame(y_train_set._e.numpy(), columns=[\"event\"])\n",
    "df_y_train[\"duration\"] = y_train_set._t.numpy()\n",
    "df_y_val = pd.DataFrame(y_val_set._e.numpy(), columns=[\"event\"])\n",
    "df_y_val[\"duration\"] = y_val_set._t.numpy()\n",
    "df_y_test = pd.DataFrame(y_test_set._e.numpy(), columns=[\"event\"])\n",
    "df_y_test[\"duration\"] = y_test_set._t.numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max time train 24.0\n"
     ]
    }
   ],
   "source": [
    "assert check_ipcw_calc(y_survival[\"train\"], y_survival[\"val\"]), \"Error, IPCW calculation failed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}