macromicro_explore_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "from IPython import get_ipython\n",
    "ipython = get_ipython()\n",
    "if '__IPYTHON__' in globals():\n",
    "    ipython.magic('load_ext autoreload')\n",
    "    ipython.magic('autoreload 2')\n",
    "\n",
    "import argparse\n",
    "import os\n",
    "from glob import glob\n",
    "import pickle\n",
    "from itertools import compress\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.metrics import r2_score\n",
    "from scipy.io import loadmat\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "from mymodels.macromicro_conv_model import load_matlab, shifted_zscore, print_metric, zscore\n",
    "\n",
    "def minmax(x, inverse=False):\n",
    "    xmin = np.array([-1,-1,6.8,-1])\n",
    "    xmax = np.array([4000.,4000.,8.,4000.])\n",
    "    \n",
    "    a = xmax - xmin\n",
    "\n",
    "    if inverse:\n",
    "        x = x*a + xmin\n",
    "    else:\n",
    "        x = (x-xmin)/a\n",
    "\n",
    "    return x\n",
    "\n",
    "# normalize_data = minmax\n",
    "# revert_data = lambda x: minmax(x, inverse=True)\n",
    "\n",
    "# normalize_data = shifted_zscore\n",
    "# revert_data = lambda x: shifted_zscore(x, inverse=True)\n",
    "\n",
    "# normalize_data = zscore\n",
    "# revert_data = lambda x: zscore(x, inverse=True)\n",
    "\n",
    "normalize_data = lambda x: x\n",
    "revert_data = lambda x: x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 9.26681881,  9.26681856, -7.31488353,  9.26681886])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# shifted_zscore(np.array([2.89606,2.89606,2.89606,2.89606])/4, inverse=True)\n",
    "shifted_zscore(np.array([2500,2500,7.5,2500])/4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "npdt = np.float32\n",
    "\n",
    "names = ['DA', '5HT', 'pH', 'NE']\n",
    "# prefix = '/mnt/nfs/proj/in-vitro/iterate/results_014/model_style_008/training-A-RBE-97Hz/'\n",
    "prefix = '/mnt/nfs/proj/in-vitro/iterate/results_014/model_style_008/training-B-RBE025-97Hz/'\n",
    "\n",
    "probes = ['MMA003W01R04', 'MMA004W01R04', 'MMA013W01R04', 'MMA018W01R04', 'MMA019W01R04', 'MMA022W01R04', 'MMA023W01R04', 'MMA024W01R04', 'MMA025W01R04']\n",
    "# good_probes = ['MMA003W01R04', 'MMA004W01R04', 'MMA013W01R04', 'MMA019W01R04', 'MMA022W01R04', 'MMA024W01R04', 'MMA025W01R04']\n",
    "good_probes = probes\n",
    "filter_files = lambda x : x.find('500kohm') == -1\n",
    "\n",
    "# probes = ['MMA003W01R04', 'MMA004W01R04', 'MMA013W01R04', 'MMA018W01R04', 'MMA019W01R04', 'MMA022W01R04'] # 500 kohms\n",
    "# good_probes = probes\n",
    "# filter_files = lambda x : x.find('500kohm') > -1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of voltammograms files 36\n",
      "after filter 36\n",
      "\n",
      "number of probe MMA003W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA004W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA013W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA018W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA019W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA022W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA023W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA024W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n",
      "number of probe MMA025W01R04s files: 4\n",
      "(39600, 999)\n",
      "(39600, 4)\n",
      "(39600, 999) (39600, 4)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "nrecords_per_session = 300\n",
    "\n",
    "voltammograms = glob(prefix + '/*MMA*/voltammograms.mat')\n",
    "print('number of voltammograms files %d'%len(voltammograms))\n",
    "if not filter_files is None:\n",
    "    voltammograms = list(compress(voltammograms, [filter_files(x) for x in voltammograms]))\n",
    "print('after filter %d'%len(voltammograms))\n",
    "print()\n",
    "\n",
    "jitter = None\n",
    "# jitter = 50\n",
    "\n",
    "probe_data = {}\n",
    "last_conc = None\n",
    "for (iprobe, probe) in enumerate(probes):\n",
    "    probe_files = voltammograms.copy()\n",
    "    probe_files = list(compress(probe_files, [x.find(probe) > -1 for x in probe_files]))\n",
    "    print('number of probe %s''s files: %d'%(probe, len(probe_files)), flush=True)\n",
    "    probe_x, probe_y = load_matlab(probe_files, nrecords_per_session, split=1., asnumpy=True, proj_y=normalize_data)\n",
    "\n",
    "    print(probe_x.shape, probe_y.shape)\n",
    "    ul, ulidx = np.unique(probe_y, return_index=True, axis=0)\n",
    "    if last_conc is not None:\n",
    "        if np.any(ul.flatten() - last_conc.flatten()):\n",
    "            print('%s different'%probe)\n",
    "        last_conb = ul\n",
    "    else:\n",
    "        last_conc = ul\n",
    "    \n",
    "    if jitter is not None:\n",
    "        for (ix, x) in enumerate(probe_x):\n",
    "            j = np.random.randint(-jitter, jitter)\n",
    "            y = np.roll(x, j)\n",
    "            if j > 0:\n",
    "                y[:j] = 0\n",
    "            elif j < 0:\n",
    "                y[j:] = 0\n",
    "            probe_x[ix,:] = y\n",
    "\n",
    "    probe_data[probe] = (probe_x, probe_y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd24718045a845f0b8099c328455adca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x2aabd3c9d2d0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "fig, axs = plt.subplots(tight_layout=True, figsize=(12, 5))\n",
    "    \n",
    "# only_probes = ['MMA003W01R04', 'MMA004W01R04', 'MMA013W01R04']\n",
    "only_probes = good_probes\n",
    "\n",
    "plot_data = np.zeros((list(probe_data.values())[0][0].shape[1], len(only_probes)))\n",
    "for (iprobe, probe) in enumerate(only_probes):\n",
    "    if only_probes is None or probe in only_probes:\n",
    "        probe_x, probe_y = probe_data[probe]\n",
    "        ul, ulidx = np.unique(probe_y, return_index=True, axis=0)\n",
    "        plot_data[:,iprobe] = np.mean(probe_x[ulidx,:], axis=0)\n",
    "\n",
    "axs.plot(plot_data)\n",
    "# axs.set_ylim(-50, 50)\n",
    "\n",
    "# for shift in range(0,900,5):\n",
    "#     axs.plot(plot_data[:,shift:shift+40], label=probe)\n",
    "#     plt.draw()\n",
    "#     time.sleep(.5)\n",
    "axs.legend(only_probes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1230ad3ec2a640d9bd1d4a43f37e3f63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b35b886fc9f4253b190111ddd2bb04a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d20e3b5f0d34379aac88a49f009d7bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d84c461ca0e4091b681dd468c0624a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92ae594acf654ba18673a9d2e8e6558b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0f046bc2d204343ad726143245f2101",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2f4394742694c87870ca95f1fb63168",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c09dc607d3f45c78f94bba7aa2697c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d38ca85487ce499bbd4d6f8cc9cc8906",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for (iprobe, probe) in enumerate(good_probes):\n",
    "    probe_x, probe_y = probe_data[probe]\n",
    "\n",
    "    ul, ulidx = np.unique(probe_y, return_index=True, axis=0)\n",
    "    \n",
    "    fig, axs = plt.subplots(tight_layout=True, figsize=(12, 5))\n",
    "\n",
    "    for idx in ulidx:\n",
    "        axs.plot(probe_x[idx,:])\n",
    "    axs.set_title(probe)\n",
    "#     axs[iprobe].set_ylim(-50, 50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "284787a86b9948e88dae7b63daaf6807",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MMA003W01R04 [-9.4523779e+03 -9.4524736e+03  4.2099996e+00 -9.4524160e+03]\n",
      "MMA003W01R04 [1.0397461e+03 1.0397556e+03 3.1908679e-01 1.0397499e+03]\n",
      "\n",
      "MMA004W01R04 [-9.4526260e+03 -9.4525010e+03  4.2099996e+00 -9.4528340e+03]\n",
      "MMA004W01R04 [1.0397694e+03 1.0397570e+03 3.1908679e-01 1.0397903e+03]\n",
      "\n",
      "MMA013W01R04 [-9.4526797e+03 -9.4525918e+03  4.2099996e+00 -9.4525947e+03]\n",
      "MMA013W01R04 [1.0397756e+03 1.0397668e+03 3.1908679e-01 1.0397671e+03]\n",
      "\n",
      "MMA018W01R04 [-9.4526650e+03 -9.4525146e+03  4.2099996e+00 -9.4527832e+03]\n",
      "MMA018W01R04 [1.0397733e+03 1.0397583e+03 3.1908679e-01 1.0397852e+03]\n",
      "\n",
      "MMA019W01R04 [-9.4526260e+03 -9.4525010e+03  4.2099996e+00 -9.4528340e+03]\n",
      "MMA019W01R04 [1.0397694e+03 1.0397570e+03 3.1908679e-01 1.0397903e+03]\n",
      "\n",
      "MMA022W01R04 [-9.4525205e+03 -9.4525527e+03  4.2099996e+00 -9.4525352e+03]\n",
      "MMA022W01R04 [1.0397604e+03 1.0397635e+03 3.1908679e-01 1.0397618e+03]\n",
      "\n",
      "MMA023W01R04 [-9.4523213e+03 -9.4524355e+03  4.2099996e+00 -9.4524072e+03]\n",
      "MMA023W01R04 [1.0397405e+03 1.0397518e+03 3.1908679e-01 1.0397490e+03]\n",
      "\n",
      "MMA024W01R04 [-9.4527861e+03 -9.4526719e+03  4.2099996e+00 -9.4526719e+03]\n",
      "MMA024W01R04 [1.0397863e+03 1.0397748e+03 3.1908679e-01 1.0397748e+03]\n",
      "\n",
      "MMA025W01R04 [-9.4525410e+03 -9.4526006e+03  4.2099996e+00 -9.4524375e+03]\n",
      "MMA025W01R04 [1.0397617e+03 1.0397677e+03 3.1908679e-01 1.0397513e+03]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fig, axs = plt.subplots(2, 2, sharey=False, sharex=False, tight_layout=True)\n",
    "fig.set_size_inches(10,10)\n",
    "\n",
    "last_probe_y = None\n",
    "for probe in good_probes:\n",
    "    _ , probe_y = probe_data[probe]\n",
    "\n",
    "#     print(probe, np.mean(probe_y, axis=0))\n",
    "#     print(probe, np.std(probe_y, axis=0))\n",
    "    \n",
    "    std = np.std(probe_y, axis=0)\n",
    "    print(probe, np.mean(probe_y, axis=0)-10*std)\n",
    "    print(probe, std)\n",
    "\n",
    "    print()\n",
    "\n",
    "#     zeros = normalize_data(np.zeros(4))\n",
    "    for ic in range(0,4):\n",
    "        idx = np.unravel_index(ic, axs.shape)\n",
    "        x = probe_y[:,ic]\n",
    "        axs[idx].hist(x, alpha=.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "537.5801684400197"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(288992.4375)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "464.06932735443115"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(693.100830078125 + 639.6287231445312 + 7.405788421630859 + 516.1419677734375)/4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}