scripts/06-rgm-metrics-knn-accuracy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import sparse\n",
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "import time\n",
    "import memory_profiler\n",
    "%load_ext memory_profiler\n",
    "\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "                <script type=\"application/javascript\" id=\"jupyter_black\">\n",
       "                (function() {\n",
       "                    if (window.IPython === undefined) {\n",
       "                        return\n",
       "                    }\n",
       "                    var msg = \"WARNING: it looks like you might have loaded \" +\n",
       "                        \"jupyter_black in a non-lab notebook with \" +\n",
       "                        \"`is_lab=True`. Please double check, and if \" +\n",
       "                        \"loading with `%load_ext` please review the README!\"\n",
       "                    console.log(msg)\n",
       "                    alert(msg)\n",
       "                })()\n",
       "                </script>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import black\n",
    "import jupyter_black\n",
    "\n",
    "jupyter_black.load(line_length=79)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from pubmed_landscape_src.metrics import (\n",
    "    knn_accuracy,\n",
    "    knn_accuracy_years,\n",
    "    chance_knn_accuracy,\n",
    "    knn_accuracy_years_chance,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "variables_path = Path(\"../results/variables\")\n",
    "figures_path = Path(\"../results/figures\")\n",
    "berenslab_data_path = Path(\"/gpfs01/berens/data/data/pubmed_processed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import pubmed_landscape_src.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mType:\u001b[0m        module\n",
       "\u001b[0;31mString form:\u001b[0m <module 'pubmed_landscape_src.metrics' from '/gpfs01/berens/user/rgonzalesmarquez/phd/pubmed-landscape/pubmed_landscape_src/metrics.py'>\n",
       "\u001b[0;31mFile:\u001b[0m        ~/phd/pubmed-landscape/pubmed_landscape_src/metrics.py\n",
       "\u001b[0;31mDocstring:\u001b[0m   <no docstring>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "?pubmed_landscape_src.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 18038185984 bytes == 0x28de994000 @ \n"
     ]
    }
   ],
   "source": [
    "tfidf_features_updated_reparsed = sp.sparse.load_npz(variables_path / \"tfidf_features_updated_reparsed.npz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 49649164288 bytes == 0x4604000 @ \n"
     ]
    }
   ],
   "source": [
    "svd_data_updated_reparsed = np.load(\n",
    "    variables_path / \"svd_data_updated_reparsed.npy\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsne_tfidf_reparsed=np.load(variables_path / \"tsne_tfidf_reparsed.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 127101853696 bytes == 0x439a2000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 122636.70 MiB, increment: 117918.22 MiB\n",
      "CPU times: user 271 ms, sys: 1min 3s, total: 1min 4s\n",
      "Wall time: 1min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "%%memit\n",
    "embeddings_reparsed_all=np.load(berenslab_data_path / 'embeddings/embeddings_reparsed_all.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsne_bert_reparsed=np.load(variables_path / \"bert/tsne_reparsed.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new - should actually give the same result as colors_new\n",
    "colors_new = np.load(variables_path / \"colors_new.npy\", allow_pickle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_year = np.load(variables_path / \"date_year_reparsed.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# kNN accuracies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## low-dim space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_accuracy_tfidf_bert_tsne_reparsed_new_colors= knn_accuracy([tsne_tfidf_reparsed, tsne_bert_reparsed] , colors_new, subset_size=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5064, 0.6261]\n"
     ]
    }
   ],
   "source": [
    "print(knn_accuracy_tfidf_bert_tsne_reparsed_new_colors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_tfidf_bert_tsne_reparsed_new_colors\",knn_accuracy_tfidf_bert_tsne_reparsed_new_colors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#knn_accuracy_tfidf_bert_tsne_reparsed_new_colors=np.load(variables_path / \"knn_accuracy_tfidf_bert_tsne_reparsed_new_colors.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## mid-dim space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 16724328448 bytes == 0x120e876000 @ \n",
      "tcmalloc: large alloc 16723132416 bytes == 0x15f4606000 @ \n",
      "tcmalloc: large alloc 16724328448 bytes == 0x120e876000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 82492.72 MiB, increment: 8371.35 MiB\n",
      "CPU times: user 17min 51s, sys: 2min 42s, total: 20min 33s\n",
      "Wall time: 3min 38s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "%%memit\n",
    "knn_accuracy_svd_reparsed_new_colors= knn_accuracy([svd_data_updated_reparsed] , colors_new, subset_size=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.6]\n"
     ]
    }
   ],
   "source": [
    "print(knn_accuracy_svd_reparsed_new_colors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.6]\n"
     ]
    }
   ],
   "source": [
    "print(knn_accuracy_svd_reparsed_new_colors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_svd_reparsed_new_colors\", knn_accuracy_svd_reparsed_new_colors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## high-dim space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 43768053760 bytes == 0x1eab51c000 @ \n",
      "tcmalloc: large alloc 43737333760 bytes == 0x28de994000 @ \n",
      "tcmalloc: large alloc 43768053760 bytes == 0x1eab51c000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 209513.21 MiB, increment: 86876.51 MiB\n",
      "CPU times: user 7h 56min 6s, sys: 2h 53min 52s, total: 10h 49min 59s\n",
      "Wall time: 1h 28min 48s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "%%memit\n",
    "knn_accuracy_bert_n_5000 = knn_accuracy([embeddings_reparsed_all] , colors_new, subset_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.697]\n"
     ]
    }
   ],
   "source": [
    "print(knn_accuracy_bert_n_5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_bert_n_5000\", knn_accuracy_bert_n_5000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "knn_accuracy_tfidf_n_5000 = knn_accuracy([tfidf_features_updated_reparsed] , colors_new, subset_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.6522]\n"
     ]
    }
   ],
   "source": [
    "print(knn_accuracy_tfidf_n_5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_tfidf_n_5000\", knn_accuracy_tfidf_n_5000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 1256.66 MiB, increment: 0.16 MiB\n",
      "CPU times: user 8.04 s, sys: 463 ms, total: 8.51 s\n",
      "Wall time: 8.67 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "%%memit\n",
    "knn_accuracy_chance = chance_knn_accuracy([tsne_bert_reparsed] , colors_new, subset_size=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0431]\n"
     ]
    }
   ],
   "source": [
    "print(knn_accuracy_chance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_chance\",knn_accuracy_chance)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Root-mean-squared error of kNN prediction of publication year"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## low-dim space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### t-SNE BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2h 30min 59s, sys: 1h 58min 28s, total: 4h 29min 28s\n",
      "Wall time: 56min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "knn_accuracy_years_bert_tsne = knn_accuracy_years([tsne_bert_reparsed], date_year, k=10, subset_size=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10.22518753])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_accuracy_years_bert_tsne\n",
    "np.sqrt(knn_accuracy_years_bert_tsne)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_years_bert_tsne\", knn_accuracy_years_bert_tsne)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### t-SNE TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2h 8min 4s, sys: 1h 56min 29s, total: 4h 4min 34s\n",
      "Wall time: 55min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "knn_accuracy_years_tfidf_tsne = knn_accuracy_years([tsne_tfidf_reparsed], date_year, k=10, subset_size=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11.16393528])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_accuracy_years_tfidf_tsne\n",
    "np.sqrt(knn_accuracy_years_tfidf_tsne)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_years_tfidf_tsne\", knn_accuracy_years_tfidf_tsne)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## high-dim space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 127071133696 bytes == 0x33100c0000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2d 1h 29min 52s, sys: 6h 8min 38s, total: 2d 7h 38min 31s\n",
      "Wall time: 8h 17min 8s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "knn_accuracy_years_bert = knn_accuracy_years([embeddings_reparsed_all], date_year, subset_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8.4263444])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_accuracy_years_bert\n",
    "np.sqrt(knn_accuracy_years_bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_years_bert\", knn_accuracy_years_bert)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 127071133696 bytes == 0x33100c0000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2d 1h 29min 52s, sys: 6h 8min 38s, total: 2d 7h 38min 31s\n",
      "Wall time: 8h 17min 8s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "knn_accuracy_years_tfidf = knn_accuracy_years([tfidf_features_updated_reparsed], date_year, subset_size=5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8.84692749])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_accuracy_years_tfidf\n",
    "np.sqrt(knn_accuracy_years_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_years_tfidf\", knn_accuracy_years_tfidf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture cap\n",
    "%%time\n",
    "knn_accuracy_years_chance_bert = knn_accuracy_years_chance([tsne_bert_reparsed], date_year, k=10, subset_size=5000, rs=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open( variables_path / 'verbose_knn_accuracy_years_chance_bert.txt', 'w') as f:\n",
    "    f.write(cap.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[12.44069974]\n"
     ]
    }
   ],
   "source": [
    "print(np.sqrt(knn_accuracy_years_chance_bert))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(variables_path / \"knn_accuracy_years_chance_bert\", knn_accuracy_years_chance_bert)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}