02-Context_Compatibility.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Context Compatibility"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "# per default - the preprocessing step for jaccard and clustering is deactivated\n",
    "do_preprocessing = False"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% \n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## SemMedDB Connection"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "connected\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "import psycopg2\n",
    "\n",
    "\n",
    "pg_cred = {\n",
    "  \"POSTGRES_DB\":   \"\",\n",
    "  \"POSTGRES_HOST\" : \"\",\n",
    "  \"POSTGRES_PORT\" : \"\",\n",
    "  \"POSTGRES_USER\" : \"\",\n",
    "  \"POSTGRES_PW\" :  \"\"\n",
    "}\n",
    "\n",
    "connection_str = \"dbname='{}' user='{}' host='{}' port='{}' password='{}'\".format(\n",
    "        pg_cred[\"POSTGRES_DB\"],\n",
    "        pg_cred[\"POSTGRES_USER\"],\n",
    "        pg_cred[\"POSTGRES_HOST\"],\n",
    "        pg_cred[\"POSTGRES_PORT\"],\n",
    "        pg_cred[\"POSTGRES_PW\"])\n",
    "\n",
    "conn = psycopg2.connect(connection_str)\n",
    "print('connected')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## PubMed Metadatabase Connection"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "source": [
    "pg_cred_meta = {\n",
    "  \"POSTGRES_DB\":   \"\",\n",
    "  \"POSTGRES_HOST\" : \"\",\n",
    "  \"POSTGRES_PORT\" : \"\",\n",
    "  \"POSTGRES_USER\" : \"\",\n",
    "  \"POSTGRES_PW\" :  \"\"\n",
    "}\n",
    "\n",
    "connection_str_meta = \"dbname='{}' user='{}' host='{}' port='{}' password='{}'\".format(\n",
    "        pg_cred_meta[\"POSTGRES_DB\"],\n",
    "        pg_cred_meta[\"POSTGRES_USER\"],\n",
    "        pg_cred_meta[\"POSTGRES_HOST\"],\n",
    "        pg_cred_meta[\"POSTGRES_PORT\"],\n",
    "        pg_cred_meta[\"POSTGRES_PW\"])\n",
    "\n",
    "if do_preprocessing:\n",
    "    conn_meta = psycopg2.connect(connection_str_meta)\n",
    "    print('connected')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% \n",
     "is_executing": false
    }
   },
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "connected\n"
     ],
     "output_type": "stream"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Queries"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "# only causes with at least 3 occurrences\n",
    "mv_cause = \"mv_cause_2\"\n",
    "mv_ddi_gen = \"mv_ddi_gen\"\n",
    "mv_ddi_function = \"mv_ddi_function\"\n",
    "\n",
    "\n",
    "query_cause = \"SELECT pmid, subject_cui, object_cui FROM {}\".format(mv_cause)\n",
    "query_ddi_gen = \"SELECT pmid, subject_cui, object_cui FROM {} \".format(mv_ddi_gen)\n",
    "query_ddi_gen_1_dg = \"SELECT pmid, subject_cui, object_cui FROM {} WHERE (object_semtype = 'gngm' OR object_semtype = 'aapp') AND (subject_semtype = 'clnd' OR subject_semtype = 'phsu' OR subject_semtype ='sbst')\".format(mv_ddi_gen)\n",
    "query_ddi_gen_2_gd = \"SELECT pmid, subject_cui, object_cui FROM {} WHERE (subject_semtype = 'gngm' OR subject_semtype = 'aapp') AND (object_semtype = 'clnd' OR object_semtype = 'phsu' OR object_semtype ='sbst')\".format(mv_ddi_gen)\n",
    "\n",
    "\n",
    "query_ddi_function_1_dg = \"SELECT pmid, subject_cui, object_cui FROM {} WHERE (object_semtype = 'gngm' OR object_semtype = 'aapp') AND (subject_semtype = 'clnd' OR subject_semtype = 'phsu' OR subject_semtype ='sbst')\".format(mv_ddi_function)\n",
    "query_ddi_function_2_gf = \"SELECT pmid, subject_cui, object_cui FROM {} WHERE \\\n",
    "(subject_semtype = 'gngm' OR subject_semtype = 'aapp') AND \\\n",
    "(object_semtype = 'biof' OR object_semtype = 'phsf' OR object_semtype ='orgf' OR object_semtype = 'menp' \\\n",
    "OR object_semtype = 'ortf' OR object_semtype = 'celf' OR object_semtype ='moft' OR object_semtype = 'genf' \\\n",
    "OR object_semtype = 'patf' OR object_semtype = 'dsyn' OR object_semtype = 'mobd' OR object_semtype = 'neop' \\\n",
    "OR object_semtype = 'comd' OR object_semtype = 'emod')\".format(mv_ddi_function)\n",
    "\n",
    "query_cause_correct = \"SELECT pmid, subject_cui, object_cui FROM {}\".format(mv_cause)\n",
    "query_ddi_correct = \"SELECT '0', d1, d2 FROM mv_ddi_correct\"\n",
    "\n",
    "query_mesh_desc = \"SELECT AM.pmid, MH.descriptorui \\\n",
    "FROM pubmed_metadata.meshheading MH JOIN pubmed_metadata.article_meshheading AM \\\n",
    "ON MH.meshid = AM.meshid WHERE AM.pmid IN (SELECT pmid FROM t_pmid_with_pred)\"\n",
    "\n",
    "query_doc_chemical_desc = \"SELECT pmid, chemicalui FROM pubmed_metadata.article_chemical \\\n",
    "WHERE pmid IN (SELECT pmid FROM t_pmid_with_pred)\"\n",
    "\n",
    "query_doc_titles = \"SELECT pmid, title FROM pubmed_metadata.pubmedarticle \\\n",
    "WHERE pmid IN (SELECT pmid FROM t_pmid_with_pred)\"\n",
    "\n",
    "query_doc_author_ids = \"\"\"SELECT pmid, authorid FROM pubmed_metadata.article_author\n",
    "WHERE pmid IN (SELECT pmid FROM t_pmid_with_pred)\"\"\"\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "---\n",
    "# Preprocessing of Document Pair Similarity"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "from model.knowledgegraph import load_kg_facts\n",
    "from model.contextcompatibility import load_kg_facts_with_doc_index, preprocess_document_pairs_for_experiment\n",
    "\n",
    "\n",
    "experiments = [\"cause\", \"ddi_gene\"]\n",
    "metadata_name = [\"chemical\", \"title\", \"authors\", \"mesh\"]\n",
    "metadata_to_process =  [query_doc_chemical_desc, query_doc_titles, query_doc_author_ids, query_mesh_desc]\n",
    "requires_splitting = [False, True, False, True]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Preprocess Cause Experiment"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "if do_preprocessing:\n",
    "    print('loading cause facts....')\n",
    "    idx_subject, idx_object, fact2docs = load_kg_facts_with_doc_index(conn, query_cause)\n",
    "    preprocess_document_pairs_for_experiment(experiments[0], metadata_to_process, metadata_name, \n",
    "                                             idx_subject, idx_object, fact2docs, fact2docs)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Preprocess DDI Gene Experiment"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "if do_preprocessing:\n",
    "    print('loading ddi gene facts...')\n",
    "    _, idx_pmid_dg_o, fact2docsRel1 = load_kg_facts_with_doc_index(conn, query_ddi_gen_1_dg)\n",
    "    idx_pmid_gd_s, _, fact2docsRel2 = load_kg_facts_with_doc_index(conn, query_ddi_gen_2_gd)\n",
    "    \n",
    "    preprocess_document_pairs_for_experiment(experiments[1], metadata_to_process, metadata_name, \n",
    "                                             idx_pmid_gd_s, idx_pmid_dg_o, fact2docsRel1, fact2docsRel2)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "---\n",
    "# Experiments with Jaccard"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "from model.contextcompatibility import do_experiment_with_context_pairs\n",
    "thresholds_to_check = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Causes Experiment with Document Pairs"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "loading correct cause facts...\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_cause_chemical.pkl ...\n",
      "7978099 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "95037 of 7978099 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "75055 of 1652754 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "65846 of 1016528 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "53908 of 612316 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "44424 of 415578 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "41937 of 390933 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "29335 of 206587 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "26786 of 189893 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "26340 of 187619 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "26151 of 186721 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "26150 of 186713 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_cause_title.pkl ...\n",
      "7978099 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "95037 of 7978099 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "93733 of 6794609 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "74990 of 1621720 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "36519 of 221296 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "15289 of 50030 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "8853 of 22318 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "6402 of 13920 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "5879 of 12317 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "5732 of 11952 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "5701 of 11866 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "5699 of 11864 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_cause_authors.pkl ...\n",
      "7978099 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "95037 of 7978099 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "21826 of 75827 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "14545 of 40151 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "10717 of 26085 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "8829 of 20243 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "8030 of 18111 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "6797 of 14718 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "6385 of 13591 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "6209 of 13158 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "6104 of 12911 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "6101 of 12900 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_cause_mesh.pkl ...\n",
      "7978099 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "95037 of 7978099 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "93687 of 6287737 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "84649 of 3162930 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "61449 of 992139 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "33800 of 231706 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "15982 of 58657 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "7763 of 18191 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "5980 of 12634 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "5686 of 11845 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "5608 of 11633 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "5604 of 11623 derived facts are correct\n",
      "============================================================\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "print('loading correct cause facts...')\n",
    "idx_correct_cause, _ = load_kg_facts(conn, query_cause)\n",
    "\n",
    "cause_results = do_experiment_with_context_pairs(experiments[0], metadata_to_process, metadata_name,\n",
    "                                                 thresholds_to_check, idx_correct_cause)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "chemical\n",
      "7978099\t95037\n",
      "1652754\t75055\n",
      "1016528\t65846\n",
      "612316\t53908\n",
      "415578\t44424\n",
      "390933\t41937\n",
      "206587\t29335\n",
      "189893\t26786\n",
      "187619\t26340\n",
      "186721\t26151\n",
      "186713\t26150\n",
      "title\n",
      "7978099\t95037\n",
      "6794609\t93733\n",
      "1621720\t74990\n",
      "221296\t36519\n",
      "50030\t15289\n",
      "22318\t8853\n",
      "13920\t6402\n",
      "12317\t5879\n",
      "11952\t5732\n",
      "11866\t5701\n",
      "11864\t5699\n",
      "authors\n",
      "7978099\t95037\n",
      "75827\t21826\n",
      "40151\t14545\n",
      "26085\t10717\n",
      "20243\t8829\n",
      "18111\t8030\n",
      "14718\t6797\n",
      "13591\t6385\n",
      "13158\t6209\n",
      "12911\t6104\n",
      "12900\t6101\n",
      "mesh\n",
      "7978099\t95037\n",
      "6287737\t93687\n",
      "3162930\t84649\n",
      "992139\t61449\n",
      "231706\t33800\n",
      "58657\t15982\n",
      "18191\t7763\n",
      "12634\t5980\n",
      "11845\t5686\n",
      "11633\t5608\n",
      "11623\t5604\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "from model.contextcompatibility import print_tab_seperated, show_precision_recall_curve\n",
    "\n",
    "print_tab_seperated(cause_results)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "chemical\n",
      "authors\n",
      "mesh\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3hUZfbA8e9JaBI6AUQCBBCE0JVihQiKoC4oooJlBRVWEUX96aprQ1xWd8FVZFkFESuooCsEG4gSEAEpSu9CgADB0Dshyfn98U5IgGQyJJnMTHI+zzNPMnfu3Dm5Yk7edl5RVYwxxpichAU6AGOMMcHNEoUxxhivLFEYY4zxyhKFMcYYryxRGGOM8apEoAM4V5GRkRodHR3oMIwxJqQsWbJkt6pWy8t7Qy5RREdHs3jx4kCHYYwxIUVEtuT1vdb1ZIwxxitLFMYYY7yyRGGMMcarkBujMMYUbSdPniQxMZHjx48HOpSQVKZMGaKioihZsmSBXdMShTEmqCQmJlK+fHmio6MRkUCHE1JUlT179pCYmEi9evUK7LrW9WSMCSrHjx+natWqliTyQESoWrVqgbfGLFEYY4KOJYm888e9s0RhjDHGK0sUxhiTi759+/L5558X+HXj4uJ49dVX8/Te6Ohodu/eXcARZc8ShTEmtE2YANHREBbmvk6YEOiIfNa9e3eefvrpQIeRK78mChHpKiLrRGSjiJx1N0TkdRFZ6nmsF5H9/ozHGFPETJgAAwbAli2g6r4OGJDvZPHhhx/SokULWrZsyd133w3AnDlzuPzyy6lfv/5prYvhw4fTtm1bWrRowYsvvghAQkICjRs3pm/fvjRq1Ig777yTmTNncsUVV9CwYUMWLlwIwPvvv8+gQYMA2LVrFzfffDMtW7akZcuWzJs3D4CbbrqJSy65hKZNmzJ27Nh8/Vx5pqp+eQDhwO9AfaAUsAyI8XL+w8D43K57ySWXqDGm6Fq9evXpBzp2PPsxerR7rXZtVZciTn9UrepeT04++725WLlypTZs2FCTk5NVVXXPnj16zz33aK9evTQtLU1XrVqlDRo0UFXV6dOna//+/TU9PV3T0tL0hhtu0NmzZ+vmzZs1PDxcly9frmlpaXrxxRdrv379ND09XadMmaI9evRQVdX33ntPH3roIVVVve222/T1119XVdXU1FTdv3//qc9XVT169Kg2bdpUd+/eraqqdevWPRVjrvdQVYHFmsff5/5cR9EO2KiqmwBE5FOgB7A6h/P7AC/6MR5jTFGTmJj98T178nzJH3/8kVtvvZXIyEgAqlSpAri/7MPCwoiJiWHXrl0AzJgxgxkzZtC6dWsADh8+zIYNG6hTpw716tWjefPmADRt2pTOnTsjIjRv3pyEhIRsP/fDDz8EIDw8nIoVKwLw5ptv8uWXXwKwbds2NmzYQNWqVfP88+WFPxNFLWBblueJQPvsThSRukA94MccXh8ADACoU6dOwUZpjAlu8fE5v1anjutuOlPduu5rZKT395+D0qVLn/re/YHuvj7zzDP85S9/Oe3chISE084PCws79TwsLIzU1FSfPjM+Pp6ZM2cyf/58ypYtS2xsbEBWrAfLYHZv4HNVTcvuRVUdq6ptVLVNtWp5KqdujCmKhg2DsmVPP1a2rDueR506dWLy5Mns8bRK9u7dm+O51113HePHj+fw4cMAbN++nT/++CNPn9u5c2feeustANLS0jhw4AAHDhygcuXKlC1blrVr17JgwYI8XTu//Nmi2A7UzvI8ynMsO72Bh/wYizGmKLrzTvf12Wdh61bXwhg2LPN4HjRt2pRnn32Wjh07Eh4efqpbKTtdunRhzZo1XHbZZQCUK1eOjz/+mPDw8HP+3JEjRzJgwADeffddwsPDeeutt+jatStvv/02TZo04aKLLuLSSy/N88+VH5LRhCrwC4uUANYDnXEJYhFwh6quOuO8xsB3QD31IZg2bdqobVxkTNG1Zs0amjRpEugwQlp291BElqhqm7xcz29dT6qaCgwCpgNrgEmqukpEhopI9yyn9gY+9SVJGGOMKXx+rR6rqt8A35xx7IUzng/xZwzGGGPyJ1gGs40xxgQpSxTGGGO8skRhjDHGK0sUxhhjvLJEYYwx+TRlyhRWr86sThQbG0tRmsZvicIYE/J27oSOHSEpKTCff2aiyA9fy3sUJksUxpiQ9/LLMHeu+1pQsivvXa5cuVOvf/755/Tt25d58+YRFxfHk08+SatWrfj9998BmDx5Mu3ataNRo0b89NNPgNsPvF+/fjRv3pzWrVsza9YswJUb7969O506daJz587s3LmTDh060KpVK5o1a3bq/YHi13UUxhiTX7Gx3l8/cQIWL4b0dHjvPfjtN7clRd++sHs39Op1+vm+1ggcP348VapU4dixY7Rt25Zbbrkl2/Muv/xyunfvzo033kivLB+WmprKwoUL+eabb3jppZeYOXMmo0ePRkRYsWIFa9eupUuXLqxfvx6AX3/9leXLl1OlShVee+01rrvuOp599lnS0tI4evSob0H7iSUKY0xIy9izCCAtLftisnmRXXnvc9GzZ08ALrnkklNlxefOncvDDz8MQOPGjalbt+6pRHHttdeeKmnetm1b7r33Xk6ePMlNN91Eq1atCuJHyjNLFMaYoOatBbBzJ9Sv7xIEQEoK7NsHXbu653mtMp5TeW8ROXVObuW+M8qKh4eH+zTuEBERcer7Dh06MGfOHL7++mv69u3L448/zp///Odz/0EKiI1RGGNC1ssvuy6nrNLS8j9WkVN57xo1arBmzRrS09NPtTYAypcvz6FDh3K97lVXXcUEzzat69evZ+vWrVx00UVnnbdlyxZq1KhB//79uf/++/n111/z9wPlkyUKY0zImj/ftSKySkkBz3bTeda1a1dSU1Np0qQJTz/99Kny3q+++io33ngjl19+OTVr1jx1fu/evRk+fDitW7c+NZidnYEDB5Kenk7z5s25/fbbef/990/b4ChDfHw8LVu2pHXr1nz22WcMHjw4fz9QPvmtzLi/WJlxY4o2KzOefyFTZtwYY0zRYInCGGOMV5YojDHGeGWJwhhjjFeWKIwxxnhlicIYY4xXliiMMcbPEhISaNasWaDDyDO/lvAQka7ASCAcGKeqr2Zzzm3AEECBZap6hz9jMsYUHeePOJ9dR3addbxGRA2SnghQzfEiyG8tChEJB0YD3YAYoI+IxJxxTkPgGeAKVW0KPOqveIwxRU92ScLbcV8lJCTQuHFj+vbtS6NGjbjzzjuZOXMmV1xxBQ0bNmThwoUcOXKEe++9l3bt2tG6dWumTp0KwKpVq2jXrh2tWrWiRYsWp4oJpqWl0b9/f5o2bUqXLl04duxYvmIsTP5sUbQDNqrqJgAR+RToAWTd3aM/MFpV9wGo6h9+jMcYE2Ie/e5RliYtzdN7Y9+PzfZ4q/Nb8UbXN3J9/8aNG5k8eTLjx4+nbdu2TJw4kblz5xIXF8c//vEPYmJi6NSpE+PHj2f//v20a9eOa665hrfffpvBgwdz5513kpKSQlpaGrt27WLDhg188sknvPPOO9x222188cUX3HXXXXn62QqbPxNFLWBblueJQPszzmkEICI/47qnhqjqd2deSEQGAAMA6tSp45dgjTEmq3r16tG8eXMAmjZtSufOnRERmjdvTkJCAomJicTFxTFixAjAVZPdunUrl112GcOGDSMxMZGePXvSsGHDU9fLKBeetfR4KAh0mfESQEMgFogC5ohIc1Xdn/UkVR0LjAVX66mwgzTGBEZuf/nLS5Lja/F94/P12VmL9YWFhZ16HhYWRmpqKuHh4XzxxRdnVX9t0qQJ7du35+uvv+b6669nzJgx1K9f/7TrhYeHh1TXkz9nPW0Hamd5HuU5llUiEKeqJ1V1M7AelziMMSaoXXfddYwaNYqMwqq//fYbAJs2baJ+/fo88sgj9OjRg+XLlwcyzALhz0SxCGgoIvVEpBTQG4g745wpuNYEIhKJ64ra5MeYjDFFSI2IGud0vCA9//zznDx5khYtWtC0aVOef/55ACZNmkSzZs1o1aoVK1euDOiGQwXFr2XGReR64A3c+MN4VR0mIkOBxaoaJ267qNeArkAaMExVP/V2TSszbkzRZmXG86+gy4z7dYxCVb8Bvjnj2AtZvlfgcc/DGGNMELKV2cYYY7yyRGGMCTqhtvNmMPHHvbNEYYwJKmXKlGHPnj2WLPJAVdmzZw9lypQp0OsGeh2FMcacJioqisTERJKTkwMdSkgqU6YMUVFRBXpNSxTGmKBSsmRJ6tWrF+gwTBbW9WSMMcYrSxTGGGO8skRhjDHGK0sUxhhjvLJEYYwxxitLFMYYY7yyRGGMMcYrSxTGGGO8skRhjDHGK0sUxhhjvLJEYYwxxitLFMYYY7yyRGGMMcYrSxTGGGO8skRhjDHGK78mChHpKiLrRGSjiDydzet9RSRZRJZ6Hvf7Mx5jjDHnzm8bF4lIODAauBZIBBaJSJyqrj7j1M9UdZC/4jDGGJM//mxRtAM2quomVU0BPgV6+PHzjDHG+IE/E0UtYFuW54meY2e6RUSWi8jnIlI7uwuJyAARWSwii20fXWOMKVyBHsyeBkSragvge+CD7E5S1bGq2kZV21SrVq1QAzTGmOLOb2MUwHYgawshynPsFFXdk+XpOOBffoznlPCXwkkn/azjYYSR9mJaYYRgjDEhw58tikVAQxGpJyKlgN5AXNYTRKRmlqfdgTV+jOeU7JKEt+PGGFOc+a1FoaqpIjIImA6EA+NVdZWIDAUWq2oc8IiIdAdSgb1AX3/FY4wxJm9EVQMdwzlp06aNLl68OF/XkJckx9dqRNQg6YmkfF3fGGOCjYgsUdU2eXlvoAezg86uI7sCHYIxxgQVSxTGGGO8skSRjeXLAx2BMcYEj2KZKGpE1PD6+p/+VEiBGGNMCCiWiSK3weqtW61VYYwxGYplogAvrYrD7vgddxRiMMYYE8SKbaJIeiIJfVFpNUVhiMK0Me6FLyYAsGoVJNksWWOMKb6JIsNvv8GDD0LJ1X92rYkr/3nqtQcfhI4dLWEYY4q3Yp8oAObPh5PHysCCR6HB91DzVwDi4mDuXHj55QAHaIwxAWSJAteqUIV93z9A+VLluf3Nf/G//0F6unu89x4sW2atC2NM8WSJIotKZSrxQJsHmLx6Mp//+Dvh4e748ePQq5e1LowxxZNPtZ5EpDRwCxBNlkKCqjrUb5HloCBqPXmz49AO6r1Rj7TF95EW99+zXj/vPNi0Cc4/328hGGNMgctPrSdfq8dOBQ4AS4ATefmgUHHxmItJSU+Bi99yjwyHa8CIJNLSXKti9OjAxWiMMYXJ10QRpapd/RpJkMixKGA5dzwlBcaNg+eft1aFMaZ48HWMYp6INPdrJCEkPd3GKowxxYevLYorgb4ishnX9SSAeva6LnZSU2HePDh2zI1ZGGNMUeZroujm1yhCRNZx/7ffhhYt3EyoGt5rDBpjTEjzqetJVbcAlYA/eR6VPMeKrdatYccO6NYN1q+3NRbGmKLLp0QhIoOBCUB1z+NjEXnYn4EFSk7FAiNKRpz2vH17+OILWLECrr7a1lgYY4ouX9dRLAcuU9UjnucRwPxAjFH4ex1Fdu6Pu5/3l77Pwv4Lubjmxae9NmoUPPKI+97WWBhjglVh7JktQFqW52meY7kF1lVE1onIRhF52st5t4iIikiefgh/G37tcKpFVOO+uPs4mXbytNfWrOHUCu4TJ2BooS9BNMYY//I1UbwH/CIiQ0RkCLAAeNfbG0QkHBiNGwiPAfqISEw255UHBgO/nEPcharyeZX57/X/ZWnSUkbMG3Hq+M6drg5UmieFpqfD2LHuuDHGFBW+Dmb/G+gH7PU8+qnqG7m8rR2wUVU3qWoK8CnQI5vzXgb+CRz3OeoAuLnJzfSK6cVLs19i3e51gBuTSE8//by0NLjhhgAEaIwxfuI1UYhIBc/XKkAC8LHnscVzzJtawLYszxM9x7Je/2Kgtqp+nUscA0RksYgsTk5OzuVj/WdUt1GULVmW+6fdT7qmM3++W6l9psOHCz82Y4zxl9xaFBM9X5cAi7M8Mp7nmYiEAf8G/i+3c1V1rKq2UdU21apVy8/H5sv55c7n39f9m7lb5/L24rdPlSc/87F+vTu/kMfcjTHGL7wmClW90fO1nqrWz/Kop6r1c7n2dqB2ludRnmMZygPNgHgRSQAuBeKCdUA7wz0t7+Ha+tfy1Myn2Hpga47nTZsGbdu6r8YYE8p8XUdxhWdKLCJyl4j8W0Tq5PK2RUBDEaknIqWA3kBcxouqekBVI1U1WlWjcQPk3VU1qP8OFxHG/mksqsoDXz1ATtOLu3VzU2e72Zp2Y0yI83XW01vAURFpiesq+h34yNsbVDUVGARMB9YAk1R1lYgMFZHu+Yg54KIrRTOs0zC+3fgtE1dMzPacEiVg0CD3dedOmDq1kIM0xpgC4uuCu19V9WIReQHYrqrvZhzzf4inC8SCu+ykpadx5XtXsmHPBlY/tJrqEdVzPPe+++CDD2DSJOjZsxCDNMYYj8JYcHdIRJ4B7gK+9gxEl8zLBxYV4WHhvNv9XQ6lHGLwd4O9njtyJLRrB717wzffFFKAxhhTQHxNFLfjyovfp6pJuIHp4X6LKkTEVIvh2aue5dOVnzJtXc6j1uXKuQTRvDnccgvMmlWIQRpjTD751PUUTIKl6ylDSloKl4y9hH3H9rFq4CoqlqmY47m7d0NsLCQkwMSJ8Npr8NlnVhvKGON/fut6EpG5nq+HRORglschETmYlw8sakqFl+Ld7u+y8/BOnpr5lNdzIyPh++/hggugVy/46SerOGuMCX65raO40vO1vKpWyPIor6oVCifE4NeuVjseu/QxxiwZw+yE2V7PrVnTtSZSU93ivHfftX0sjDHBzdd1FJd6ivdlPC8vIu39F1boGXr1UOpXrs/90+7n2MljXs8dP95NmwU4edJaFcaY4HYu6yiyVjA64jlmPMqWLMs7f3qHjXs3MiR+SI7nZVScPempVp6e7p4vXXp2gUFjjAkGPu9HoVlGvVU1Hd/32y42OtXrxP2t72fE/BEs2bEk23Nyqjh71VXw0EOFEKQxxpwjXxPFJhF5RERKeh6DgU3+DCxUDe8ynBoRNbLd5AjItuJsSgpUrAj9+hVSkMYYcw58TRQPAJfjivolAu2BAf4KKpRVKlOJ/97wX5btWsbweWcvNcmp4mxioluUB/DKK1Z51hgTPHzduOgPVe2tqtVVtYaq3qGqf/g7uFB1U+ObuDXmVl6a/RJrd689p/ceOOB2yevQAT7/3E8BGmPMOfB11lMjEflBRFZ6nrcQkef8G1poG9VtFBElI7g/zm1y5KuKFeGXX6B1a7j1Vhg2zLU4jDEmUHztenoHeAY4CaCqy3Flw00OapSrwevXvc7P237mrUXnNkGsenX44Qe480547jm4+244HtQbxRpjijJfE0VZVV14xrHUgg6mqPlzyz/TpUEXnv7habbs33JO7y1TBj76CP7+d5gwATp3hj+ss88YEwC+JordItIAUAAR6QXs9FtURYSIMObGMW6To69z3uQo5/fDs8/C5MluELx9e1i50k/BGmNMDnxNFA8BY4DGIrIdeBQ3E8rkIrpSNP/o/A++2/gdE1ZMyNM1evWCOXPgxAm44w5bmGeMKVy5Vo/17D3RS1UnebZDDVPVQ4USXTaCrXqsL9LS07jqvatYt2cdax5a43WTI28SE+HgQYiJcclCxD2MMSY3ft24yLMK+6+e748EMkmEqvCwcMZ1H8fhlMM88u0jeb5OVJRLEqpum9WBA21GlDHG/3ztepopIk+ISG0RqZLx8GtkRUxMtRieu+o5Plv1GXHr4vJ9vYoVoUIFa1EYY/zP1z2zN+MZyM5KVev7IyhvQrHrKUNKWgptxrZhz7E9rB642usmR75QdYliyRKXNBo2LKBAjTFFTmHsmR0DjAaWAUuBUUBTHwLrKiLrRGSjiDydzesPiMgKEVkqInNFJOZcgg81GZscJR1O4q/f/zXf1xNxYxV9+7oZUfHx+b6kMcacxddE8QHQBHgTlyRiPMdyJCLhuOTSzXN+n2wSwURVba6qrYB/Af8+h9hDUttabXn80scZ++tY4hPi8329sDCYOtVtp3rttTBuXP5jNMaYrHxNFM1U9X5VneV59Aea5fKedsBGVd2kqinAp0CPrCeoatbtVCPIpnurKHrp6pdoULkB/af15+jJo/m+Xv36riptp07Qvz888YQrXW6MMQXB10Txq4hcmvHEs7tdbgMFtYBtWZ4neo6dRkQeEpHfcS2KbKcEicgAEVksIouTk5N9DDl4+brJ0bmoWBG+/trNhnrtNbjpJjh0yG2U1LGjbbdqjMk7XxPFJcA8EUkQkQRgPtDWM76wPD8BqOpoVW0APAVkW2hQVceqahtVbVOtWrX8fFzQuLre1fS/uD+vzX+NxTsKZnC+RAkYNQr+8x/49lu48kr4619h7lzbbtUYk3e+znqq6+11VT2rkJGIXAYMUdXrPM+f8Zz7Sg6fEQbsU1WvU4FCedbTmfYf30/M6BiqRVRjcf/FlAwvWWDXnjEDbrkFDns2sD3vPNi0yY1lGGOKH7/PelLVLd4eObxtEdBQROqJSClctdnTFhCISNYJnTcAG/LyQ4SqSmUq8dYNb7F813L+9fO/CvTaXbrAjTdCeLh7npbmjs2eXaAfY4wpBnztejpnqpoKDAKmA2uASaq6SkSGikh3z2mDRGSViCwFHgfu8Vc8wapH4x7c1vQ2hs4ZyprkNQV23Z07YcqUzEHtlBRYscINegMkJ8Nbb8H27QX2kcaYIqqEPy+uqt8A35xx7IUs3w/25+eHilmbZ5GSlkLMf0+fPVwjogZJT+RtFPrll88uHliqFCQkuO+//96VABk4ENq2hR493AB4TIyt9jbGnM5vLQrju+Sj2c/k2nVkV56vOX++a0VklZLids8D6NMHVq92+3OHhbkNkpo1c6u7n3gCfvrJptgaYxyfBrODSVEazM4gL+X8J7y+WDj/fXbsgGnT3OK9H35wSaVmTdi8GUqXziwXYowJTfkZzPZr15MJHRdcAH/5i3scOgTffQcbN7okAXD99a5b6rXXAhunMabwWddTkOvzRR+2HdiW+4kFqHx5uPVWeOYZ91wVGjeGup5J0sePu3Ihr73mkokxpmizRBHkpqydQuPRjfn7nL9zPPV4QGIQgddfh0c86+YTE92sqSeecGMazZq5MY5Fi2z3PWOKIksUQaBGRI0cj699aC3XN7ye52c9T8zoGKasnXLOe28XtAsvhKVL3fjFG29A9erw6qvQrh3Uru1mUk2ffvZgujEmNNlgdoj4cfOPPPLtI6xKXsU19a9hZNeRxFQLnqrse/e6WlNTprgkceQIrF/vWhxbt7paVBXzt/2GMSYfCmM/ChNgnep1YukDSxnVbRSLdyymxVsteOy7x9h/fH+gQwOgShW4+2744gvYvRtmzszcSOnJJ6F588xtWzPKihhjQoO1KELQ7qO7ee7H5xi7ZCyRZSP5R+d/0K9VP8LDwgMdWrYWLoRt21ztKVU3KF69euYiv2bNbOqtMf5mLYpiJrJsJG/f+DZLBizhosiL6D+tP+3HtWfetnmBDi1b7dq5JAFu3OKhh6BkSXjhBWjRAho0gMcfd3WoUlMDG6sx5myWKEJY65qtmdN3DhN7TiTpcBJXjL+Cu7+8mx2HdgQ6tByVLg1PPeVWju/YAWPGQJMmMHo0xMa66rZ9+8LyfBWvN8YUJEsUIU5E6NO8D2sHreXZq55l0qpJNBrViFfnvsqJ1BOBDs+rmjVhwAA3CL57N0yeDN26udXhBw64c5Ytg3ffhaP53wjQGJNHNkZRxPy+93f+b8b/MXXdVBpUbsAbXd/ghoY3ICE0CHDypCuPHhYGzz8PI0a4RBIRAQsWQNWqmQPlxhjf2BiFOaVBlQZM6T2F6XdNp2R4Sf70yZ+4YeINrNu9LtCh+axkSZckAIYOdcULIyLc80cegUaNoGlT+NvfXJFDW+RnjH9ZoiiiujTowvIHlvP6da/z87afafZWM56c8SQHTxwMdGjnRATq1ct8PmkSjBwJNWrAv/4Fl14KUVHwwAOuPtWJ4O5tMyYkWddTMfDHkT/42w9/Y/xv46keUZ1Xr3mVP7f8M2ES2n8n7N0L33zjFvl9951b5Fe+PAwbBg8/HOjojAku1vVkvKoeUZ1x3cfxy/2/UK9yPfpN7cdl717Gwu0LAx1avlSpAnfdBZ9/7sYwvvoKevd2020B1q51xQtXrAhsnMaEOksUxUjbWm35+d6f+fCmD9l6YCvtx7Wn39R+JB3O2y56waRMGbjhBhg71pVEB0hKcgUMK1Vyz6dNc2Mey5dnrhI3xuTOEkUxEyZh3N3ybtYPWs9TVzzFhOUTaDSqESPmjSAlrWhV8YuNhTVrXKFCgLlzYcgQaNkS6teHxx6D+Hhb5GdMbmyMopjbsGcDj01/jK83fE2jqo0Y2XUkXS/sGuiw/CYpKXMnv5kz3eB3lSpw442upMh112XOsDKmKAnaMQoR6Soi60Rko4g8nc3rj4vIahFZLiI/iEhdf8ZjztawakO+uuMrvr7ja1SVbhO60f2T7mzcWzR3JDr/fOjf341n7N7txjduuMElj1tugZ49M8/dHxz1Fo0JOL+1KEQkHFgPXAskAouAPqq6Oss5VwO/qOpREXkQiFXV271d11oU/pOSlsLIBSMZOmcoKWkpPH7p4zzb4VnKlSoX6ND87uRJ1zUVHg4dOsCePW7l+KhRbntY2zPchLpgbVG0Azaq6iZVTQE+BXpkPUFVZ6lqRnGGBUCUH+MxuSgVXoonr3iS9YPW06dZH179+VUu+s9FfLz844BvluRvJUvC1Ve7JAEuMTz3HFxxhXv+7bduz/BnnrFFfqb48WeiqAVk3ew50XMsJ/cB32b3gogMEJHFIrI4OTm5AEM02alZvibv3/Q+8++bzwXlL+DuL+/myveuZMmOJYEOrdBERrrqts2auefnnQcXXADDh2cu8vvLX1wCsUV+pqgLillPIgILVR8AABgMSURBVHIX0AYYnt3rqjpWVduoaptq1aoVbnDF2KVRl/LL/b8wvvt4Nu7dSNt32tI/rj9/HPkj0KEVuquvdoPfycnw8ceupTFxopuKGxkJt94KEybYtFtTNPkzUWwHamd5HuU5dhoRuQZ4Fuiuqva3WZAJkzD6te7H+kHreezSx3h/2fs0GtWINxa8wcm0k4EOr9BVrgx33ukq3SYnu8q3d9zhxjeGDcscx/juO7cFrDFFgT8Hs0vgBrM74xLEIuAOVV2V5ZzWwOdAV1Xd4Mt1bTA7sNbuXsvg7wYz4/cZNIlswsiuI7m2wbWBDivg0tNh506oVcttzlStmmtljBvnWhkrV9pOfiawgnIwW1VTgUHAdGANMElVV4nIUBHp7jltOFAOmCwiS0Ukzl/xmILROLIx3935HVN7T+VE2gm6fNyFmz+7mU37NgU6tIAKC3NJAqBUKVi0CJ72TAhfutTt5FevHjz6KMyaZYv8TGixBXcmz46nHuf1+a/z95/+Tlp6Gk9e/iRPX/k0EaVsxVpW+/bB//7nFvnNmOEGvytXPn2RX7miPwPZBFh+WhSWKEy+JR5M5KmZTzFxxUSiKkQx/Nrh3N709pDaLKmwHDniksWUKW7R3969bnvYa66B8eOhevVAR2iKqqDsejLFR1SFKCb0nMBP/X6iWtlq9PmiDx3f78jSpKWBDi3oRETAzTfDBx/Arl2uG+rBB933Vaq4c0aPdntuGBMsLFGYAnNlnStZ1H8RY28cy5rda7hk7CU8+NWD7D66O9ChBaUSJVzhwtdfd2MaJUq44z/8AN9/n3neqFFuC1hb5GcCxbqejF/sO7aPIfFDGL1oNBVKV2Do1UN5oM0DlAgrEejQQkJKihsU37fP1adKSXFf//QnuOkm6NTJlVY3xlfW9WSCTuXzKjOy20iWPrCU1jVb8/C3D9N6TGtmbZ4V6NBCQqlS7mvlyq5bauJEV17k009dEcPISOjVyy3+27cvsLGaos9aFMbvVJUv137J49MfZ8uBLdwacyvDrx1O3UpWLPhcnTjh9tCYMgXi4mDHDlfIcPp06NzZdU+F2Z9/JhvWojBBTUTo2aQnax5aw9DYoXy1/isaj27MS/EvcezksUCHF1JKl3bTad96C7ZtcwUKn3oK2rZ1r7/2GrRuDUePer+OMefCWhSm0G09sJUnv3+SSasmESZhpOvZo7Q1ImqQ9ETob9Fa2D77zA2Ejxvnng8c6AbJe/RwXVclSwY2PhM41qIwBW/gQNenIZL5iIx0le/ONGECREe7Po/o6OzPyaJOxTp81uszZt0zK9skAbDryK78/wzF0O23ZyYJVTd+8c47bp1G9epw112uTtWhQ4GN04QWSxTmbAMHur6NM+dj7tkD/fplJgJV9/2AAbBli3u+ZYt7nkuyAIiNjvX6+vJdy3NMJCZ3IvDJJ24nvy+/dLOlvvsObrvN5fzrr4cxY9z2sMZ4Y11P5mwlSkBaWs6vi7g+jD593Mjqli1nn1O5Mvz8MzRp4vWj5CXvq7ernFeFjnU7EhsdS2x0LM2qNyNM7O+bvEpNhXnzXDmRKVNg0yZ44w0YPBgOHoTt26FxYyteWBRZ15MpWN6SBLiWw2OPuWJFOdXS3rfP9XEAHDjgRlynToU/fN/L4oObPqDHRT1YmrSUwd8NpuXbLak2vBo9P+vJm7+8aS2OPChRwo1VvPYabNwIK1a4MungEkdMjCtiCC5x5PZPwRQP1qIwZ8utRVG3LiQkuO+jo7NvUdSqBUuWQI0abllxhw5uY2qACy+Eyy+Hv/4V+bxZjh+jL2b+29yyfwuzt8wmPiGe+IR4Nu/fDLgWR4e6HYit61oczWs0txZHHu3c6epP3XefG2568EFXzDBjkV/nzm6nPxOarCigKVgZYxTZKVkS3nvP7d4DmWMUWedjli0LY8dmngNw7JhLHPPmZT5mzOD8mV2zHbjObdbT1gNbmZ3gSRxb4k+VObfEUXCmTXP/eb/5xg1+ly3rpubedJNb9Fe1aqAjNOfCEoUpeAMHupHOrAPaVau6anVZEwC43ybPPuu6oerUcVu9nXnOmTL+3Ym45cb/+Ifb3UfV/TnbvDnMmQMVKriSq2XLeu04t8ThPxmL/KZOdY+MRX5XXQX9+2d2XZngZonCFA0HDrgVZPPmwbp1bsoOuEHzWbNcd9Xll7sNqy++2K0+y0FOiaNymcp0jO5oiSOP0tNdwzBjMPzGG+HVV10tqldecdNvGzQIdJQmO5YoTNH2ySfw7bcugfz+uzvWpo0ruQpuw+qGDd14SA4scfhHaqob0lq4EC67zE3D7d7dDVtt2AAdO9oiv2BhicIUH7t2wfz57k/bnj3doHvlyq4TvX79zFbHNde45JEDSxwFLznZ9RSWLg0vvwwvvAAVK7rxjB49oGtX97oJDEsUpvhKT8/srsp4JCXBc8+531aHD8OIES55tG/vfnPBWeMqW4c8xuzWVc5OHCfC6LgpndiDlYnt9QTN+z1ticMHR4+6UiJTp7pB8d27XUXcTp3cYHj37lCzZqCjLF4sURiTQdVN3S1Vyk3R/eUXlyTS091geLNmUK2aSyjHj2e+r2xZN3h/110AbHv/TWaPfpL4C1KIj4bfPbvPVQ6LoGOja63FcQ7S0tztnjLFJY6M3sPrr4evvw5sbMVJ0CYKEekKjATCgXGq+uoZr3cA3gBaAL1V9fPcrmmJwpyzQ4dcJ3pGi2P69MxZV9kJD3fjHTt2nDq0rQLMjob4xmWIb1ed3w+7hYaVy1R2s6o8K8db1GhhicMLVVi92iUNVdfwU3XdUr17uwoxxj+CMlGISDiwHrgWSAQWAX1UdXWWc6KBCsATQJwlClMowsJyThTPP+/+BH7lFa/JZNuW5czev4z4We8Rv38pv7MXsMSRFwcPuvpTt90G997rFv4995zrorrmGlvkV1CCNVFcBgxR1es8z58BUNVXsjn3feArSxSmUOS0mtyXFefnn+/qX2QsHhg4EN5+m23l1bU46gnxjUrxe8QJACqXrECHqhcTG3M9sQ2vtcThgx9+cPMUDh7MXOTXo4ebimuL/PIuWGs91QK2ZXme6Dl2zkRkgIgsFpHFycnJBRKcKcaGDXO/gbIqW9Ydz+2cESNOX2H23//C4cPUjv+Vu/76MeOaPsPG9EFsfXQrH938ET03lWbl6nge+/GvtB7TmsgXynDTsOa8seANliYtJf3Ecbw6xxLuRUHnzm4G1fTp0Lev6zXs29eVSY+Nhddfd8UMTeHxZ4uiF9BVVe/3PL8baK+qg7I5932sRWEKky+ryfOy4vxMa9bAihVsWz2f2dvnEZ+ynvioVH4vdRhws6o6JJUmlrrERralxUUdCLv0Mmja1PfyKEWc6umL/FaudGsz9uyB8uXdjKqqVa3ibW6s68mYUKLKtoOJzE6IJ37qSOJPrMtMHMegQ1g9Yrs/QuyD/6TFsiTCzvxfNGsXWTG0aRP8+iv06uWeX3mlW5/xzTfuue0bnr1gTRQlcIPZnYHtuMHsO1R1VTbnvo8lClOMJR5MZPamWcSv+or4XQvZeCgB8CSOLRCb4B4tduESh6r7k/qDD9yeHzExULt2sfwN+dFHbpHfbbe52pMXXuhWhPfoAd262SK/DEGZKABE5Hrc9NdwYLyqDhORocBiVY0TkbbAl0Bl4DiQpKpNvV3TEoUpDhIPJjL7plbEl9tDfDRs9AziVjoGHXaVJnbAK8TujqBFj78QnvG/cNmybtehN9909bD27nX7fzRoUGzqaOzaBX/7G8TFuS6pkiVPX+R3wQWBjjBwgjZR+IMlClNsZBmjSKwAs+tC/IXhxLeJZONJV5q9UumKdCjfjNgTFxC7oyQtVu4mfPgIV333gw/cKHCJEu7P7CZN3GPwYDcyrFpkO/bT0lyllylT3CNjkV/bti5p3Hef19JgRZIlCmOKqhwG1BMPJp5Wq2rj3o0AVCpTKbOsepmLaLEimfC169yg+po1blu77dvdb8lXXnGr0TMSSMbjssvcosMiImORX0aZ9IULXeKoX98Nkh896hpgRb3XzhKFMcWcT4kjOpYWlRsTXrqMa0lMnQqTJrnfouvWuQ7+UqXc/h8lSrjNtNeuPT2JREVl3wopiBlihWTXrszWxO23w08/QWKiSxQrVrieujNnRhcFliiMMafxOXHUaEF4WDjnjzg/+50GjwhJw7P8jmja1M1PBbcnesmSrpXy4oshOY334EFXDv2SS9xsqTp13NBOly6ui+rGGyEyMtBRFgxLFMYYr7Yf3H7anuMb9m4AMhNH3Lq4HN+rDyRldl2B20wb3LhHRud/dmrVcqvbQ6QbKz3d7Y+VMa6R0cq48ko3g6pHj9DelMkShTHmnOSUOLIzrc80oitFE10pmnKlymW+cPCg65pq3z7nDzrvPDd1t1kz97jmGmjVqgB/Ev9Qhd9+y6x4u3y5O96smVucf911gY0vLyxRGGPyRV7ybfZT1fOqnkoapx6PDyV6QzJ190P5lKwnV4V77nEd/ytXump/L73kdjTau9f9id6smZuhlZFIqlTxzw+YT5s3Z64Mf+UVN94/b54bmhkyxFWuD3aWKIwx+eItUSy4bwEJ+xMyHwcS2LJ/Cwn7EziWeuy0c6sehej9UPdQONHtuxB9abfMhJJenvKlyrsEsnGjm7q7cqXbKz3Dhx/C3Xe7mVkzZrgkEhMTlKPL48bBM8+43rWyZd02sCdOuEV+GftjBRNLFMaYfPGWKPTF7H9HqCrJR5Nd8pj2EQlffUwC+0moWYaEelVI0H1nJZIq51XJTBwVo4muVJfotPJEJx2n7sbdVLi5t9vC9pNPMosvirjBgWbN4F//cq8fPeoG0jMWEgZo1lXGnuHgBsC//96FdPXVrsHUvbubKBYMLFEYY/Ilx1lPETVIeiIpT9c8LZGc8dhywLVIjp48etp7TiWSinWpqxWJPhhG9M5jRG9IJnrZFip884MrVTJihEsMjRtDRAQsXgwnT2ZeKACzrtLS3IaKGeMa69e7423auBlUPXq4SWOBWuNoicIYE3JUld1Hd5+dSA5kfn9mIqlcpvKpbqzoHUeJ3n6U6AVrid6bTvR+qHAiy8llykD//q410qCBm6XVuHGh/Xxr12YmjQUL3LFHH3Vl0lXdLKvCnBBmicIYU+TkKZEcc2Mkpx7HSlN31wn3/Xk1qbjZs73tCy+4wfWMJJLx8NPgws6dMG2aG2658kpYtcoVLpw0ydWiKgz5SRQlCjoYY4wpCCJCtYhqVIuoRttabc96XVXZc2wPCVc1J+F4EgmVOPVYXxWmXygcLZm1ibGTSv/0tEiO7SF6926iVxzLTCoXXkLFuZ4/Qp97zv25n9ESadDA1cfKY79RzZqubFeGsDA36J3RwPnoI7d+MWORX/XqefoYv7FEYYwJSSJCZNlIIh8fQZtsNnjSMWPY07NrtmMkG+qd5PvKezlyMusVl2QmkuRNRG89SPR0qHvAk0iu6UWljya7U4cMcaVoMxJJVBTnv17L53GeJk1ccshw/DgsW+ZaHSKu9lTGIr+GDQvsluWZdT0ZY0JfHmY9nWqRZAywe6b8ZnRtbd63mSMnj5z2noqlKxJdoQ7Rc1cSvU8zWyOHS9D6/tScPyuHmWOnxwNLl2au11i2zB2PiXEJ4+abXfXbvLIxCmOMKWCqyt5je7MfI9mXQMK+zRxOPZL7hYAfvq1OrUF/I+q2+4k4dNzVCskYF8lhZ6WEhMyKt3PmuLGMGTPca/Pnw8UXuw2bfGWJwhhjCtmZiaTX5F4+va9SeDlq7TxM1EGIOgi10soSdV4Novo+Qq02nYg6VpLK2/ciDRu6Jd8i7N3rNmJq1MjtRVWzphuPf/FFSElxvW6Nx3mf4myJwhhjAszbosVZ98wi8WAi2w9uJ3FfAok71rH9wDYSj+0iSQ+hZ7y1zEmXRKKOhFOL8kSVqU5U97uoVacZ1Y+ex6bfanHpVRfR8KJSfPWV65ZKfS6XgfYxoDvO/CTf2GC2Mcb4WWx0bI6vnUw7SdLhJJdIDm0ncfsaEn//je17t5B4NImf0/exvcR6Ti54ARZkvi98IlxwrASRqVVocO9FrPNj/JYojDGmANSIqJFj1483JcNLUrtibWpXrO0OxADXnn5Ouqaz++hul0xWzCNx1XwS92xie8pOEk/u4eh58wrop8iedT0ZY0yoU0WG5rKXaz66nvy6S6yIdBWRdSKyUUSezub10iLymef1X0Qk2p/xGGNMkeTnAlJ+SxQiEg6MBrrhGlN9RCTmjNPuA/ap6oXA68A//RWPMcYUZbl1ceWHP8co2gEbVXUTgIh8CvQAVmc5pwcwxPP958B/REQ01PrDjDEmwLKr8lvymfNJLXP2uMm58meiqAVsy/I8EThzz8RT56hqqogcAKoCu7OeJCIDgIxKKSdEZKVfIg49kZxxr4oxuxeZ7F5ksnuR6aK8vjEkZj2p6lhgLICILM7ropGixu5FJrsXmexeZLJ7kUlE8jwLyJ+D2duB2lmeR3mOZXuOiJQAKgJ7/BiTMcaYc+TPRLEIaCgi9USkFNAbiDvjnDjgHs/3vYAfbXzCGGOCi9+6njxjDoOA6UA4MF5VV4nIUGCxqsYB7wIfichGYC8umeRmrL9iDkF2LzLZvchk9yKT3YtMeb4XIbfgzhhjTOHy64I7Y4wxoc8ShTHGGK+CNlFY+Y9MPtyLx0VktYgsF5EfRKRuIOIsDLndiyzn3SIiKiJFdmqkL/dCRG7z/NtYJSITCzvGwuLD/yN1RGSWiPzm+f/k+kDE6W8iMl5E/shprZk4b3ru03IRudinC6tq0D1wg9+/A/WBUsAyIOaMcwYCb3u+7w18Fui4A3gvrgbKer5/sDjfC8955YE5uKLMbQIddwD/XTQEfgMqe55XD3TcAbwXY4EHPd/HAAmBjttP96IDcDGwMofXrwe+BQS4FPjFl+sGa4viVPkPVU0BMsp/ZNUD+MDz/edAZxE/V8YKjFzvharOUtWMneUX4NasFEW+/LsAeBlXN+x4YQZXyHy5F/2B0aq6D0BV/yjkGAuLL/dCgYw9RysCOwoxvkKjqnNwM0hz0gP4UJ0FQCURqZnbdYM1UWRX/qNWTueoaiqQUf6jqPHlXmR1H+4vhqIo13vhaUrXVtWvCzOwAPDl30UjoJGI/CwiC0Ska6FFV7h8uRdDgLtEJBH4Bni4cEILOuf6+wQIkRIexjcichfQBugY6FgCQUTCgH8DfQMcSrAoget+isW1MueISHNV3R/QqAKjD/C+qr4mIpfh1m81U9X0QAcWCoK1RWHlPzL5ci8QkWuAZ4HuqnqikGIrbLndi/JAMyBeRBJwfbBxRXRA25d/F4lAnKqeVNXNwHpc4ihqfLkX9wGTAFR1PlAGVzCwuPHp98mZgjVRWPmPTLneCxFpDYzBJYmi2g8NudwLVT2gqpGqGq2q0bjxmu6qWhS3RPTl/5EpuNYEIhKJ64raVJhBFhJf7sVWoDOAiDTBJYrkQo0yOMQBf/bMfroUOKCqO3N7U1B2Pan/yn+EHB/vxXCgHDDZM56/VVW7ByxoP/HxXhQLPt6L6UAXEVkNpAFPqmqRa3X7eC/+D3hHRB7DDWz3LYp/WIrIJ7g/DiI94zEvAiUBVPVt3PjM9cBG4CjQz6frFsF7ZYwxpgAFa9eTMcaYIGGJwhhjjFeWKIwxxnhlicIYY4xXliiMMcZ4ZYnCmEIkItEZlT1FJFZEvgp0TMbkxhKFMT7wLFCy/19MsWT/8I3Jgeev/3Ui8iGwErhbROaLyK8iMllEynnOaysi80RkmYgsFJHynvf+5Dn3VxG5PLA/jTF5F5Qrs40JIg1xpWI2Av8DrlHVIyLyFPC4iLwKfAbcrqqLRKQCcAz4A7hWVY+LSEPgE1zBRmNCjiUKY7zboqoLRORG3IY3P3vKpJQC5gMXATtVdRGAqh4EEJEI4D8i0gpXPqNRIII3piBYojDGuyOerwJ8r6p9sr4oIs1zeN9jwC6gJa6LtyhvomSKOBujMMY3C4ArRORCcC0GEWkErANqikhbz/HyWcre7/Tsd3A3rlidMSHJEoUxPlDVZNyGSJ+IyHJct1Njz9abtwOjRGQZ8D2uhPV/gXs8xxqT2TIxJuRY9VhjjDFeWYvCGGOMV5YojDHGeGWJwhhjjFeWKIwxxnhlicIYY4xXliiMMcZ4ZYnCGGOMV/8Pls83vUekyKAAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cause_kg_correct_facts = 95037\n",
    "\n",
    "cause_results_new = cause_results.copy()\n",
    "del cause_results_new[\"title\"]\n",
    "\n",
    "show_precision_recall_curve(cause_results_new, cause_kg_correct_facts, 'prec_recall_jaccard_cause.pdf')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## DDI Gene Experiment with Document Pairs"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "loading correct interactions from semmed...\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_ddi_gene_chemical.pkl ...\n",
      "753899 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "55370 of 753899 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "31432 of 221670 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "17819 of 78987 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "9477 of 29045 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "5752 of 14282 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "4253 of 9656 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "2454 of 4381 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "1759 of 2927 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "1540 of 2545 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "1446 of 2385 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "1445 of 2384 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_ddi_gene_title.pkl ...\n",
      "753899 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "55370 of 753899 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "37558 of 337349 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "13719 of 59848 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "3417 of 8072 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "1319 of 2177 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "996 of 1475 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "924 of 1339 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "918 of 1330 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "911 of 1316 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "909 of 1311 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "909 of 1311 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_ddi_gene_authors.pkl ...\n",
      "753899 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "55370 of 753899 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "3168 of 6452 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "2404 of 4407 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "1861 of 3160 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "1576 of 2600 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "1444 of 2310 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "1256 of 1912 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "1111 of 1675 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "1064 of 1586 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "1035 of 1534 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "1035 of 1534 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "loading data from file: preprocessed_context_compatibility/fact2maxjaccard_ddi_gene_mesh.pkl ...\n",
      "753899 jaccard coefficients loaded\n",
      "checking threshold: 0.0\n",
      "55370 of 753899 derived facts are correct\n",
      "checking threshold: 0.1\n",
      "35319 of 279697 derived facts are correct\n",
      "checking threshold: 0.2\n",
      "16249 of 66771 derived facts are correct\n",
      "checking threshold: 0.3\n",
      "5791 of 13754 derived facts are correct\n",
      "checking threshold: 0.4\n",
      "2328 of 3936 derived facts are correct\n",
      "checking threshold: 0.5\n",
      "1344 of 1973 derived facts are correct\n",
      "checking threshold: 0.6\n",
      "1013 of 1441 derived facts are correct\n",
      "checking threshold: 0.7\n",
      "930 of 1336 derived facts are correct\n",
      "checking threshold: 0.8\n",
      "919 of 1322 derived facts are correct\n",
      "checking threshold: 0.9\n",
      "912 of 1314 derived facts are correct\n",
      "checking threshold: 1.0\n",
      "911 of 1313 derived facts are correct\n",
      "============================================================\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "print('loading correct interactions from semmed...')\n",
    "idx_subjects_correct_ddi_gene, _ = load_kg_facts(conn, query_ddi_correct)\n",
    "\n",
    "ddi_gen_results = do_experiment_with_context_pairs(experiments[1], metadata_to_process, metadata_name,\n",
    "                  thresholds_to_check, idx_subjects_correct_ddi_gene)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## DDI Gene Experiment"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "chemical\n",
      "753899\t55370\n",
      "221670\t31432\n",
      "78987\t17819\n",
      "29045\t9477\n",
      "14282\t5752\n",
      "9656\t4253\n",
      "4381\t2454\n",
      "2927\t1759\n",
      "2545\t1540\n",
      "2385\t1446\n",
      "2384\t1445\n",
      "title\n",
      "753899\t55370\n",
      "337349\t37558\n",
      "59848\t13719\n",
      "8072\t3417\n",
      "2177\t1319\n",
      "1475\t996\n",
      "1339\t924\n",
      "1330\t918\n",
      "1316\t911\n",
      "1311\t909\n",
      "1311\t909\n",
      "authors\n",
      "753899\t55370\n",
      "6452\t3168\n",
      "4407\t2404\n",
      "3160\t1861\n",
      "2600\t1576\n",
      "2310\t1444\n",
      "1912\t1256\n",
      "1675\t1111\n",
      "1586\t1064\n",
      "1534\t1035\n",
      "1534\t1035\n",
      "mesh\n",
      "753899\t55370\n",
      "279697\t35319\n",
      "66771\t16249\n",
      "13754\t5791\n",
      "3936\t2328\n",
      "1973\t1344\n",
      "1441\t1013\n",
      "1336\t930\n",
      "1322\t919\n",
      "1314\t912\n",
      "1313\t911\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "print_tab_seperated(ddi_gen_results)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "chemical\n",
      "authors\n",
      "mesh\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3iUVfbA8e9JQgsEpEkPk9A7SAdRBAQ0CKiIKKCIkrWA7lp2VX6WFVHXsvayiAFdsYAumE1QERXpC1EQBKQmhEASeighpN3fH3dSTRlCJjNJzud55snMO+/MnHnFnNx2rhhjUEoppQrj4+kAlFJKeTdNFEoppYqkiUIppVSRNFEopZQqkiYKpZRSRfLzdAAXqkGDBsbhcHg6DKWUKld+/vnno8aYhiV5bblLFA6Hg6ioKE+HoZRS5YqI7C/pa7XrSSmlVJE0USillCqSJgqllFJFKndjFEqpii0tLY24uDhSUlI8HUq5VL16dZo3b06VKlVK7T01USilvEpcXBwBAQE4HA5ExNPhlCvGGI4dO0ZcXBxBQUGl9r7a9aSU8iopKSnUr19fk0QJiAj169cv9daYJgqllNfRJFFy7rh2miiUUkoVSROFUkoVY8qUKXzxxRel/r7h4eG88MILJXqtw+Hg6NGjpRxRwcr1YHbjlxuTeDbxD8cb1WxEwsMJHohIKVXmFiyAmTMhNhYCA2H2bJg40dNRuWT06NGMHj3a02EUy60tChEZKSI7RWSPiDxawPOvishm522XiJy8kPcvKEkUdVwpVcEsWAChobB/Pxhjf4aG2uMX4aOPPqJr165069aNyZMnA7By5UoGDBhAcHBwntbFSy+9RO/evenatStPPfUUADExMbRv354pU6bQtm1bJk6cyPLlyxk4cCBt2rRhw4YNAMyfP5/p06cDkJiYyPXXX0+3bt3o1q0ba9euBWDs2LH07NmTTp06MWfOnIv6XiXlthaFiPgCbwNXA3HARhEJN8ZszzrHGPOXXOfPAHq4Kx6lVDk1ePAfj40fD/feC489BsnJeZ9LToYHHrCtiqNHYdy4vM+vWFHkx23bto1nn32WtWvX0qBBA44fP86DDz5IfHw8q1ev5vfff2f06NGMGzeOZcuWsXv3bjZs2IAxhtGjR7Ny5UoCAwPZs2cPixYtIiwsjN69e/PJJ5+wevVqwsPDee6551iyZEmez73//vu58sorWbx4MRkZGZw5cwaAsLAw6tWrx7lz5+jduzc33ngj9evXv8CLeHHc2fXUB9hjjNkHICKfAWOA7YWcfwvwlBvjUUpVNHFxBR8/dqzEb/nDDz9w00030aBBAwDq1asH2L/sfXx86NixI4mJttdi2bJlLFu2jB497N+4Z86cYffu3QQGBhIUFESXLl0A6NSpE0OHDkVE6NKlCzExMQV+7kcffQSAr68vderUAeCNN95g8eLFABw4cIDdu3dXqETRDDiQ63Ec0LegE0WkJRAE/FDI86FAKEBgYGDpRqmU8m5FtQACA213U34tW9qfDRoU24JwVbVq1bLvG2Oyfz722GP86U9/ynNuTExMnvN9fHyyH/v4+JCenu7SZ65YsYLly5ezbt06/P39GTx4sEdWrHvLrKcJwBfGmIyCnjTGzDHG9DLG9GrYsETl1JVSFdHs2eDvn/eYv789XkJDhgxh0aJFHHO2So4fP17ouSNGjCAsLCy7m+jgwYMcPny4RJ87dOhQ3n33XQAyMjJISkoiKSmJunXr4u/vz++//8769etL9N4Xy50tioNAi1yPmzuPFWQCcF9pfrj8XXT2k1IVXdbsplKc9dSpUydmzpzJlVdeia+vb3a3UkGGDx/Ojh076N+/PwC1atXi448/xtfX94I/9/XXXyc0NJQPPvgAX19f3n33XUaOHMl7771Hhw4daNeuHf369Svx97oYktWEKvU3FvEDdgFDsQliI3CrMWZbvvPaA98AQcaFYHr16mWyNi6Svxe/AtE85Z7vp5Ryjx07dtChQwdPh1GuFXQNReRnY0yvkryf27qejDHpwHTgW2AHsNAYs01EnhGR3BOHJwCfuZIklFJKlT23LrgzxiwFluY79mS+x0+X9P198CGTzJK+XCmllAu8ZTC7RDKeymDTGG2IKKWUO5XrRAEwaZKnI1BKqYqtXCeKzZth2zbgTKMCn29Us+DjSimlXFeuiwJmtyZezjUFdvSd0H4JB/98hKZNynUeVEopr1Cuf5Pu3VvAwQMDwf84Dz+/q8zjUUpVTkuWLGH79pzqRIMHDyZrGn9FUK4Txblz0L17voMHBgDw0741ZR+QUsoj4uPhyishwUPra/MniovhanmPslSuEwXA0qVQvXquA0fbQXJ9arZf67GYlFJla9YsWL3a/iwtBZX3rlWrVvbzX3zxBVOmTGHt2rWEh4fzyCOP0L17d/Y6uzoWLVpEnz59aNu2LatWrQLsfuB33HEHXbp0oUePHvz444+ALTc+evRohgwZwtChQ4mPj+eKK66ge/fudO7cOfv1nlKuxyjA/sPIzLOUQuDAAHbXX0NCAjRu7KnIlFKloaAq47mdPw9RUfb3wLx5sGmT3ZJiypQSVRnPVlB574IMGDCA0aNHM2rUKMbl+rD09HQ2bNjA0qVL+fvf/87y5ct5++23ERG2bt3K77//zvDhw9m1y3aT//LLL2zZsoV69erxyiuvMGLECGbOnElGRgbJ+Uupl7Fy36JYtw5SU/MdPDAAGuzk8WfLZptApZTnZO1ZBJCRUXAx2ZJ444036NatG/369csu730hbrjhBgB69uyZXVZ89erVTHLOwmnfvj0tW7bMThRXX311dknz3r17M2/ePJ5++mm2bt1KQEBA6XypEir3LYpNm6BHDztVNlvsQABW7F0HXOeRuJRSpaOoFkB8PAQH2wQB9o/GEydg5Ej7uKRVxgsr7y2SU1+uuHLfWWXFfX19XRp3qFmzZvb9K664gpUrVxIZGcmUKVN48MEHue222y78i5SSct+iAJssjIHp06FOHUje04sqPlUY/5AOaCtVkf2x69kmjYsdqyisvHejRo3YsWMHmZmZ2ZsJAQQEBHD69Oli33fQoEEscG7TumvXLmJjY2nXrt0fztu/fz+NGjVi2rRp3HXXXfzyyy8X94UuUoVIFFlatoSkJDh/tgaXNbmMtQd0QFupiqygrufUVFh7kf/rjxw5kvT0dDp06MCjjz6aXd77hRdeYNSoUQwYMIAmTZpknz9hwgReeuklevTokT2YXZB7772XzMxMunTpws0338z8+fPzbHCUZcWKFXTr1o0ePXrw+eef88ADD1zcF7pIbisz7i65y4znt2OHbV2MGQNPrH6Qd6PeJenRJKr6Vi3jKJVSJaVlxi9euSkz7gkdOsBVV8G110LHgIGkpKewKX6Tp8NSSqlyrUIlisxMuPdeWLUKVn9qF96tOaDjFEopdTEqVKJISIAlS+zA9sIPmhAYEKTjFEopdZHK/fTY3J59FqpUgbQ0O/Oh+pGBrDHLMcbkmdamlFLKdRWmRREfb1dlpqXZx6mpEP3TABLOJBBzMsajsSmlVHlWYRLFrFmQeT7vPDmzrw+g4xRKKXUxKkyiWPfvPaSavNNg0w93x+d8LdbEaqJQSnlOTEwMnTt39nQYJebWMQoRGQm8DvgCc40xLxRwznjgacAAvxpjbi3JZ2060waAxYzhBpYAUMOcp8+BVNZuXQqjSvQVlFJerPHLjUk8m/iH441qNiLhYQ/VHK+A3NaiEBFf4G3gGqAjcIuIdMx3ThvgMWCgMaYT8OeL/dwIRmFzDmTgQ/KBq9l6Ppakj96/2LdWSnmZgpJEUcddFRMTQ/v27ZkyZQpt27Zl4sSJLF++nIEDB9KmTRs2bNjA2bNnmTp1Kn369KFHjx589dVXAGzbto0+ffrQvXt3unbtml1MMCMjg2nTptGpUyeGDx/OuXPnLirGsuS2ldki0h942hgzwvn4MQBjzPO5znkR2GWMmevq+xa6MluEeBoTzD5SqGGPPdwYaulfG0qVJ7lXFf/5mz+zOWFzoef+tP+nQp+7suWVBR7v3rg7r418rcgYYmJiaN26NZs2baJTp0707t2bbt268cEHHxAeHs68efPo2LEjHTt2ZNKkSZw8eZI+ffqwadOm7JIfEydOJDU1lYyMDBITE2ndujVRUVF0796d8ePHM3r06OxKsqWttFdmu7PrqRlwINfjOKBvvnPaAojIGmz31NPGmG/yv5GIhAKhAIGBgQV/2tChzPr+BjLJNQ22gCQBF//XhlKq4gsKCqJLly4AdOrUiaFDhyIidOnShZiYGOLi4ggPD+fll18GbDXZ2NhY+vfvz+zZs4mLi+OGG26gTZs22e/X3bklZ+7S4+WBp9dR+AFtgMFAc2CliHQxxpzMfZIxZg4wB2yLosB3Wr6cdQF7SD1TvcCnlVLlT3F/+cvfC18ftWLKiov67NzF+nx8fLIf+/j4kJ6ejq+vL19++eUfqr926NCBvn37EhkZybXXXsu//vUvgoOD87yfr69vuep6cuesp4NAi1yPmzuP5RYHhBtj0owx0cAubOIokU2nW2M+XsChGq3wwfv2nVVKVRwjRozgzTffJKv7ftMmW1du3759BAcHc//99zNmzBi2bNniyTBLhTsTxUagjYgEiUhVYAIQnu+cJdjWBCLSANsVte+iPnXiRGYN+BofyldVXKXUhWtUs9EFHS9NTzzxBGlpaXTt2pVOnTrxxBNPALBw4UI6d+5M9+7d+e233zy64VBpcWuZcRG5FngNO/4QZoyZLSLPAFHGmHCxdTVeAUYCGcBsY8xnRb1nUWXGIWfHq5QU4OnCm6XmKU0kSnkjLTN+8crTYDbGmKXA0nzHnsx13wAPOm+lIs+OV2caFTrrSSmllGs8PZhd6vLsePVyzhTYbt1SOX57a9rWb8vy25Z7JjillCqHKkwJjyxZ+2cbA1u2QMtG51jOUDY/8V/u6XUP30d/z/Yj2z0dplKqCOVt501v4o5rV+ESRW4OB/QcUI2a/sDkydx1zeNUS4e3F3h2/1mlVOGqV6/OsWPHNFmUgDGGY8eOUb166S4TqHBdT7kFBMCXN34K/10J6ek0BCb8Bh92XM5zH71PndumeTpEpVQ+zZs3Jy4ujiNHjng6lHKpevXqNG/evFTf062zntyhuFlPf+BwcHB/GvU4Tg1SiGoKvUPh9fV1uf/r4+4LVCmlvMjFzHqq0F1PAOv2N6U5B/mOqwHodQj6HYC32pwg02QW82qllFIVPlFcFniUGiTzA0Oyj03fALvrw3d7v/NgZEopVT5U+ERR7bmnuNxnHd8wgitZQQKNuCm6Bo186/Dmhjc9HZ5SSnm9Cp8omDiRIePqsZMOrOJyZvn+narvvU/owPtZunspe4/v9XSESinl1Sp+ogC6TekBgMGXeRmTSRh4I3f3uhtfH1/ejXrXw9EppZR3qxSJwrnxFGB3vZt1XwJNA5pyQ4cb+GDTB5xNPeu54JRSystV+EQRHw8ffpjzOJXqzFvenIQEmNFnBidTTvLJ1k88F6BSSnm5Cp8o8hQJdMrAj1mzYGCLgXRr1I03N7ypq0CVUqoQFT5R5CkS6JSaCmtXZyAizOgzg62Ht7IqdpVnAlRKKS9X4RNF7iKBP/0Ep0+kY1o62DTsrwDc0uUW6lavq1NllVKqEBU+UeR2xRVQ6xI/6NgRliwBY/Cv4s9dl93F4h2LiTsV5+kQlVLK61SqRJGaascs/hv8AOzbB7/9BsA9ve4h02TyXtR7Ho5QKaW8T6VKFFWqQFgYrMy8HERsqwIIqhvEde2uY87Pczifft7DUSqllHepVIlCBLZvh5feqQn9+2cnCoDpvadzJPkIC7ct9GCESinlfSr0fhQFqVHDeeepp/LMmx0WPIx29dvx1sa3mNxtsmeCU0opL+TWFoWIjBSRnSKyR0QeLeD5KSJyREQ2O293uTMesLOfbrgB/vb9cBg5MncsTO8znQ0HN7Dh4AZ3h6GUUuWG2xKFiPgCbwPXAB2BW0SkYwGnfm6M6e68zXVXPDlxwfnz8OWXYLZshTlzsp+7vdvtBFQN4K0Nb7k7DKWUKjfc2aLoA+wxxuwzxqQCnwFj3Ph5LgsJgb17oc/IuiTc/TQcOwZAQLUAbu92O59v+5zDZw97NkillPIS7kwUzYADuR7HOY/ld6OIbBGRL0SkRUFvJCKhIhIlIlGlsY/utdfanz/HN2OWmQkREdnP3dfnPlIzUnn/5/cv+nOUUqoi8PSsp/8CDmNMV+A74MOCTjLGzDHG9DLG9GrYsOFFf2i1arYLyiDMYyoJn63Ifq59g/ZcHXw170a9S1pG2kV/llJKlXfuTBQHgdwthObOY9mMMceMMVkLF+YCPd0YT7ZZs8DH+c3PUZ3R395L7M5z2c/P6DODg6cP8tXOrwp5B6WUqjzcmSg2Am1EJEhEqgITgPDcJ4hIk1wPRwM73BgPYMuOz5sHGRnZUbDR9KZl++pscE52urbNtTgucWj9J6WUwo2JwhiTDkwHvsUmgIXGmG0i8oyIjHaedr+IbBORX4H7gSnuiidLQWXHq1Qx9O0r9HS2Z1560ZcW8fexcv9KtiRucXdISinl1dy64M4YsxRYmu/Yk7nuPwY85s4Y8iuo7HhamnD+PPj62sfx8VA3cSo1ujzJWxveYmTaHAYPhnr1yjJSpZTyDp4ezC5zucuOZ992/M6mzG7w448AvP46LPm0HhO7TOTfv37MjRNP0LQp3HILfPfdH1skSilVkVW6RFGgFi1g1648tZ9EYHqf6aRknOMvH4URGgrffgvDh0NQkK0AEh3twZiVUqqMaKIAqFkTrr46e4+KLN0ad2NQ4CC+in+HV1/L4NAh+OwzaN/ejnUEB8OwYfDJJ3a1t1JKVUSaKLKMHQuxsbB5c57D0/tMZ9+JfXy952uqV4ebb7Yti5gYeOYZu8L7rrtyEsWJE3lyjVJKlXuaKLJcd51dXLF4cZ7D17e/nqYBTf9Q/ykwEJ54wiaKn3+G2rXt8WHDYOLEsgpaKaXcTxNFloYN4dFHoXfvPIer+Fbh7p538+3eb9l1bNcfXubjAx062PuZmTBtGtx4o3186hTceit8803udRtKKVW+iCln/SS9evUyUVFRZfqZiWcSafFqC+7pdQ+vX/O6y69bv942VI4ehWbNYMoUuOMOaNXKfbEqpVRBRORnY0yvkrxWWxT5xcbaObS5NKrViPGdxjP/1/mcPn/a5bfq1w8OHoQvvoBu3eD556F1axg8GD76CJKTSzl2pZRyA00U+V1+ud0m1ccHHA5YsACw9Z9OnT/Fv7f8+4LermpV2xUVGWlz0HPP2eRx++3QuDHcfbd2SymlvJsmitwWLICEBDuFyRjYvx9CQ2HBAvo060Ovpr14a8NblLS7rlkzeOwxu2Rj5Uq7096hQzkrwhcvhsO6DYZSystoosht5kxIy1daPDkZZs5ERJjRZwY7ju7gh+gfLupjRGDQIJg/H75yFqg9dgzGj4dXXrGPMzMhPf2iPkYppUqFS4lCRKqJyK0i8riIPJl1c3dwZS42tsjj4zuNp4F/A97aWHpbpYrYn/Xrw5YtcP/99vGyZXbB+KOPws6dpfZxSil1wVxtUXyF3cY0HTib61axBAYWeby6X3VCLwslfGc4+0/uL/WP79DBdk8B1K1rZ+q+/LJdCX755RAWBmfOlPrHKqVUkVxNFM2NMTcbY140xrySdXNrZJ4wezb4++c95u9vjzvd3etuAN6NetetofTtC+HhcOAA/OMfdortnXfaAfA774Q1a3QFuFKqbLi0jkJE5gBvGmO2uj+korl9HcWCBXasIjbWtiRuuAH8/ODFFwFo/HJjEs8m/uFljWo2IuHhBLeFZYwtkR4WBp9/blsWISF5tvtWSqlClcU6isuBn0Vkp4hsEZGtIlIxd/SZONEWcsrMtD/9/OCll+Bd24IoKEkUdby0iMCAATB3rt0vIywsp1RISgpMmED2Dn1KKVWaXN246Bq3RuHNnn8etm+HGTOgXTtPRwNArVp2hXeWrOm206bZxwcOwOnT0LGjZ+JTSlUsLrUojDH7gUuA65y3S5zHKj5fX1tHvH17GDfO09EUqGtX21N21VX28WuvQadOdt3g++/bmlNKKVVSrk6PfQBYAFzqvH0sIjPcGZhXqV3bjiz7FH25PFk3y88vJ7y//tXOlkpKsusFmzSxdaZWrtQBcKXUhXN1jOJOoK8x5knnntf9gGnuC8sLBQfbhQ5FmLx4MufSzpVRQIVr1Ageegi2bbOFCSdNgv/8B668Etq2zSkjopRSrnA1UQiQuyJRhvNY0S8SGekcAN8jIo8Wcd6NImJEpEQj8mWmaVMa+dYp8KlaUo1Ptn7CFfOvIO5UXBkHVjARO832X/+yA+AffmjXacycaZMF2BZGaqpn41RKeTdXB7PnAf8TkaxdfcYCHxT1AhHxBd4GrgbigI0iEm6M2Z7vvADgAeB/FxK4pyTMvQT2J/3xiZaN+e+3bzLxPxPpNacX/7n5PwxoMaDsAyxEzZpw2232tmcPVKlij69ZA9dfb4sW9unj2RiVUt7J1cHsfwJ3AMedtzuMMa8V87I+wB5jzD5jTCrwGXZ1d36zgH8AKS5H7UlFlPm4rt11rL9rPQHVArjqw6sI2xRWtrG5qHVraNnS3g8IgOHDc2ZILVwI770HJ096Lj6llHcpMlGISG3nz3pADPCx87bfeawozYADuR7HOY/lfv/LgBbGmMhi4ggVkSgRiTpy5EgxH+tmxZT56NiwIxvu2sBgx2DuDL+T+7++n7SMtIJf4wW6dbNrDGvVso+//BLuuccOgE+aBD/+aJeUKKUqr+JaFJ84f/4MROW6ZT0uMRHxAf4JPFTcucaYOcaYXsaYXg0bNryYj714BZX58PGBZ5/Nfli3Rl0ib43kwX4P8uaGNxm5YCTHko+VcaAl89lnEBUFU6faVd9DhtgWyKxZhTemlFIVW5GJwhgzyvkzyBgTnOsWZIwJLua9DwItcj1u7jyWJQDoDKwQkRjsTKpwrx/QnjgR5syxfTcitiUxd6798zvXqLCfjx+vjHiFD8d+yJrYNfR+vzdbEz1eAaVYItCzJ7z9th0A/+QTO+HrySftPk4jRsCqVZ6OUilVllxdRzFQRGo6708SkX+KSCF9MNk2Am1EJEhEqgITgPCsJ40xScaYBsYYhzHGAawHRhtjynZD7JLIXeZj/367TPr0aVvi1VkTKstt3W7jpyk/kZKeQv8P+rN4x+KC39ML1agBt9wCy5dDdLRNFr//DseP2+fj42HzZs/GqJRyP1enx74LJItIN2xX0V6gyD1BjTHpwHTgW2AHsNAYs01EnhGR0RcRs3eqUQNatYK//S1n7qlT3+Z9iQqNovOlnblh4Q38fcXfyTTlq+Pf4YCnn7YJY9Qoe+y99+Cyy+wufaCL+ZSqqFydHptujDEiMgZ4yxjzgYjcWdyLjDFLgaX5jhW44ZExZrCLsXgnPz/497/tz5kz7fZ0T+Z81aYBTVkxZQV3R9zN0z89zZbDW/hw7IfUqlrLg0FfuNyL0x94AHr0gKZN7eMbboBq1ez4xtChOVu8KqXKN1dbFKdF5DFgEhDpHIiu4r6wyik/P7u/6e23w1NPwauv5nm6ul915o2Zx6sjXmXJ70sY8MEAok9EeybWUlCvHowda+8bY1sdy5bZcYzgYHsJosvv11NKObmaKG4GzgN3GmMSsAPTL7ktqvLM19fWAH/00ZzformICH/u92e+mfgNcafi6P1+74veg9sbiNi8eOiQ3S+jQwc7Uyo42LYuFiyAc56vbqKUKgGXNi7yJm7fuKi0ZWbC4sW2X0byVj3Zc3wPYz4bw86jO3lt5Gvc1/s+RIqtjFJuxMbasiHz5tmWRZ06sGQJDB7s6ciUqnzctnGRiKx2/jwtIqdy3U6LiBavdsWiRbY8+UMP/WG0t3W91qy7cx0hbUOY8fUMpv13GufTz3so0NIXGAhPPGFLhvzwA4wZY0uig00Yr71mh3KUUt6tuHUUlzt/Bhhjaue6BRhjapdNiOXc+PF21PfVV+3PfMmidrXaLL55Mf836P/4YNMHDPloCAln3Lelqif4+Ni9Mj780I5rgK0tNXduzoD3tm2QkVH4eyilPMfVdRT9nMX7sh4HiEhf94VVgWR13j/4ILz5Jtx33x9qYviID7OGzGLhuIVsTthM7/d7E3WoHHWvlcD779uChCJw9qzdZKllSzthbM8eT0enlMrtQtZRnMn1+KzzmHKFiN1J6K9/tQPd27YVeNpNnW5izdQ1+Iovg+YN4pOtnxR4XkVRx1mxvWpVO1mse3d44QVo08aOY3z0kU0iSinPcnk/CpNr1NsYk4nrazAU2GTxwgvw66/QpUuhp3Vv3J2N0zbSt1lfJv5nIn/77m9kZFbsPpkqVexYf0SE3e87a2Ol22+3xQlDQ+0GTOVs3oVSFYariWKfiNwvIlWctweAfe4MrEISgXbt7P35823pjwI65hvWbMh3k7/jnl738OLaF7nu0+s4mVI56n43bQqPPQa7dtmtW2+80U6t7d8f9u6152jCUKpsuZoo7gYGYIv6xQF9gVB3BVUpxMXZZHHbbQVO/aniW4V3Qt7hvZD3+G7fd/Sd25edR3eWfZweIgKDBtmptQkJdoZx69b2uYkTYVrl2ohXKY9ydeOiw8aYCcaYS40xjYwxtxpjDrs7uArt//7Pliz/5BNbebaQeaJ/6vUnfrjtB06cO0GfuX1YuntpgedVZAEBedcuBgfnbLyUmWkrvP/+u2diU6oycHXWU1sR+V5EfnM+7ioi/+fe0CqBxx+31WY//9yWaS1kh6BBLQcRFRpFq7qtGPXJKF5c8yLlbaFkaXr2WZtnAX77zRYr7NDBFu8NC4MzZ4p8uVLqArna9fQ+8BiQBmCM2YItG64u1iOPwD//aQe4P/3UFkzy8bE/FyzIPi2wTiCrp65mfKfx/G3535i0eBLn0rQmRteuthfvxRfh6FG4805o3Nj+XLNGxzOUKg2uJgp/Y8yGfMd0TW1p+ctfbIny0FC7v4Ux9mdoaJ5k4V/FnzPr5CQAACAASURBVE9v/JTnhjzHp1s/5fJ5l3Mg6UARb1w5NG5s8+2OHTY5TJhg9/6+/HJo3x7+8Q9IKR87sivllVxNFEdFpBVgAERkHBDvtqgqo5kzITk577HkZHs8FxHhsUGPEX5LOLuP7ab3+71ZE7umDAP1XiIwYIBd8R0fbwfCL73U7ptRtao957ffIM17tzBXyiu5VBRQRIKBOdiZTyeAaGCiMWa/e8P7o3JXFNBVPj4F95OIFDp2sePIDkZ/Npr9J/fzTsg73HXZXW4Osnw6dQpq17Y71TZrZjdemjfP01EpVbbcVhTQ+eY+QC9jzDCgIdDeGHO5J5JEhRZYyM6yvr72N10BOjTswIa7NnBV0FVM++80ZiydQVqG/rmcX21nVTJfXzsj+b777ONdu6BfP7sFeiGXWCmFC4nCuQr7r877Z40xp90eVWU0ezb4++c9Vq0aDB+e85sua7PqXOrWqEvkrZE81P8h3tr4FiM+HsHR5KNlEHD54+sLISHQy/k3VWKi3er8T3+y4xy33w4//aQD4Erl5+oYxXIReVhEWohIvaybWyOrbCZOtH/atmxpu5tatoQPPrBlVgG2b4cWLeyobVJSnpf6+fjx8vCX+WjsR6w9sJbe7/dma+JWD3yJ8mXQIDtmsX49TJ5sF/UNHmxrTc2ebWdTKaVcH6OIxjmQnZsxJtgdQRWlwo5RFOfwYbvuIiwMGja0BZHuuCPvJtbAhoMbuP7z60lKSeLDsR9yY8cbPRRw+ZOcDF9+aS/xihX20o4bZ5e5KFXeuXWMwqkj8DbwK7AZeBPo5EJgI0Vkp4jsEZFHC3j+bhHZKiKbRWS1iHS8kOArlUsvtdN5Nm60f/LedZed/5mvVlSfZn2ImhZFl0ZdGLdoHE/9+BSZpuDBcJWXv79tWfz4oy11/vjj9lKD7Y567jldAa4qJ1dbFAuBU0DWpP5bgTrGmPFFvMYX2AVcja0PtRG4xRizPdc5tY0xp5z3RwP3GmNGFhVLpW1R5GYMfPaZ7Rt55BF77MgR29JwSklP4Z7Ie5i/eT7Xt7+eD8d+SEC1gELeUBVn7167+vutt+zylrNn7TTbSy7xdGRKuaYsWhSdjTF3GWN+dN6mAZ2LeU0fYI8xZp8xJhX4DBiT+4SsJOFUkwK6t1QBRGzJj6wksWyZHdN45pnstRjV/aoTNjqM10a8RvjOcAaEDWDfCS34W1KtWsGhQ3YoCexufU2a2DJdP/xQ6AxmpSoEVxPFLyLSL+uBc3e74v6sbwbkXjYc5zyWh4jcJyJ7gReB+wt6IxEJFZEoEYk6cuSIiyFXIu3a2cUBTz1l/+xduBCMQUR4oN8DfDPpGw6eOkjv93vz/b7vPR1tudWgAdSsae9fcYUtExIZCUOH2kTyzDMQG+vZGJVyB1cTRU9grYjEiEgMsA7o7Rxf2HIxARhj3jbGtAL+BhRYaNAYM8cY08sY06thru4V5dSypU0OK1ZA3bpw881w003ZTw8LHsbGaRtpUqsJIz4ewRv/e6NSFxUsDZ07226o+HhbALh1a5unHQ47o/nzz7VsiKo4XB2jaFnU8wUtvhOR/sDTxpgRzsePOc99vpDP8AFOGGPqFPVZOkZRjIwMO63W39/2i2RkwIkT0KABp8+fZvLiyXy18yumdp/KOyHvUM2vmqcjrjBiYmyX1Lx5tlTXjTfCF194OiqlrIsZo8AY45YbdqvUfUAQUBU7Y6pTvnPa5Lp/HRBV3Pv27NnTqAvw3nvG1KljzKuvGpOaajIyM8wTPzxheBrTf25/E3863tMRVjgZGcZ8950xa9faxwcOGNOzpzHr1nk2LlW5ufL7tbCbq11PJUlA6cB04FtgB7DQGLNNRJ5xznACmC4i20RkM/AgcLu74qm0rrzS7iP6l79A1674fLuMZ656hkU3LeLXxF/pNacXGw9u9HSUFYqPDwwbZi872B36wM5wBjvDedmyAnfBVcorudT15E2066kEjIGlS22y2L0b7r8fXn+dXxN+ZcxnY0g4k8Dc0XOZ1HWSpyOtFCZOtOMaLVrAlCn2FlzmS1dVZVMW02NVeSZiixz99hu8/DJcdx0A3Wq1YuPN39OveT8mL57MI8seISNT/8x1t7AwO9jdsaPdra9VKztzasECOKd7USkvpImiMqlaFR56yPaLADz7LA279ee7tAnc2/MeXl73MqM+HcXJlJOejbOCq1YNxo+Hb76xg96zZkF0tJ170KQJ3HMPbLmouYRKlS5NFJXZTTdB27ZUCb2Ht59cz7/aP8L3+76nz/t9+P2o1qooCy1a2P2/9+yxpUNGj7Yzp5Yvt8+fO2cX3SvlSZooKrOePWHVKrtX95EjhE54iR/87uRkykn6zu1L5K5IT0dYafj42Mq1H31k12ZMm2aPL1xoN1vascOj4alKThNFZSdiN5neuROefJLLx8wgKjSKVgEtue7T63hh9Qu6OK+M1akDAc6yXP37wxNP2L2/wa7+njnTtkCUKis660kVKHnCjUyt9jWfB5/jls63MHf0XPyr+Bf/QuVWkyfbGVOZmbaMyNSpthR6VmkRpQqjs55UqfO/9wE+3dKW55fDZ1s/ZdDbPYlN0kJGnvbvf8OBA7bk+aFDdmptkya2ou369bo7n3IPbVGowjnLgUTOeYRbrz7F6eqCKaDAb6OajUh4OMEDAVZuxsDq1Xa67cKFtnBwhw7w9ttw1VWejk55G21RKPfw9YXQUEKW72e9ubPAJAGQeDaxjANTYIeXBg2ytaXi4+H99+34RoMG9vnNm+G//4X0dM/Gqco/TRSqeJdcQocX5no6ClWE2rXtpofr1kGXLvbYe+/ZMY20NPv49GnPxafKN00UqlR89ftXnEk94+kwVC5vvmlnP9eoYbupeve2u+eGhcEZ/U+lLoAmClUqxn4+lvov1mfkxyN5839v6m56XqBKlZzWRVqa3Wjp2DH7s3Fj+3PNGh0AV8XTRKFKxQ+3fMP03tOJObKb+7+5n1ZvtKLjX6ryyAhhRb/GpH38oadDrNSqVrU7527fDmvX2p10Fy60LYz27eEf/7DjHEoVRGc9KZc1frlxgQPX2bOezp2Ddu3YUyeDyDqJRAZnsMIBab5QJwVGNOhLyLB7uab1NTSsqTsVetrZs7Boke2KWrXKzl1Yt852UamK52JmPWmiUKXHGPtn6sSJ2ZstnK4Ky4Mhoi0sbe9Lgn8GgtC3eV9C2oQwqu0oujXqhoh4OPjKbdcuW9H28cdtwnjxRTh+HJ5/3s6uUuWfJgrlXXx8Cuz4zhTYtHsVEQe+J3J3JBsP2Q2TmgU0I6RNCCFtQxgaNJSaVXWZsafdey8kJsKXX9rH4eF2D6w6RW5UrLyZJgrlXRwOWz87v1q17KjqkCEQEkLCkD58nfIbkbsjWbZ3GadTT1PNtxpXBV1lE0ebEILqBpV5+MrKzLQ5PzYWWra0s6fGjbOD4FdcoS2N8kYThfIuCxbYmhLJyTnH/P3taGpSEkRE5FS1GzoUli8nNSOVVTEridyzlMjdkew6tguAjg07ZieNAS0GUMW3ige+UOVmjN2+NSzMFho+dcputnTHHXD77dC8uacjVK7QRKG8z4IFtsxpbCwEBsLs2XbsIsuuXTZhGGM3UzIGLrvM1qAYNYrd/doQeXgNEbsiWLl/JWmZaVxS/RJGtBrBqLajGNl6JA38G3ju+1VSycm2OyosDFassC2O4cNtccIxY+zsKuWdNFGo8u/MGZg+3e7tfeSI/Q00YAA8/jinhgxk+b7lROyKYOnupSSeTUQQ+jXvx6i2owhpE0LXRl11QLyM7d0L8+fb27FjdnptnTq2xVG7tqejU/l5baIQkZHA64AvMNcY80K+5x8E7gLSgSPAVGNMAZ3bOTRRVHAZGRAVZVsbkZHw5JMwdixs2wbvvENmyLX80uESIvZ/R+TuSKIO2X8LzWs3z+6iGho8VEuil6GMDLs+I2txX8+e0Lmz3alPeQ+vTBQi4gvsAq4G4oCNwC3GmO25zrkK+J8xJllE7gEGG2NuLup9NVFUMsbYUdPPP7f9G8nJdrxj2DAICSF+zFC+PvRT9oD4mdQzVPOtxpCgIdkzqRyXODz9LSqNzExbvbZxY7vT7qlTMGMG3HabrWjro0t8PcZbE0V/4GljzAjn48cAjDHPF3J+D+AtY8zAot5XE0UllpJiO8azWhuHDtk+j1q1YMUKzlf1YVXDc0Tu/YaI3RHsOW4HzDs17JS9ZqN/i/74+fh59ntUImvXQkgInDxpJ8NlDYC3bOnpyCofb00U44CRxpi7nI8nA32NMdMLOf8tIMEY82wBz4UCoQCBgYE99xc09VJVLsbkzNsEO56xbh00bAjXXAOjRrGrdzCR8SuJ2G0HxNMz06lbvS4jWo9gVBs7IF7fv75nv0clkJICS5bABx/A99/bY8OG2Qbi2LFQvbpn46ssyn2iEJFJwHTgSmPM+aLeV1sUqkDHj8O339rWxjff2MchIfYxcGr3b3yX9jsRuyNZunsph88exkd87IB4m1GEtA2hy6VddEDczWJi7NjFvHl2qU3dujBpErz2mnZLuZu3JgqXup5EZBjwJjZJHC7ufTVRqGKlp9t9QX19oX9/u8S4cWNo3RpCQsgMuZaf29QiIvpbIndH8nP8zwC0qN0ie1xjSNAQHRB3o8xM+PFHO8325Enbkwh2o6UBA6C+NvRKnbcmCj/sYPZQ4CB2MPtWY8y2XOf0AL7Atjx2u/K+mijUBUtKsus6IiNt38f583ZcY9EiGDmS+FOHWLrn6+wB8bNpZ6nuVz1nQLxNCC0v0U51d8mar3D0qM3nDz1kq9lmZtrnfH09HWHF4JWJAkBErgVew06PDTPGzBaRZ4AoY0y4iCwHugBZBY5jjTGji3pPTRTqoiQnww8/2C6pxx+3iwHnzYN334WQEM5fM5yV9U4RuedrInZFsPfEXgA6X9o5e0C8X/N+OiDuJr/+alsTzZvbHsTQUJgyxd6Cgz0dXfnmtYnCHTRRqFL3n//Ayy/b7ipj7J+1ISGYd99lV9I+InZFELk7klWxq7IHxEe2Hpm9QrxejXqe/gYV0tq1MGuWHXoyxk6vnToVbrjBzpBWF0YThVKl4cgR+2dsRAQkJMBPP9njzzwDl1xC0tVXsCxzN5HOAfEjyUfwER/6N++fvUK886WddUC8lB04kDMAvm+fXfV96602afTqpcUJXaWJQqnSltVxbgz06WNXi4PdDm7UKDLH38TGJobI3ZFE7o7kl/hfAAisE5g9rjEkaAg1qtTw4JeoWDIzYeVKOwD+xRd2n6yxY2HxYk9HVj5oolDK3fbutYPhERG2pfHII/Dss3aRwKJFHLqiB0uPrSdiVwTL9y3nbNpZavjVyLNCPLBOoKe/RYWRlGQX69esaWtNpqTAtGnwl7/Y2pLqjzRRKFWWzpyxM6fq14dly2DECNv66Ncve0D8p9rHiXC2Nvad2AdAl0u75BkQ9/XR6TylZfNmW8X2009t5fq4ONviaNPG05F5D00USnlKZiZs2mRbGhEROV1UGzdCr16YxER2ph7K3tVvdexq0jPTqVejnh0QbzOKEa1H6IB4KUhNBT8/u3DvoYfgn/+0GyxNnWo3XKpZyTdO1EShlLdISLDTdCZPtr+x7r3XjsI6d/U7OexylqX9nj0gfjT5KD7iw8AWA7O7qDo17KQD4hfp0CE7AB4WZvfICgiAm2+2SaNfv8o5AK6JQilvtWaNXdgXEWHHOcAWOvruOzIyM9h4cAORu5cSsTuCzQmbAWhZp2V20rjKcZUOiF8EY2D1apswFi60y2g6dLAJY/JkaNTI0xGWHU0USnk7Y/Lu6vfwwzm7+nXqBCEhHBzYhaVH1hGx2w6IJ6clU8OvBkODh2bPpGpRp4Wnv0m5dfq0TRYffGDrR957ry2Jboyt+lKlgu+yq4lCqfLozBm47z67q9/Ro7ZWhXNXv5Rhg/kp5iciIl4hMu5HogPSAehaNZCQPhMZ1XYUfZv11QHxEtqxA2rUsKXPV6+2e2dERNhNlyqqi0kUWodAKU+pVct2pGdkwIYNOdNvU1Ko7ledEV9uZsT//cgb6ensaAiRbSCyfRwvpv6D51c/T/0a9bNXiI9oNYK6Nep6+huVGx065Nz394dBg6BdO/v4iy/s9Nvx4+3YhtIWhVLeJ2uxX8OGtqWRz4mgJiwLf5XITZ+x9MCPHEtLwld8GRg4MLuLqmPDjjogXkI33GAX8dWsaZPF1KkwcGD5HwDXrielKiIfH5s08hOx03L/+lcyXn6JDc0gslsNItoJv9ZOBsBxiYOQxlcQ4riaqy67keo6IO4yY2zZr7Aw+Owz20PYtq1NGLfdBk2aeDrCktFEoVRF5HDY3X3ya9nS7gAUHW1Xmu3ZY2dU7d3LgbPxLH1jBpG7I1n++1LO+WTgnwpDjwYQkh5ESNPBNJ/1OgCNX7yUxHNH/vD2jWo2IuHhBPd+t3Li7FnbFRUWZsuH+PraDRQff9xudVKeaKJQqiJasMDW2U5Ozjnm7w9z5ti6FcU4t2YFKzYsIvLoGiLNbmKq2ffp1qgbo9qOYvaq2YW+1jxlbF0M3ac02+7ddknMhx/CO+/AmDF22cyxY3bimrfTRKFURbVgAcycafcHDwyE2bNdShL5GWPYfmR7dhHDNbFryDAZhZ6f+WQm4nDYOhitW0OrVvY2aJCtkWHftPx33JdAerr92r6+8NRTthT6oUO2Or03XxJNFEqpC3Li3AnqvVh42ZCaVWriSK+FI7kqQcczcRw8iyPmJEFDbsTx4hzq+gUgjRpBixY5SSQrkZSHP69LyZEjtkbkuHH28bhxttE3daotH+JN+4BrolBKXTD5e+F/+v6575+JSYoh+kQ00SejOXX+VJ7na1etjSO5Ko7TvjiOpBIUewrH0QwcE+8j6OHZ1DmRbJNG7iTSqpWtn9G4sbu/mkcYA9Onw8cfw6lTdke+O+6wu/M1b+7p6DRRKKVKoKhEYZ7K+3vhZMpJok9EE3MyhuiT9mfW/egT0ZxNO5vn/Euq1saR5EPQSXDEnyMo4TyOk+D462wck2YQsH0PzJhhk0furq3Onct99b7kZLtpYlgY/Pij7YoaMcK2MkaPhmrVPBOXJgql1AVr/HJjEs8m/uH4hc56MsZw/NzxvAnkRDQxSTn3z6Wfy/Oa+lXq4DiWQdDRdBzxKThOYpPKP8NoOXw8NVeth/fe+2OLpEUL7+rPKca+fXYAfP58W/q8Xj1YssQ2tsqaJgqllNcyxnAk+UhOAsnVGsm6fz7jfJ7XXOpbB0diCo7DqQQdN7Y1chKCPv+Wll2voHrkt7B8ed4kEhzstbO0MjJsuPPn2/pS9erBV1/BwYN2YptfGdTI8NpEISIjgdcBX2CuMeaFfM9fAbwGdAUmGGO+KO49NVEoVbFkmkwSzyQWmECiT0SzP2k/aZlpeV7TmACCDp3FcSzTJpAT4DglOH7aQuClbaj25RK7viR3t1Zd7ypxMmWK3bbkt99s99TOnXajJXc1mLwyUYiIL7ALuBqIAzYCtxhjtuc6xwHUBh4GwjVRKKXyyzSZxJ+O/2PX1skYYo7vJfZ0HOm5pvoKQtP0GjgOJdsEktUaqdUcx1c/0aJ2C6osdG66nZVEmjb1SJfW8eO2dXH2rF3xXa9ezgB4y5al+1nemij6A08bY0Y4Hz8GYIx5voBz5wMRmiiUUhcqPTOdQ6cP5U0gSTFEH91DzPF9HEhOIJPM7PN9xIfmZ/1wHE7NGRs560dQh4E4Xv+QZrWb4ffFf2zRxlat7Ap5N49Ap6bmDIAvX26PDRtmB8DHji2dHjVvTRTjgJHGmLucjycDfY0x0ws4dz5FJAoRCQVCAQIDA3vuL6isgVJKFSAtI424U3F5u7aO7yM6cScxSTEcTD2GIef3oJ+PHy1OZOI4npmTSHzq4xgYQtAjs2lSqwm+kUvtnNdWraB27VKNd/9+u/p73jxbqeWSS+way6lT7fYlJVXhE0Vu2qJQSpWm1IxUYpNi83ZrJewg5sgeos8cID7jZJ7zq/hUIfBoWk63Vqo/QTWa4Bg5Acet99K4egN8/rfBJpHGjUu8VDsz006vDQuDL7+E8+dtSa/g4JKtAPfW/SgOArm342ruPKaUUl6jqm9VWtdrTet6rQt8PiU9hdik2JxurRP7iN6/mZjj0fz3XDyJnAH2QsJs+OdsqvlUpWVWt9YZPxy+9XHUak7Q9VNxDL2RS6UWkpBgS7IUsa2ej4+tljJ0KLz1lu2SCg62z02ebBsy77zjhgtSAHcmio1AGxEJwiaICcCtbvw8pZQqddX9qtO2flva1m9b4PPJacnsP7k/p1vr8C5i9v1MdFIsv6Qd4ahPIpAIm3+GzfdRw6cajkTnAsTM2gRVa4SjTkscN00jqMcQ6psaCORZeFi3rt2FL0vTpnYIBWzL46WX7PNZiaS0uXt67LXY6a++QJgxZraIPANEGWPCRaQ3sBioC6QACcaYIgvFaNeTUqo8OZN6JrtbK+ZkDNFxvxGzJ4qY5INEZx7nhG9qnvNrik0kQclVcfjUw+HfjKAGrXDcFIojqAd1fWshVapk9z1t3mzHLoyBwYPhzjvt5kv+/nnj8MoxCnfRRKGUqkiSUpLYn7Sf6BP7iDm5n5i9UUTv/ZmY84lE+yRxyi9vld/aVMNxOA1HRi2C/BriqB1IrZpd2Zw+mchPWxOzsw61a8Mtt8Cilo05nupcff8vMIdMiQZMNFEopZSXMsZwMuUkMYd3En32IDFJ+4nZuoqY/b8SnX6E6CpnOVsl7+/wgHR/qhxrzMnjncnsEJ7zxEUkijJYOK6UUqokRIS6NepSt2U/emQd7P9g9vPGGI6dOUzMvl+IqXLWDriv/5qYczvYW2M5O0spDk0USilVTokIDQIa0aDbNWQPPgx8JOf5IioEX4jyU4ZRKaWUR2iiUEopVSRNFEopVUH5pTQqnfcplXdRSinlddKez9mASp6Wn0v6PtqiUEopVSRNFEoppYqkiUIppVSRNFEopZQqkiYKpZRSRdJEoZRSqkiaKJRSShVJE4VSSqkiaaJQSilVJE0USimliqSJQimlVJE0USillCqSJgqllFJFcmuiEJGRIrJTRPaIyKMFPF9NRD53Pv8/EXG4Mx6llFIXzm2JQkR8gbeBa4COwC0i0jHfaXcCJ4wxrYFXgX+4Kx6llFIl484WRR9gjzFmnzEmFfgMGJPvnDHAh877XwBDRaR0NnlVSilVKty5cVEz4ECux3FA38LOMcaki0gSUB84mvskEQkFQp0Pz4vIb26JuPxpQL5rVYnptcih1yKHXosc7Ur6wnKxw50xZg4wB0BEoowxvTwcklfQa5FDr0UOvRY59FrkEJGokr7WnV1PB4EWuR43dx4r8BwR8QPqAMfcGJNSSqkL5M5EsRFoIyJBIlIVmACE5zsnHLjdeX8c8IMxxrgxJqWUUhfIbV1PzjGH6cC3gC8QZozZJiLPAFHGmHDgA+DfIrIHOI5NJsWZ466YyyG9Fjn0WuTQa5FDr0WOEl8L0T/glVJKFUVXZiullCqSJgqllFJF8tpEoeU/crhwLR4Uke0iskVEvheRlp6IsywUdy1ynXejiBgRqbBTI125FiIy3vlvY5uIfFLWMZYVF/4fCRSRH0Vkk/P/k2s9Eae7iUiYiBwubK2ZWG84r9MWEbnMpTc2xnjdDTv4vRcIBqoCvwId851zL/Ce8/4E4HNPx+3Ba3EV4O+8f09lvhbO8wKAlcB6oJen4/bgv4s2wCagrvPxpZ6O24PXYg5wj/N+RyDG03G76VpcAVwG/FbI89cCXwMC9AP+58r7emuLQst/5Cj2WhhjfjTGJDsfrseuWamIXPl3ATALWzcspSyDK2OuXItpwNvGmBMAxpjDZRxjWXHlWhigtvN+HeBQGcZXZowxK7EzSAszBvjIWOuBS0SkSXHv662JoqDyH80KO8cYkw5klf+oaFy5Frndif2LoSIq9lo4m9ItjDGRZRmYB7jy76It0FZE1ojIehEZWWbRlS1XrsXTwCQRiQOWAjPKJjSvc6G/T4ByUsJDuUZEJgG9gCs9HYsniIgP8E9giodD8RZ+2O6nwdhW5koR6WKMOenRqDzjFmC+MeYVEemPXb/V2RiT6enAygNvbVFo+Y8crlwLRGQYMBMYbYw5X0axlbXirkUA0BlYISIx2D7Y8Ao6oO3Kv4s4INwYk2aMiQZ2YRNHRePKtbgTWAhgjFkHVMcWDKxsXPp9kp+3Jgot/5Gj2GshIj2Af2GTREXth4ZiroUxJskY08AY4zDGOLDjNaONMSUuhubFXPl/ZAm2NYGINMB2Re0ryyDLiCvXIhYYCiAiHbCJ4kiZRukdwoHbnLOf+gFJxpj44l7klV1Pxn3lP8odF6/FS0AtYJFzPD/WGDPaY0G7iYvXolJw8Vp8CwwXke1ABvCIMabCtbpdvBYPAe+LyF+wA9tTKuIfliLyKfaPgwbO8ZingCoAxpj3sOMz1wJ7gGTgDpfetwJeK6WUUqXIW7uelFJKeQlNFEoppYqkiUIppVSRNFEopZQqkiYKpZRSRdJEoVQZEhFHVmVPERksIhGejkmp4miiUMoFzgVK+v+LqpT0H75ShXD+9b9TRD4CfgMmi8g6EflFRBaJSC3neb1FZK2I/CoiG0QkwPnaVc5zfxGRAZ79NkqVnFeuzFbKi7TBlorZA/wHGGaMOSsifwMeFJEXgM+Bm40xG0WkNnAOOAxcbYxJEZE2wKfYgo1KlTuaKJQq2n5jzHoRGYXd8GaNs0xKVWAd0A6IN8ZsBDDGnAIQkZrAWyLSHVs+o60ngleqNGiiUKpoZ50/BfjOGHNL7idFpEshr/sLkAh0w3bxVuRNlFQFp2MUSrlmPTBQRFqDbTGISFtgJ9BERHo7jwfkKnsf79zvYDK2WJ1S5ZImeHzU5QAAAGhJREFUCqVcYIw5gt0Q6VMR2YLtdmrv3HrzZuBNEfkV+A5bwvod4HbnsfbktEyUKne0eqxSSqkiaYtCKaVUkTRRKKWUKpImCqWUUkXSRKGUUqpImiiUUkoVSROFUkqpImmiUEopVaT/BxyUINH7Xix6AAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ddi_gene_kg_correct_facts = 55370\n",
    "\n",
    "ddi_gen_results_new = ddi_gen_results.copy()\n",
    "del ddi_gen_results_new[\"title\"]\n",
    "\n",
    "show_precision_recall_curve(ddi_gen_results_new, ddi_gene_kg_correct_facts, 'prec_recall_jaccard_ddi_gene.pdf')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "---\n",
    "# Clustering"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "query_doc_titles = \"SELECT pmid, title FROM pubmed_metadata.pubmedarticle \\\n",
    "WHERE pmid IN (SELECT pmid FROM t_pmid_with_pred)\"\n",
    "\n",
    "\n",
    "query_doc_abstracts = \"\"\"\n",
    "SELECT pmid, array_to_string(abstracts, ' ') FROM pubmed_metadata.pubmedarticle  \n",
    "WHERE abstracts IS NOT NULL AND array_length(abstracts, 1) > 0 AND\n",
    "pmid IN (SELECT pmid FROM t_pmid_with_pred)\"\"\" "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "from model.librarygraph import load_lg_facts\n",
    "from model.experiments import do_cause_experiment_with_context, do_ddi_gene_experiment_with_context\n",
    "from model.clustering import load_dumped_clusters, combine_document_clusters, preprocess_and_dump_clustering"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "clusters_to_check = [2, 5, 10, 15, 30, 45, 60, 100, 200, 300, 400, 500, 1000, 2000, 4000, 6000, 10000, 20000, 50000, 100000]\n",
    "clustering_types = [\"clustering_kmeans_title\", \"clustering_kmeans_abstract\"]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "if do_preprocessing:\n",
    "    preprocess_and_dump_clustering(conn_meta, query_doc_titles, clustering_types[0], clusters_to_check)\n",
    "    preprocess_and_dump_clustering(conn_meta, query_doc_abstracts, clustering_types[1], clusters_to_check)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Cause Experiment"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "load facts from semmeddb and building index...\n",
      "load facts from semmeddb and building index...\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 2\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k2.pkl...\n",
      "2 cluster contexts are loaded\n",
      "computing context clusters... [2 / 2]\n",
      "deriving new facts...\n",
      "7936775 facts derived\n",
      "verifying facts in semmeddb...\n",
      "95000 of 7936775 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 5\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k5.pkl...\n",
      "5 cluster contexts are loaded\n",
      "computing context clusters... [5 / 5]\n",
      "deriving new facts...\n",
      "7918004 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94986 of 7918004 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 10\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k10.pkl...\n",
      "10 cluster contexts are loaded\n",
      "computing context clusters... [10 / 10]\n",
      "deriving new facts...\n",
      "7914607 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94924 of 7914607 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 15\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k15.pkl...\n",
      "15 cluster contexts are loaded\n",
      "computing context clusters... [15 / 15]\n",
      "deriving new facts...\n",
      "7555056 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94429 of 7555056 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 30\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k30.pkl...\n",
      "30 cluster contexts are loaded\n",
      "computing context clusters... [30 / 30]\n",
      "deriving new facts...\n",
      "7602534 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94510 of 7602534 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 45\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k45.pkl...\n",
      "45 cluster contexts are loaded\n",
      "computing context clusters... [45 / 45]\n",
      "deriving new facts...\n",
      "6704104 facts derived\n",
      "verifying facts in semmeddb...\n",
      "92726 of 6704104 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 60\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k60.pkl...\n",
      "60 cluster contexts are loaded\n",
      "computing context clusters... [60 / 60]\n",
      "deriving new facts...\n",
      "7480457 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94234 of 7480457 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 100\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k100.pkl...\n",
      "100 cluster contexts are loaded\n",
      "computing context clusters... [100 / 100]\n",
      "deriving new facts...\n",
      "7714827 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94587 of 7714827 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 200\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k200.pkl...\n",
      "200 cluster contexts are loaded\n",
      "computing context clusters... [200 / 200]\n",
      "deriving new facts...\n",
      "3776098 facts derived\n",
      "verifying facts in semmeddb...\n",
      "86351 of 3776098 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 300\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k300.pkl...\n",
      "299 cluster contexts are loaded\n",
      "computing context clusters... [299 / 299]\n",
      "deriving new facts...\n",
      "6332995 facts derived\n",
      "verifying facts in semmeddb...\n",
      "92227 of 6332995 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 400\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k400.pkl...\n",
      "400 cluster contexts are loaded\n",
      "computing context clusters... [400 / 400]\n",
      "deriving new facts...\n",
      "6185806 facts derived\n",
      "verifying facts in semmeddb...\n",
      "91788 of 6185806 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 500\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k500.pkl...\n",
      "500 cluster contexts are loaded\n",
      "computing context clusters... [500 / 500]\n",
      "deriving new facts...\n",
      "2564974 facts derived\n",
      "verifying facts in semmeddb...\n",
      "81463 of 2564974 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 1000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k1000.pkl...\n",
      "998 cluster contexts are loaded\n",
      "computing context clusters... [998 / 998]\n",
      "deriving new facts...\n",
      "5856369 facts derived\n",
      "verifying facts in semmeddb...\n",
      "91056 of 5856369 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 2000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k2000.pkl...\n",
      "1990 cluster contexts are loaded\n",
      "computing context clusters... [1990 / 1990]\n",
      "deriving new facts...\n",
      "5042214 facts derived\n",
      "verifying facts in semmeddb...\n",
      "89073 of 5042214 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 4000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k4000.pkl...\n",
      "3990 cluster contexts are loaded\n",
      "computing context clusters... [3990 / 3990]\n",
      "deriving new facts...\n",
      "4300333 facts derived\n",
      "verifying facts in semmeddb...\n",
      "86592 of 4300333 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 6000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k6000.pkl...\n",
      "5976 cluster contexts are loaded\n",
      "computing context clusters... [5976 / 5976]\n",
      "deriving new facts...\n",
      "3763408 facts derived\n",
      "verifying facts in semmeddb...\n",
      "84135 of 3763408 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 10000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k10000.pkl...\n",
      "9981 cluster contexts are loaded\n",
      "computing context clusters... [9981 / 9981]\n",
      "deriving new facts...\n",
      "2486307 facts derived\n",
      "verifying facts in semmeddb...\n",
      "77845 of 2486307 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 20000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k20000.pkl...\n",
      "19967 cluster contexts are loaded\n",
      "computing context clusters... [19967 / 19967]\n",
      "deriving new facts...\n",
      "1990999 facts derived\n",
      "verifying facts in semmeddb...\n",
      "73225 of 1990999 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 50000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k50000.pkl...\n",
      "49954 cluster contexts are loaded\n",
      "computing context clusters... [49954 / 49954]\n",
      "deriving new facts...\n",
      "1075800 facts derived\n",
      "verifying facts in semmeddb...\n",
      "62219 of 1075800 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 100000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k100000.pkl...\n",
      "99950 cluster contexts are loaded\n",
      "computing context clusters... [99950 / 99950]\n",
      "deriving new facts...\n",
      "890000 facts derived\n",
      "verifying facts in semmeddb...\n",
      "56300 of 890000 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 2\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k2.pkl...\n",
      "2 cluster contexts are loaded\n",
      "computing context clusters... [2 / 2]\n",
      "deriving new facts...\n",
      "7597395 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94753 of 7597395 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 5\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k5.pkl...\n",
      "5 cluster contexts are loaded\n",
      "computing context clusters... [5 / 5]\n",
      "deriving new facts...\n",
      "7244456 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94604 of 7244456 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 10\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k10.pkl...\n",
      "10 cluster contexts are loaded\n",
      "computing context clusters... [10 / 10]\n",
      "deriving new facts...\n",
      "7573101 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94656 of 7573101 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 15\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k15.pkl...\n",
      "15 cluster contexts are loaded\n",
      "computing context clusters... [15 / 15]\n",
      "deriving new facts...\n",
      "7294232 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94437 of 7294232 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 30\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k30.pkl...\n",
      "30 cluster contexts are loaded\n",
      "computing context clusters... [30 / 30]\n",
      "deriving new facts...\n",
      "7177428 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94162 of 7177428 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 45\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k45.pkl...\n",
      "45 cluster contexts are loaded\n",
      "computing context clusters... [45 / 45]\n",
      "deriving new facts...\n",
      "7208165 facts derived\n",
      "verifying facts in semmeddb...\n",
      "94134 of 7208165 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 60\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k60.pkl...\n",
      "60 cluster contexts are loaded\n",
      "computing context clusters... [60 / 60]\n",
      "deriving new facts...\n",
      "5183946 facts derived\n",
      "verifying facts in semmeddb...\n",
      "91773 of 5183946 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 100\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k100.pkl...\n",
      "100 cluster contexts are loaded\n",
      "computing context clusters... [100 / 100]\n",
      "deriving new facts...\n",
      "6658499 facts derived\n",
      "verifying facts in semmeddb...\n",
      "93507 of 6658499 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 200\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k200.pkl...\n",
      "198 cluster contexts are loaded\n",
      "computing context clusters... [198 / 198]\n",
      "deriving new facts...\n",
      "6354094 facts derived\n",
      "verifying facts in semmeddb...\n",
      "93425 of 6354094 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 300\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k300.pkl...\n",
      "300 cluster contexts are loaded\n",
      "computing context clusters... [300 / 300]\n",
      "deriving new facts...\n",
      "3380666 facts derived\n",
      "verifying facts in semmeddb...\n",
      "86814 of 3380666 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 400\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k400.pkl...\n",
      "400 cluster contexts are loaded\n",
      "computing context clusters... [400 / 400]\n",
      "deriving new facts...\n",
      "5996888 facts derived\n",
      "verifying facts in semmeddb...\n",
      "92599 of 5996888 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 500\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k500.pkl...\n",
      "500 cluster contexts are loaded\n",
      "computing context clusters... [500 / 500]\n",
      "deriving new facts...\n",
      "5752645 facts derived\n",
      "verifying facts in semmeddb...\n",
      "92185 of 5752645 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 1000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k1000.pkl...\n",
      "999 cluster contexts are loaded\n",
      "computing context clusters... [999 / 999]\n",
      "deriving new facts...\n",
      "5563621 facts derived\n",
      "verifying facts in semmeddb...\n",
      "91492 of 5563621 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 2000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k2000.pkl...\n",
      "1994 cluster contexts are loaded\n",
      "computing context clusters... [1994 / 1994]\n",
      "deriving new facts...\n",
      "5551689 facts derived\n",
      "verifying facts in semmeddb...\n",
      "91130 of 5551689 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 4000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k4000.pkl...\n",
      "3981 cluster contexts are loaded\n",
      "computing context clusters... [3981 / 3981]\n",
      "deriving new facts...\n",
      "4659867 facts derived\n",
      "verifying facts in semmeddb...\n",
      "88937 of 4659867 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 6000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k6000.pkl...\n",
      "5978 cluster contexts are loaded\n",
      "computing context clusters... [5978 / 5978]\n",
      "deriving new facts...\n",
      "4278644 facts derived\n",
      "verifying facts in semmeddb...\n",
      "87311 of 4278644 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 10000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k10000.pkl...\n",
      "9971 cluster contexts are loaded\n",
      "computing context clusters... [9971 / 9971]\n",
      "deriving new facts...\n",
      "2511950 facts derived\n",
      "verifying facts in semmeddb...\n",
      "80883 of 2511950 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 20000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k20000.pkl...\n",
      "19958 cluster contexts are loaded\n",
      "computing context clusters... [19958 / 19958]\n",
      "deriving new facts...\n",
      "1346615 facts derived\n",
      "verifying facts in semmeddb...\n",
      "72315 of 1346615 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 50000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k50000.pkl...\n",
      "49951 cluster contexts are loaded\n",
      "computing context clusters... [49951 / 49951]\n",
      "deriving new facts...\n",
      "789544 facts derived\n",
      "verifying facts in semmeddb...\n",
      "62628 of 789544 derived facts are correct\n",
      "============================================================\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 100000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k100000.pkl...\n",
      "99952 cluster contexts are loaded\n",
      "computing context clusters... [99952 / 99952]\n",
      "deriving new facts...\n",
      "517052 facts derived\n",
      "verifying facts in semmeddb...\n",
      "54491 of 517052 derived facts are correct\n",
      "============================================================\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "print('load facts from semmeddb and building index...')\n",
    "idx_cause = load_lg_facts(conn, query_cause)\n",
    "print('load facts from semmeddb and building index...')\n",
    "idx_correct_cause, _ = load_kg_facts(conn, query_cause)\n",
    "\n",
    "results_type_clustering_cause = {}\n",
    "for c in clustering_types:\n",
    "    results_clustering_cause = []\n",
    "    for k in clusters_to_check:\n",
    "        print('='*60)\n",
    "        print('cluster context: {}'.format(c))\n",
    "        print('cluster k: {}'.format(k))\n",
    "        cluster2docs = load_dumped_clusters(c, k)\n",
    "        # now do experiment\n",
    "        idx_contexts = combine_document_clusters(cluster2docs, idx_cause)\n",
    "        correct, obtained = do_cause_experiment_with_context(idx_contexts, idx_correct_cause)\n",
    "        results_clustering_cause.append((k, correct, obtained))\n",
    "        print('='*60)\n",
    "    results_type_clustering_cause[c] = results_clustering_cause\n",
    "\n",
    "del idx_cause\n",
    "del idx_correct_cause"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "============================================================\n",
      "clustering_kmeans_title\n",
      "2\t7936775\t95000\n",
      "5\t7918004\t94986\n",
      "10\t7914607\t94924\n",
      "15\t7555056\t94429\n",
      "30\t7602534\t94510\n",
      "45\t6704104\t92726\n",
      "60\t7480457\t94234\n",
      "100\t7714827\t94587\n",
      "200\t3776098\t86351\n",
      "300\t6332995\t92227\n",
      "400\t6185806\t91788\n",
      "500\t2564974\t81463\n",
      "1000\t5856369\t91056\n",
      "2000\t5042214\t89073\n",
      "4000\t4300333\t86592\n",
      "6000\t3763408\t84135\n",
      "10000\t2486307\t77845\n",
      "20000\t1990999\t73225\n",
      "50000\t1075800\t62219\n",
      "100000\t890000\t56300\n",
      "============================================================\n",
      "clustering_kmeans_abstract\n",
      "2\t7597395\t94753\n",
      "5\t7244456\t94604\n",
      "10\t7573101\t94656\n",
      "15\t7294232\t94437\n",
      "30\t7177428\t94162\n",
      "45\t7208165\t94134\n",
      "60\t5183946\t91773\n",
      "100\t6658499\t93507\n",
      "200\t6354094\t93425\n",
      "300\t3380666\t86814\n",
      "400\t5996888\t92599\n",
      "500\t5752645\t92185\n",
      "1000\t5563621\t91492\n",
      "2000\t5551689\t91130\n",
      "4000\t4659867\t88937\n",
      "6000\t4278644\t87311\n",
      "10000\t2511950\t80883\n",
      "20000\t1346615\t72315\n",
      "50000\t789544\t62628\n",
      "100000\t517052\t54491\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "for c, results_clustering_cause in results_type_clustering_cause.items():\n",
    "    print(\"=\"*60)\n",
    "    print(c)\n",
    "    for k, correct, obtained in results_clustering_cause:\n",
    "        print('{}\\t{}\\t{}'.format(k, obtained, correct))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "jaccard title\n",
      "clustering titles\n",
      "clustering abstracts\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3iUVfbA8e9JCMQgAtJ7QEFaCiQ0sYCgIiK6a0NAhXVlLVjWlRXLz7ooK6ysrlhwsS4KiuLiwtooa6GYBFAB6QYINURAQgsh5/fHnSSTkAxDkslkkvN5nnlm3jqH15iT+973niuqijHGGFOcsGAHYIwxpmKzRGGMMcYnSxTGGGN8skRhjDHGJ0sUxhhjfKoW7ABOVf369TU6OjrYYRhjTEhJSUnZo6oNSnJsyCWK6OhokpOTgx2GMcaEFBHZXNJj7daTMcYYnyxRGGOM8ckShTHGGJ9Cro/CGOMcO3aMtLQ0jhw5EuxQTAUSGRlJ8+bNiYiIKLNzWqIwJkSlpaVRq1YtoqOjEZFgh2MqAFUlIyODtLQ0WrduXWbntVtPxoSoI0eOUK9ePUsSJo+IUK9evTJvZVqiMCaEWZIwhQXiZ8IShTHGGJ8sURhjSuzcc88N6vcvXLiQQYMGnbB+xYoVzJ07N2959uzZjB8/HoCPP/6Y1atX520bMWIEM2fODHywIcwShTFVxbRpEB0NYWHufdq0Up9y0aJFpT7HqTh+/Lhf+xVOFIMHD2bs2LHAiYnCnFxAE4WIDBCRtSKyQUTGFrF9kois8LzWici+QMZjTJU1bRqMGgWbN4Oqex81qtTJ4vTTTyczM5N+/frRtWtXYmJi+Pe//523/e233yY2Npa4uDhuvPFGAHbt2sVvfvMb4uLiiIuLy0s2V111FQkJCXTq1IkpU6YU+I4//elPxMXFsXjxYj799FPat29P165d+eijj06IKSsri0cffZQZM2YQHx/PjBkzePPNNxk9ejSLFi1i9uzZjBkzhvj4eDZu3Fjg2JSUFC688EISEhK49NJL2bFjR6muT6WhqgF5AeHARqANUB34HujoY/+7gNdPdt6EhAQ1xqiuXr264IoLLzzxNXmy29aihapLEQVf9eq57enpJx7rh5o1a+qxY8d0//79ntOk61lnnaU5OTm6cuVKbdu2raanp6uqakZGhqqqXnfddTpp0iRVVc3OztZ9+/YV2H7o0CHt1KmT7tmzR1VVAZ0xY4aqqh4+fFibN2+u69at05ycHL322mv18ssvPyGuN954Q++8884il2+++Wb94IMP8rblLmdlZWmvXr109+7dqqo6ffp0HTlypF/XoaI54WdDVYFkLeHv80COo+gObFDVTQAiMh24EiiuzXcD8FgA4zGm6kpLK3p9RkapT62qPPTQQ3z11VeEhYWxbds2du3axfz587n22mupX78+AGeeeSYA8+fP5+233wYgPDyc2rVrA/DCCy8wa9YsALZu3cr69eupV68e4eHhXH311QCsWbOG1q1b07ZtWwCGDx9eoPVRGmvXrmXlypVcfPHFgLvN1aRJkzI5d6gLZKJoBmz1Wk4DehS1o4i0AloD84vZPgoYBdCyZcuyjdKYymLhwuK3tWzpbjcV1qqVe69f3/fxPkybNo309HRSUlKIiIggOjr6lJ/jX7hwIV9++SWLFy8mKiqKPn365J0jMjKS8PDwEsV2KlSVTp06sXjx4oB/V6ipKJ3ZQ4CZqlpkT5WqTlHVRFVNbNCgROXUjanaxo2DqKiC66Ki3PpS2r9/Pw0bNiQiIoIFCxaw2ZOQLrroIj744AMyPK2WX375BYB+/frx8ssvA+6v9v3797N//37q1q1LVFQUa9asYcmSJUV+V/v27UlNTc3rW3jvvfeK3K9WrVocOHDglLadc845pKen5yWKY8eOsWrVKn8vQ6UWyESxDWjhtdzcs64oQ4Ci/4sbY0pv2DCYMsW1IETc+5Qpbn0piAjDhg0jOTmZmJgY3n77bdq3bw9Ap06dePjhh7nwwguJi4vjvvvuA+D5559nwYIFxMTEkJCQwOrVqxkwYADZ2dl06NCBsWPH0rNnzyK/LzIykilTpnD55ZfTtWtXGjZsWOR+ffv2ZfXq1Xmd2d6GDBnChAkT6NKlS4HO7OrVqzNz5kweeOAB4uLiiI+PL/enuioqcX0cATixSDVgHdAPlyCSgKGquqrQfu2BT4HW6kcwiYmJahMXGQM//fQTHTp0CNr3Z2Rk0LVr17wWhKk4ivrZEJEUVU0syfkC1qJQ1WxgNPAZ8BPwvqquEpEnRWSw165DgOn+JAljTMWwfft2evXqxf333x/sUEw5CGj1WFWdC8wttO7RQsuPBzIGY0zZa9q0KevWrQt2GKacVJTObGOMMRWUJQpjjDE+WaIwxhjjkyUKY4wxPlmiMMaUqccff5yJEyee8nH79u3jpZdeKtF3Dhw4kH37Sl9TNDU1lXfffTdvOTk5mbvvvhtwo8e9x1WU9N8ZiixRGFOF7NgBF14IO3cGO5ITlSRRqCo5OTnMnTuXOnXqlDqGwokiMTGRF154ATgxUVQlliiMqUKeegq++ca9l4Wiyoh769OnD7kDZPfs2UN0dDQAq1atonv37sTHxxMbG8v69esZO3YsGzduJD4+njFjxgAwYcIEunXrRmxsLI895mqGpqamcs4553DTTTfRuXNntm7dSnR0NHv27CE1NZUOHTpw66230qlTJy655BIOHz4MQFJSErGxsXnn79y58wnxjh07lq+//pr4+HgmTZqUNzFSamoqr7zyCpMmTSI+Pp6vv/66wHEbN25kwIABJCQkcP7557NmzRoAPvjgAzp37kxcXBwXXHBB2Vz0IAjoOApjTPnp08f39qNHITkZcnLgjTdg+XI3JcWIEbBnD1xzTcH9T1YjcNWqVfzlL39h0aJF1K9fP6+Wkz9eeeUV7rnnHoYNG0ZWVhbHjx9n/PjxrFy5khUrVgDw+eefs379er777jtUlcGDB/PVV1/RsmVL1q9fz1tvvVVkqY/169fz3nvv8dprr3Hdddfx4YcfMnz4cEaOHMlrr71Gr1698iYxKmz8+PFMnDiR//znP55r4C5CdHQ0t912G6effnreIMN58+blHTdq1CheeeUV2rZty9KlS7njjjuYP38+Tz75JJ999hnNmjUrk1tjwWKJwpgqInfOIoDjx4suJnsqiisj7o9evXoxbtw40tLS+O1vf5tXNtzb559/zueff06XLl0AyMzMZP369bRs2ZJWrVoVWw+qdevWxMfHA5CQkEBqair79u3jwIED9OrVC4ChQ4fmJYPSyszMZNGiRVx77bV5644ePQpA7969GTFiBNdddx2//e1vy+T7gsEShTGVhK8WwI4d0KaNSxAAWVmwdy8MGOCWS1Fl3Kdq1aqRk5MDUKD0+NChQ+nRowdz5sxh4MCBvPrqq7Rp06bAsarKgw8+yB/+8IcC61NTU6lZs2ax31mjRo28z+Hh4Xm3ngIlJyeHOnXq5LWEvL3yyissXbqUOXPmkJCQQEpKCvXq1QtoPIFgfRTGVAFPPeVuOXk7frx0fRXFlRH3Fh0dTUpKCgAzZ87MW79p0ybatGnD3XffzZVXXskPP/xwQvnvSy+9lNdff53MzEwAtm3bxu7du0sUa506dahVqxZLly4FYPr06UXuV5Ly5GeccQatW7fmgw8+AFyC+/777wHXd9GjRw+efPJJGjRowNatW084PhRYojCmCli82LUivGVlQWke4imujLi3+++/n5dffpkuXbqwZ8+evPXvv/8+nTt3Jj4+npUrV3LTTTdRr149evfuTefOnRkzZgyXXHIJQ4cOpVevXsTExHDNNdcU+0vcH1OnTuXWW28lPj6egwcP5s2s5y02Npbw8HDi4uKYNGlSgW1XXHEFs2bNKrIze9q0aUydOpW4uDg6deqUN2/4mDFjiImJoXPnzpx77rnExcWVOP5gCliZ8UCxMuPGOMEuMx5qMjMzOf300wHXab1jxw6ef/75IEcVGGVdZtz6KIwxVcKcOXN45plnyM7OplWrVrz55pvBDilkWKIwxlQJ119/Pddff32wwwhJ1kdhjDHGJ0sUxhhjfLJEYYwxxidLFMYYY3yyRGGMKVOhXGa8OKmpqUUWESzOxx9/zOrVq8vs+59++ukyO1dJBDRRiMgAEVkrIhtEpMgqXCJynYisFpFVIvJuUfsYY0qn8cTGyBNywqvxxMbBDi1PRSgzXlZ8JYrs7OxTPl+lTRQiEg5MBi4DOgI3iEjHQvu0BR4EeqtqJ+DeQMVjTFW26+CuU1rvr8pWZjwzM5N+/frRtWtXYmJi8kZYg/sFP2zYMDp06MA111zDoUOHAFeavGPHjsTGxnL//fezaNEiZs+ezZgxY4iPj2fjxo306dOHe++9l8TERJ5//nk++eQTevToQZcuXejfvz+7du3K+/6RI0cSExNDbGwsH374IWPHjuXw4cPEx8czbNgwDh48yOWXX05cXBydO3dmxowZpfpv6I9AjqPoDmxQ1U0AIjIduBLwTrO3ApNVdS+AqpaskIsxVdy9n97Lip0nFqXzR583+xS5Pr5xPH8f8Pdij6uMZcYjIyOZNWsWZ5xxBnv27KFnz54MHjwYgLVr1zJ16lR69+7N7373O1566SVGjhzJrFmzWLNmDSLCvn37qFOnDoMHD2bQoEFc41W7PSsrKy9p7t27lyVLliAi/POf/+TZZ5/lb3/7G0899RS1a9fmxx9/zNvv6quv5sUXX8y7Lh9++CFNmzZlzpw5AOzfv9/v615Sgbz11AzwroCV5lnnrR3QTkS+FZElIjKgqBOJyCgRSRaR5PT09ACFa4w5FaUtM/7000/z17/+lc2bN3PaaaedsI93mfGuXbuyZs0a1q9fD1AmZcaLoqo89NBDxMbG0r9/f7Zt25b3136LFi3o3bs3AMOHD+ebb76hdu3aREZGcsstt/DRRx8RFRVV7L/Ze7BfWloal156KTExMUyYMIFVq1YB8OWXX3LnnXfm7Ve3bt0TzhMTE8MXX3zBAw88wNdff11kzaqyFuyR2dWAtkAfoDnwlYjEqGqBXilVnQJMAVfrqbyDNKai8/WXP4A8IcVuWzhiYRlHky/UyoxPmzaN9PR0UlJSiIiIIDo6Oi9ukYLXUESoVq0a3333HfPmzWPmzJm8+OKLzJ8/v8hze8d81113cd999zF48GAWLlzI448/7neM7dq1Y9myZcydO5dHHnmEfv368eijj/p9fEkEskWxDWjhtdzcs85bGjBbVY+p6s/AOlziMMZUcJWxzPj+/ftp2LAhERERLFiwgM1esztt2bKFxYsXA/Duu+9y3nnnkZmZyf79+xk4cCCTJk3KKy/uq1x57vc0a+ZusLz11lt56y+++GImT56ct7x3714AIiIiOHbsGADbt28nKiqK4cOHM2bMGJYtW3bK1+NUBTJRJAFtRaS1iFQHhgCzC+3zMa41gYjUx92K2hTAmIypkhrVbHRK6/1RGcuMDxs2jOTkZGJiYnj77bdp37593rZzzjmHyZMn06FDB/bu3cvtt9/OgQMHGDRoELGxsZx33nk899xzAAwZMoQJEybQpUsXNm7ceML3PP7441x77bUkJCTk3boDeOSRR9i7d2/ePNsLFiwA3FSrsbGxDBs2jB9//DHvQYAnnniCRx55pMTXxF8BLTMuIgOBvwPhwOuqOk5EngSSVXW2uLbc34ABwHFgnKoWneo9rMy4MY6VGT81Vma8gpYZV9W5wNxC6x71+qzAfZ6XMcYEjJUZL7lgd2YbY0y5sDLjJWclPIwJYaE2Q6UJvED8TFiiMCZERUZGkpGRYcnC5FFVMjIyiIyMLNPz2q0nY0JU8+bNSUtLwwahGm+RkZE0b968TM9picKYEBUREUHr1q2DHYapAuzWkzHGGJ8sURhjjPHJEoUxxhifLFEYY4zxyRKFMcYYnyxRGGOM8ckShTHGGJ8sURhjjPHJEoUxxhifLFEYY4zxyRKFMcYYnyxRGGOM8ckShTHGGJ8sURhjjPHJEoUxxhifApooRGSAiKwVkQ0iMraI7SNEJF1EVnhevw9kPMYYY05dwCYuEpFwYDJwMZAGJInIbFVdXWjXGao6OlBxGGOMKZ1Atii6AxtUdZOqZgHTgSsD+H3GGGMCIJCJohmw1Ws5zbOusKtF5AcRmSkiLYo6kYiMEpFkEUm2+YGNMaZ8Bbsz+xMgWlVjgS+At4raSVWnqGqiqiY2aNCgXAM0xpiqLpCJYhvg3UJo7lmXR1UzVPWoZ/GfQEIA4zHGGFMCgUwUSUBbEWktItWBIcBs7x1EpInX4mDgpwDGU1D//iCS/6pRA6ZNK7evN8aYUBGwp55UNVtERgOfAeHA66q6SkSeBJJVdTZwt4gMBrKBX4ARgYqngP79Yd68guuysmD4cPd52LByCcMYY0KBqGqwYzgliYmJmpycXLqTiBS/rVUrSE0t3fmNMaaCEZEUVU0sybHB7syueLZsCXYExhhToViiKKxly2BHYIwxFUrVTBT9+hW/LTPTOrWNMcZL1UwUX35ZfLLIyIBRoyxZGGOMR9VMFOCSharrvC7s0CF4+OHyj8kYYyqgqpsochXXeW2d2sYYA1iiKL7z2jq1jTEGsEQB48ZBVFTBdVFRbr0xxhhLFAwbBlOmuL4KEWjeHF5+2UZnG2OMhyUKcEkhNRVWr4b9+yHMLosxxuSy34je2rVzfRPjx0NOTrCjMcaYCsGvooAiUgO4Goj2PkZVnwxMWEESFgZjx8KNN8KcOXDFFcGOyBhjgs7fFsW/cdOYZgMHvV6Vz5AhEB0NzzzjxlkYY0wV52+Z8eaqOiCgkVQU1arB/ffD6NHwww8QFxfsiIwxJqj8bVEsEpGYgEZSkfzud5CSYknCGGPwv0VxHjBCRH4GjgICqGeu68rntNOga1f3WdX3/BXGGFPJ+ZsoLgtoFBXVXXfBvn3wzjvBjsQYY4LGr1tPqroZqANc4XnV8ayr3KKi4N13YcOGYEdijDFB41eiEJF7gGlAQ8/rXyJyVyADqxDuvRciImDChGBHYowxQeNvZ/YtQA9VfVRVHwV6ArcGLqwKokkTGDEC3nwTduwIdjTGGBMU/iYKAY57LR/3rPN9kMgAEVkrIhtEZKyP/a4WERWREk38HVBjxkB2NkyaFOxIjDEmKPxNFG8AS0XkcRF5HFgCTPV1gIiEA5NxHeEdgRtEpGMR+9UC7gGWnkLc5eess+D3v3d9FWFhbjCezX5njKlC/O3Mfg4YCfzieY1U1b+f5LDuwAZV3aSqWcB03Ojuwp4C/goc8Tvq8jRtGvzrX7Btm3tUdvNmmyrVGFOl+EwUInKG5/1MIBX4l+e12bPOl2bAVq/lNM867/N3BVqo6pyTxDFKRJJFJDk9Pf0kX1vGHn7YTY3qzaZKNcZUIScbR/EuMAhIAbwLH4lnuU1Jv1hEwoDngBEn21dVpwBTABITE8u3AJNNlWqMqeJ8JgpVHeR5b12Cc28DWngtN/esy1UL6AwsFDfyuTEwW0QGq2pyCb4vMFq2dLebCmvR4sR1xhhTCfk7jqK3iNT0fB4uIs+JyMkmlU4C2opIaxGpDgwBZuduVNX9qlpfVaNVNRrXQV6xkgQUPVUqQPv25R+LMcYEgb9PPb0MHBKROOBPwEbAZ10LVc0GRgOfAT8B76vqKhF5UkQGlyLm8lV4qtRWrWDgQEhPh4OVs9K6McZ4E/VjzgURWaaqXUXkUWCbqk7NXRf4EAtKTEzU5OQgNzpycuDwYahZM7hxGGOMn0QkRVVLNFbN3xbFARF5EBgOzPF0REeU5AsrhbAwlyQyM+H6611JcmOMqaT8TRTX48qL36KqO3Ed01YAKTMTlixxU6Zu3Xry/Y0xJgT5O+Bup6o+p6pfe5a3qOrbgQ0tBDRu7ObWPngQBg2CAweCHZExxpS5kw24+8bzfkBEfvV6HRCRX8snxAquc2f44ANYtcrdhsrODnZExhhTpnwmClU9z/NeS1XP8HrVUtUzyifEEHDJJfDyy66vIjU12NEYY0yZ8nccRU9P8b7c5Voi0iNwYYWgW2+FNWvg7LODHYkxxpSpUxlHkem1fNCzznirW9c9OvvQQ/Dxx8GOxhhjyoTf81Go14ALVc3B//m2q5asLJg3D4YOhWCP9zDGmDLgb6LYJCJ3i0iE53UPsCmQgYWsyEiYPRsaNnSPzVrxQGNMiPM3UdwGnIsr6pcG9ABGBSqokNeoEcyd60ZvX345/GoPiBljQpe/4yh2q+oQVW2oqo1Udaiq7g50cCGtY0eYORN+/hmWVszJ+4wxxh/+PvXUTkTmichKz3KsiDwS2NAqgf79XaK4+OJgR2KMMSXm762n14AHgWMAqvoDrmy4OZkGDdz79Onw95PNHmuMMRWPv4kiSlW/K7TOhiD7S9U9LnvfffDRR8GOxhhjTom/iWKPiJyFZzpUEbkG2BGwqCobEXjjDejRA4YPh+8K51xjjKm4/E0UdwKvAu1FZBtwL+5JKOOv006Df//bFRK84gor9WGMCRknTRSeuScSVbU/0ABor6rnqWoRE0kbnxo2dNVms7Jg1qxgR2OMMX456ehqVc0RkT/jpjK1uT9Lq0MHV2m2adNgR2KMMX7x99bTlyJyv4i0EJEzc18Bjawyy00SKSkweLCbhzssDKKjYdq0oIZmjDGF+Vuv6XpcR/Ydhda3Kdtwqpinn4ZPPslf3rwZRnkGvA8bFpyYjDGmEH9bFB2BycD3wArgH0Cnkx0kIgNEZK2IbBCRsUVsv01EfhSRFSLyjYh0PJXgQ15RRQMPHYKHHy7/WIwxphj+Joq3gA7AC7gk0dGzrlgiEo5LLpd59r+hiETwrqrGqGo88Czw3CnEHvqKm2fbCgkaYyoQf289dVZV71/yC0Rk9UmO6Q5sUNVNACIyHbgSyDtOVb2r5dXEM06jymjZ0t1uKmr97t3uKSljjAkyf1sUy0SkZ+6CZ3a7k0220Azw/pM5zbOuABG5U0Q24loUdxd1IhEZJSLJIpKcnp7uZ8ghYNw4iIoquC4qCu65x3VwjxxZfKvDGGPKib+JIgFYJCKpIpIKLAa6efoXfihNAKo6WVXPAh4Aiiw0qKpTVDVRVRMb5NZOqgyGDYMpU1xSEHHvU6bAiBEwejS8+y60awdjx8K+fcGO1hhTRYnXxHXF7yTSytf2ogbfiUgv4HFVvdSz/KBn32eK+Y4wYK+q1vb1XYmJiZpcVWaO27wZ/u//4F//crehNm6EmjWDHZUxJgSJSIqqJpbkWL/6KEo4CjsJaCsirXETHg0BhnrvICJtVXW9Z/FyYD0mX6tW8Pbbrpjg4sX5SWLhQrjgAjf2whhjAixgv2lUNRsYDXwG/IQb2b1KRJ4UkcGe3UaLyCoRWQHcB9wcqHhCWnw83H67+5yUBH37Qrdubm5uY4wJML9uPVUkVerWU1FyclzfxSOPuFtTAwbAX/8KsbHBjswYU4GV5taT3bsINWFhrlT5mjUwcaKbZrVvXzc/tzHGBIAlilAVGQl/+pPr4P7wQ1fGPCcH/vY32Ls32NEZYyoRSxShrm5d6NPHfV6yBMaMgbPOcgnjyJGghmaMqRwsUVQm554Ly5dD9+5w//1wzjnu0dqcnGBHZowJYZYoKpu4OPj0U/jiC6hXD554Ao4fD3ZUxpgQZomisurf31WnnT8fIiLg4EEYOhRWrAh2ZMaYEGOJojILC4MWLdznlStdS6NrV7jxxqKLERpjTBEsUVQVPXrApk3w5z/DzJmuhtSYMXDsWLAjM8ZUcJYoqpI6dWD8eFi3zt2GWrYMqnmquITYwEtjTPmxRFEVtWgBb7zhbkWJwLZt0LEjvPOOPSFljDmBJYqqLCLCve/d6woO3nST68P4/PPgxmWMqVAsURjo3Bm++w7eew9+/RUuvdTVkMrODnZkxpgKwBKFccLCYMgQ+Okn+Pvf3a2o3P6LjIzgxmaMCSpLFKagGjXcVKzPPeeWU1KgeXNXV+qXX4IbmzEmKCxRGN+aNHFPSE2a5GpIPfusVao1poqxRGF8a9oUpk6F7793taQeeAASEqwsiDFViF9ToRpDTAzMmeOmYf35ZwgPd2Mvvv0Wevd2j9kaYyola1GYU9OnD4wc6T7PmQPnnw8XX+wG7xljKiVLFKbkLrkEnn/eFRpMSIBhw1xrwxhTqViiMCVXvTrcfbebZe+hh2DWLNe6sNHdxlQqAU0UIjJARNaKyAYRGVvE9vtEZLWI/CAi80SkVSDjMQFSuzaMGwfr18Obb7oxGUePwj/+YU9IGVMJBCxRiEg4MBm4DOgI3CAiHQvtthxIVNVYYCbwbKDiMeWgWTM47zz3ee5c19po184lD3tKypiQFcgWRXdgg6puUtUsYDpwpfcOqrpAVQ95FpcAzQMYjylPv/mNe0KqSRPX+d2lC/z3v1al1pgQFMhE0QzY6rWc5llXnFuA/xa1QURGiUiyiCSnp6eXYYgmoC68EJYuhRkz4NAheOyxYEdkjCmBCtGZLSLDgURgQlHbVXWKqiaqamKDBg3KNzhTOiJw3XWwejV8+KFbTk93rYxNm4IdnTHGD4FMFNuAFl7LzT3rChCR/sDDwGBVPRrAeEwwVa+ePy1rUpJrZbRvD/feC3v2BDc2Y4xPgUwUSUBbEWktItWBIcBs7x1EpAvwKi5J7A5gLKYiGTgQNmyAm292T0addZabec/6L4ypkAKWKFQ1GxgNfAb8BLyvqqtE5EkRGezZbQJwOvCBiKwQkdnFnM5UNk2bwmuvwY8/ur6MZcvyy4BYwjCmQhENsf8pExMTNTk5OdhhmLJ29Kgrcb56tatWO26ca3lYDSljyoSIpKhqYkmOrRCd2cZQo4Z7/+UXOHgQBg2Ciy5y/RnGmKCyRGEqlvPOc62KF1+EVauge3cYMcLdjpo2DaKj3cjv6Gi3bIwJOLv1ZCquX3+FiRNdSfOzz4ZRo9x4jFxRUTBliitGaIzxqTS3nixRmNAQHQ2bN5+4vlUrSE0t72iMCTnWR2Eqvy1bil6/eTNs3+4+Z2eXXzzGVCGWKExoaNmy6PVRUdC4sfs8erQbxHf77cmunPEAABGeSURBVPD++7DbhuYYUxYsUZjQMG6cSwrecvsowjw/xr16ucF706bB9ddDo0Zw2WX5+1vJc2NKxObMNqEht8P64YfdbaiWLV3y8O7Ivvlm98rOhpQUWLDAlQ4B99TU2WdDgwbQt697XXAB1KlT/v8WY0KMdWabquHoUZgwwSWPRYvgyBHXEnnmGfjzn918GYcOQa1awY7UmICwzmxjTqZGDXjkEZg3D/budXNl/N//udtV4EqI1K0LPXvCgw/C55+7gX/GGGtRGAPAzz/DP//pWhxJSe72VUQEfPONG/S3bx9ERrqXMSGoNC0K66MwBqB1a9fnAZCZCd9+65JG585u3cSJ7tWrV34fR48e+X0gxlRi1qIwxh+LFrmJlxYsgBUrXOd4o0ZuDEdYmOtgb9LEtUKMqYCsRWFMoJ17rnuBK1z41Vewc2f+o7mDBrnbV+efn9/i6NLFlR8xJsRZi8KYsvDhhzB/vmtx/PSTW3fTTfDWW+7zjz9Cp075icWYcmYtCmOC7eqr3Qtgxw73VFWzZm45NRViY91TVRde6FobF13kEofNt2FCgCUKY8pakyZwww35y2eeCe+841ob8+fDxx+79dOnuxHk6emQkQHnnGOJw1RIliiMCbQzzoDhw90LXAtjwQLXqgB47z245x5Xsyq3f6NvX1eOxBKHqQAsURhT3qKjYeTI/OXf/AZOO80ljwULXOIIC3Od5rVr03h8fXYdzTjhNI1qNmLn/TvLL25TZVmiMCbYWrSAW291L1VYuxZ++AFq1wYoMkkA7Dq4qzyjNFVYQB/BEJEBIrJWRDaIyNgitl8gIstEJFtErglkLMaEBBFXKv266/w/5tNPYZclDRM4AUsUIhIOTAYuAzoCN4hIx0K7bQFGAO8GKg5jKrNpS19jx/UDXf9Gx45w550wc6brHDemjASyRdEd2KCqm1Q1C5gOXOm9g6qmquoPQE4A4zCm0hr+6Sia3qd0fqIR91xwiE++mcqvw6+Ff/3L7ZCR4Z6y2rs3uIGakBbIPopmwFav5TSgR0lOJCKjgFEALYub6cyYKij51mTm/TyPLzd9yZTwr3mhyVHCJZzu1d+h3/w99P9Z6PmHp6iRIxAfn/9E1UUXnTgRlDHFCNjIbE+fwwBV/b1n+Uagh6qOLmLfN4H/qOrMk53XRmabqqbxxMZFdlwXfurpSPYRFm9dnJc4krYnkaM5nBZWgwtyWtBvYw79/7eVuK3HCNu4yRVCXLrUtTbOOw9OP708/1mmnFXUkdnbgBZey80964wxp8DfR2Ajq0XSt3Vf+rbuy18u+gv7juzjf6n/y0scf+YnOAvqRdTmouQH6JfRj/4vz6HNW58g1apBt26utdGvX/4YD2MIbIuiGrAO6IdLEEnAUFVdVcS+b2ItCmMCavuB7cz/eT5fbvqSeT/PI+3XNABa1WhE/0ON6bfyIBfN20Sjlh1pfHO6X60YEzpK06IIaFFAERkI/B0IB15X1XEi8iSQrKqzRaQbMAuoCxwBdqpqJ1/ntERhTOmpKusy1uW1NhakLmDfkX0AxNRux4/71xV/7FUrIC6uvEI1ZaTCJopAsERhTNk7nnOcZTuW5SWOeT/PK3bfnGMPIX8ZB4cPwwsvQJ8+kJAA1Wz8bkVmicIYU6bkieJrTNWLPJOeLXrRI6cJPR/7J923Qe3qtfLn4hg6FJo2LcdojT8qame2MaYc7NgBQ4bAjBlu3F2gXdn+KpZsW8Lc9LnoTSAI7Y+F03PLN/ScPpee8U3o1HgI4d8luaeq+vZ1U8raXBwhy1oUxoS4O+6AV1+F226DyZPL5py+WhT6mPudsf/IfpK2J7EkbQlL0pawdNtS9hzaA0DNiJp0y6pPz28202Mb9DxYl8bdL3JJY9QomzI2COzWkzFV1I4d0KYNHDkCkZGu+Gz79q6eYGkqlDd8tjHph0/tqSdVZdPeTQUSx/Idy8nWbABaHQin564Ietwxjp7Ne9FlTgqREuGSR9u2VlI9wCxRGFNF3XEHTJ0KWVnuzk6OpxhOeDjUq1fwVb++uzX1l7+4fVasgGPH3PAJcIVrc39Xl1Ur5fCxwyzfuZylaUtdAtmyiC2Z7rHciBwhfofSM821OHq0OZ82g25CcmcKxP/BhubkLFEYUwV5tyZyRUTAQw+5BLBnjyv15P0KD4ctW9y+V10Fmza5iubgBmevXu1aI5s3u8RRrRpMmAA9e7qWSp06ZRD3gR0s3baUJVsXs2TDQpLSV3BIswCofzySnh3607NpD3rM+o6L635S7Hlyb4EZ/1hntjFV0FNP5bcgcom4mVX9aQWMHw8HDuQvDxkCa9bAZ5/lr8vOhj/+MX/5kkvyt8+Y4RJVbovEX01qNeGq9ldxVfur4GLIzslm1e5VLElbzJLUb1m6K4X/rPuPG13lr2nT4OGHXRZs2RLGjYNhw04tMFMsa1EYE6K6dHG3jwqLj4fly0t2zqJaKZGRMGUK7NzpWhS58yudeaabGvyll1xC6d0bzj7btTzat4cOHVzXQ40apx7HviP7+G7rEi5997Ji9xl9sBPdzr6Qbkfqcc6DEwk7dDh/Y1SUC9qSRR679WSMKRPefR65qleH3/++YCtFFXbvhuPH3ZCJjAw3JfhPP7nbVrnCwlztwQ4dXPL47W+hVy//4/H19FXNLDhY3X2udRQStkPidui2Hbptg+jaLZHUzcUeX9XYrSdjTJlYvLhgkgC3vGhRwXUi0KhR/nK9evDf/7rPhw652VzXrHGvn35y71984aYL79XLbb/gAnjrLRgwwLVkVqxwCaVlS/+GXKz9wxH2rfuIpDFDSWoKSc3ghR6Q5fmtVv/gFhLvOpNu1VrSrV4M3c66gMZDfm9PV5WAtSiMMeXi+HHXyR4ZCRs3uj6SP/7RTcz3zjtw001uv9NOg3btXNL4pH1jDlLENK+Zjbjj0E7XyomOzmvGZIXDD40guSkktaxGUpsarIo6SI4n8TQ/ozmJTRPptmQr3TYeIbFBLHXbdHL3yDp0gJgYoHI+bWW3nowxIe3XX93TV7mtj9yWSGqqu80FriHwxRcwaJDrQ6lWDdavh+hvp7H71ofh8CHqs4cwtEAfxcGsgyzf9C1J6/9HUtbPLNmSxM+/bsj77rMzPLerqrWk27PT6NK4C6c/U/zcHKH6tJUlCmNMpXT4MKxbl588tm51rY/c22O33QYvvwy39VvPq/PbEk42jcL30KRlBE061aNJEwq8rrgCRo+GV97ay2W/S+G865JI2rqY77Z8x7ajrgURJmHkaPGzM+tj6k7SuLHrvW/b1r3Xru3z3xLsVoolCmNMpVfUE1mnnebGgmzaBMuWuX0Kv9LTXaskKsq1QM46y51DBLZvd7/vL7gAvl6+E5omQdNk6PNksXE0jWpO/dRfaLHvEI0yodFBaJQJNc6/gZfXjeIfT9Sh8+wZ1G0bi7Rt6xJJ7dp+lUUJJEsUxphKz98nsgrLzoZdu+CXX1zrI/cc4eHwhz+4Yz/80CWV8HDP+h3F/1I/5/AIUvfspGbDnUTW3s7uIxlkc/yE/SKOQ8OD0DgTGmVFMDf62Cn/m8uytWGJwhhT6ZV23IivFknhqru+/vo/7a/K+PGu37tvX/g5NYezY/aiUbvQqF2E195Jn4HbOZq9mQNZ2ziQs5NfZQ97zix+Mihf9DEt9rZVUYpLLvZ4rDGm0ivpIMJcRY1kP37crS/cImlUs1HRv5gzG3H8uHu89+673arqEWHEta3H8uX1gI4cB+Z977aJuLIqOTnAIyV7LPejK9uxq6t/SQJg18FdRSe6JiSUKAAsURhjqgh/x4gAeX+RF9WKyaLgMWFh7gktb6ed5vpDmjXLb8lklzDuq7uuL+GRZcdmEjHGVAnLl7tO7cIvXy0Vf44prqXy9NOFtmc2oiQSvvu+RMeVJUsUxhhTCidrqeRtn7gTHj/1PuH/TI0tfZClFNBEISIDRGStiGwQkbFFbK8hIjM825eKSHQg4zHGmLJ2slbH8uWuwz3PqbQsMhvx1FNlGm6JBKyPQkTCgcnAxUAakCQis1V1tddutwB7VfVsERkC/BW4PlAxGWNMMBS8vVX0467FPdW1KB6qDWhEdqT/HdplLZCd2d2BDaq6CUBEpgNXAt6J4krgcc/nmcCLIiIaas/sGmNMKfl+qqtgcjkhqTwaDmHFjyYvrUAmimbAVq/lNKBHcfuoaraI7AfqAXu8dxKRUcAoz+JREVkZkIhDT30KXasqzK5FPrsW+arGtSh+ILm3c0p6+pB4PFZVpwBTAEQkuaSDRiobuxb57Frks2uRz65FPhEp8UjlQHZmbwNaeC0396wrch8RqQbUBjICGJMxxphTFMhEkQS0FZHWIlIdGALMLrTPbOBmz+drgPnWP2GMMRVLwG49efocRgOfAeHA66q6SkSeBJJVdTYwFXhHRDYAv+CSyclMCVTMIciuRT67FvnsWuSza5GvxNci5IoCGmOMKV82MtsYY4xPliiMMcb4VGEThZX/yOfHtbhPRFaLyA8iMk9EWgUjzvJwsmvhtd/VIqIiUmkfjfTnWojIdZ6fjVUi8m55x1he/Ph/pKWILBCR5Z7/TwYGI85AE5HXRWR3cWPNxHnBc51+EJGufp1YVSvcC9f5vRFoA1QHvgc6FtrnDuAVz+chwIxgxx3Ea9EXiPJ8vr0qXwvPfrWAr4AlQGKw4w7iz0VbYDlQ17PcMNhxB/FaTAFu93zuCKQGO+4AXYsLgK7AymK2DwT+CwjQE1jqz3kraosir/yHqmYBueU/vF0JvOX5PBPoJyIlmxmkYjvptVDVBap6yLO4BDdmpTLy5+cC4Clc3bAjRWyrLPy5FrcCk1V1L4Cq7i7nGMuLP9dCgTM8n2sD28sxvnKjql/hniAtzpXA2+osAeqISJOTnbeiJoqiyn80K24fVc0Gcst/VDb+XAtvt+D+YqiMTnotPE3pFqo6pzwDCwJ/fi7aAe1E5FsRWSIiA8otuvLlz7V4HBguImnAXOCu8gmtwjnV3ydAiJTwMP4RkeFAInBhsGMJBhEJA54DRgQ5lIqiGu72Ux9cK/MrEYlR1X1BjSo4bgDeVNW/iUgv3PitzqoauEp6lUhFbVFY+Y98/lwLRKQ/8DAwWFWPllNs5e1k16IW0BlYKCKpuHuwsytph7Y/PxdpwGxVPaaqPwPrcImjsvHnWtwCvA+gqouBSFzBwKrGr98nhVXURGHlP/Kd9FqISBfgVVySqKz3oeEk10JV96tqfVWNVtVoXH/NYFUtcTG0Csyf/0c+xrUmEJH6uFtRm8ozyHLiz7XYAvQDEJEOuESRXq5RVgyzgZs8Tz/1BPar6o6THVQhbz1p4Mp/hBw/r8UE4HTgA09//hZVHRy0oAPEz2tRJfh5LT4DLhGR1cBxYIyqVrpWt5/X4k/AayLyR1zH9ojK+IeliLyH++Ogvqc/5jEgAkBVX8H1zwwENgCHgJF+nbcSXitjjDFlqKLeejLGGFNBWKIwxhjjkyUKY4wxPlmiMMYY45MlCmOMMT5ZojCmHIlIdG5lTxHpIyL/CXZMxpyMJQpj/OAZoGT/v5gqyX7wjSmG56//tSLyNrASuFFEFovIMhH5QERO9+zXTUQWicj3IvKdiNTyHPu1Z99lInJucP81xpRchRyZbUwF0hZXKmYD8BHQX1UPisgDwH0iMh6YAVyvqkkicgZwGNgNXKyqR0SkLfAermCjMSHHEoUxvm1W1SUiMgg34c23njIp1YHFwDnADlVNAlDVXwFEpCbwoojE48pntAtG8MaUBUsUxvh20PMuwBeqeoP3RhGJKea4PwK7gDjcLd7KPImSqeSsj8IY/ywBeovI2eBaDCLSDlgLNBGRbp71tbzK3u/wzHdwI65YnTEhyRKFMX5Q1XTchEjvicgPuNtO7T1Tb14P/ENEvge+wJWwfgm42bOuPfktE2NCjlWPNcYY45O1KIwxxvhkicIYY4xPliiMMcb4ZInCGGOMT5YojDHG+GSJwhhjjE+WKIwxxvj0/yGxZVmvu8GLAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cause_kg_correct_facts = 95037\n",
    "\n",
    "cluster_cause_results_new = {}\n",
    "cluster_cause_results_new[\"jaccard title\"] = cause_results[\"title\"]\n",
    "\n",
    "cl_cause_titles_results = []\n",
    "for _, obtained, correct in results_type_clustering_cause[clustering_types[0]]:\n",
    "    cl_cause_titles_results.append((obtained, correct))\n",
    "    \n",
    "cl_cause_abstracts_results = []\n",
    "for _, obtained, correct in results_type_clustering_cause[clustering_types[1]]:\n",
    "    cl_cause_abstracts_results.append((obtained, correct))\n",
    "    \n",
    "cluster_cause_results_new[\"clustering titles\"] = cl_cause_titles_results\n",
    "cluster_cause_results_new[\"clustering abstracts\"] = cl_cause_abstracts_results\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "show_precision_recall_curve(cluster_cause_results_new, cause_kg_correct_facts, 'prec_recall_cluster_cause.pdf')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## DDI Gene"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "load facts from semmeddb and building index...\n",
      "loading correct interactions from semmed...\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 2\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k2.pkl...\n",
      "2 cluster contexts are loaded\n",
      "computing context clusters... [2 / 2]\n",
      "computing context clusters... [2 / 2]\n",
      "deriving new facts...\n",
      "739540 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 54767 of 739540 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 5\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k5.pkl...\n",
      "5 cluster contexts are loaded\n",
      "computing context clusters... [5 / 5]\n",
      "computing context clusters... [5 / 5]\n",
      "deriving new facts...\n",
      "709175 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 53619 of 709175 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 10\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k10.pkl...\n",
      "10 cluster contexts are loaded\n",
      "computing context clusters... [10 / 10]\n",
      "computing context clusters... [10 / 10]\n",
      "deriving new facts...\n",
      "572939 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 47001 of 572939 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 15\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k15.pkl...\n",
      "15 cluster contexts are loaded\n",
      "computing context clusters... [15 / 15]\n",
      "computing context clusters... [15 / 15]\n",
      "deriving new facts...\n",
      "381976 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 38355 of 381976 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 30\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k30.pkl...\n",
      "30 cluster contexts are loaded\n",
      "computing context clusters... [30 / 30]\n",
      "computing context clusters... [30 / 30]\n",
      "deriving new facts...\n",
      "397188 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 39166 of 397188 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 45\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k45.pkl...\n",
      "45 cluster contexts are loaded\n",
      "computing context clusters... [45 / 45]\n",
      "computing context clusters... [45 / 45]\n",
      "deriving new facts...\n",
      "187738 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 24892 of 187738 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 60\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k60.pkl...\n",
      "60 cluster contexts are loaded\n",
      "computing context clusters... [60 / 60]\n",
      "computing context clusters... [60 / 60]\n",
      "deriving new facts...\n",
      "318839 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 34267 of 318839 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 100\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k100.pkl...\n",
      "100 cluster contexts are loaded\n",
      "computing context clusters... [100 / 100]\n",
      "computing context clusters... [100 / 100]\n",
      "deriving new facts...\n",
      "406546 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 38815 of 406546 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 200\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k200.pkl...\n",
      "200 cluster contexts are loaded\n",
      "computing context clusters... [200 / 200]\n",
      "computing context clusters... [200 / 200]\n",
      "deriving new facts...\n",
      "94377 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 16882 of 94377 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 300\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k300.pkl...\n",
      "299 cluster contexts are loaded\n",
      "computing context clusters... [299 / 299]\n",
      "computing context clusters... [299 / 299]\n",
      "deriving new facts...\n",
      "160380 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 23135 of 160380 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 400\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k400.pkl...\n",
      "400 cluster contexts are loaded\n",
      "computing context clusters... [400 / 400]\n",
      "computing context clusters... [400 / 400]\n",
      "deriving new facts...\n",
      "179923 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 24496 of 179923 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 500\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k500.pkl...\n",
      "500 cluster contexts are loaded\n",
      "computing context clusters... [500 / 500]\n",
      "computing context clusters... [500 / 500]\n",
      "deriving new facts...\n",
      "69471 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 13783 of 69471 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 1000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k1000.pkl...\n",
      "998 cluster contexts are loaded\n",
      "computing context clusters... [998 / 998]\n",
      "computing context clusters... [998 / 998]\n",
      "deriving new facts...\n",
      "153479 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 21937 of 153479 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 2000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k2000.pkl...\n",
      "1990 cluster contexts are loaded\n",
      "computing context clusters... [1990 / 1990]\n",
      "computing context clusters... [1990 / 1990]\n",
      "deriving new facts...\n",
      "121477 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 18539 of 121477 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 4000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k4000.pkl...\n",
      "3990 cluster contexts are loaded\n",
      "computing context clusters... [3990 / 3990]\n",
      "computing context clusters... [3990 / 3990]\n",
      "deriving new facts...\n",
      "88339 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 15413 of 88339 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 6000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k6000.pkl...\n",
      "5976 cluster contexts are loaded\n",
      "computing context clusters... [5976 / 5976]\n",
      "computing context clusters... [5976 / 5976]\n",
      "deriving new facts...\n",
      "65953 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 13115 of 65953 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 10000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k10000.pkl...\n",
      "9981 cluster contexts are loaded\n",
      "computing context clusters... [9981 / 9981]\n",
      "computing context clusters... [9981 / 9981]\n",
      "deriving new facts...\n",
      "35580 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 8774 of 35580 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 20000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k20000.pkl...\n",
      "19967 cluster contexts are loaded\n",
      "computing context clusters... [19967 / 19967]\n",
      "computing context clusters... [19967 / 19967]\n",
      "deriving new facts...\n",
      "26573 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 7220 of 26573 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 50000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k50000.pkl...\n",
      "49954 cluster contexts are loaded\n",
      "computing context clusters... [49954 / 49954]\n",
      "computing context clusters... [49954 / 49954]\n",
      "deriving new facts...\n",
      "14428 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 4733 of 14428 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_title\n",
      "cluster k: 100000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_title.k100000.pkl...\n",
      "99950 cluster contexts are loaded\n",
      "computing context clusters... [99950 / 99950]\n",
      "computing context clusters... [99950 / 99950]\n",
      "deriving new facts...\n",
      "9540 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 3584 of 9540 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 2\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k2.pkl...\n",
      "2 cluster contexts are loaded\n",
      "computing context clusters... [2 / 2]\n",
      "computing context clusters... [2 / 2]\n",
      "deriving new facts...\n",
      "668832 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 51538 of 668832 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 5\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k5.pkl...\n",
      "5 cluster contexts are loaded\n",
      "computing context clusters... [5 / 5]\n",
      "computing context clusters... [5 / 5]\n",
      "deriving new facts...\n",
      "441478 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 42885 of 441478 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 10\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k10.pkl...\n",
      "10 cluster contexts are loaded\n",
      "computing context clusters... [10 / 10]\n",
      "computing context clusters... [10 / 10]\n",
      "deriving new facts...\n",
      "498525 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 45104 of 498525 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 15\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k15.pkl...\n",
      "15 cluster contexts are loaded\n",
      "computing context clusters... [15 / 15]\n",
      "computing context clusters... [15 / 15]\n",
      "deriving new facts...\n",
      "400895 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 39907 of 400895 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 30\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k30.pkl...\n",
      "30 cluster contexts are loaded\n",
      "computing context clusters... [30 / 30]\n",
      "computing context clusters... [30 / 30]\n",
      "deriving new facts...\n",
      "320669 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 35242 of 320669 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 45\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k45.pkl...\n",
      "45 cluster contexts are loaded\n",
      "computing context clusters... [45 / 45]\n",
      "computing context clusters... [45 / 45]\n",
      "deriving new facts...\n",
      "332892 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 35512 of 332892 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 60\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k60.pkl...\n",
      "60 cluster contexts are loaded\n",
      "computing context clusters... [60 / 60]\n",
      "computing context clusters... [60 / 60]\n",
      "deriving new facts...\n",
      "245636 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 30380 of 245636 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 100\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k100.pkl...\n",
      "100 cluster contexts are loaded\n",
      "computing context clusters... [100 / 100]\n",
      "computing context clusters... [100 / 100]\n",
      "deriving new facts...\n",
      "352100 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 37350 of 352100 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 200\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k200.pkl...\n",
      "198 cluster contexts are loaded\n",
      "computing context clusters... [198 / 198]\n",
      "computing context clusters... [198 / 198]\n",
      "deriving new facts...\n",
      "350408 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 36793 of 350408 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 300\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k300.pkl...\n",
      "300 cluster contexts are loaded\n",
      "computing context clusters... [300 / 300]\n",
      "computing context clusters... [300 / 300]\n",
      "deriving new facts...\n",
      "124003 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 20125 of 124003 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 400\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k400.pkl...\n",
      "400 cluster contexts are loaded\n",
      "computing context clusters... [400 / 400]\n",
      "computing context clusters... [400 / 400]\n",
      "deriving new facts...\n",
      "246708 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 29702 of 246708 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 500\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k500.pkl...\n",
      "500 cluster contexts are loaded\n",
      "computing context clusters... [500 / 500]\n",
      "computing context clusters... [500 / 500]\n",
      "deriving new facts...\n",
      "235503 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 29302 of 235503 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 1000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k1000.pkl...\n",
      "999 cluster contexts are loaded\n",
      "computing context clusters... [999 / 999]\n",
      "computing context clusters... [999 / 999]\n",
      "deriving new facts...\n",
      "194008 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 25500 of 194008 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 2000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k2000.pkl...\n",
      "1994 cluster contexts are loaded\n",
      "computing context clusters... [1994 / 1994]\n",
      "computing context clusters... [1994 / 1994]\n",
      "deriving new facts...\n",
      "202643 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 25608 of 202643 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 4000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k4000.pkl...\n",
      "3981 cluster contexts are loaded\n",
      "computing context clusters... [3981 / 3981]\n",
      "computing context clusters... [3981 / 3981]\n",
      "deriving new facts...\n",
      "133240 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 20453 of 133240 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 6000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k6000.pkl...\n",
      "5978 cluster contexts are loaded\n",
      "computing context clusters... [5978 / 5978]\n",
      "computing context clusters... [5978 / 5978]\n",
      "deriving new facts...\n",
      "111553 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 18359 of 111553 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 10000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k10000.pkl...\n",
      "9971 cluster contexts are loaded\n",
      "computing context clusters... [9971 / 9971]\n",
      "computing context clusters... [9971 / 9971]\n",
      "deriving new facts...\n",
      "71496 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 13905 of 71496 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 20000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k20000.pkl...\n",
      "19958 cluster contexts are loaded\n",
      "computing context clusters... [19958 / 19958]\n",
      "computing context clusters... [19958 / 19958]\n",
      "deriving new facts...\n",
      "41224 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 10153 of 41224 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 50000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k50000.pkl...\n",
      "49951 cluster contexts are loaded\n",
      "computing context clusters... [49951 / 49951]\n",
      "computing context clusters... [49951 / 49951]\n",
      "deriving new facts...\n",
      "24422 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 7263 of 24422 derived facts are correct\n",
      "============================================================\n",
      "cluster context: clustering_kmeans_abstract\n",
      "cluster k: 100000\n",
      "load cluster contexts from preprocessed_context_compatibility/clustering_kmeans_abstract.k100000.pkl...\n",
      "99952 cluster contexts are loaded\n",
      "computing context clusters... [99952 / 99952]\n",
      "computing context clusters... [99952 / 99952]\n",
      "deriving new facts...\n",
      "16813 facts derived\n",
      "verifying facts in semmeddb...\n",
      "derived 5569 of 16813 derived facts are correct\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "print('load facts from semmeddb and building index...')\n",
    "idx_pmid_dg = load_lg_facts(conn, query_ddi_gen_1_dg)\n",
    "idx_pmid_gd = load_lg_facts(conn, query_ddi_gen_2_gd)\n",
    "print('loading correct interactions from semmed...')\n",
    "ddi_gen_idx_subjects_correct, _ = load_kg_facts(conn, query_ddi_correct)\n",
    "\n",
    "results_type_clustering_ddi_gene = {}\n",
    "for c in clustering_types:\n",
    "    results_clustering_ddi_gene = []\n",
    "    for k in clusters_to_check:\n",
    "        print('='*60)\n",
    "        print('cluster context: {}'.format(c))\n",
    "        print('cluster k: {}'.format(k))\n",
    "        cluster2docs = load_dumped_clusters(c, k)\n",
    "        # now do experiment\n",
    "        c_idx_pmid_dg = combine_document_clusters(cluster2docs, idx_pmid_dg)\n",
    "        c_idx_pmid_gd = combine_document_clusters(cluster2docs, idx_pmid_gd)\n",
    "        correct, obtained = do_ddi_gene_experiment_with_context(c_idx_pmid_dg, c_idx_pmid_gd, ddi_gen_idx_subjects_correct)    \n",
    "        results_clustering_ddi_gene.append((k, correct, obtained))\n",
    "        \n",
    "    results_type_clustering_ddi_gene[c] = results_clustering_ddi_gene\n",
    "        \n",
    "del idx_pmid_dg\n",
    "del idx_pmid_gd\n",
    "del ddi_gen_idx_subjects_correct"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "============================================================\n",
      "clustering_kmeans_title\n",
      "2\t739540\t54767\n",
      "5\t709175\t53619\n",
      "10\t572939\t47001\n",
      "15\t381976\t38355\n",
      "30\t397188\t39166\n",
      "45\t187738\t24892\n",
      "60\t318839\t34267\n",
      "100\t406546\t38815\n",
      "200\t94377\t16882\n",
      "300\t160380\t23135\n",
      "400\t179923\t24496\n",
      "500\t69471\t13783\n",
      "1000\t153479\t21937\n",
      "2000\t121477\t18539\n",
      "4000\t88339\t15413\n",
      "6000\t65953\t13115\n",
      "10000\t35580\t8774\n",
      "20000\t26573\t7220\n",
      "50000\t14428\t4733\n",
      "100000\t9540\t3584\n",
      "============================================================\n",
      "clustering_kmeans_abstract\n",
      "2\t668832\t51538\n",
      "5\t441478\t42885\n",
      "10\t498525\t45104\n",
      "15\t400895\t39907\n",
      "30\t320669\t35242\n",
      "45\t332892\t35512\n",
      "60\t245636\t30380\n",
      "100\t352100\t37350\n",
      "200\t350408\t36793\n",
      "300\t124003\t20125\n",
      "400\t246708\t29702\n",
      "500\t235503\t29302\n",
      "1000\t194008\t25500\n",
      "2000\t202643\t25608\n",
      "4000\t133240\t20453\n",
      "6000\t111553\t18359\n",
      "10000\t71496\t13905\n",
      "20000\t41224\t10153\n",
      "50000\t24422\t7263\n",
      "100000\t16813\t5569\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "for c, results_clustering_ddi_gene in results_type_clustering_ddi_gene.items():\n",
    "    print(\"=\"*60)\n",
    "    print(c)\n",
    "    for k, correct, obtained in results_clustering_ddi_gene:\n",
    "        print('{}\\t{}\\t{}'.format(k, obtained, correct))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "jaccard title\n",
      "clustering titles\n",
      "clustering abstracts\n"
     ],
     "output_type": "stream"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3yUVfb48c8hJISEEmoiNaBIDYQiSBMUXVEUFCwouMK6suqiu7q4YvkpyuLXXfu6KKKCsosUsaHYBRWlBhSliBQDBAKEbmghyfn9cSfJQJLJpEwm5bxfr+c1M0+9eQhzcp9777miqhhjjDH5qRLsAhhjjCnbLFAYY4zxyQKFMcYYnyxQGGOM8ckChTHGGJ+qBrsAhVW/fn2NjY0NdjGMMaZcWbVq1T5VbVCUY8tdoIiNjSUhISHYxTDGmHJFRLYV9Vh79GSMMcYnCxTGGGN8skBhjDHGp3LXRmGMcU6dOkVSUhInTpwIdlFMGRIeHk6TJk0IDQ0tsXNaoDCmnEpKSqJmzZrExsYiIsEujikDVJX9+/eTlJREixYtSuy89ujJmHLqxIkT1KtXz4KEySYi1KtXr8RrmRYojCnHLEiYMwXid8IChTHGGJ8sUBhjiqxXr15Bvf5XX33FFVdckWv9Dz/8wEcffZT9ef78+TzxxBMAvPfee6xfvz5726hRo5g3b17gC1uOWaAwprKYORNiY6FKFfc6c2axT7lkyZJin6MwMjIy/NrvzEAxePBgxo8fD+QOFKZgAQ0UIjJQRDaKyGYRGZ/H9mdF5AfP8ouIHApkeYyptGbOhDFjYNs2UHWvY8YUO1jUqFGD1NRUBgwYQJcuXYiLi+P999/P3j5jxgw6duxIp06duOmmmwDYs2cPV199NZ06daJTp07Zweaqq66ia9eutG/fnqlTp552jb/97W906tSJpUuX8sknn9CmTRu6dOnCO++8k6tMaWlpPPzww8yZM4f4+HjmzJnD66+/ztixY1myZAnz58/n3nvvJT4+ni1btpx27KpVq+jXrx9du3bl0ksvJTk5uVj3p8JQ1YAsQAiwBWgJhAFrgHY+9r8TmFbQebt27arGGNX169efvqJfv9zL5MluW9Omqi5EnL7Uq+e2p6TkPtYPkZGReurUKT18+LDnNCl69tlna2Zmpq5du1ZbtWqlKSkpqqq6f/9+VVW97rrr9Nlnn1VV1fT0dD106NBp248dO6bt27fXffv2qaoqoHPmzFFV1ePHj2uTJk30l19+0czMTL322mt10KBBuco1ffp0/fOf/5zn55tvvlnfeuut7G1Zn9PS0rRnz566d+9eVVWdPXu2jh492q/7UNbk+t1QVSBBi/h9HshxFN2Bzaq6FUBEZgNDgPzqfDcAjxT6KhdfDF9+mfN5wAD44otCn8aYCi0pKe/1+/cX+9SqygMPPMA333xDlSpV2LlzJ3v27GHhwoVce+211K9fH4C6desCsHDhQmbMmAFASEgItWvXBuDf//437777LgA7duxg06ZN1KtXj5CQEIYNGwbAzz//TIsWLWjVqhUAI0eOPK32URwbN25k7dq1XHLJJYB7zHXWWWeVyLnLu0AGisbADq/PSUCPvHYUkeZAC2BhPtvHAGMAmjVrlrPhzCAB7vPFF1uwMJXPV1/lv61ZM/e46UzNm7vX+vV9H+/DzJkzSUlJYdWqVYSGhhIbG1vofvxfffUVX3zxBUuXLiUiIoL+/ftnnyM8PJyQkJAila0wVJX27duzdOnSgF+rvCkrjdnDgXmqmmdLlapOVdVuqtqtQQOvdOpnBomC1htTWU2aBBERp6+LiHDri+nw4cM0bNiQ0NBQFi1axDZPQLrooot466232O+ptRw4cACAAQMG8NJLLwHur/bDhw9z+PBh6tSpQ0REBD///DPLli3L81pt2rQhMTExu21h1qxZee5Xs2ZNfvvtt0Jta926NSkpKdmB4tSpU6xbt87f21ChBTJQ7ASaen1u4lmXl+FA3v/ixpjiGzECpk51NQgR9zp1qltfDCLCiBEjSEhIIC4ujhkzZtCmTRsA2rdvz4MPPki/fv3o1KkT99xzDwDPP/88ixYtIi4ujq5du7J+/XoGDhxIeno6bdu2Zfz48Zx//vl5Xi88PJypU6cyaNAgunTpQsOGDfPc78ILL2T9+vXZjdnehg8fzpNPPknnzp1Pa8wOCwtj3rx53HfffXTq1In4+PhS79VVVolr4wjAiUWqAr8AA3ABYiVwo6quO2O/NsAnQAv1ozDdunXT7ImLfI1ADNDPZUxZsWHDBtq2bRu06+/fv58uXbpk1yBM2ZHX74aIrFLVbkU5X8BqFKqaDowFPgU2AHNVdZ2IPCYig712HQ7M9idIGGPKhl27dtGzZ0/GjRsX7KKYUhDQ7LGq+hHw0RnrHj7j84QiX2DAgLzbIwYMKPIpjTEFa9SoEb/88kuwi2FKSVlpzC6azZsLt94YY0yhle9AsX174dYbY4wptPIdKLzHVPiz3hhjTKGV70ARwL7hxhhjnPIdKM7sG96sWYn0DTfGFN2ECRN46qmnCn3coUOHePHFF4t0zcsvv5xDh4qfUzQxMZE333wz+3NCQgJ33XUX4EaPe4+rKOrPWR6V70ABLijMneveT5liQcIYH5KToV8/2L072CXJrSiBQlXJzMzko48+IioqqthlODNQdOvWjX//+99A7kBRmZT/QAHQoIEbYGcpgY3xaeJE+PZb91oS8koj7q1///5kDZDdt28fsbGxAKxbt47u3bsTHx9Px44d2bRpE+PHj2fLli3Ex8dz7733AvDkk09y3nnn0bFjRx55xOUMTUxMpHXr1vz+97+nQ4cO7Nixg9jYWPbt20diYiJt27bl1ltvpX379vzud7/j+PHjAKxcuZKOHTtmn79Dhw65yjt+/HgWL15MfHw8zz77bPbESImJiUyZMoVnn32W+Ph4Fi9efNpxW7ZsYeDAgXTt2pW+ffvy888/A/DWW2/RoUMHOnXqxAUXXFAyNz0IAjqOotRkZXi0QGEqsf79fW8/eRISEiAzE6ZPh++/d1NSjBoF+/bBNdecvn9BOQLXrVvHP/7xD5YsWUL9+vWzczn5Y8qUKfzlL39hxIgRpKWlkZGRwRNPPMHatWv54YcfAPjss8/YtGkTK1asQFUZPHgw33zzDc2aNWPTpk288cYbeab62LRpE7NmzeKVV17huuuu4+2332bkyJGMHj2aV155hZ49e2ZPYnSmJ554gqeeeooPP/zQcw/cTYiNjeW2226jRo0a2YMMv/QawzVmzBimTJlCq1atWL58OXfccQcLFy7kscce49NPP6Vx48Yl8mgsWCpGoAgPh6goCxTG+JA1ZxFARkbeyWQLI7804v7o2bMnkyZNIikpiaFDh2anDff22Wef8dlnn9G5c2cAUlNT2bRpE82aNaN58+b55oNq0aIF8fHxAHTt2pXExEQOHTrEb7/9Rs+ePQG48cYbs4NBcaWmprJkyRKuvfba7HUnT54EoHfv3owaNYrrrruOoUOHlsj1gqFiBApwtYqy+ODVmFLiqwaQnAwtW7oAAZCWBgcPwsCB7nMxsoz7VLVqVTIzMwFOSz1+44030qNHDxYsWMDll1/Oyy+/TMuWLU87VlW5//77+dOf/nTa+sTERCIjI/O9ZrVq1bLfh4SEZD96CpTMzEyioqKya0LepkyZwvLly1mwYAFdu3Zl1apV1KtXL6DlCYSK0UYBcMcdcNVVwS6FMWXSxInukZO3jIzitVXkl0bcW2xsLKtWrQJg3rx52eu3bt1Ky5YtueuuuxgyZAg//vhjrvTfl156KdOmTSM1NRWAnTt3snfv3iKVNSoqipo1a7J8+XIAZs+ened+RUlPXqtWLVq0aMFbb70FuAC3Zs0awLVd9OjRg8cee4wGDRqwY8eOXMeXBxUnUIwdCyNHBrsUxpRJS5e6WoS3tDQoTiee/NKIexs3bhwvvfQSnTt3Zt++fdnr586dS4cOHYiPj2ft2rX8/ve/p169evTu3ZsOHTpw77338rvf/Y4bb7yRnj17EhcXxzXXXJPvl7g/XnvtNW699Vbi4+M5evRo9sx63jp27EhISAidOnXi2WefPW3blVdeybvvvptnY/bMmTN57bXX6NSpE+3bt8+eN/zee+8lLi6ODh060KtXLzp16lTk8gdTwNKMB8ppaca9paW5+nWzZr7TjxtTQQQ7zXh5k5qaSo0aNQDXaJ2cnMzzzz8f5FIFRrlJM17qXngBYmPh8OFgl8QYUwYtWLCA+Ph4OnTowOLFi3nooYeCXaRyo2I1ZoOrVZTAwBtjTMVy/fXXc/311we7GOVSxalR2FgKY4wJiIoTKGJi3KsFCmOMKVEVJ1BYjcIYYwKi4gSKBQtc28Tf/+4atWfODHaJjDGmQqgYgWLmTJe05tAhl6Ng2zb32YKFMaWuPKcZz09iYmKeSQTz895777F+/foSu/7jjz9eYucqioAGChEZKCIbRWSziOSZhUtErhOR9SKyTkTezGufAj34IBw7dvq6Y8fcemMMMU/FII9KriXmqZhgFy1bWUgzXlJ8BYr09PRCn6/CBgoRCQEmA5cB7YAbRKTdGfu0Au4Heqtqe+CvRbqYzZ1tjE97ju4p1Hp/VbQ046mpqQwYMIAuXboQFxeXPcIa3Bf8iBEjaNu2Lddccw3HPH+cjh8/nnbt2tGxY0fGjRvHkiVLmD9/Pvfeey/x8fFs2bKF/v3789e//pVu3brx/PPP88EHH9CjRw86d+7MxRdfzJ49e7KvP3r0aOLi4ujYsSNvv/0248eP5/jx48THxzNixAiOHj3KoEGD6NSpEx06dGDOnDnF+jf0RyDHUXQHNqvqVgARmQ0MAbzD7K3AZFU9CKCqRUvk0qxZ3qkwbe5sU0n89ZO/8sPu3Enp/NH/9f55ro+Piee5gc/le1xFTDMeHh7Ou+++S61atdi3bx/nn38+gwcPBmDjxo289tpr9O7dmz/84Q+8+OKLjB49mnfffZeff/4ZEeHQoUNERUUxePBgrrjiCq7xyt2elpaWHTQPHjzIsmXLEBFeffVV/vWvf/H0008zceJEateuzU8//ZS937Bhw/jPf/6TfV/efvttGjVqxIIFCwA4XAqDjAP56Kkx4J0BK8mzztu5wLki8p2ILBORgXmdSETGiEiCiCSkpKTk3sHmzjam1BU3zfjjjz/OP//5T7Zt20b16tVz7eOdZrxLly78/PPPbNq0CaBE0oznRVV54IEH6NixIxdffDE7d+7M/mu/adOm9O7dG4CRI0fy7bffUrt2bcLDw7nlllt45513iDjze8iL92C/pKQkLr30UuLi4njyySdZt24dAF988QV//vOfs/erU6dOrvPExcXx+eefc99997F48eI8c1aVtGCPzK4KtAL6A02Ab0QkTlVPa5VS1anAVHC5nnKdJWv60wcecI+bIiPh5ZdtWlRTafj6yx9AHs0//9lXo74q4dLkKG9pxmfOnElKSgqrVq0iNDSU2NjY7HLLGTnkRISqVauyYsUKvvzyS+bNm8d//vMfFi5cmOe5vct85513cs899zB48GC++uorJkyY4HcZzz33XFavXs1HH33EQw89xIABA3j44Yf9Pr4oAlmj2Ak09frcxLPOWxIwX1VPqeqvwC+4wFF4I0a4x09XXw3R0RYkjAmwiphm/PDhwzRs2JDQ0FAWLVrENq9H2tu3b2fp0qUAvPnmm/Tp04fU1FQOHz7M5ZdfzrPPPpudXtxXuvKs6zRu7B6wvPHGG9nrL7nkEiZPnpz9+eDBgwCEhoZy6tQpAHbt2kVERAQjR47k3nvvZfXq1YW+H4UVyECxEmglIi1EJAwYDsw/Y5/3cLUJRKQ+7lHU1mJddcIEmH/mZYyp3KIjowu13h8VMc34iBEjSEhIIC4ujhkzZtCmTZvsba1bt2by5Mm0bduWgwcPcvvtt/Pbb79xxRVX0LFjR/r06cMzzzwDwPDhw3nyySfp3LkzW7ZsyXWdCRMmcO2119K1a9fsR3cADz30EAcPHsyeZ3vRokWAm2q1Y8eOjBgxgp9++im7I8Cjjz5aKskNA5pmXEQuB54DQoBpqjpJRB4DElR1vri63NPAQCADmKSqeYd6j3zTjBtTyVia8cKxNONFTzMe0DYKVf0I+OiMdQ97vVfgHs9Sct56y81PYY+fjDEeCxYs4P/+7/9IT0+nefPmvP7668EuUrkR7MbswHj1Vdi1ywKFMSabpRkvuoqRwuNMF1wAa9eCp5HNmIqqvM1QaQIvEL8TFTNQ9O3rXr/7LrjlMCaAwsPD2b9/vwULk01V2b9/P+Hh4SV63or56Kl7dwgLg8WLwTOq0piKpkmTJiQlJZHnIFRTaYWHh9OkSZMSPWfFDBTh4S5Y5NEtzZiKIjQ0lBYtWgS7GKYSqJiBAuDTT3On9TDGGFNoFbONAixIGGNMCam4gSIzE66/Hp7znQPHGGOMbxU3UFSpAps2WToPY4wppoobKMCNp1i2zI3SNsYYUyQVO1D07QvHj0MpZFc0xpiKquIHCoBvvgluOYwxphyr2IGiYUO45hqoVy/YJTHGmHKr4o6jyPLWW8EugTHGlGsVu0aR5dQpOHYs2KUwxphyqeIHir17ISoKpk8PdkmMMaZcqviBokEDqFvXJQg0xhhTaBU/UIi48RTffAOWjtkYYwqt4gcKcN1kk5Nh69Zgl8QYY8qdyhMowB4/GWNMEQQ0UIjIQBHZKCKbRWR8HttHiUiKiPzgWf4YkIK0bQuPP+7mqDDGGFMoARtHISIhwGTgEiAJWCki81V1/Rm7zlHVsYEqB+ASBN5/f0AvYYwxFVUgaxTdgc2qulVV04DZwJAAXs+3o0fhk0/gwIGgFcEYY8qjQAaKxsAOr89JnnVnGiYiP4rIPBFpmteJRGSMiCSISEKR5wdetw4uuww+/7xoxxtjTCUV7MbsD4BYVe0IfA68kddOqjpVVbuparcGDRoU7UqdO0NkpDVoG2NMIQUyUOwEvGsITTzrsqnqflU96fn4KtA1YKUJDYWePS1QGGNMIQUyUKwEWolICxEJA4YDp003JyJneX0cDGwIYHlcN9mffoKDBwN6GWOMqUgCFihUNR0YC3yKCwBzVXWdiDwmIoM9u90lIutEZA1wFzAqUOUB3AhtVfjuu4BexhhjKhLRcpbWolu3bpqQkFC0g0+cgA0boGNHCAkp2YIZY0wZJiKrVLVbUY6t+PNReAsPd43axhhj/BbsXk+lb80aGDvW5qcwxhg/Vb5AsXMnTJ4My5cHuyTGGFMuVL5A0auXSz1u3WSNMcYvlS9QREVBp05ufgpjjDEFqnyBAtx4iqVL3VzaxhhjfKq8gaJuXdixo+B9jTGmkqucgWLYMNi+HVq2DHZJjDGmzKtc4yiyVKmc8dEYY4qi8n5jTp0K8fGQmRnskhhjTJlWeQNFWJgbfLchsHkIjTGmvPPr0ZOIVAOGAbHex6jqY4EpVim44AL3+s030L59cMtijDFlmL81ivdx05imA0e9lvKrRQto1MgG3hljTAH8bcxuoqoDA1qS0iYCzZrB3Lkwe7Z7P2kSjBgR7JIZY0yZ4m+NYomIxAW0JKVt5kxYvRoyMtwcFdu2wZgxbr0xxphs/gaKPsAqEdkoIj+KyE8i8mMgCxZwDz4IaWmnrzt2zK03xhiTzd9HT5cFtBTBsH174dYbY0wl5VeNQlW3AVHAlZ4lyrOu/GrWrHDrjTGmkvIrUIjIX4CZQEPP8j8RuTOQBQu4SZMgIiL3+g4dbBCeMcZ48beN4hagh6o+rKoPA+cDtwauWKVgxAg3Ort585weUBdfDAsWwB/+4Bq4jTHG+B0oBMjw+pzhWef7IJGBngbwzSIy3sd+w0RERaRIE38X2YgRkJjoahDbtsFnn8HEidCtmwsexhhj/G7Mng4sF5F3PZ+vAl7zdYCIhACTgUuAJGCliMxX1fVn7FcT+AsQ/LlJReChh3I+f/sttG4NDRoEr0zGGBNk/jZmPwOMBg54ltGq+lwBh3UHNqvqVlVNA2bjRnefaSLwT+CE36UuDUePwtCh0Lu3q3UYY0wl5TNQiEgtz2tdIBH4n2fZ5lnnS2PAe2agJM867/N3AZqq6oICyjFGRBJEJCElJaWAy5aQyEh4911ISYGePV0CQWOMqYQKqlG86XldBSR4LVmfi0xEqgDPAH8raF9Vnaqq3VS1W4PSfAzUu7d7/FS1qksi+NVXpXdtY4wpI3wGClW9wvPaQlVbei0tVLWg6eF2Ak29PjfxrMtSE+gAfCUiibieVPNLvUG7IO3bw5Il0LixywlljDGVjL9pxnsDP6jqUREZCXQBnlNVX8OYVwKtRKQFLkAMB27M2qiqh4H6Xtf4ChinqsWqqQRE06YuWNSo4T4fOQK1agW3TMYYU0r87R77EnBMRDrhHhVtAf7r6wBVTQfGAp8CG4C5qrpORB4TkcHFKHOJS06Gfv1g924fO0VFuUdQ+/ZB587w8MM21sIYUyn4GyjSVVVxvZb+o6qTcY+OfFLVj1T1XFU9W1UnedY9rKrz89i3f7BqExMnuqaIiRP92DkqCi680O38pz9BenrAy2eMMcHkb6D4TUTuB0YCCzwN0aGBK1bpSU6G6dPdmLvp0wuoVYCrVbzyissy+8orcM01cPx4qZTVGGOCwd9AcT1wErhFVXfjGqafDFipStHEiXDypHufkeFnrUIE/vEPeOEFmD8f7rsvoGU0xphgEi1nz9m7deumCQkl84QqORlatoQTXkP9qleHrVshJsbPk3zwAfTqBfXqlUiZjDEmEERklaoWqVdpQQPuvvW8/iYiR7yW30TkSFEuWJZMnJg7UazftYosV17pgsTJkzBqFGzYUJJFNMaYoCtoHEUfz2tNVa3ltdRU1XLfP3Tp0tyT3KWluZ6whbZjB3zyCfTp405sjDEVhL/zUZzvSd6X9bmmiPQIXLFKx/ffux6uqq5m0bOnGzKxbFkRTnbOOS7C1K0LAwbAhx+WeHmNMSYYCjOOItXr81HPugpDBB591FUMpk0r4klatoTvvoN27eCqq+DNNws+xhhjyji/56NQr1ZvVc3E/xTl5cbFF7t26WnTijGWrmFDlxNq6FCIiyvJ4hljTFD4Gyi2ishdIhLqWf4CbA1kwYJBBGbOhK+/Lua8RTVqwNy5LlCouiy0Nr2qMaac8jdQ3Ab0wuVsSgJ6AGMCVahgio11U2mfOpW7obtIFi1ytYsbbsgZsGGMMeWIvxMX7VXV4araUFWjVfVGVd0b6MIFS0oKtGkDL79cAie76CJ48klXw7j8cpdQ0BhjyhF/s8eei2u8jlbVDiLSERisqv8IaOmCpH59GDjQBQtvMU/FsOfonlz7R0dGs3ucj9wf48ZBdDT84Q8u++DHHxdiRJ8xxgSXv4+eXgHuB04BqOqPuLThFZIITJ4Ml1xyembZvIIE5L/+NDfd5EZxb9kCJTSy3BhjSoO/PZciVHWFnN7CW+HTph44AEOGwKpVntHaDYt5woED4ddfc9J9HD4MtWsXt5jGGBNQ/tYo9onI2YACiMg1QHLASlVGLFoEK1fmZJYtEVlB4rPPXMv555+X0ImNMSYw/A0UfwZeBtqIyE7gr7ieUBXal19CFc8dKpEeUN46dIBmzWDQIBuYZ4wp0wp89OSZe6Kbql4sIpFAFVX9LfBFCy7veSrAJQv0JS0jjbCQMP8v0KgRfPONe7Y1YgTs2QN33130AhtjTIAUWKPwjML+u+f90coQJCDvzLKkRue7/7C5wziRfiLf7XmqXdslEhw2DO65xz2OMsaYMsbfxuwvRGQcMAeX5wkAVT0QkFKVAXllluWp3YjAwoXQv3/O6pdWvsQdH93BlbOu5L3r3yMyLNL/C4WHw5w58NZbrpuVMcaUMYWZ4e4O4GsgwWupsLwzy2Yt+/a5sRWDB8Pq1Tn73n7e7bw+5HUW/rqQgTMHcuRkIQfVhYTA8OGuX+7GjTByJBw9WvBxxhhTCvwNFO2AycAa4AfgBaB9QQeJyEAR2Sgim0VkfB7bbxORn0TkBxH5VkTaFabwpa1ePfd0qE4deOyx07fdHH8zs4fNZlnSMgbMGMCB40WsbK1aBbNmuRHd+/YVv9DGGFNMfk2FKiJzgSPATM+qG4Haqnqdj2NCgF+AS3D5oVYCN6jqeq99aqnqEc/7wcAdqjrQV1lKcirUovr1V5ckNjKPJ0wfbPyAwbMH53lcgSO4s7z/vqthNG8On37qXo0xphgCNhWqlw6q+kdVXeRZbgU6FHBMd2Czqm5V1TRgNjDEe4esIOERiWecRlnXooULEkeOwJ/+5AbmZbmy9ZX5HufXCG5wPaE+/9z1hOrZEzZtKmaJjTGm6PwNFKtF5PysD57Z7Qr6s74xsMPrc5Jn3WlE5M8isgX4F3BXXicSkTEikiAiCSkpKX4WOfDWrXNDIIo0I15B+vSBxYuhb19o0iQAFzDGGP/4Gyi6AktEJFFEEoGlwHme9oUfi1MAVZ2sqmcD9wEP5bPPVFXtpqrdGjRoUJzLlaiePd1jqMsvD9AFOnRwPaKqV4dDh1xXWmOMKWX+BoqBQAugn2dp4Vl3BZDfs5adQFOvz0086/IzG7jKz/KUGfXru9d58+DmmwsemFdkjz7qItJLFWoGWmNMOeDXOApV3VaEc68EWolIC1yAGI5rBM8mIq1UNesB/CCg3D6MT0yEGTPc5Ha+kgeO/Wgszw18jqpVCjmT7KRJsHkz3HGHS2U7YUIxp+Ezxhj/+FujKDRVTQfGAp8CG4C5qrpORB7z9HACGCsi60TkB+Ae4OZAlSfQxo2Dv/8dXnwRIjXvEdwRoRFMXjmZK2ddyeEThwt3gYgIN6Xq6NGub+5tt0F6hU/ga4wpA/zqHluWlIXusflRhT/+EaZNg3//G665xvVynTMnZ56iV1e/yu0Lbqd1vdZ8eOOHxEbFFv4iDz0E//0vrFhhEyAZY/xSnO6xFihKWHo6XHstvPceDBjgUpXfdpubCCnLwl8XMmzuMMJCwnjv+vfo2bRn4S908KAb+ZeR4UZx16pVcj+EMabCKY1xFMZPVau6gdU9e7o05VlzWWo+zLoAABv+SURBVOz2Gmd3UYuLWHbLMmqG1eTCNy5k9trZhb9QnTru9W9/g969ISmpZH4AY4w5gwWKAAgPh3btctqa09I8M+R5aV2/Ncv/uJweTXpww9s38NjXj1Gk2t3gwbBtG/TqBRs2FL/wxhhzBgsUAZCcDDNnuuYEcE+HzqxVANSLqMdnIz/j5k4388hXj3DTuzcVPlX5RRfB11+7aNSnj0t7a4wxJcgCRQDkNZdFRgZcf737TvdWrWo1pg+ZzqSLJjHzp5kMmDGAlKOFHH3euTMsWQJ168KVV8JvlWLKEGNMKbFAEQB5zWWRlgbLl8OUKbn3FxEe6PsAc6+Zy+rk1fR4tQfrU9bn3tGXli3hu+9g7lyoWbPohTfGmDNYr6dSdOSIa7eoWRN27HAdlY4dO70L7YqdKxg8azB7j+5F88iR6HcG2tdeg717Yfx4G5hnjLFeT+VFrVouSGRmwtChcN558Ne/wrff5jR2d2/cnRW3rsgzSICfGWhV3XzcDzzgLpBrTldjjPGfBYogqFIFnnnG5fmbOzd3F9pmtZsV7wIi7oR33+1G/t1wA5w8WfyCG2MqJQsUQdK3r8vxl/VU6PhxuOWWnJ5SxZYVjZ580kWjQYMs5YcxpkgKmZnOlJTkZNcu4R0YPvoIevSAp5/2fWxaRhphIWH+XWjcOIiOdgPyqto/tzGm8KxGESR5daENCXGTIV1wge9ju7/Sne+Tv/f/YjfdBPff794vW+ZqGrGxrtYRG+sGfRhjTD4sUARJXl1oMzLgnHPgn/+E2iF5Z6CNCo9iz9E9dH+1Ow8vepi0jLQ898tTRgZcfbVL+7Ftm6vObNsGY8ZYsDDG5Mu6x5ZxL7zg2qR/+gnatnXrDhw/wN2f3s2MNTPo0LAD04dMp1sjP3u9NW4Mu3blXt+8uZtUwxhTIVn32Arsuutcm0WbNu7zhx+CHqvLG1e9wYc3fMiB4wc4/9XzeeDLB/xL/5GcnPf67dvhxx9h69YSbFE3xlQEVqMoRw4fhkaNXFvGuHGuppEReoi/ffo3pv0wjbb12zJ9yHR6NOmR/0liY93jpjM1bw6tW8Nnn7mL9Onjlv79IS4uUD+SMaaUWI2ikqhdG1auhIsvhkcegbPPhtenRDH50tf4eMTH/Jb2G72m9eLvn/+d46eO532SSZPcbHneIiLc+mefdVP09e/vGlHuusstWV57DRYvhhOFTFxojCnfVLVcLV27dlWjuny56oABqqDatKnqa6+p7k89rGPmj1EmoK1faK3fbf8u74P/9z/V5s1VRdzr//6X936Jiapr1rj3R4+qVq3qLhgWptqnj+r48aoJCYH48YwxJQxI0CJ+71qNopzq3h2++MItMTFusF7vbrX4XdrLfH7T55xIP0GfaX2459N7OHbq2OkHjxjhGq4zM93riBF5X6R5c+jY0b2PiHDtG++9B3feCadOwVNPuVQhAHv2wNixbnBIXo3lxphyy9ooKgBV9/394INuwqR58+C3k79x3xf38VLCS5xT9xymDZ5G3+Z9S/bCR4+6Lre1arn86YMGuXUALVq44ecPPQStWpXsdY0xhVZm2yhEZKCIbBSRzSIyPo/t94jIehH5UUS+FJHmgSxPRSXihkf89BNMnerW7fy1Jr88+yL/HbCQjMwM+r3ej8Z//Atbd7gv8uRk6Ncv92RKhRIZmTNXd79+LnnVypWuraNzZ/j4YzeoD+DNN10hn3nG7XPqVDEubIwpTQGrUYhICPALcAmQBKwEblDV9V77XAgsV9VjInI70F9Vr/d1XqtR+Ofjj1079JIlUL12Kuc/+ADrarxArYyWhEQe5OCJg7mO8TuFub+yfrdEXAT75z9d91twQaZnT5e3JDTU7Wvp0I0JmLJao+gObFbVraqaBswGhnjvoKqLVDXrAfoyoEkAy1OpXHYZbNwIDRrAkX01WP/0v2H61xw5LHkGCfAzhXlhiOR8+Y8ZA1u2uJxTs2fD6NGuNhIa6rYPHQrnn+/6/b7/PuzfX7JlMcYUWSADRWNgh9fnJM+6/NwCfJzXBhEZIyIJIpKQklLIaUIrsaynPo8+6nm/7QJ46ceglonGjd2csC+8AG+/nbO+Rw8XNF54Aa66CurXh5Ejc7bv3m0DAY0JkjKRTlRERgLdgH55bVfVqcBUcI+eSrFo5V5yMsyY4dqcATgV4XP/E+knCK8aHviCnWn8eLecOAEJCW42p7POctuOH4dmzVwW3L593UDAvn2hffucaGiMCZhABoqdQFOvz008604jIhcDDwL9VNVm1ylheWWp9aXBkw0Y1GoQw9oO47JWl1EjrEbgCpeX8PCcUeFZMjJcA/nixa531axZbv1TT7kEh0eOwNq10LUrVKtWuuU1phII5J9jK4FWItJCRMKA4cB87x1EpDPwMjBYVfcGsCyVVl5Zan0Z3v4GPtm4kOvmXUf9f9VnyOwhzFgzg4PH827XgBLqQeVLjRrw5z+7to2kJNfW8cYbMHiw275wIfTuDVFRriAPPQSffuomJDfGFFtAx1GIyOXAc0AIME1VJ4nIY7gRgvNF5AsgDsjKVLddVQf7Oqf1eiq+mKdi8my4jo6M5qmY3dz0+wyenvctidXfZt66d0g+upOqVapyUYuLGNpmKFe1uYroGjlp0O+4A15+GW67DSZPLs2fxOPgQVi0yD2uWrwYvv/e1UI2bHDZFJcvhx07XC0lJiYIBTQm+IrT68kG3JnTZGS4NuZrr3UdlkaNzmTGFwnE3/A2R5q8zZaDWxCEdjX60OrUMKr8cjUL3mzGyZNQvbrr/Rr07+LUVDdB04AB7ocYMwZeecVtO+cc177Rty+MGmVdck2lYYHCBMTSpdCrl3tfpQo0aqzsPPUT2uYdaPsORP/kNu48DzYMJWTjMK7s3Yp33w1emfN06hSsXp1T4/j2W6hTBzZtctuffBLCwlzw6NTJpec1poKxQGEC4tZbYdo01xgu4rLV3nCDy8hxzjmwn1+4+v53ST/3bWi80h20pwP3XDaUm7sPI65hHGc9fVa+j7lKdHBfYWRmwt69OVWf885zPa0AatZ0AwFvvBFuvjk45TMmACxQmBKXnAwtW56eUfzMR0t33OEyj6elAbW3Q5v3kPZvo00Xgygta5/D1sOb872GPnL6756vtpOAB5UdO3JqHIsXw8CBrqaRlgaXXuqyMPbt66pYdesGtizGBIAFClPiTgsCHmFh8Mc/5jRYd+4MP/yQ+9j2PfaQcc77JFZ/hxNNPs33GlOvmEpUeFT20v3V7vnue2ZQCbjMTPe8LSnJTTOYkJCTn6pDB/jXv9zwd0s9YsoJCxSmxOUXBOLjXaeigixf7r5ft/+h5L5Eg/q46vhxWLEip9bx8MOudvHRR3D77TkN5H36uMnNbSCgKWOKEyjKxMhsU/b4Ewx86dHDnaPeC/nvs+PuHRw6cSh7uXLWlT7PWeK5qAqjenU3RqPfGckDoqJyJgeZOdOtq1sXVq1y084eOODaPbJyWhlTDlmgMAFT0KP8JrWa0KRWOc8D2auXW1Rh82ZX41i+HJp6khI8+KAbHHj++Tm1jvPPd4MIjSknrH5sAio6MjrP9dUz3HpVeOIJ/2swjy9+nC+2fsGhE4cA36PCAz5i3JuI6w42ejRMmZLTxXbYMNd97NAh+Mc/4JJLXBfcLCtWuB5YxpRh1kZhSoV343hICPzud+7x/p49rhfVCy/AP07k3espP+fWOxeSurPp6/O4unt3Zj4df1pCw6CPGD/TkSNucMqRI25Eoyo0aeKmjm3dOifZYf/+bhpaY0qQNWabMq2grrZHj7rvzBo14NdfoeWM/BvAw549QHR8AhfdtJLdISv4dO0KqOkywFStUpWWEZ3ITDqPsL3d2fBld3RvG0KrhrB6teusdGa5hg9303wHZTS5qhtBntUl99tvXc1j7FgXOdPTXaTr3Rvi4mwgoCkWCxSmTPOnq623/MZT1KoSzZgju9myxWUkf/11l5kjvfpOQpqvoONlK0irv5INh1eSGXbEHXSyBiR3hZ3daRl+HoPiuzN8YDN69BDuvDOnxvHQQ0EOGuC65K5f7zLonnOOex7XpYvbVru2awvp29cVtEWLIBXSlFcWKEyZVtyutnnxVUvJyMyk5XmbSKu/AhqvcKPGY76Hqp5IdbQhF557Hotndyd9W3fC95/H9YPr8UaDGKhRhkaRq8K2bacPBNywAT77zLV1JCTAO+/kDASsXbv0y2jKDQsUptLxVUtRzb0tNDyNIWN+pMfQlXy5YQXLd67gYMgGEM/v/4GWUHdrvtcL6hgOb/v2ue621aq56tDYse4RlQh07OiCxsSJrtuuMV4sUJhKx1ctBXzXYLJrI5m/QaNVrtbRaCW0n+fzmgsvUGJjXc/XqoXsWO4rPQnkPUbEr+B09KjrOZVV41i71qUjqVoVHn/cTZyeNRCwdWsbRV6J2YA7U+kUZ0Bg9qx/aTUhsb9bANr7/hK96OH/g4wwLr+0GkOuCCPteBjz5lTjskvCaN+mGpmnwkg/UY2YBmGEh1YjLCSMsJAwqoVUy7c3l69eXn71AIuMhAsvdAvkpB4B1zD+8cduLlyABg1g6FDXfRcs/Yjxm9UoTKWTX22ECWXvS7PYOa5U4Zdfcmocdeu6aWXB9aSKickZCNijB0T4nlPdlF9WozCmELxrI/kGjTycePAEJzNOkpaRxsl0z6vX5+070/jmuzSSdp9k1940dqecZO++NI6lnYQhtxSprIdPHKZWtVpIUf/yF3GPnFq3dg04WdLS3GjEb7+FCRNcQKlaFR57DO6/39VMDh6EevWKdl1ToViNwhiP/NoRshT1r/tDh6DO80WvrVSlGnXDoompGUPTOtGcVTOa6BrRxNSIITrSvY+OdJ/9DSr5tpmE1mH3Awfgxx/dCPJ27XIGAvbp4wYCvvmmS02yfTs0awaTJsGIEUX++UzpsMZsY0pIoObEkEeLFiiqfPEUmdX3QOQe13W3xm6iz95DyrG9ZGpmrv2rhVQ7LXBkBZLwjGjefDmGSQ9E065ZNG0mt/F53ejqDdidcTcxqRPYE5bme99U2B3/PwsWZZwFCmPKuIJqK3mJjoxm59272bXLjVj/9VfXY2v8eMjUTEbdsZ/v1uzm5f/tYU/qHh55ag+/7t1NRMM9VKvrgsvJsN0cI4VMMgpdZn1E/Q5wOjHEPcf75BP3uOq771yPq0aNcpZ69azxPIjKbBuFiAwEngdCgFdV9Ykztl8APAd0BIarqu/+icaUU961kcLWWpo2dcsFF+SsqyJVmPFSAzIyGhASEgdA1UGuveXXX+HXpe41NQWQTKi+P7tG8p/X9zB24cgCy3zz3Dv8/vk2RmUQWqsFo6+uw5y5EDNrFkyeTDIxDGc2G8Z1JqVG7hpQNDXY3fJFF0gaN4Y2vms6JjgCVqMQkRDgF+ASIAlYCdygquu99okFagHjgPn+BAqrURjjv9RU14b99ttuXF7VqjBmDLzY0I+/7I82gMiUQl4wmobVmtOjVTO+/7IpjUMasGJJHHpj/nON6ATPm8aN3YyC4CaD2rDh9BpJq1bEbLy17M3BXk6U1RpFd2Czqm4FEJHZwBAgO1CoaqJnW+4/NYwxxfbbb/D++y5IgHudPh24r+Bjp5y9l9t2+/eo6B8h9/DQ53UgKpG9tbexIHUNmc3nk1Q1DWJ9H3vD9EFEnFSqawgRn/+diNAIIupvJKJREhH71xPx8yEijp0i4uw27OnuYzzKlCkucRfAmjVQp47r/hsW5tfPYPIXyEDRGNjh9TkJ6FGUE4nIGGAMQLNmzYpfMmMqiezBhV4yMtx8IMdDfLeZ/OlPcNuj/l3n18SnCfnOnbtKFc81JdM1wkdtgz/2zPfY1ambOHbqmFtWfM6J9BPum6n1mXv+7LMM4ze8QKvVobSq14pWV15FzI6DCECDBsTccpA94em5jokOq8vum3+Chg0LP9y+EikXd0ZVpwJTwT16CnJxjCk3li49PecVuM/xH+zm++8LTi0SHRldYCN8g+rRzJzpggR4BSatAqlnucWHWm9s5OwGUL++W+rWy6R2vePUaXiMvhe5ALLvyDEyqhzjwhkX5HueZ+pv4tQHnrEit0CkVOMcrUOr4xHsCc/7EdqetAPukVeVKvD//p8bU3LsGPztb6c/9mrUyGXsrVXL589SUQUyUOwEmnp9buJZZ4wpJQWlOinoub4/z/3vuANeK8bD4/r1ISXFNUmkpMDRo1WASJo2jWT7drfPoFGeiQCvyP88U5oeQ2ttJzVsEwerbGZv+ia2pW5izf5NcCD/4wY+3o5GJ0Jp1OQXzloxmUYnQzlr8SxCksK4J/VN3soYSQx74Lnn4C9/cSmKR4xwAcY7kPTv78aVZLX7VqAeXoEMFCuBViLSAhcghgM3BvB6xpggyKvWkktqdL4p3D/++PR1x4/D/v0u32GWUaNcSvnf55/gl1tGVwVaepZLueAC+Pprt81XN99fQyP5vsou9u+aS8bOWW7ltVlbL6FFel3OjWxOo+pvc9b7a2iUFsZZ5xyg0e4dNFp7iLN2HyUmFcLmzHOBYuFCGDw4d43kttvcdLkHD7oswI0aQWRkwMbulKSAjqMQkctx3V9DgGmqOklEHgMSVHW+iJwHvAvUAU4Au1W1va9zWq8nY8quQMw94i2/L9WGEdEsHbabffvIXqKi3Pc1FDDgcUJWDSADIvYR3mAXkTG7OJieTGbkLkKikok+excRMbs4HpLM7tTdZGjucSlRVevRMLIxZ0ltGicfoVlqOk0OnKTR7lQabTvIWa/NIabvZYT9b5aLfAC1aiH3HMm3aDoq0TXIV6uWa5uvGRrz2mYD7owxxgdfgWLxAD0twKSkwIcfwpYtrt0lLMx9T0+aBHfeCZs2Z3Bu/D6oucuzJOe8r5Gcva5Krd15DnSsW7Uep/Y3pMbxetQ8XptfWizIt2w6AY5RnZ114miyZgHVm9aHDz6AFSu44+vrefnb9tw2dC+Tn093NRTP467T5ovvNRMefJBu27aRoFqk52EWKIwxFV5hHu8UNMd7ZqYbn+JrOXoUzuuRQWy7FH5KTOZfL+2ia79kIqJ3sWbrLr5cnsyJsF2cqpZMZo38m261yat89V0oF07/PQs/z+DCi0OYM3gmoz+4muNUBwQhkz58S8w1fYmqI4T+mMDUFfGka1Wqh5xka5VWxJzaQTewQGGMMSWhsHO8F5ev2o4+ouzeDV984Wa/jY6G1avhD6OVn9ZCZqYgojSseZw6jSI4dAj27s4gkxBXbk7wR15jMmOLFSiqFOknM8aYCiq/LsVLlgSnPDExMHKkCxIAZ50FG38RMjPdd76qcORUBIsWuSASFh6SfWwa4UxnNLuJLlYZysU4CmOMKS0l0eheGPmNVckay3Km/AZRTpzoeubm2kYVJvL/gLFFLqMFCmOMCaLCdoEtqMaTaxvhLKEXIRSdBQpjjClHCl3jmZnV66no17Q2CmOMqchGjIDERFbBqqKewgKFMcYYnyxQGGOM8ckChTHGGJ8sUBhjjPHJAoUxxhifLFAYY4zxyQKFMcYYnyxQGGOM8ckChTHGGJ8sUBhjjPHJAoUxxhifLFAYY4zxyQKFMcYYnwIaKERkoIhsFJHNIjI+j+3VRGSOZ/tyEYkNZHmMMcYUXsAChYiEAJOBy4B2wA0i0u6M3W4BDqrqOcCzwD8DVR5jjDFFE8gaRXdgs6puVdU0YDYw5Ix9hgBveN7PAwaISJEm/zbGGBMYgZzhrjGww+tzEtAjv31UNV1EDgP1gH3eO4nIGGCM5+NJEVkbkBKXP/U5415VYnYvcti9yGH3Ikfroh5YLqZCVdWpwFQAEUlQ1W5BLlKZYPcih92LHHYvcti9yCEiCUU9NpCPnnYCTb0+N/Gsy3MfEakK1Ab2B7BMxhhjCimQgWIl0EpEWohIGDAcmH/GPvOBmz3vrwEWqqoGsEzGGGMKKWCPnjxtDmOBT4EQYJqqrhORx4AEVZ0PvAb8V0Q2AwdwwaQgUwNV5nLI7kUOuxc57F7ksHuRo8j3QuwPeGOMMb7YyGxjjDE+WaAwxhjjU5kNFJb+I4cf9+IeEVkvIj+KyJci0jwY5SwNBd0Lr/2GiYiKSIXtGunPvRCR6zy/G+tE5M3SLmNp8eP/SDMRWSQi33v+n1wejHIGmohME5G9+Y01E+ffnvv0o4h08evEqlrmFlzj9xagJRAGrAHanbHPHcAUz/vhwJxglzuI9+JCIMLz/vbKfC88+9UEvgGWAd2CXe4g/l60Ar4H6ng+Nwx2uYN4L6YCt3vetwMSg13uAN2LC4AuwNp8tl8OfAwIcD6w3J/zltUahaX/yFHgvVDVRap6zPNxGW7MSkXkz+8FwERc3rATpVm4UubPvbgVmKyqBwFUdW8pl7G0+HMvFKjleV8b2FWK5Ss1qvoNrgdpfoYAM9RZBkSJyFkFnbesBoq80n80zm8fVU0HstJ/VDT+3Atvt+D+YqiICrwXnqp0U1VdUJoFCwJ/fi/OBc4Vke9EZJmIDCy10pUuf+7FBGCkiCQBHwF3lk7RypzCfp8A5SSFh/GPiIwEugH9gl2WYBCRKsAzwKggF6WsqIp7/NQfV8v8RkTiVPVQUEsVHDcAr6vq0yLSEzd+q4OqZga7YOVBWa1RWPqPHP7cC0TkYuBBYLCqniylspW2gu5FTaAD8JWIJOKewc6voA3a/vxeJAHzVfWUqv4K/IILHBWNP/fiFmAugKouBcJxCQMrG7++T85UVgOFpf/IUeC9EJHOwMu4IFFRn0NDAfdCVQ+ran1VjVXVWFx7zWBVLXIytDLMn/8j7+FqE4hIfdyjqK2lWchS4s+92A4MABCRtrhAkVKqpSwb5gO/9/R+Oh84rKrJBR1UJh89aeDSf5Q7ft6LJ4EawFue9vztqjo4aIUOED/vRaXg5734FPidiKwHMoB7VbXC1br9vBd/A14RkbtxDdujKuIfliIyC/fHQX1Pe8wjQCiAqk7Btc9cDmwGjgGj/TpvBbxXxhhjSlBZffRkjDGmjLBAYYwxxicLFMYYY3yyQGGMMcYnCxTGGGN8skBhTCkSkdiszJ4i0l9EPgx2mYwpiAUKY/zgGaBk/19MpWS/+Mbkw/PX/0YRmQGsBW4SkaUislpE3hKRGp79zhORJSKyRkRWiEhNz7GLPfuuFpFewf1pjCm6Mjky25gypBUuVcxm4B3gYlU9KiL3AfeIyBPAHOB6VV0pIrWA48Be4BJVPSEirYBZuISNxpQ7FiiM8W2bqi4TkStwE95850mTEgYsBVoDyaq6EkBVjwCISCTwHxGJx6XPODcYhTemJFigMMa3o55XAT5X1Ru8N4pIXD7H3Q3sATrhHvFW5EmUTAVnbRTG+GcZ0FtEzgFXYxCRc4GNwFkicp5nfU2vtPfJnvkObsIlqzOmXLJAYYwfVDUFNyHSLBH5EffYqY1n6s3rgRdEZA3wOS6F9YvAzZ51bcipmRhT7lj2WGOMMT5ZjcIYY4xPFiiMMcb4ZIHCGGOMTxYojDHG+GSBwhhjjE8WKIwxxvhkgcIYY4xP/x9keIZMyPFfAAAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ddi_gene_kg_correct_facts = 55370\n",
    "\n",
    "cluster_ddi_gene_results_new = {}\n",
    "cluster_ddi_gene_results_new[\"jaccard title\"] = ddi_gen_results[\"title\"]\n",
    "\n",
    "cl_ddig_titles_results = []\n",
    "for _, obtained, correct in results_type_clustering_ddi_gene[clustering_types[0]]:\n",
    "    cl_ddig_titles_results.append((obtained, correct))\n",
    "    \n",
    "cl_ddig_abstracts_results = []\n",
    "for _, obtained, correct in results_type_clustering_ddi_gene[clustering_types[1]]:\n",
    "    cl_ddig_abstracts_results.append((obtained, correct))\n",
    "    \n",
    "cluster_ddi_gene_results_new[\"clustering titles\"] = cl_ddig_titles_results\n",
    "cluster_ddi_gene_results_new[\"clustering abstracts\"] = cl_ddig_abstracts_results\n",
    "\n",
    "\n",
    "\n",
    "show_precision_recall_curve(cluster_ddi_gene_results_new, ddi_gene_kg_correct_facts, 'prec_recall_cluster_ddi_gene.pdf')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}