notebooks/0a. Parse data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cdcb8f3c",
   "metadata": {},
   "source": [
    "# Crawl dataset with all submissions info\n",
    "OpenReview Venue Crawling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b15b9499",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import time\n",
    "import pandas as pd\n",
    "import multiprocessing as mp\n",
    "from multiprocessing import Pool\n",
    "from tqdm import tqdm\n",
    "from tqdm.notebook import tqdm\n",
    "import requests"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba524d57",
   "metadata": {},
   "source": [
    "## Crawl list of all submissions\n",
    "Here we scrape the _notes_ , (list of all submissions) using OpenReview's API, way faster than Selenium-based scraping.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "37d02c61",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = '../data/'\n",
    "venue = 'ICLR.cc/2023/Conference'\n",
    "venue_short = 'iclr2023'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a451ae95",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_conference_notes(venue, blind_submission=False):\n",
    "    \"\"\"\n",
    "    Get all notes of a conference (data) from OpenReview API.\n",
    "    If results are not final, you should set blind_submission=True.\n",
    "    \"\"\"\n",
    "\n",
    "    blind_param = '-/Blind_Submission' if blind_submission else ''\n",
    "    offset = 0\n",
    "    notes = []\n",
    "    while True:\n",
    "        print('Offset:', offset, 'Data:', len(notes))\n",
    "        url = f'https://api.openreview.net/notes?invitation={venue}/{blind_param}&offset={offset}'\n",
    "        response = requests.get(url)\n",
    "        data = response.json()\n",
    "        if len(data['notes']) == 0:\n",
    "            break\n",
    "        offset += 1000\n",
    "        notes.extend(data['notes'])\n",
    "    return notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0c6974ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Offset: 0 Data: 0\n",
      "Offset: 1000 Data: 1000\n",
      "Offset: 2000 Data: 2000\n",
      "Offset: 3000 Data: 3000\n",
      "Offset: 4000 Data: 3851\n",
      "Number of submissions: 3851\n"
     ]
    }
   ],
   "source": [
    "raw_notes = get_conference_notes(venue, blind_submission=True)\n",
    "print(\"Number of submissions:\", len(raw_notes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d074dc0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>original</th>\n",
       "      <th>number</th>\n",
       "      <th>cdate</th>\n",
       "      <th>mdate</th>\n",
       "      <th>ddate</th>\n",
       "      <th>tcdate</th>\n",
       "      <th>tmdate</th>\n",
       "      <th>tddate</th>\n",
       "      <th>forum</th>\n",
       "      <th>...</th>\n",
       "      <th>content.resubmission</th>\n",
       "      <th>content.student_author</th>\n",
       "      <th>content.Please_choose_the_closest_area_that_your_submission_falls_into</th>\n",
       "      <th>content.paperhash</th>\n",
       "      <th>content.pdf</th>\n",
       "      <th>content.supplementary_material</th>\n",
       "      <th>content._bibtex</th>\n",
       "      <th>content.venue</th>\n",
       "      <th>content.venueid</th>\n",
       "      <th>content.TL;DR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RUzSobdYy0V</td>\n",
       "      <td>pmo4AKuE4-p</td>\n",
       "      <td>6620</td>\n",
       "      <td>1663850590815</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1663850590815</td>\n",
       "      <td>1675279442105</td>\n",
       "      <td>None</td>\n",
       "      <td>RUzSobdYy0V</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Social Aspects of Machine Learning (eg, AI saf...</td>\n",
       "      <td>adebayo|quantifying_and_mitigating_the_impact_...</td>\n",
       "      <td>/pdf/c91f0bb05e5ac8a99867a69b17cb3772ccf04628.pdf</td>\n",
       "      <td>/attachment/151652f4d981a49f9dfa81be992839a243...</td>\n",
       "      <td>@inproceedings{\\nadebayo2023quantifying,\\ntitl...</td>\n",
       "      <td>ICLR 2023 poster</td>\n",
       "      <td>ICLR.cc/2023/Conference</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N3kGYG3ZcTi</td>\n",
       "      <td>kVYulJycT2K</td>\n",
       "      <td>6611</td>\n",
       "      <td>1663850589829</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1663850589829</td>\n",
       "      <td>1675279442101</td>\n",
       "      <td>None</td>\n",
       "      <td>N3kGYG3ZcTi</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Deep Learning and representational learning</td>\n",
       "      <td>zhuang|suppression_helps_lateral_inhibitionins...</td>\n",
       "      <td>/pdf/bc66a3bbb804a7158ba77a4de9f91a196e8eaf9a.pdf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@misc{\\nzhuang2023suppression,\\ntitle={Suppres...</td>\n",
       "      <td>Submitted to ICLR 2023</td>\n",
       "      <td>ICLR.cc/2023/Conference</td>\n",
       "      <td>Improving feature learning with lateral inhibi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tmIiMPl4IPa</td>\n",
       "      <td>RAIF4RUF0T</td>\n",
       "      <td>6610</td>\n",
       "      <td>1663850589709</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1663850589709</td>\n",
       "      <td>1675279442055</td>\n",
       "      <td>None</td>\n",
       "      <td>tmIiMPl4IPa</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Machine Learning for Sciences (eg biology, phy...</td>\n",
       "      <td>tran|factorized_fourier_neural_operators</td>\n",
       "      <td>/pdf/d80c8a332eea03754e31236157306d3491a3b832.pdf</td>\n",
       "      <td>/attachment/dc36a32060af8da1f81e6570a28b4890b1...</td>\n",
       "      <td>@inproceedings{\\ntran2023factorized,\\ntitle={F...</td>\n",
       "      <td>ICLR 2023 poster</td>\n",
       "      <td>ICLR.cc/2023/Conference</td>\n",
       "      <td>An efficient and scalable neural PDE solver us...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mhnHqRqcjYU</td>\n",
       "      <td>ix_LR-W0OM2</td>\n",
       "      <td>6603</td>\n",
       "      <td>1663850588877</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1663850588877</td>\n",
       "      <td>1675279442026</td>\n",
       "      <td>None</td>\n",
       "      <td>mhnHqRqcjYU</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Deep Learning and representational learning</td>\n",
       "      <td>narshana|dfpc_data_flow_driven_pruning_of_coup...</td>\n",
       "      <td>/pdf/491bdb807d29b7113e662ff30f32501632e3a1d3.pdf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@inproceedings{\\nnarshana2023dfpc,\\ntitle={{DF...</td>\n",
       "      <td>ICLR 2023 poster</td>\n",
       "      <td>ICLR.cc/2023/Conference</td>\n",
       "      <td>We propose a novel data-free algorithm to acce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sZI1Oj9KBKy</td>\n",
       "      <td>vRziu1jJDu</td>\n",
       "      <td>6601</td>\n",
       "      <td>1663850588630</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1663850588630</td>\n",
       "      <td>1675279441672</td>\n",
       "      <td>None</td>\n",
       "      <td>sZI1Oj9KBKy</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Deep Learning and representational learning</td>\n",
       "      <td>murti|tvsprune_pruning_nondiscriminative_filte...</td>\n",
       "      <td>/pdf/41f9c167358fdf50559b697f2a7ee4b4d3b0f9a7.pdf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@inproceedings{\\nmurti2023tvsprune,\\ntitle={{T...</td>\n",
       "      <td>ICLR 2023 poster</td>\n",
       "      <td>ICLR.cc/2023/Conference</td>\n",
       "      <td>We use the total variation distance between th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id     original  number          cdate mdate ddate         tcdate  \\\n",
       "0  RUzSobdYy0V  pmo4AKuE4-p    6620  1663850590815  None  None  1663850590815   \n",
       "1  N3kGYG3ZcTi  kVYulJycT2K    6611  1663850589829  None  None  1663850589829   \n",
       "2  tmIiMPl4IPa   RAIF4RUF0T    6610  1663850589709  None  None  1663850589709   \n",
       "3  mhnHqRqcjYU  ix_LR-W0OM2    6603  1663850588877  None  None  1663850588877   \n",
       "4  sZI1Oj9KBKy   vRziu1jJDu    6601  1663850588630  None  None  1663850588630   \n",
       "\n",
       "          tmdate tddate        forum  ... content.resubmission  \\\n",
       "0  1675279442105   None  RUzSobdYy0V  ...                        \n",
       "1  1675279442101   None  N3kGYG3ZcTi  ...                        \n",
       "2  1675279442055   None  tmIiMPl4IPa  ...                        \n",
       "3  1675279442026   None  mhnHqRqcjYU  ...                        \n",
       "4  1675279441672   None  sZI1Oj9KBKy  ...                        \n",
       "\n",
       "  content.student_author  \\\n",
       "0                          \n",
       "1                          \n",
       "2                          \n",
       "3                          \n",
       "4                          \n",
       "\n",
       "  content.Please_choose_the_closest_area_that_your_submission_falls_into  \\\n",
       "0  Social Aspects of Machine Learning (eg, AI saf...                       \n",
       "1        Deep Learning and representational learning                       \n",
       "2  Machine Learning for Sciences (eg biology, phy...                       \n",
       "3        Deep Learning and representational learning                       \n",
       "4        Deep Learning and representational learning                       \n",
       "\n",
       "                                   content.paperhash  \\\n",
       "0  adebayo|quantifying_and_mitigating_the_impact_...   \n",
       "1  zhuang|suppression_helps_lateral_inhibitionins...   \n",
       "2           tran|factorized_fourier_neural_operators   \n",
       "3  narshana|dfpc_data_flow_driven_pruning_of_coup...   \n",
       "4  murti|tvsprune_pruning_nondiscriminative_filte...   \n",
       "\n",
       "                                         content.pdf  \\\n",
       "0  /pdf/c91f0bb05e5ac8a99867a69b17cb3772ccf04628.pdf   \n",
       "1  /pdf/bc66a3bbb804a7158ba77a4de9f91a196e8eaf9a.pdf   \n",
       "2  /pdf/d80c8a332eea03754e31236157306d3491a3b832.pdf   \n",
       "3  /pdf/491bdb807d29b7113e662ff30f32501632e3a1d3.pdf   \n",
       "4  /pdf/41f9c167358fdf50559b697f2a7ee4b4d3b0f9a7.pdf   \n",
       "\n",
       "                      content.supplementary_material  \\\n",
       "0  /attachment/151652f4d981a49f9dfa81be992839a243...   \n",
       "1                                                NaN   \n",
       "2  /attachment/dc36a32060af8da1f81e6570a28b4890b1...   \n",
       "3                                                NaN   \n",
       "4                                                NaN   \n",
       "\n",
       "                                     content._bibtex           content.venue  \\\n",
       "0  @inproceedings{\\nadebayo2023quantifying,\\ntitl...        ICLR 2023 poster   \n",
       "1  @misc{\\nzhuang2023suppression,\\ntitle={Suppres...  Submitted to ICLR 2023   \n",
       "2  @inproceedings{\\ntran2023factorized,\\ntitle={F...        ICLR 2023 poster   \n",
       "3  @inproceedings{\\nnarshana2023dfpc,\\ntitle={{DF...        ICLR 2023 poster   \n",
       "4  @inproceedings{\\nmurti2023tvsprune,\\ntitle={{T...        ICLR 2023 poster   \n",
       "\n",
       "           content.venueid                                      content.TL;DR  \n",
       "0  ICLR.cc/2023/Conference                                                NaN  \n",
       "1  ICLR.cc/2023/Conference  Improving feature learning with lateral inhibi...  \n",
       "2  ICLR.cc/2023/Conference  An efficient and scalable neural PDE solver us...  \n",
       "3  ICLR.cc/2023/Conference  We propose a novel data-free algorithm to acce...  \n",
       "4  ICLR.cc/2023/Conference  We use the total variation distance between th...  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw = pd.json_normalize(raw_notes)\n",
    "# set index as first column\n",
    "# df_raw.set_index(df_raw.columns[0], inplace=True)\n",
    "df_raw.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1de767e9",
   "metadata": {},
   "source": [
    "## (optional) older crawled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "8fafbef2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>keywords</th>\n",
       "      <th>ratings</th>\n",
       "      <th>confidences</th>\n",
       "      <th>withdraw</th>\n",
       "      <th>review_lengths</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kRvZ2PcsxjJj</td>\n",
       "      <td>Quantum reinforcement learning</td>\n",
       "      <td>['quantum reinforcement learning', 'multi-agen...</td>\n",
       "      <td>[1, 1, 1, 1]</td>\n",
       "      <td>[5, 5, 5, 5]</td>\n",
       "      <td>1</td>\n",
       "      <td>[45, 49, 25, 283]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RUzSobdYy0V</td>\n",
       "      <td>Quantifying and Mitigating the Impact of Label...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[5, 6, 8]</td>\n",
       "      <td>[4, 3, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[443, 274, 401]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N3kGYG3ZcTi</td>\n",
       "      <td>Suppression helps: Lateral Inhibition-inspired...</td>\n",
       "      <td>['Lateral Inhibition', 'Convolutional Neural N...</td>\n",
       "      <td>[3, 5, 3, 1]</td>\n",
       "      <td>[5, 5, 5, 5]</td>\n",
       "      <td>0</td>\n",
       "      <td>[333, 360, 362, 304]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tmIiMPl4IPa</td>\n",
       "      <td>Factorized Fourier Neural Operators</td>\n",
       "      <td>['fourier transform', 'fourier operators', 'pd...</td>\n",
       "      <td>[8, 6, 3, 8, 3]</td>\n",
       "      <td>[5, 4, 4, 2, 2]</td>\n",
       "      <td>0</td>\n",
       "      <td>[203, 142, 323, 520, 635]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mhnHqRqcjYU</td>\n",
       "      <td>DFPC: Data flow driven pruning of coupled chan...</td>\n",
       "      <td>['Pruning', 'Data Free', 'Model Compression']</td>\n",
       "      <td>[8, 6, 6]</td>\n",
       "      <td>[3, 2, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[302, 90, 257]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                                              title  \\\n",
       "0  kRvZ2PcsxjJj                    Quantum reinforcement learning    \n",
       "1   RUzSobdYy0V  Quantifying and Mitigating the Impact of Label...   \n",
       "2   N3kGYG3ZcTi  Suppression helps: Lateral Inhibition-inspired...   \n",
       "3   tmIiMPl4IPa                Factorized Fourier Neural Operators   \n",
       "4   mhnHqRqcjYU  DFPC: Data flow driven pruning of coupled chan...   \n",
       "\n",
       "                                            keywords          ratings  \\\n",
       "0  ['quantum reinforcement learning', 'multi-agen...     [1, 1, 1, 1]   \n",
       "1                                                 []        [5, 6, 8]   \n",
       "2  ['Lateral Inhibition', 'Convolutional Neural N...     [3, 5, 3, 1]   \n",
       "3  ['fourier transform', 'fourier operators', 'pd...  [8, 6, 3, 8, 3]   \n",
       "4      ['Pruning', 'Data Free', 'Model Compression']        [8, 6, 6]   \n",
       "\n",
       "       confidences  withdraw             review_lengths  \n",
       "0     [5, 5, 5, 5]         1          [45, 49, 25, 283]  \n",
       "1        [4, 3, 3]         0            [443, 274, 401]  \n",
       "2     [5, 5, 5, 5]         0       [333, 360, 362, 304]  \n",
       "3  [5, 4, 4, 2, 2]         0  [203, 142, 323, 520, 635]  \n",
       "4        [3, 2, 3]         0             [302, 90, 257]  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read data from old version \n",
    "df_old = pd.read_csv(DATA_PATH + 'iclr2023_20221120.csv')\n",
    "df_old.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "5fd840eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of papers (including old): 4874\n"
     ]
    }
   ],
   "source": [
    "papers_ids = df_old['id'].values\n",
    "print(\"Number of papers (including old):\", len(papers_ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa1c02af",
   "metadata": {},
   "source": [
    "## Crawl forums of each submission\n",
    "Here we scrape the forums of each submissions, it can be pretty fast thanks to:\n",
    "- OpenReview's API (we use requests)\n",
    "- Multiprocessing to parallelize the scraping of each paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "139f9b4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create multiprocessing pool of requests over index of dataframe\n",
    "\n",
    "extra = \"trash=true&details=replyCount%2Cwritable%2Crevisions%2Coriginal%2Coverwriting%2Cinvitation%2Ctags\"\n",
    "\n",
    "def get_paper_data(paper_id, extra='', timeout=5):\n",
    "    try:\n",
    "        url = f\"https://api.openreview.net/notes?forum={paper_id}&{extra}\"\n",
    "        response = requests.get(url, timeout=timeout)\n",
    "        data = response.json()\n",
    "        return data\n",
    "    except requests.exceptions.Timeout:\n",
    "        print(f\"Error for paper {paper_id}: Request timed out\")\n",
    "        return None\n",
    "    except:\n",
    "        print(f\"Error for paper {paper_id}: General error\")\n",
    "        return None\n",
    "\n",
    "def retry_get_paper_data(paper_id, extra='', timeout=5, retries=10):\n",
    "    for i in range(retries):\n",
    "        data = get_paper_data(paper_id, extra, timeout)\n",
    "        if data is not None:\n",
    "            return data\n",
    "    print(f\"Error for paper {paper_id}: All {retries} attempts failed\")\n",
    "    return None\n",
    "\n",
    "def get_paper_data_multi(paper_ids, ratio=0.8):\n",
    "    num_processes = int(ratio*mp.cpu_count())\n",
    "    with Pool(num_processes) as p:\n",
    "        data = list(tqdm(p.imap(retry_get_paper_data, paper_ids), total=len(paper_ids)))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "969f4cfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content.title</th>\n",
       "      <th>content.keywords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RUzSobdYy0V</td>\n",
       "      <td>Quantifying and Mitigating the Impact of Label...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N3kGYG3ZcTi</td>\n",
       "      <td>Suppression helps: Lateral Inhibition-inspired...</td>\n",
       "      <td>[Lateral Inhibition, Convolutional Neural Netw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tmIiMPl4IPa</td>\n",
       "      <td>Factorized Fourier Neural Operators</td>\n",
       "      <td>[fourier transform, fourier operators, pde, na...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mhnHqRqcjYU</td>\n",
       "      <td>DFPC: Data flow driven pruning of coupled chan...</td>\n",
       "      <td>[Pruning, Data Free, Model Compression]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sZI1Oj9KBKy</td>\n",
       "      <td>TVSPrune - Pruning Non-discriminative filters ...</td>\n",
       "      <td>[Structured pruning, model compression]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id                                      content.title  \\\n",
       "0  RUzSobdYy0V  Quantifying and Mitigating the Impact of Label...   \n",
       "1  N3kGYG3ZcTi  Suppression helps: Lateral Inhibition-inspired...   \n",
       "2  tmIiMPl4IPa                Factorized Fourier Neural Operators   \n",
       "3  mhnHqRqcjYU  DFPC: Data flow driven pruning of coupled chan...   \n",
       "4  sZI1Oj9KBKy  TVSPrune - Pruning Non-discriminative filters ...   \n",
       "\n",
       "                                    content.keywords  \n",
       "0                                                 []  \n",
       "1  [Lateral Inhibition, Convolutional Neural Netw...  \n",
       "2  [fourier transform, fourier operators, pde, na...  \n",
       "3            [Pruning, Data Free, Model Compression]  \n",
       "4            [Structured pruning, model compression]  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter df with only id, title, url and keywords\n",
    "df_raw_filtered = df_raw[['id', 'content.title', 'content.keywords']]\n",
    "df_raw_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e32b36c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c7d2c047573490faff2d9c548400582",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ids = list(df_raw_filtered['id'])\n",
    "ids = df_old['id'].values # use old ids to get data from old papers\n",
    "data = get_paper_data_multi(ids, ratio=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "667bdae8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get only notes\n",
    "notes = [d['notes'] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "55385088",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_data(item, \n",
    "                review_keys=['summary_of_the_paper', 'strength_and_weaknesses', 'clarity,_quality,_novelty_and_reproducibility', 'summary_of_the_review'],\n",
    "                decision=True):\n",
    "    \"\"\"Filter only ratings, confidence, withdraw status and decisions\"\"\"\n",
    "    # parse each note\n",
    "    withdraw = 0\n",
    "    # filter meta note\n",
    "    meta_note = [d for d in item if 'Paper' not in d['invitation']]\n",
    "    # check withdrawn\n",
    "    withdraw = 1 if 'Withdrawn_Submission' in meta_note[0]['invitation'] else 0\n",
    "    # decision\n",
    "    if decision:\n",
    "        try:\n",
    "            if withdraw == 0:\n",
    "                decision_note = [d for d in item if 'Decision' in d['invitation']]\n",
    "                decision = decision_note[0]['content']['decision']\n",
    "            else:\n",
    "                decision = ''\n",
    "        except:\n",
    "            decision = ''\n",
    "    # filter reviewer comments\n",
    "    comment_notes = [d for d in item \\\n",
    "                     if 'Official_Review' in d['invitation'] and 'recommendation' in d['content'].keys()]\n",
    "    comment_notes = sorted(comment_notes, key=lambda d: d['number'])[::-1]\n",
    "    ratings = [int(note['content']['recommendation'].split(':')[0]) for note in comment_notes]\n",
    "    confidences = [int(note['content']['confidence'].split(':')[0]) for note in comment_notes]\n",
    "    review_lengths = [sum(len(note['content'][key].split()) for key in review_keys) for note in comment_notes] # review lengths\n",
    "\n",
    "    data = {'ratings': ratings, 'confidences': confidences, 'withdraw': withdraw, 'review_lengths': review_lengths}\n",
    "    if decision: data['decision'] = decision\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e92dd456",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddb6ca6c50ee4ef0a4d9b9d4392f0de6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4874 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# filter data in a pool of processes\n",
    "with Pool(8) as p:\n",
    "    filtered_notes = list(tqdm(p.imap(filter_data, notes), total=len(notes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "0939f79d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ratings</th>\n",
       "      <th>confidences</th>\n",
       "      <th>withdraw</th>\n",
       "      <th>review_lengths</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1, 1, 1, 1]</td>\n",
       "      <td>[5, 5, 5, 5]</td>\n",
       "      <td>1</td>\n",
       "      <td>[45, 49, 25, 283]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[5, 6, 8]</td>\n",
       "      <td>[4, 3, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[443, 274, 401]</td>\n",
       "      <td>Accept: poster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[3, 6, 3, 1]</td>\n",
       "      <td>[5, 5, 5, 5]</td>\n",
       "      <td>0</td>\n",
       "      <td>[333, 360, 362, 304]</td>\n",
       "      <td>Reject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[8, 6, 5, 8, 6]</td>\n",
       "      <td>[5, 4, 4, 2, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[203, 142, 323, 520, 752]</td>\n",
       "      <td>Accept: poster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[8, 6, 6]</td>\n",
       "      <td>[3, 2, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[302, 90, 257]</td>\n",
       "      <td>Accept: poster</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ratings      confidences  withdraw             review_lengths  \\\n",
       "0     [1, 1, 1, 1]     [5, 5, 5, 5]         1          [45, 49, 25, 283]   \n",
       "1        [5, 6, 8]        [4, 3, 3]         0            [443, 274, 401]   \n",
       "2     [3, 6, 3, 1]     [5, 5, 5, 5]         0       [333, 360, 362, 304]   \n",
       "3  [8, 6, 5, 8, 6]  [5, 4, 4, 2, 3]         0  [203, 142, 323, 520, 752]   \n",
       "4        [8, 6, 6]        [3, 2, 3]         0             [302, 90, 257]   \n",
       "\n",
       "         decision  \n",
       "0             NaN  \n",
       "1  Accept: poster  \n",
       "2          Reject  \n",
       "3  Accept: poster  \n",
       "4  Accept: poster  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create dataframe\n",
    "ratings = pd.DataFrame(filtered_notes)\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c6602e7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content.title</th>\n",
       "      <th>content.keywords</th>\n",
       "      <th>ratings</th>\n",
       "      <th>confidences</th>\n",
       "      <th>withdraw</th>\n",
       "      <th>review_lengths</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RUzSobdYy0V</td>\n",
       "      <td>Quantifying and Mitigating the Impact of Label...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1, 1, 1, 1]</td>\n",
       "      <td>[5, 5, 5, 5]</td>\n",
       "      <td>1</td>\n",
       "      <td>[45, 49, 25, 283]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N3kGYG3ZcTi</td>\n",
       "      <td>Suppression helps: Lateral Inhibition-inspired...</td>\n",
       "      <td>[Lateral Inhibition, Convolutional Neural Netw...</td>\n",
       "      <td>[5, 6, 8]</td>\n",
       "      <td>[4, 3, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[443, 274, 401]</td>\n",
       "      <td>Accept: poster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tmIiMPl4IPa</td>\n",
       "      <td>Factorized Fourier Neural Operators</td>\n",
       "      <td>[fourier transform, fourier operators, pde, na...</td>\n",
       "      <td>[3, 6, 3, 1]</td>\n",
       "      <td>[5, 5, 5, 5]</td>\n",
       "      <td>0</td>\n",
       "      <td>[333, 360, 362, 304]</td>\n",
       "      <td>Reject</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mhnHqRqcjYU</td>\n",
       "      <td>DFPC: Data flow driven pruning of coupled chan...</td>\n",
       "      <td>[Pruning, Data Free, Model Compression]</td>\n",
       "      <td>[8, 6, 5, 8, 6]</td>\n",
       "      <td>[5, 4, 4, 2, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[203, 142, 323, 520, 752]</td>\n",
       "      <td>Accept: poster</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sZI1Oj9KBKy</td>\n",
       "      <td>TVSPrune - Pruning Non-discriminative filters ...</td>\n",
       "      <td>[Structured pruning, model compression]</td>\n",
       "      <td>[8, 6, 6]</td>\n",
       "      <td>[3, 2, 3]</td>\n",
       "      <td>0</td>\n",
       "      <td>[302, 90, 257]</td>\n",
       "      <td>Accept: poster</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id                                      content.title  \\\n",
       "0  RUzSobdYy0V  Quantifying and Mitigating the Impact of Label...   \n",
       "1  N3kGYG3ZcTi  Suppression helps: Lateral Inhibition-inspired...   \n",
       "2  tmIiMPl4IPa                Factorized Fourier Neural Operators   \n",
       "3  mhnHqRqcjYU  DFPC: Data flow driven pruning of coupled chan...   \n",
       "4  sZI1Oj9KBKy  TVSPrune - Pruning Non-discriminative filters ...   \n",
       "\n",
       "                                    content.keywords          ratings  \\\n",
       "0                                                 []     [1, 1, 1, 1]   \n",
       "1  [Lateral Inhibition, Convolutional Neural Netw...        [5, 6, 8]   \n",
       "2  [fourier transform, fourier operators, pde, na...     [3, 6, 3, 1]   \n",
       "3            [Pruning, Data Free, Model Compression]  [8, 6, 5, 8, 6]   \n",
       "4            [Structured pruning, model compression]        [8, 6, 6]   \n",
       "\n",
       "       confidences  withdraw             review_lengths        decision  \n",
       "0     [5, 5, 5, 5]         1          [45, 49, 25, 283]             NaN  \n",
       "1        [4, 3, 3]         0            [443, 274, 401]  Accept: poster  \n",
       "2     [5, 5, 5, 5]         0       [333, 360, 362, 304]          Reject  \n",
       "3  [5, 4, 4, 2, 3]         0  [203, 142, 323, 520, 752]  Accept: poster  \n",
       "4        [3, 2, 3]         0             [302, 90, 257]  Accept: poster  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge with df_raw_filtered\n",
    "df_final = pd.concat([df_raw_filtered, ratings], axis=1)\n",
    "df_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d83b433",
   "metadata": {},
   "source": [
    "## Save filtered dataset \n",
    "We will be saving a smaller version of the dataset in csv format with the data we need for our analysis - this can also be saved directly in Github"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f2ab086e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save dataframe as csv\n",
    "# rename title\n",
    "df_final.rename(columns={'content.title': 'title'}, inplace=True)\n",
    "#rename keywords\n",
    "df_final.rename(columns={'content.keywords': 'keywords'}, inplace=True)\n",
    "df_final.to_csv(f'{DATA_PATH}{venue_short}_{time.strftime(\"%Y%m%d\")}.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d32cd65",
   "metadata": {},
   "source": [
    "## Saving full crawled dataset\n",
    "\n",
    "Note that this dataset is raw and contains everyting; so it will be pretty large (>100 MBs)!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "90c63c49",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_893738/1720629586.py:5: PerformanceWarning: \n",
      "your performance may suffer as PyTables will pickle object types that it cannot\n",
      "map directly to c-types [inferred_type->mixed-integer,key->axis0] [items->None]\n",
      "\n",
      "  df.to_hdf(f'{DATA_PATH}{venue_short}_data_full_{time.strftime(\"%Y%m%d\")}.h5', key='df', mode='w')\n",
      "/tmp/ipykernel_893738/1720629586.py:5: PerformanceWarning: \n",
      "your performance may suffer as PyTables will pickle object types that it cannot\n",
      "map directly to c-types [inferred_type->mixed,key->block2_values] [items->Index([                                                                    'id',\n",
      "                                                                     'original',\n",
      "                                                                        'mdate',\n",
      "                                                                        'ddate',\n",
      "                                                                       'tddate',\n",
      "                                                                        'forum',\n",
      "                                                                      'replyto',\n",
      "                                                                   'invitation',\n",
      "                                                                   'signatures',\n",
      "                                                                      'readers',\n",
      "                                                                   'nonreaders',\n",
      "                                                                      'writers',\n",
      "                                                                'content.title',\n",
      "                                                            'content.authorids',\n",
      "                                                              'content.authors',\n",
      "                                                             'content.keywords',\n",
      "                                                             'content.abstract',\n",
      "                                                        'content.anonymous_url',\n",
      "                                           'content.no_acknowledgement_section',\n",
      "                                                       'content.code_of_ethics',\n",
      "                                                'content.submission_guidelines',\n",
      "                                                         'content.resubmission',\n",
      "                                                       'content.student_author',\n",
      "       'content.Please_choose_the_closest_area_that_your_submission_falls_into',\n",
      "                                                            'content.paperhash',\n",
      "                                                                  'content.pdf',\n",
      "                                               'content.supplementary_material',\n",
      "                                                              'content._bibtex',\n",
      "                                                                'content.venue',\n",
      "                                                              'content.venueid',\n",
      "                                                                'content.TL;DR',\n",
      "                                                                              0,\n",
      "                                                                              1,\n",
      "                                                                              2,\n",
      "                                                                              3,\n",
      "                                                                              4,\n",
      "                                                                              5,\n",
      "                                                                              6,\n",
      "                                                                              7,\n",
      "                                                                              8,\n",
      "                                                                              9,\n",
      "                                                                             10,\n",
      "                                                                             11,\n",
      "                                                                             12,\n",
      "                                                                             13,\n",
      "                                                                             14,\n",
      "                                                                             15,\n",
      "                                                                             16,\n",
      "                                                                             17,\n",
      "                                                                             18,\n",
      "                                                                             19,\n",
      "                                                                             20,\n",
      "                                                                             21,\n",
      "                                                                             22,\n",
      "                                                                             23,\n",
      "                                                                             24,\n",
      "                                                                             25,\n",
      "                                                                             26,\n",
      "                                                                             27,\n",
      "                                                                             28,\n",
      "                                                                             29,\n",
      "                                                                             30,\n",
      "                                                                             31,\n",
      "                                                                             32,\n",
      "                                                                             33,\n",
      "                                                                             34,\n",
      "                                                                             35,\n",
      "                                                                             36,\n",
      "                                                                             37,\n",
      "                                                                             38,\n",
      "                                                                             39,\n",
      "                                                                             40,\n",
      "                                                                             41,\n",
      "                                                                             42,\n",
      "                                                                             43,\n",
      "                                                                             44,\n",
      "                                                                             45,\n",
      "                                                                             46,\n",
      "                                                                             47,\n",
      "                                                                             48,\n",
      "                                                                             49,\n",
      "                                                                             50,\n",
      "                                                                             51,\n",
      "                                                                             52,\n",
      "                                                                             53,\n",
      "                                                                             54,\n",
      "                                                                             55,\n",
      "                                                                             56,\n",
      "                                                                             57,\n",
      "                                                                             58,\n",
      "                                                                             59,\n",
      "                                                                             60,\n",
      "                                                                             61,\n",
      "                                                                             62,\n",
      "                                                                             63,\n",
      "                                                                             64,\n",
      "                                                                             65,\n",
      "                                                                             66],\n",
      "      dtype='object')]\n",
      "\n",
      "  df.to_hdf(f'{DATA_PATH}{venue_short}_data_full_{time.strftime(\"%Y%m%d\")}.h5', key='df', mode='w')\n",
      "/tmp/ipykernel_893738/1720629586.py:5: PerformanceWarning: \n",
      "your performance may suffer as PyTables will pickle object types that it cannot\n",
      "map directly to c-types [inferred_type->mixed-integer,key->block2_items] [items->None]\n",
      "\n",
      "  df.to_hdf(f'{DATA_PATH}{venue_short}_data_full_{time.strftime(\"%Y%m%d\")}.h5', key='df', mode='w')\n"
     ]
    }
   ],
   "source": [
    "# Save dataframe as hdf5\n",
    "notes_df = pd.DataFrame([n['notes'] for n in data])\n",
    "count_df = pd.DataFrame({'notes_count': [n['count'] for n in data]})\n",
    "df = pd.concat([df_raw, notes_df, count_df], axis=1)\n",
    "df.to_hdf(f'{DATA_PATH}{venue_short}_data_full_{time.strftime(\"%Y%m%d\")}.h5', key='df', mode='w')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2.7.12 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}