Eval_Finetuning_10-fold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0dd146e7-a609-42dc-ba20-8c70699b0511",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shjiang/miniconda3/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2023-09-25 18:04:58.317224: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-09-25 18:04:58.369487: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-09-25 18:04:59.377519: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer, get_cosine_schedule_with_warmup\n",
    "from seqeval.metrics import classification_report\n",
    "\n",
    "from torch import cuda\n",
    "device = 'cuda' if cuda.is_available() else 'cpu'\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "392fc23e-4821-4588-91b2-e68953dd2610",
   "metadata": {},
   "source": [
    "## Launche NER classifer and evaluate the prediction - CCRoberta-ep10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "62e4e3e5-e52e-4291-9d05-6ed9f62e8653",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 33min 27s, sys: 48.3 s, total: 1h 34min 15s\n",
      "Wall time: 12min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#df_test_data = pd.read_csv('data/test_GPT+labels.csv')\n",
    "df_test_data = pd.read_csv('GPT_results/Human_corrected_annotations+gpt_res.csv')\n",
    "\n",
    "stat_list = []\n",
    "for trainset_num in range(1, 11):\n",
    "\n",
    "    eval_list = []\n",
    "    model_name = f'ner_model/allenai/scibert_scivocab_uncased_ft_3ep_train_size_10240_trainset_{trainset_num}'\n",
    "    model = AutoModelForTokenClassification.from_pretrained(model_name)\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_name,truncation=True, model_max_length=512)\n",
    "    pipe = pipeline(task=\"token-classification\", model=model, tokenizer=tokenizer, aggregation_strategy=\"first\")\n",
    "    \n",
    "    for index, row in df_test_data.iterrows():\n",
    "    \n",
    "        #let's remove repeated terms, keeping 1616 unique out of 1660 terms\n",
    "        expected_list = set(row['plain_text_term'].split(';'))\n",
    "        while '' in expected_list:\n",
    "            expected_list.remove('')\n",
    "\n",
    "        #when using test_GPT+labels.csv\n",
    "        #extracted_list = pipe(row['sentence'])\n",
    "        \n",
    "        #when using test_GPT+labels.csv\n",
    "        extracted_list = pipe(row['plain_text_def'])\n",
    "        \n",
    "        extracted_list = [x['word'].strip() for x in extracted_list] \n",
    "        while '' in extracted_list:\n",
    "            extracted_list.remove('')\n",
    "        \n",
    "        num_TP = 0\n",
    "        num_too_long = 0\n",
    "        num_cut_off = 0\n",
    "        num_split_term = 0\n",
    "        TP_list = []\n",
    "        ST_list = [] \n",
    "        \n",
    "        for expected in expected_list:\n",
    "            for extracted in extracted_list:\n",
    "                \n",
    "                if extracted.casefold() == expected.casefold():\n",
    "                    num_TP = num_TP + 1\n",
    "                    TP_list.append(expected)\n",
    "                elif extracted.casefold() in expected.casefold():\n",
    "                    num_cut_off = num_cut_off + 1\n",
    "                elif expected.casefold() in extracted.casefold():\n",
    "                    num_too_long = num_too_long + 1\n",
    "                    \n",
    "            expected_no_space = expected.replace(\" \",\"\")\n",
    "            extracted_no_space = (\"\".join(extracted_list)).replace(\" \",\"\")\n",
    "            if expected_no_space.casefold() in extracted_no_space.casefold(): # including TPs\n",
    "                num_split_term = num_split_term + 1\n",
    "                ST_list.append(expected)\n",
    "        \n",
    "        num_TP = num_TP - (len(TP_list) - len(set(TP_list)))\n",
    "        num_split_term = num_split_term - (len(ST_list) -len(set(ST_list)))\n",
    "        \n",
    "        eval_list.append({'True Term Num' : len(expected_list),\n",
    "                            'Extracted Term Num': len(extracted_list),\n",
    "                            'TP': num_TP,\n",
    "                            'Cut Off': num_cut_off,\n",
    "                            'Too Long': num_too_long,\n",
    "                            'Split Term': num_split_term,\n",
    "                            'extracted': '###'.join(extracted_list)})\n",
    "    df_eval = pd.DataFrame(eval_list)\n",
    "    df_eval['expected'] = df_test_data['plain_text_term']    \n",
    "    df_eval.to_csv(f'GPT_results/scibert_scivocab_uncased_ft_3ep_train_size_10240_trainset_{trainset_num}_first_eval.csv', index=False)\n",
    "\n",
    "    num_T = df_eval['True Term Num'].sum()\n",
    "    num_Ex = df_eval['Extracted Term Num'].sum()\n",
    "    num_ST = df_eval['Split Term'].sum()\n",
    "    precision = num_ST / num_Ex\n",
    "    recall = num_ST / num_T\n",
    "    stat_list.append({\n",
    "        \"model_name\": model_name,\n",
    "        \"True Term Num\": num_T,\n",
    "        \"Extracted Term Num\": num_Ex,\n",
    "        \"True positive\": df_eval['TP'].sum(),\n",
    "        \"True positive + split terms\": num_ST,\n",
    "        \"Too Long\": df_eval['Too Long'].sum(),\n",
    "        \"Cut Off\": df_eval['Cut Off'].sum(),\n",
    "        \"precision /correct rate\": precision,\n",
    "        \"recall\": recall,\n",
    "        \"f1\": 2 * precision * recall / ( precision + recall )\n",
    "    })\n",
    "df_stat = pd.DataFrame(stat_list)\n",
    "df_stat.to_csv(f'GPT_results/scibert_scivocab_uncased_ft_3ep_train_size_10240_first_eval_stat.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e1baf534-f4a2-4690-90c3-78114e2e7493",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>True Term Num</th>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <th>True positive</th>\n",
       "      <th>True positive + split terms</th>\n",
       "      <th>Too Long</th>\n",
       "      <th>Cut Off</th>\n",
       "      <th>precision /correct rate</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2534</td>\n",
       "      <td>936</td>\n",
       "      <td>1277</td>\n",
       "      <td>153</td>\n",
       "      <td>1735</td>\n",
       "      <td>0.503946</td>\n",
       "      <td>0.822809</td>\n",
       "      <td>0.625061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2182</td>\n",
       "      <td>829</td>\n",
       "      <td>1110</td>\n",
       "      <td>123</td>\n",
       "      <td>1483</td>\n",
       "      <td>0.508708</td>\n",
       "      <td>0.715206</td>\n",
       "      <td>0.594537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2543</td>\n",
       "      <td>902</td>\n",
       "      <td>1248</td>\n",
       "      <td>153</td>\n",
       "      <td>1731</td>\n",
       "      <td>0.490759</td>\n",
       "      <td>0.804124</td>\n",
       "      <td>0.609524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2471</td>\n",
       "      <td>922</td>\n",
       "      <td>1256</td>\n",
       "      <td>144</td>\n",
       "      <td>1655</td>\n",
       "      <td>0.508296</td>\n",
       "      <td>0.809278</td>\n",
       "      <td>0.624410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2428</td>\n",
       "      <td>929</td>\n",
       "      <td>1251</td>\n",
       "      <td>151</td>\n",
       "      <td>1570</td>\n",
       "      <td>0.515239</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>0.628643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2584</td>\n",
       "      <td>925</td>\n",
       "      <td>1297</td>\n",
       "      <td>177</td>\n",
       "      <td>1703</td>\n",
       "      <td>0.501935</td>\n",
       "      <td>0.835696</td>\n",
       "      <td>0.627176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2432</td>\n",
       "      <td>902</td>\n",
       "      <td>1225</td>\n",
       "      <td>138</td>\n",
       "      <td>1683</td>\n",
       "      <td>0.503701</td>\n",
       "      <td>0.789304</td>\n",
       "      <td>0.614960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2476</td>\n",
       "      <td>914</td>\n",
       "      <td>1272</td>\n",
       "      <td>184</td>\n",
       "      <td>1500</td>\n",
       "      <td>0.513732</td>\n",
       "      <td>0.819588</td>\n",
       "      <td>0.631579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2655</td>\n",
       "      <td>961</td>\n",
       "      <td>1283</td>\n",
       "      <td>139</td>\n",
       "      <td>1759</td>\n",
       "      <td>0.483239</td>\n",
       "      <td>0.826675</td>\n",
       "      <td>0.609936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ner_model/allenai/scibert_scivocab_uncased_ft_...</td>\n",
       "      <td>1552</td>\n",
       "      <td>2525</td>\n",
       "      <td>895</td>\n",
       "      <td>1292</td>\n",
       "      <td>217</td>\n",
       "      <td>1588</td>\n",
       "      <td>0.511683</td>\n",
       "      <td>0.832474</td>\n",
       "      <td>0.633799</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          model_name  True Term Num  \\\n",
       "0  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "1  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "2  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "3  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "4  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "5  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "6  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "7  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "8  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "9  ner_model/allenai/scibert_scivocab_uncased_ft_...           1552   \n",
       "\n",
       "   Extracted Term Num  True positive  True positive + split terms  Too Long  \\\n",
       "0                2534            936                         1277       153   \n",
       "1                2182            829                         1110       123   \n",
       "2                2543            902                         1248       153   \n",
       "3                2471            922                         1256       144   \n",
       "4                2428            929                         1251       151   \n",
       "5                2584            925                         1297       177   \n",
       "6                2432            902                         1225       138   \n",
       "7                2476            914                         1272       184   \n",
       "8                2655            961                         1283       139   \n",
       "9                2525            895                         1292       217   \n",
       "\n",
       "   Cut Off  precision /correct rate    recall        f1  \n",
       "0     1735                 0.503946  0.822809  0.625061  \n",
       "1     1483                 0.508708  0.715206  0.594537  \n",
       "2     1731                 0.490759  0.804124  0.609524  \n",
       "3     1655                 0.508296  0.809278  0.624410  \n",
       "4     1570                 0.515239  0.806057  0.628643  \n",
       "5     1703                 0.501935  0.835696  0.627176  \n",
       "6     1683                 0.503701  0.789304  0.614960  \n",
       "7     1500                 0.513732  0.819588  0.631579  \n",
       "8     1759                 0.483239  0.826675  0.609936  \n",
       "9     1588                 0.511683  0.832474  0.633799  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6393f7d5-e815-4efd-aa97-194e1b341fe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True Term Num</th>\n",
       "      <td>1552.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <td>2483.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive</th>\n",
       "      <td>911.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive + split terms</th>\n",
       "      <td>1251.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Too Long</th>\n",
       "      <td>157.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cut Off</th>\n",
       "      <td>1640.700000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision /correct rate</th>\n",
       "      <td>0.504124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.806121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.619962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       0\n",
       "True Term Num                1552.000000\n",
       "Extracted Term Num           2483.000000\n",
       "True positive                 911.500000\n",
       "True positive + split terms  1251.100000\n",
       "Too Long                      157.900000\n",
       "Cut Off                      1640.700000\n",
       "precision /correct rate         0.504124\n",
       "recall                          0.806121\n",
       "f1                              0.619962"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(df_stat.mean(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d3473752-4f6c-47e4-b965-b046af985483",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True Term Num</th>\n",
       "      <td>1552.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <td>2481.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive</th>\n",
       "      <td>863.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive + split terms</th>\n",
       "      <td>1201.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Too Long</th>\n",
       "      <td>171.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cut Off</th>\n",
       "      <td>1648.900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision /correct rate</th>\n",
       "      <td>0.485267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.774227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.595982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       0\n",
       "True Term Num                1552.000000\n",
       "Extracted Term Num           2481.200000\n",
       "True positive                 863.200000\n",
       "True positive + split terms  1201.600000\n",
       "Too Long                      171.200000\n",
       "Cut Off                      1648.900000\n",
       "precision /correct rate         0.485267\n",
       "recall                          0.774227\n",
       "f1                              0.595982"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat_sci_2048 = pd.read_csv('GPT_results/scibert_scivocab_uncased_ft_5ep_train_size_2048_first_eval_stat.csv')\n",
    "pd.DataFrame(df_stat_sci_2048.mean(numeric_only=True))\n",
    "#pd.DataFrame(df_stat_10_3.std(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32f9a8a6-5633-4f7e-8b25-0d460c97f667",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_stat_10_3 = pd.read_csv('GPT_results/cc_math_roberta_ep10_ft_3ep_train_size_10240_first_eval_stat.csv')\n",
    "#pd.DataFrame(df_stat_10_3.mean(numeric_only=True))\n",
    "pd.DataFrame(df_stat_10_3.std(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "ff59aaf3-b2c5-41a5-9143-089c028416e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_70688/2642871097.py:3: FutureWarning: The default value of numeric_only in DataFrame.std is deprecated. In a future version, it will default to False. In addition, specifying 'numeric_only=None' is deprecated. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  pd.DataFrame(df_stat_01_3.std())\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True Term Num</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <td>333.799274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive</th>\n",
       "      <td>88.298798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive + split terms</th>\n",
       "      <td>96.693617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Too Long</th>\n",
       "      <td>37.247670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cut Off</th>\n",
       "      <td>104.628656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision /correct rate</th>\n",
       "      <td>0.057420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.062303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.042937</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      0\n",
       "True Term Num                  0.000000\n",
       "Extracted Term Num           333.799274\n",
       "True positive                 88.298798\n",
       "True positive + split terms   96.693617\n",
       "Too Long                      37.247670\n",
       "Cut Off                      104.628656\n",
       "precision /correct rate        0.057420\n",
       "recall                         0.062303\n",
       "f1                             0.042937"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat_01_3 = pd.read_csv('GPT_results/cc_math_roberta_ep01_ft_3ep_train_size_10240_first_eval_stat.csv')\n",
    "#pd.DataFrame(df_stat_01_3.mean(numeric_only=True))\n",
    "pd.DataFrame(df_stat_01_3.std(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "ac5b661c-56e8-4a01-ab4c-23fb813c2bc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_70688/4170808313.py:3: FutureWarning: The default value of numeric_only in DataFrame.std is deprecated. In a future version, it will default to False. In addition, specifying 'numeric_only=None' is deprecated. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  pd.DataFrame(df_stat_ro_3.std())\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True Term Num</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <td>85.684499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive</th>\n",
       "      <td>29.896860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive + split terms</th>\n",
       "      <td>24.253293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Too Long</th>\n",
       "      <td>24.680402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cut Off</th>\n",
       "      <td>27.790686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision /correct rate</th>\n",
       "      <td>0.023374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.015627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.010583</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     0\n",
       "True Term Num                 0.000000\n",
       "Extracted Term Num           85.684499\n",
       "True positive                29.896860\n",
       "True positive + split terms  24.253293\n",
       "Too Long                     24.680402\n",
       "Cut Off                      27.790686\n",
       "precision /correct rate       0.023374\n",
       "recall                        0.015627\n",
       "f1                            0.010583"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat_ro_3 = pd.read_csv('GPT_results/roberta-base_ft_3ep_train_size_10240_first_eval_stat.csv')\n",
    "#pd.DataFrame(df_stat_ro_3.mean(numeric_only=True))\n",
    "pd.DataFrame(df_stat_ro_3.std(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "fdd01f84-306e-4758-bc59-6621b18e7998",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True Term Num</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <td>693.232044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive</th>\n",
       "      <td>95.847622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive + split terms</th>\n",
       "      <td>132.830552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Too Long</th>\n",
       "      <td>65.555405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cut Off</th>\n",
       "      <td>233.797253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision /correct rate</th>\n",
       "      <td>0.081552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.085587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.044332</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      0\n",
       "True Term Num                  0.000000\n",
       "Extracted Term Num           693.232044\n",
       "True positive                 95.847622\n",
       "True positive + split terms  132.830552\n",
       "Too Long                      65.555405\n",
       "Cut Off                      233.797253\n",
       "precision /correct rate        0.081552\n",
       "recall                         0.085587\n",
       "f1                             0.044332"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat_01_5 = pd.read_csv('GPT_results/cc_math_roberta_ep01_ft_5ep_train_size_2048_first_eval_stat_corrected.csv')\n",
    "#print(df_stat_ro_3.mean(numeric_only=True))\n",
    "pd.DataFrame(df_stat_01_5.std(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "178d070c-e5f3-4cd0-9ed0-99d568865ae9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_70688/3069313623.py:4: FutureWarning: The default value of numeric_only in DataFrame.std is deprecated. In a future version, it will default to False. In addition, specifying 'numeric_only=None' is deprecated. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  pd.DataFrame(df_stat_10_5.std())\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True Term Num</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <td>261.143128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive</th>\n",
       "      <td>100.494113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive + split terms</th>\n",
       "      <td>120.302628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Too Long</th>\n",
       "      <td>64.220886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cut Off</th>\n",
       "      <td>80.113531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision /correct rate</th>\n",
       "      <td>0.060666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.077515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.051962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      0\n",
       "True Term Num                  0.000000\n",
       "Extracted Term Num           261.143128\n",
       "True positive                100.494113\n",
       "True positive + split terms  120.302628\n",
       "Too Long                      64.220886\n",
       "Cut Off                       80.113531\n",
       "precision /correct rate        0.060666\n",
       "recall                         0.077515\n",
       "f1                             0.051962"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat_10_5 = pd.read_csv('GPT_results/cc_math_roberta_ep10_ft_5ep_train_size_2048_first_eval_stat.csv')\n",
    "#pd.DataFrame(df_stat_10_5.mean(numeric_only=True))\n",
    "#print(df_stat_10_5.mean(numeric_only=True))\n",
    "pd.DataFrame(df_stat_10_5.std(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "5be904aa-a3e1-46a1-9138-cb6207fa3311",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_70688/2768062811.py:3: FutureWarning: The default value of numeric_only in DataFrame.std is deprecated. In a future version, it will default to False. In addition, specifying 'numeric_only=None' is deprecated. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  pd.DataFrame(df_stat_ro_5.std())\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True Term Num</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <td>291.651161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive</th>\n",
       "      <td>86.461038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True positive + split terms</th>\n",
       "      <td>118.986694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Too Long</th>\n",
       "      <td>86.441759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cut Off</th>\n",
       "      <td>46.124108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision /correct rate</th>\n",
       "      <td>0.049217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.076667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.031259</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      0\n",
       "True Term Num                  0.000000\n",
       "Extracted Term Num           291.651161\n",
       "True positive                 86.461038\n",
       "True positive + split terms  118.986694\n",
       "Too Long                      86.441759\n",
       "Cut Off                       46.124108\n",
       "precision /correct rate        0.049217\n",
       "recall                         0.076667\n",
       "f1                             0.031259"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat_ro_5 = pd.read_csv('GPT_results/roberta-base_ft_5ep_train_size_2048_first_eval_stat_corrected.csv')\n",
    "#pd.DataFrame(df_stat_ro_5.mean(numeric_only=True))\n",
    "pd.DataFrame(df_stat_ro_5.std(numeric_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54f43848-b202-4c42-8729-038f2001044f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6502595023873526, 0.7186881188118812, 0.6810759139456813)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat['precision /correct rate'].mean(), df_stat['recall'].mean(), df_stat['f1'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a994243a-48a2-4154-af4d-62aba8b10aeb",
   "metadata": {},
   "source": [
    "## Update test set... and eval res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "0f6d3827-d0f3-428c-b986-d875dd805cef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n",
      "111\n",
      "198\n",
      "271\n",
      "336\n",
      "337\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "447\n",
      "448\n",
      "600\n",
      "867\n",
      "868\n",
      "869\n",
      "870\n",
      "871\n",
      "881\n"
     ]
    }
   ],
   "source": [
    "rm_list = []\n",
    "df_old_res = pd.read_csv('GPT_results/gpt-4_res+eval.csv')\n",
    "df_updated_test_data = pd.read_csv('GPT_results/Human_corrected_annotations+gpt_res.csv')\n",
    "\n",
    "old_gpt4_list = df_old_res['gpt-4 terms'].to_list()\n",
    "old_tt_list = df_old_res['plain_text_term'].to_list()\n",
    "\n",
    "new_gpt4_list = df_updated_test_data['gpt-4 terms'].to_list()\n",
    "new_tt_list = df_updated_test_data['plain_text_term'].to_list()\n",
    "\n",
    "j = 0\n",
    "for i in range(0, 999):\n",
    "    while old_gpt4_list[j] != new_gpt4_list[i]:\n",
    "        print(j)\n",
    "        rm_list.append(j)\n",
    "        j = j+1       \n",
    "    j = j + 1\n",
    "\n",
    "update_list=[]\n",
    "for i in range(0, 999):\n",
    "    while old_gpt4_list[i] != new_gpt4_list[i]:\n",
    "        del old_gpt4_list[i], old_tt_list[i]\n",
    "\n",
    "    if old_tt_list[i] == new_tt_list[i]:\n",
    "        continue\n",
    "    else:\n",
    "        update_list.append({'idx':i, 'expected': new_tt_list[i], 'old_expected': old_tt_list[i]})\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "007f3794-4c41-4ed1-961d-c80ac3616621",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'idx': 365, 'expected': 'Poincaré lemma', 'old_expected': '(A Poincaré'},\n",
       " {'idx': 368, 'expected': 'non-regular', 'old_expected': 'non-'},\n",
       " {'idx': 496,\n",
       "  'expected': 'Bounded difference assumption',\n",
       "  'old_expected': '(Bounded difference assumption)'},\n",
       " {'idx': 505, 'expected': 'reduction', 'old_expected': 'reduction;i.e.'},\n",
       " {'idx': 514,\n",
       "  'expected': 'substring;length',\n",
       "  'old_expected': 'i;substring;length'},\n",
       " {'idx': 583,\n",
       "  'expected': 'The Dehn-Sommerville Equations',\n",
       "  'old_expected': '(The Dehn-Sommerville Equations)'},\n",
       " {'idx': 584,\n",
       "  'expected': \"McMullen's Upper Bound Theorem\",\n",
       "  'old_expected': \"(McMullen's Upper Bound Theorem)\"},\n",
       " {'idx': 585,\n",
       "  'expected': 'The Generalized Lower Bound Theorem',\n",
       "  'old_expected': '(The Generalized Lower Bound Theorem)'},\n",
       " {'idx': 586,\n",
       "  'expected': 'The Generalized Dehn-Sommerville Equations',\n",
       "  'old_expected': '(The Generalized Dehn-Sommerville Equations)'},\n",
       " {'idx': 587,\n",
       "  'expected': 'Bayer and Ehrenborg',\n",
       "  'old_expected': '(Bayer and Ehrenborg)'},\n",
       " {'idx': 978,\n",
       "  'expected': 'Erdős-Szekeres on-line game;̋ Erdős-Szekeres on-line number',\n",
       "  'old_expected': 'The Erdő;̋E̋r̋d̋ő̋'}]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "update_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "3bd8a0cb-2df4-4118-9009-fdce85fde20f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "999\n",
      "999\n",
      "999\n",
      "999\n",
      "999\n",
      "999\n",
      "999\n",
      "999\n",
      "999\n",
      "999\n"
     ]
    }
   ],
   "source": [
    "for trainset_num in range(1, 11):\n",
    "\n",
    "    df_eval = pd.read_csv(f'GPT_results/cc_math_roberta_ep10_ft_3ep_train_size_10240_trainset_{trainset_num}_first_eval.csv')\n",
    "    df_eval = df_eval.drop([ df_eval.index[i] for i in rm_list])\n",
    "    print(len(df_eval))\n",
    "\n",
    "    for x in update_list:\n",
    "        index_ici = x['idx']\n",
    "        expected = x['expected']\n",
    "        df_eval['expected'] = df_eval['expected'].replace(x['old_expected'], x['expected'])\n",
    "    \n",
    "    df_eval.to_csv(f'GPT_results/cc_math_roberta_ep10_ft_3ep_train_size_10240_trainset_{trainset_num}_first_eval_tmp.csv',index=False)\n",
    "\n",
    "     \n",
    "\n",
    "stat_list = []\n",
    "\n",
    "for trainset_num in range(1, 11):\n",
    "\n",
    "    df_res = pd.read_csv(f'GPT_results/cc_math_roberta_ep10_ft_3ep_train_size_10240_trainset_{trainset_num}_first_eval_tmp.csv')\n",
    "    eval_list =[]\n",
    "\n",
    "    for index, row in df_res.iterrows():\n",
    "\n",
    "        #let's remove repeated terms, keeping 1616 unique out of 1660 terms\n",
    "        expected_list = set(row['expected'].split(';'))\n",
    "        while '' in expected_list:\n",
    "            expected_list.remove('')\n",
    "\n",
    "        extracted_string = row['extracted']#['gpt-3.5-turbo terms']\n",
    "        if extracted_string != extracted_string: #nan\n",
    "            extracted_list = []\n",
    "        else:\n",
    "            extracted_list = extracted_string.split('###')\n",
    "            extracted_list = [x.strip() for x in extracted_list] \n",
    "            while '' in extracted_list:\n",
    "                extracted_list.remove('')\n",
    "\n",
    "        num_TP = 0\n",
    "        num_too_long = 0\n",
    "        num_cut_off = 0\n",
    "        num_split_term = 0\n",
    "        TP_list = []\n",
    "        ST_list = [] \n",
    "\n",
    "        for expected in expected_list:\n",
    "            for extracted in extracted_list:\n",
    "\n",
    "                if extracted.casefold() == expected.casefold():\n",
    "                    num_TP = num_TP + 1\n",
    "                    TP_list.append(expected)\n",
    "                elif extracted.casefold() in expected.casefold():\n",
    "                    num_cut_off = num_cut_off + 1\n",
    "                elif expected.casefold() in extracted.casefold():\n",
    "                    num_too_long = num_too_long + 1\n",
    "\n",
    "            expected_no_space = expected.replace(\" \",\"\")\n",
    "            extracted_no_space = (\"\".join(extracted_list)).replace(\" \",\"\")\n",
    "            if expected_no_space.casefold() in extracted_no_space.casefold(): # including TPs\n",
    "                num_split_term = num_split_term + 1\n",
    "                ST_list.append(expected)\n",
    "\n",
    "        num_TP = num_TP - (len(TP_list) - len(set(TP_list)))\n",
    "        num_split_term = num_split_term - (len(ST_list) -len(set(ST_list)))\n",
    "\n",
    "        eval_list.append({'True Term Num' : len(expected_list),\n",
    "                            'Extracted Term Num': len(extracted_list),\n",
    "                            'TP': num_TP,\n",
    "                            'Cut Off': num_cut_off,\n",
    "                            'Too Long': num_too_long,\n",
    "                            'Split Term': num_split_term})\n",
    "    df_eval = pd.DataFrame(eval_list)\n",
    "    df_eval['expected'] = df_res['expected']\n",
    "    df_eval['extracted'] = df_res['extracted']\n",
    "    df_eval.to_csv(f'GPT_results/cc_math_roberta_ep10_ft_3ep_train_size_10240_trainset_{trainset_num}_first_eval_corrected.csv', index=False)\n",
    "    num_T = df_eval['True Term Num'].sum()\n",
    "    num_Ex = df_eval['Extracted Term Num'].sum()\n",
    "    num_ST = df_eval['Split Term'].sum()\n",
    "    precision = num_ST / num_Ex\n",
    "    recall = num_ST / num_T\n",
    "    stat_list.append({\n",
    "        \"True Term Num\": num_T,\n",
    "        \"Extracted Term Num\": num_Ex,\n",
    "        \"True positive\": df_eval['TP'].sum(),\n",
    "        \"True positive + split terms\": num_ST,\n",
    "        \"Too Long\": df_eval['Too Long'].sum(),\n",
    "        \"Cut Off\": df_eval['Cut Off'].sum(),\n",
    "        \"precision /correct rate\": precision,\n",
    "        \"recall\": recall,\n",
    "        \"f1\": 2 * precision * recall / ( precision + recall )\n",
    "    })\n",
    "df_stat = pd.DataFrame(stat_list)\n",
    "df_stat.to_csv(f'GPT_results/cc_math_roberta_ep10_ft_3ep_train_size_10240_first_eval_stat_corrected.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "71d94f31-bca7-4781-8dbd-05191f517768",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6516446172588075, 0.742590206185567, 0.6924612342211107)"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stat['precision /correct rate'].mean(), df_stat['recall'].mean(), df_stat['f1'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ce8a386c-a7eb-43c2-90be-451a4c5ba74c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7314987977558107"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# GPT-4\n",
    "p = 0.6248288452761296\n",
    "r = 0.8820876288659794\n",
    "2 * p * r / (p + r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4a1db62d-0088-406f-b65c-a6e0c78a7484",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.31314479912610754"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# GPT-3.5\n",
    "p = 0.19291161956034095\n",
    "r = 0.8311855670103093\n",
    "2 * p * r / (p + r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7f0d955d-c818-49e3-ae60-4ef1e8fb548a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7301304863582444"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#roberta_base testset 10 11367\n",
    "p = 0.7010250569476082\n",
    "r = 0.7617574257425742\n",
    "2 * p * r / (p + r)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eb0f1ef8-b129-4732-9358-ec409b293302",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7216890595009596"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#roberta_base testset 1 11366\n",
    "p = 0.7470198675496689\n",
    "r = 0.698019801980198\n",
    "2 * p * r / (p + r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65d8a8d9-d93d-48e6-b831-d6394e326f30",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}