03_evaluate_alephbert.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Re-evaluation\n",
    "Perform two evaluations:\n",
    "1. Strict morpheme evaluation\n",
    "1. Token evaluation (morpheme labels are extended to the token level heuristically)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.034028Z",
     "start_time": "2019-03-13T07:20:22.687626Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.382406Z",
     "start_time": "2019-03-13T07:20:24.037019Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:27.996727Z",
     "start_time": "2019-03-13T07:20:24.385088Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_context('paper')\n",
    "sns.set_style('white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/nlp/danb/NER')\n",
    "\n",
    "import bclm\n",
    "import ne_evaluate_mentions as nem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create BIOSE files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def jsonl_to_biose(in_path, out_path, bioul_to_biose=True):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            sent = json.loads(line)\n",
    "            for word, tag in zip(sent['words'], sent['tags']):\n",
    "                if bioul_to_biose:\n",
    "                    tag = tag.replace('L-', 'E-').replace('U-', 'S-')\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            of.write('\\n')\n",
    "            sents+=1\n",
    "    print (sents)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_54360\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_54360\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_54360\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_54360\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_54360\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_54360\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_44184\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_44184\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_20423\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_20423\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_44184\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_44184\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_44184\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_44184\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_80520\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_80520\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_27916\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/single_27916\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_20423\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_20423\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_20423\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_20423\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_80520\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_80520\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_80520\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_80520\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_27916\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_27916\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_27916\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/morph_27916\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_54360\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_54360\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_44184\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_44184\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_20423\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_20423\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_80520\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_80520\n",
      "706\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_27916\n",
      "500\n",
      "output/predict_alephbert/bert-small-wordpiece-oscar-52000-10/multi_27916\n",
      "706\n"
     ]
    }
   ],
   "source": [
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    for folder in os.scandir(trans):\n",
    "        if not '.ipynb' in folder.name:\n",
    "            for file in os.scandir(folder):\n",
    "                if '.json' in file.name and not '.ipynb' in file.name:\n",
    "                    output_path = file.path.replace('.json', '.bmes')\n",
    "                    if not os.path.exists(output_path):\n",
    "                        print (folder.path)\n",
    "                        jsonl_to_biose(file.path, output_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tasks\n",
    "1. Strict morpheme evaluation\n",
    "   1. morph - as they are now\n",
    "   1. token - evaluate token mentions against gold morpheme mentions\n",
    "   1. multi - a) token vs gold morph b) yap/pruned vs gold morph\n",
    "1. Token evaluation \n",
    "   1. morph - extend heuristically and evaluate against gold token mentions\n",
    "   1. token - as it is now\n",
    "   1. multi - as it is now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_biose_count(path, sent_id_shift=1):\n",
    "    sents = nem.read_file_sents(path, fix_multi_tag=False, sent_id_shift=sent_id_shift)\n",
    "    bc = []\n",
    "    for i, sent in sents.iteritems():\n",
    "        for j, (tok, bio) in enumerate(sent):\n",
    "            bc.append([i, j+1, tok, bio, len(bio.split('^'))])\n",
    "\n",
    "    bc = pd.DataFrame(bc, columns=['sent_id', 'token_id', 'token_str', \n",
    "                                   'biose', 'biose_count'])\n",
    "    return bc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_valid_edges(lattices, bc,\n",
    "                    non_o_only=True, keep_all_if_no_valid=True):\n",
    "    valid_edges = []\n",
    "    for (i, df), (_, biose, biose_count) in zip(lattices.groupby(['sent_id', 'token_id']), \n",
    "                                                bc[['biose', 'biose_count']].itertuples()):\n",
    "        el = df[['ID1', 'ID2']].rename(columns={'ID1': 'source', 'ID2': 'target'})\n",
    "        #min_node = [n for n,v in G.nodes(data=True) if v['since'] == 'December 2008'][0]\n",
    "\n",
    "        g = nx.from_pandas_edgelist(el, create_using=nx.DiGraph)\n",
    "        min_node = el.source.min()\n",
    "        max_node = el.target.max()\n",
    "        #print(min_node,max_node)\n",
    "        #print(biose_count)\n",
    "        if non_o_only and not '-' in biose:\n",
    "            vp = list(nx.all_simple_paths(g, min_node, max_node))\n",
    "        else:\n",
    "            vp = [path for path in nx.all_simple_paths(g, min_node, max_node, cutoff=biose_count+1) if len(path)==biose_count+1]\n",
    "        if keep_all_if_no_valid and len(vp)==0:\n",
    "             vp = nx.all_simple_paths(g, min_node, max_node)\n",
    "        for path in vp:\n",
    "            for source, target in zip(path[:-1], path[1:]):\n",
    "                valid_edges.append((i[0], i[1], source, target))\n",
    "                \n",
    "    return valid_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_lattices(df, path, cols = ['ID1', 'ID2', 'form', 'lemma', 'upostag', 'xpostag', 'feats', 'token_id']):\n",
    "    with open(path, 'w', encoding='utf8') as of:\n",
    "        for _, sent in df.groupby('sent_id'):\n",
    "            for _, row in sent[cols].iterrows():\n",
    "                of.write('\\t'.join(row.astype(str).tolist())+'\\n')\n",
    "            of.write('\\n')\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_lattices(lattices_path, ner_pred_path, output_path, keep_all_if_no_valid=True):\n",
    "    lat = bclm.read_lattices(lattices_path)\n",
    "    bc = get_biose_count(ner_pred_path, sent_id_shift=1)\n",
    "    valid_edges = get_valid_edges(lat, bc, non_o_only=False, keep_all_if_no_valid=keep_all_if_no_valid)\n",
    "    cols = ['sent_id', 'token_id', 'ID1', 'ID2']\n",
    "    pruned_lat = lat[lat[cols].apply(lambda x: tuple(x), axis=1).isin(valid_edges)]\n",
    "    to_lattices(pruned_lat, output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['52k',\n",
       " '.ipynb_checkpoints',\n",
       " '2k',\n",
       " '8k',\n",
       " '32k',\n",
       " '4k',\n",
       " '16k',\n",
       " '64k',\n",
       " '128k',\n",
       " 'heBERT',\n",
       " 'unichar_improved_52k',\n",
       " 'unichar_improved_with_hash_52k',\n",
       " 'bert-distilled-wordpiece-oscar-52000',\n",
       " 'bert-basic-wordpiece-owt-52000-10',\n",
       " 'heBERT2',\n",
       " 'bert-small-wordpiece-oscar-52000-10']"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('output/predict_alephbert/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bert-small-wordpiece-oscar-52000-10']\n"
     ]
    }
   ],
   "source": [
    "include_only = ['bert-small-wordpiece-oscar-52000-10',]\n",
    "print(include_only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52k\n",
      ".ipynb_checkpoints\n",
      "2k\n",
      "8k\n",
      "32k\n",
      "4k\n",
      "16k\n",
      "64k\n",
      "128k\n",
      "heBERT\n",
      "unichar_improved_52k\n",
      "unichar_improved_with_hash_52k\n",
      "bert-distilled-wordpiece-oscar-52000\n",
      "bert-basic-wordpiece-owt-52000-10\n",
      "heBERT2\n",
      "bert-small-wordpiece-oscar-52000-10\n",
      "multi_54360\n",
      "multi_54360\n",
      "multi_44184\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_27916\n"
     ]
    }
   ],
   "source": [
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    print(trans.name)\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    for folder in os.scandir(trans):\n",
    "        if 'multi' in folder.name and not '.ipynb' in folder.name:\n",
    "            #print(folder.name)\n",
    "            for file in os.scandir(folder):\n",
    "                if 'dummy' in file.name and '.bmes' in file.name and not '.ipynb' in file.name:\n",
    "                    if 'dev' in file.name:\n",
    "                        output_path = os.path.join(folder.path, 'dev_pruned.lat')\n",
    "                        if not os.path.exists(output_path):\n",
    "                            print(folder.name)\n",
    "                            prune_lattices(bclm.LATTICES_PATHS['dev'], \n",
    "                               file.path,\n",
    "                               output_path)\n",
    "                    elif 'test' in file.name:\n",
    "                        output_path = os.path.join(folder.path, 'test_pruned.lat')\n",
    "                        if not os.path.exists(output_path):\n",
    "                            print(folder.name)\n",
    "                            prune_lattices(bclm.LATTICES_PATHS['test'], \n",
    "                               file.path,\n",
    "                               output_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run YAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "yap_path = '/home/nlp/danb/yapproj/src/yap/yap'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export GOPATH=/home/nlp/danb/yapproj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/yapproj/src/yap/yap - invoke yap as a standalone app or as an api server\n",
      "\n",
      "Commands:\n",
      "\n",
      "    api         start api server\n",
      "    dep         runs dependency training/parsing\n",
      "    hebma       run lexicon-based morphological analyzer on raw input\n",
      "    joint       runs joint morpho-syntactic training and parsing\n",
      "    ma          run data-driven morphological analyzer on raw input\n",
      "    md          runs standalone morphological disambiguation training and parsing\n",
      "\n",
      "Use \"/home/nlp/danb/yapproj/src/yap/yap help <command>\" for more information about a command.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!{yap_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n",
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n"
     ]
    }
   ],
   "source": [
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans.name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if 'multi' in folder.name and not '.ipynb' in folder.name:\n",
    "            print(folder.name)\n",
    "            for file in os.scandir(folder):\n",
    "                if '.lat' in file.name and not '.ipynb' in file.name:\n",
    "                    base_out = '.'.join(file.name.split('.')[:-1])\n",
    "                    seg_out, map_out, conll_out = [os.path.join(folder.path, base_out+suf)\n",
    "                                                   for suf in ['.seg', '.map', '.conll']]\n",
    "                    if True:#not os.path.exists(map_out):\n",
    "                        !{yap_path} joint -in {file.path} -os {seg_out} -om {map_out} -oc {conll_out} > /dev/null 2>&1\n",
    "                    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "spdf = bclm.read_dataframe('spmrl')\n",
    "spdf = spdf[(~spdf.sent_id.isin(dropped))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "dev_gold = bclm.read_dataframe('spmrl', subset='dev')\n",
    "test_gold = spdf[spdf.set=='test']\n",
    "test_sent_id_map = (test_gold.groupby('sent_id').size()\n",
    "                    .reset_index().drop(0, axis=1).reset_index()\n",
    "                    .assign(index=lambda x: x+1).set_index('sent_id')['index'])\n",
    "test_gold['sent_id'] = test_gold.sent_id.map(test_sent_id_map)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['sent_id', 'token_id', 'form']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "use_filter = True\n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if use_filter and trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if 'multi' in folder.name and not '.ipynb' in folder.name:\n",
    "            dev_lfo = bclm.read_yap_output(treebank_set=None, tokens_path=bclm.TREEBANK_TOKEN_PATHS['dev'], \n",
    "                                                 dep_path=os.path.join(folder, 'dev_pruned.conll'),\n",
    "                                                 map_path=os.path.join(folder, 'dev_pruned.map'))\n",
    "            p,r,f_sp = bclm.evaluate_dfs(dev_gold, dev_lfo, verbose=False)\n",
    "\n",
    "            p,r,f_so = bclm.evaluate_dfs(dev_gold, dev_lfo, cols=cols, verbose=False)\n",
    "            res.append(('dev', trans_name, folder.name, f_sp, f_so))\n",
    "\n",
    "            test_lfo = bclm.read_yap_output(treebank_set=None, tokens_path=bclm.TREEBANK_TOKEN_PATHS['test'], \n",
    "                                                 dep_path=os.path.join(folder, 'test_pruned.conll'),\n",
    "                                                 map_path=os.path.join(folder, 'test_pruned.map'))\n",
    "            p,r,f_sp = bclm.evaluate_dfs(test_gold, test_lfo, verbose=False)\n",
    "\n",
    "            p,r,f_so = bclm.evaluate_dfs(test_gold, test_lfo, cols=cols, verbose=False)\n",
    "\n",
    "            res.append(('test', trans_name, folder.name, f_sp, f_so))\n",
    "#         for file in os.scandir(folder):\n",
    "#             if '.bmes' in file.name and not '.ipynb' in file.name:\n",
    "#                 if 'dev' in file.name:\n",
    "#                     prune_lattices(bclm.LATTICES_PATHS['dev'], \n",
    "#                        file.path,\n",
    "#                        os.path.join(folder.path, 'dev_pruned.lat'))\n",
    "#                 elif 'test' in file.name:\n",
    "#                     prune_lattices(bclm.LATTICES_PATHS['test'], \n",
    "#                        file.path,\n",
    "#                        os.path.join(folder.path, 'test_pruned.lat'))\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_res_df = pd.DataFrame(res, columns=['pred_set', 'trans_name', 'model', 'f_seg_pos', 'f_seg_only'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f_seg_pos</th>\n",
       "      <th>f_seg_only</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trans_name</th>\n",
       "      <th>bert-small-wordpiece-oscar-52000-10</th>\n",
       "      <th>bert-small-wordpiece-oscar-52000-10</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_set</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dev</th>\n",
       "      <td>93.425476</td>\n",
       "      <td>97.889414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>91.622853</td>\n",
       "      <td>97.715750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     f_seg_pos  \\\n",
       "trans_name bert-small-wordpiece-oscar-52000-10   \n",
       "pred_set                                         \n",
       "dev                                  93.425476   \n",
       "test                                 91.622853   \n",
       "\n",
       "                                    f_seg_only  \n",
       "trans_name bert-small-wordpiece-oscar-52000-10  \n",
       "pred_set                                        \n",
       "dev                                  97.889414  \n",
       "test                                 97.715750  "
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_res_df.groupby(['pred_set', 'trans_name']).mean().unstack()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Align Multitok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_merge_bio_labels(multitok_sents, tokmorph_sents, verbose=False):\n",
    "    new_sents = []\n",
    "    for (i, mt_sent), (sent_id, mor_sent) in zip(multitok_sents.iteritems(), tokmorph_sents.iteritems()):\n",
    "        new_sent = []\n",
    "        for (form, bio), (token_id, token_str, forms) in zip(mt_sent, mor_sent):\n",
    "            forms = forms.split('^')\n",
    "            bio = bio.split('^')\n",
    "            if len(forms) == len(bio):\n",
    "                new_forms = (1, list(zip(forms,bio)))\n",
    "            elif len(forms)>len(bio):\n",
    "                dif = len(forms) - len(bio)\n",
    "                new_forms = (2, list(zip(forms[:dif],['O']*dif)) + list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            else:\n",
    "                new_forms = (3, list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            new_sent.extend(new_forms[1])\n",
    "        new_sents.append(new_sent)\n",
    "    return new_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_multitok(ner_pred_path, tokens_path, conll_path, map_path, output_path):\n",
    "    x = nem.read_file_sents(ner_pred_path, fix_multi_tag=False)\n",
    "    prun_yo = bclm.read_yap_output(treebank_set=None, tokens_path=tokens_path, dep_path=conll_path, map_path=map_path)\n",
    "    prun_yo = bclm.get_token_df(prun_yo, fields=['form'])\n",
    "    prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str', 'form'])\n",
    "    new_sents = soft_merge_bio_labels(x, prun_sents, verbose=False)\n",
    "\n",
    "    with open(output_path, 'w') as of:\n",
    "        for sent in new_sents:\n",
    "            for form, bio in sent:\n",
    "                of.write(form+' '+bio+'\\n')\n",
    "            of.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "decode_sets = {\n",
    "    'token': {\n",
    "        'dev': '../NER/data/for_ncrf/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/for_ncrf/morph_gold_test.bmes',\n",
    "    },\n",
    "    'multitok': {\n",
    "        'dev': '../NER/data/for_ncrf/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/for_ncrf/morph_gold_test.bmes',\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n",
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "            print (folder.name)\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.bmes')\n",
    "            if True: #not os.path.exists(pruned_ner_path):\n",
    "                align_multitok(os.path.join(folder.path, 'token_gold_dev_dummy_o.bmes'), \n",
    "                               bclm.TREEBANK_TOKEN_PATHS['dev'], \n",
    "                               os.path.join(folder.path, 'dev_pruned.conll'),\n",
    "                               os.path.join(folder.path, 'dev_pruned.map'),\n",
    "                               pruned_ner_path\n",
    "                              )\n",
    "                p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], pruned_ner_path, str_join_char='')\n",
    "                res.append(('dev', trans_name, folder.name, p, r, f))\n",
    "\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.bmes')\n",
    "            if True: #not os.path.exists(pruned_ner_path):\n",
    "                align_multitok(os.path.join(folder.path, 'token_gold_test_dummy_o.bmes'), \n",
    "                               bclm.TREEBANK_TOKEN_PATHS['test'], \n",
    "                               os.path.join(folder.path, 'test_pruned.conll'),\n",
    "                               os.path.join(folder.path, 'test_pruned.map'),\n",
    "                               pruned_ner_path\n",
    "                              )\n",
    "                p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], pruned_ner_path, str_join_char='')\n",
    "                res.append(('test', trans_name, folder.name, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    }
   ],
   "source": [
    "def biose_to_o(in_path, out_path):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            if line=='\\n':\n",
    "                of.write(line)\n",
    "                sents+=1\n",
    "            else:\n",
    "                line = line.strip()\n",
    "                word, tag = line.split()\n",
    "                tag = 'O'\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            \n",
    "    \n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue    \n",
    "    print(trans.name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.bmes')\n",
    "            output_path = pruned_ner_path.replace('.bmes', '.bioul')\n",
    "            if True: #not os.path.exists(output_path):\n",
    "                biose_to_o(pruned_ner_path, output_path)\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.bmes')\n",
    "            output_path = pruned_ner_path.replace('.bmes', '.bioul')\n",
    "            if True: #not os.path.exists(output_path):\n",
    "                biose_to_o(pruned_ner_path, output_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <-- NOW RUN PREDICT ON PRUNED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n"
     ]
    }
   ],
   "source": [
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if os.path.isdir(folder) and 'morph' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.json')\n",
    "            output_path = pruned_ner_path.replace('.json', '.bmes')\n",
    "            if True:#not os.path.exists(output_path):\n",
    "                jsonl_to_biose(pruned_ner_path, output_path)\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.json')\n",
    "            output_path = pruned_ner_path.replace('.json', '.bmes')\n",
    "            if True:#not os.path.exists(output_path):\n",
    "                jsonl_to_biose(pruned_ner_path, output_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SINGLE + MULTI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set</th>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">dev</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>0.816269</td>\n",
       "      <td>0.777956</td>\n",
       "      <td>0.796625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>0.802461</td>\n",
       "      <td>0.761924</td>\n",
       "      <td>0.781648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>0.724738</td>\n",
       "      <td>0.698597</td>\n",
       "      <td>0.711411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.839158</td>\n",
       "      <td>0.762725</td>\n",
       "      <td>0.799088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.823865</td>\n",
       "      <td>0.792385</td>\n",
       "      <td>0.807789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">test</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>0.771591</td>\n",
       "      <td>0.790773</td>\n",
       "      <td>0.781056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>0.741636</td>\n",
       "      <td>0.753863</td>\n",
       "      <td>0.747688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>0.664694</td>\n",
       "      <td>0.669099</td>\n",
       "      <td>0.666880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.791587</td>\n",
       "      <td>0.779185</td>\n",
       "      <td>0.785329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.762308</td>\n",
       "      <td>0.790129</td>\n",
       "      <td>0.775935</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                p         r         f\n",
       "set  eval_unit variant prediction align                              \n",
       "dev  morph     morph   gold       -      0.816269  0.777956  0.796625\n",
       "                       hybrid     -      0.802461  0.761924  0.781648\n",
       "                       yap        -      0.724738  0.698597  0.711411\n",
       "     token     multi   tokens     -      0.839158  0.762725  0.799088\n",
       "               single  tokens     -      0.823865  0.792385  0.807789\n",
       "test morph     morph   gold       -      0.771591  0.790773  0.781056\n",
       "                       hybrid     -      0.741636  0.753863  0.747688\n",
       "                       yap        -      0.664694  0.669099  0.666880\n",
       "     token     multi   tokens     -      0.791587  0.779185  0.785329\n",
       "               single  tokens     -      0.762308  0.790129  0.775935"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = []\n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if '.ipynb' in folder.name:\n",
    "            continue\n",
    "\n",
    "        variant, seed = folder.name.split('_')\n",
    "\n",
    "        if 'single' in folder.name:    \n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_test_fix.bmes'), str_join_char='')\n",
    "            res.append(('test', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "\n",
    "        if 'multi' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_test_dummy_o.bmes'), str_join_char='')\n",
    "            res.append(('test', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "        if 'morph' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                       os.path.join(folder.path,'morph_gold_test.bmes'), str_join_char='')\n",
    "            res.append(('test', 'morph', variant, 'gold', '-', trans_name, seed, p, r, f))\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                       os.path.join(folder.path,'morph_yap_test.bmes'), str_join_char='')\n",
    "            res.append(('test', 'morph', variant, 'yap', '-', trans_name, seed, p, r, f))\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                       os.path.join(folder.path,'morph_pruned_test.bmes'), str_join_char='')\n",
    "            res.append(('test', 'morph', variant, 'hybrid', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "        #dev\n",
    "\n",
    "        if 'single' in folder.name:    \n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_dev_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_dev_fix.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "\n",
    "        if 'multi' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_dev_fix.bmes', \n",
    "                                       os.path.join(folder.path,'token_gold_dev_dummy_o.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'token', variant, 'tokens', '-', trans_name, seed, p, r, f))\n",
    "\n",
    "        if 'morph' in folder.name:\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                       os.path.join(folder.path,'morph_gold_dev.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'morph', variant, 'gold', '-', trans_name, seed, p, r, f))\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                       os.path.join(folder.path,'morph_yap_dev.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'morph', variant, 'yap', '-', trans_name, seed, p, r, f))\n",
    "            p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                       os.path.join(folder.path,'morph_pruned_dev.bmes'), str_join_char='')\n",
    "            res.append(('dev', 'morph', variant, 'hybrid', '-', trans_name, seed, p, r, f))\n",
    "    \n",
    "    \n",
    "\n",
    "ne_df = pd.DataFrame(res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'trans_name', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "ne_df.groupby(['set', 'eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Alignments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Token Level Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "o_re = re.compile('^O+$') \n",
    "s_re = re.compile('^O*SO*$|^O*BI*EO*$')\n",
    "b_re = re.compile('^O*BI*$')\n",
    "i_re = re.compile('^I+$')\n",
    "e_re = re.compile('^I*EO*$')\n",
    "def get_fixed_for_valid_biose(bio_seq):\n",
    "    if o_re.match(bio_seq):\n",
    "        return 'O'\n",
    "    if s_re.match(bio_seq):\n",
    "        return 'S'\n",
    "    if b_re.match(bio_seq):\n",
    "        return 'B'\n",
    "    if i_re.match(bio_seq):\n",
    "        return 'I'\n",
    "    if e_re.match(bio_seq):\n",
    "        return 'E'\n",
    "    raise ValueError\n",
    "    \n",
    "\n",
    "def get_fixed_for_invalid_biose(parts):\n",
    "    bio = 'O'\n",
    "    if 'S' in parts:\n",
    "        bio = 'S'\n",
    "    elif 'B' in parts and 'E' in parts:\n",
    "        bio='S'\n",
    "    elif 'E' in parts:\n",
    "        bio = 'E'\n",
    "    elif 'B' in parts:\n",
    "        bio = 'B'\n",
    "    elif 'I' in parts:\n",
    "        bio = 'I'\n",
    "    return bio\n",
    "\n",
    "valid_bio_re = re.compile('^O*BI*$|^O*BI*EO*$|^I+$|^I*EO*$|^O*SO*$')\n",
    "\n",
    "from functools import lru_cache\n",
    "\n",
    "\n",
    "def validate_biose_sequence(full_bio_seq):\n",
    "    #print(full_bio_seq)\n",
    "    bio_seq, type_seq = zip(*[('O', None) if b=='O' else b.split('-') for b in full_bio_seq])\n",
    "    bio_seq = ''.join(bio_seq)\n",
    "    valid_bio = valid_bio_re.match(bio_seq)\n",
    "    type_seq = list(filter(lambda x: x is not None, type_seq))\n",
    "    type_seq_set = set(type_seq)\n",
    "\n",
    "    if valid_bio:\n",
    "        fixed_bio = get_fixed_for_valid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "            \n",
    "    else:\n",
    "        #take the first BIOSE tag which is not O:\n",
    "        #fixed_bio = list(filter(lambda x: x!='O', full_bio_seq))[0]\n",
    "        #rough BIOSE and first category:\n",
    "        fixed_bio = get_fixed_for_invalid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "        \n",
    "    return valid_bio is not None, len(type_seq_set)<=1, fixed_bio\n",
    "\n",
    "\n",
    "@lru_cache(1000)\n",
    "def get_fixed_bio_sequence(full_bio_seq):\n",
    "    return validate_biose_sequence(full_bio_seq)[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "spdf = bclm.read_dataframe('spmrl')\n",
    "spdf = spdf[(~spdf.sent_id.isin(dropped))]\n",
    "dev_gold = spdf[spdf.set=='dev']\n",
    "test_gold = spdf[spdf.set=='test']\n",
    "test_gold['sent_id'] = test_gold.sent_id.rank(method='dense').astype(int)\n",
    "dev_yap = bclm.read_yap_output(treebank_set='dev')\n",
    "test_yap = bclm.read_yap_output(treebank_set='test')\n",
    "dev_gold_sents = bclm.get_sentences_list(dev_gold, fields=['token_id', 'token_str'])\n",
    "test_gold_sents = bclm.get_sentences_list(test_gold, fields=['token_id', 'token_str'])\n",
    "dev_yap_sents = bclm.get_sentences_list(dev_yap, fields=['token_id', 'token_str'])\n",
    "test_yap_sents = bclm.get_sentences_list(test_yap, fields=['token_id', 'token_str'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_gold_tok = (bclm.get_token_df(dev_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok = (bclm.get_token_df(test_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok['sent_id'] = test_gold_tok.sent_id.rank(method='dense').astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fixed_tok(path, orig_sents=dev_yap_sents):\n",
    "    x = nem.read_file_sents(path, fix_multi_tag=False)\n",
    "    new_sents = []\n",
    "    for (i, ner_sent), (sent_id, yap_sent) in zip(x.iteritems(), orig_sents.iteritems()):\n",
    "        for (form, bio), (token_id, token_str) in zip(ner_sent, yap_sent):\n",
    "            new_sents.append((sent_id, token_id, token_str, form, bio))\n",
    "    new_sents = pd.DataFrame(new_sents, columns=['sent_id', 'token_id', 'token_str', 'form', 'bio'])\n",
    "    new_toks = bclm.get_token_df(new_sents, fields=['bio'])\n",
    "    new_toks['fixed_bio'] = new_toks.bio.apply(lambda x: get_fixed_bio_sequence(tuple(x.split('^'))))\n",
    "    return new_toks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sents_from_df(df, sent_id_col='sent_id', \n",
    "                  group_cols=['token_str'], \n",
    "                  val_cols=['fixed_bio']):\n",
    "    sents = bclm.get_sentences_list(df, fields=group_cols+val_cols)\n",
    "    return sents\n",
    "\n",
    "def evaluate_dataframes(gold_df, pred_df, fix_multi_tag_pred=True, truncate=None, ignore_cat=False, str_join_char=' '):\n",
    "    gold_sents = sents_from_df(gold_df)\n",
    "    pred_sents = sents_from_df(pred_df)\n",
    "    gold_mentions = nem.sents_to_mentions(gold_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    pred_mentions = nem.sents_to_mentions(pred_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    return nem.evaluate_mentions(gold_mentions, pred_mentions, verbose=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hybrid\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_folder = '../NER/data/tokens_for_ncrf'\n",
    "dev_out = os.path.join(out_folder, 'dev_tokens.txt')\n",
    "test_out = os.path.join(out_folder, 'test_tokens.txt')\n",
    "token_paths = {'dev': dev_out, 'test': test_out}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prun_yo(ds, dep_path, map_path):\n",
    "\n",
    "    \n",
    "    prun_yo = bclm.read_yap_output(treebank_set=None,\n",
    "                               tokens_path=token_paths[ds],\n",
    "                               dep_path=dep_path,\n",
    "                               map_path=map_path,\n",
    "                                )\n",
    "    return prun_yo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run on all pruned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sent_list(ds, dp, mp):\n",
    "    prun_yo = get_prun_yo(ds, dp, mp)\n",
    "    return bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    }
   ],
   "source": [
    "align_tok_res = []\n",
    "for trans in os.scandir('output/predict_alephbert'): \n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if 'morph' in folder.name and not '.ipynb' in folder.name:\n",
    "            ## dev \n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_pruned_dev.bmes')\n",
    "            multi_folder = folder.path.replace('morph_', 'multi_')\n",
    "            dep_path = os.path.join(multi_folder, 'dev_pruned.conll')\n",
    "            map_path = os.path.join(multi_folder, 'dev_pruned.map')\n",
    "            out_path = os.path.join(folder.path, 'morph_pruned_dev_align_tokens.bmes')\n",
    "\n",
    "            prun_sents = get_sent_list('dev',dep_path , map_path)\n",
    "            new_toks = get_fixed_tok(file, orig_sents=prun_sents)\n",
    "\n",
    "            if True: #not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "\n",
    "            p, r, f = evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res.append(('dev', 'token', 'morph', 'hybrid', 'tokens', trans_name, seed, p, r, f))\n",
    "            \n",
    "            ## test \n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_pruned_test.bmes')\n",
    "            multi_folder = folder.path.replace('morph_', 'multi_')\n",
    "            dep_path = os.path.join(multi_folder, 'test_pruned.conll')\n",
    "            map_path = os.path.join(multi_folder, 'test_pruned.map')\n",
    "            out_path = os.path.join(folder.path, 'morph_pruned_test_align_tokens.bmes')\n",
    "\n",
    "            prun_sents = get_sent_list('test',dep_path , map_path)\n",
    "            new_toks = get_fixed_tok(file, orig_sents=prun_sents)\n",
    "\n",
    "            if True: #not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "\n",
    "            p, r, f = evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res.append(('test', 'token', 'morph', 'hybrid', 'tokens', trans_name, seed, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set   trans_name                           eval_unit  variant  prediction  align \n",
       "dev   bert-small-wordpiece-oscar-52000-10  token      morph    hybrid      tokens    0.795792\n",
       "test  bert-small-wordpiece-oscar-52000-10  token      morph    hybrid      tokens    0.770039\n",
       "Name: f, dtype: float64"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'trans_name', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align']).f.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run all gold and YAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    }
   ],
   "source": [
    "align_tok_res_yg = []\n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if 'morph' in folder.name and not '.ipynb' in folder.name:\n",
    "            ## dev \n",
    "            ## - gold\n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_gold_dev.bmes')\n",
    "            out_path = os.path.join(folder.path, 'morph_gold_dev_align_tokens.bmes')\n",
    "\n",
    "            new_toks = get_fixed_tok(file, orig_sents=dev_gold_sents)\n",
    "\n",
    "            if True: #not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "\n",
    "            p, r, f = evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res_yg.append(('dev', 'token', 'morph', 'gold', 'tokens', trans_name, seed, p, r, f))\n",
    "\n",
    "            ## - yap\n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_yap_dev.bmes')\n",
    "            out_path = os.path.join(folder.path, 'morph_yap_dev_align_tokens.bmes')\n",
    "\n",
    "            new_toks = get_fixed_tok(file, orig_sents=dev_yap_sents)\n",
    "\n",
    "            if True: #not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "            p, r, f = evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res_yg.append(('dev', 'token', 'morph', 'yap', 'tokens', trans_name, seed, p, r, f))\n",
    "\n",
    "            ## test \n",
    "            ## - gold\n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_gold_test.bmes')\n",
    "            out_path = os.path.join(folder.path, 'morph_gold_test_align_tokens.bmes')\n",
    "\n",
    "            new_toks = get_fixed_tok(file, orig_sents=test_gold_sents)\n",
    "\n",
    "            if True: #not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "\n",
    "            p, r, f = evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res_yg.append(('test', 'token', 'morph', 'gold', 'tokens', trans_name, seed, p, r, f))\n",
    "\n",
    "            ## - yap\n",
    "            variant, seed = folder.name.split('_')\n",
    "            file = os.path.join(folder.path,'morph_yap_test.bmes')\n",
    "            out_path = os.path.join(folder.path, 'morph_yap_test_align_tokens.bmes')\n",
    "\n",
    "            new_toks = get_fixed_tok(file, orig_sents=test_yap_sents)\n",
    "\n",
    "            if True: # not os.path.exists(out_path):\n",
    "                new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "                with open(out_path, 'w') as of:\n",
    "                    for i, sent in new_sents.iteritems():\n",
    "                        for tok, bio in sent:\n",
    "                            of.write(tok+' '+bio+'\\n')\n",
    "                        of.write('\\n')\n",
    "            p, r, f = evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "            align_tok_res_yg.append(('test', 'token', 'morph', 'yap', 'tokens', trans_name, seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set</th>\n",
       "      <th>trans_name</th>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">dev</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">bert-small-wordpiece-oscar-52000-10</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.825018</td>\n",
       "      <td>0.785972</td>\n",
       "      <td>0.804998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.817154</td>\n",
       "      <td>0.775551</td>\n",
       "      <td>0.795792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.770481</td>\n",
       "      <td>0.742685</td>\n",
       "      <td>0.756311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">test</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">bert-small-wordpiece-oscar-52000-10</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.775941</td>\n",
       "      <td>0.795064</td>\n",
       "      <td>0.785378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.763810</td>\n",
       "      <td>0.776395</td>\n",
       "      <td>0.770039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.736611</td>\n",
       "      <td>0.740987</td>\n",
       "      <td>0.738783</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                     p  \\\n",
       "set  trans_name                          eval_unit variant prediction align              \n",
       "dev  bert-small-wordpiece-oscar-52000-10 token     morph   gold       tokens  0.825018   \n",
       "                                                           hybrid     tokens  0.817154   \n",
       "                                                           yap        tokens  0.770481   \n",
       "test bert-small-wordpiece-oscar-52000-10 token     morph   gold       tokens  0.775941   \n",
       "                                                           hybrid     tokens  0.763810   \n",
       "                                                           yap        tokens  0.736611   \n",
       "\n",
       "                                                                                     r  \\\n",
       "set  trans_name                          eval_unit variant prediction align              \n",
       "dev  bert-small-wordpiece-oscar-52000-10 token     morph   gold       tokens  0.785972   \n",
       "                                                           hybrid     tokens  0.775551   \n",
       "                                                           yap        tokens  0.742685   \n",
       "test bert-small-wordpiece-oscar-52000-10 token     morph   gold       tokens  0.795064   \n",
       "                                                           hybrid     tokens  0.776395   \n",
       "                                                           yap        tokens  0.740987   \n",
       "\n",
       "                                                                                     f  \n",
       "set  trans_name                          eval_unit variant prediction align             \n",
       "dev  bert-small-wordpiece-oscar-52000-10 token     morph   gold       tokens  0.804998  \n",
       "                                                           hybrid     tokens  0.795792  \n",
       "                                                           yap        tokens  0.756311  \n",
       "test bert-small-wordpiece-oscar-52000-10 token     morph   gold       tokens  0.785378  \n",
       "                                                           hybrid     tokens  0.770039  \n",
       "                                                           yap        tokens  0.738783  "
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'trans_name', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Morpheme Level Eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Token Multi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hybrid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_hyb = []\n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.bmes')\n",
    "\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], pruned_ner_path, str_join_char='')\n",
    "            align_morph_res_hyb.append(('dev', 'morph', 'multi', 'tokens', 'hybrid', trans_name, seed, p, r, f))\n",
    "\n",
    "            pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.bmes')\n",
    "\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], pruned_ner_path, str_join_char='')\n",
    "            align_morph_res_hyb.append(('test', 'morph', 'multi', 'tokens', 'hybrid', trans_name, seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set</th>\n",
       "      <th>trans_name</th>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">dev</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">bert-small-wordpiece-oscar-52000-10</th>\n",
       "      <th>morph</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>hybrid</th>\n",
       "      <td>0.819594</td>\n",
       "      <td>0.744289</td>\n",
       "      <td>0.780101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.825018</td>\n",
       "      <td>0.785972</td>\n",
       "      <td>0.804998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.817154</td>\n",
       "      <td>0.775551</td>\n",
       "      <td>0.795792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.770481</td>\n",
       "      <td>0.742685</td>\n",
       "      <td>0.756311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">test</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">bert-small-wordpiece-oscar-52000-10</th>\n",
       "      <th>morph</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>hybrid</th>\n",
       "      <td>0.762503</td>\n",
       "      <td>0.749571</td>\n",
       "      <td>0.755973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.775941</td>\n",
       "      <td>0.795064</td>\n",
       "      <td>0.785378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.763810</td>\n",
       "      <td>0.776395</td>\n",
       "      <td>0.770039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.736611</td>\n",
       "      <td>0.740987</td>\n",
       "      <td>0.738783</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                     p  \\\n",
       "set  trans_name                          eval_unit variant prediction align              \n",
       "dev  bert-small-wordpiece-oscar-52000-10 morph     multi   tokens     hybrid  0.819594   \n",
       "                                         token     morph   gold       tokens  0.825018   \n",
       "                                                           hybrid     tokens  0.817154   \n",
       "                                                           yap        tokens  0.770481   \n",
       "test bert-small-wordpiece-oscar-52000-10 morph     multi   tokens     hybrid  0.762503   \n",
       "                                         token     morph   gold       tokens  0.775941   \n",
       "                                                           hybrid     tokens  0.763810   \n",
       "                                                           yap        tokens  0.736611   \n",
       "\n",
       "                                                                                     r  \\\n",
       "set  trans_name                          eval_unit variant prediction align              \n",
       "dev  bert-small-wordpiece-oscar-52000-10 morph     multi   tokens     hybrid  0.744289   \n",
       "                                         token     morph   gold       tokens  0.785972   \n",
       "                                                           hybrid     tokens  0.775551   \n",
       "                                                           yap        tokens  0.742685   \n",
       "test bert-small-wordpiece-oscar-52000-10 morph     multi   tokens     hybrid  0.749571   \n",
       "                                         token     morph   gold       tokens  0.795064   \n",
       "                                                           hybrid     tokens  0.776395   \n",
       "                                                           yap        tokens  0.740987   \n",
       "\n",
       "                                                                                     f  \n",
       "set  trans_name                          eval_unit variant prediction align             \n",
       "dev  bert-small-wordpiece-oscar-52000-10 morph     multi   tokens     hybrid  0.780101  \n",
       "                                         token     morph   gold       tokens  0.804998  \n",
       "                                                           hybrid     tokens  0.795792  \n",
       "                                                           yap        tokens  0.756311  \n",
       "test bert-small-wordpiece-oscar-52000-10 morph     multi   tokens     hybrid  0.755973  \n",
       "                                         token     morph   gold       tokens  0.785378  \n",
       "                                                           hybrid     tokens  0.770039  \n",
       "                                                           yap        tokens  0.738783  "
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'trans_name', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### YAP + GOLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_multitok_yg(ner_pred_path, prun_sents, output_path):\n",
    "    x = nem.read_file_sents(ner_pred_path, fix_multi_tag=False)\n",
    "\n",
    "    new_sents = soft_merge_bio_labels(x, prun_sents, verbose=False)\n",
    "\n",
    "    with open(output_path, 'w') as of:\n",
    "        for sent in new_sents:\n",
    "            for form, bio in sent:\n",
    "                of.write(form+' '+bio+'\\n')\n",
    "            of.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_morph = {'dev': dev_gold, 'test': test_gold}\n",
    "def get_sents_for_mult(treebank_set, gold=False, pred_set=None, \n",
    "                       dep_path=None, map_path=None):\n",
    "    if treebank_set is None:\n",
    "        prun_yo = get_prun_yo(pred_set, dep_path, map_path)\n",
    "    else:\n",
    "        if not gold:\n",
    "            prun_yo = bclm.read_yap_output(treebank_set=treebank_set)\n",
    "        else:\n",
    "            prun_yo = gold_morph[treebank_set]\n",
    "    prun_yo = bclm.get_token_df(prun_yo, fields=['form'])\n",
    "    prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str', 'form'])\n",
    "    return prun_sents\n",
    "\n",
    "dev_yap_sents_m = get_sents_for_mult('dev')\n",
    "test_yap_sents_m = get_sents_for_mult('test')\n",
    "dev_gold_sents_m = get_sents_for_mult('dev', gold=True)\n",
    "test_gold_sents_m = get_sents_for_mult('test', gold=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_yap = []\n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "            #dev\n",
    "            yap_ner_path=os.path.join(folder.path, 'morph_yap_dev.bmes')\n",
    "\n",
    "            align_multitok_yg(os.path.join(folder.path, 'token_gold_dev_dummy_o.bmes'), \n",
    "                               dev_yap_sents_m,\n",
    "                               yap_ner_path\n",
    "                              )\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], yap_ner_path, str_join_char='')\n",
    "            align_morph_res_yap.append(('dev', 'morph', 'multi', 'tokens', 'yap', trans_name, seed, p, r, f))\n",
    "            \n",
    "            #test\n",
    "            yap_ner_path=os.path.join(folder.path, 'morph_yap_test.bmes')\n",
    "\n",
    "            align_multitok_yg(os.path.join(folder.path, 'token_gold_test_dummy_o.bmes'), \n",
    "                               test_yap_sents_m,\n",
    "                               yap_ner_path\n",
    "                              )\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], yap_ner_path, str_join_char='')\n",
    "            align_morph_res_yap.append(('test', 'morph', 'multi', 'tokens', 'yap', trans_name, seed, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-small-wordpiece-oscar-52000-10\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_gold = []\n",
    "for trans in os.scandir('output/predict_alephbert'):\n",
    "    trans_name = trans.name\n",
    "    if trans.name not in include_only:\n",
    "        continue\n",
    "    print(trans_name)\n",
    "    for folder in os.scandir(trans):\n",
    "        if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "            #dev\n",
    "            gold_ner_path=os.path.join(folder.path, 'morph_gold_dev.bmes')\n",
    "\n",
    "            align_multitok_yg(os.path.join(folder.path, 'token_gold_dev_dummy_o.bmes'), \n",
    "                               dev_gold_sents_m,\n",
    "                               gold_ner_path\n",
    "                              )\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], gold_ner_path, str_join_char='')\n",
    "            align_morph_res_gold.append(('dev', 'morph', 'multi', 'tokens', 'gold', trans_name, seed, p, r, f))\n",
    "\n",
    "            #test\n",
    "            gold_ner_path=os.path.join(folder.path, 'morph_gold_test.bmes')\n",
    "\n",
    "            align_multitok_yg(os.path.join(folder.path, 'token_gold_test_dummy_o.bmes'), \n",
    "                               test_gold_sents_m,\n",
    "                               gold_ner_path\n",
    "                              )\n",
    "            p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], gold_ner_path, str_join_char='')\n",
    "            align_morph_res_gold.append(('test', 'morph', 'multi', 'tokens', 'gold', trans_name, seed, p, r, f))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ALL SCORES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb+align_morph_res_yap+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', \n",
    "                                                                                                                           'prediction', 'align', 'trans_name', \n",
    "                                                                                                                           'seed', 'p', 'r', 'f'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALL_DF_PATH = 'output/all_results_alephbert.csv'\n",
    "all_df = pd.read_csv(ALL_DF_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "min    5\n",
       "max    5\n",
       "dtype: int64"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df.groupby(['set', 'eval_unit','variant', \n",
    "                'prediction', 'align','trans_name']).size().agg([min,max])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "min    5\n",
       "max    5\n",
       "dtype: int64"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df = pd.concat([all_df, at_df, ne_df])\n",
    "all_df.groupby(['set', 'eval_unit','variant', \n",
    "                'prediction', 'align','trans_name']).size().agg([min,max])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>trans_name</th>\n",
       "      <th>bert-small-wordpiece-oscar-52000-10</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set</th>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"11\" valign=\"top\">dev</th>\n",
       "      <th rowspan=\"6\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>79.662546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>78.164751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>71.141128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">multi</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>78.991742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <td>78.010109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <td>73.582826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>80.499798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>79.579169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>75.631056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>79.908810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>80.778950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"11\" valign=\"top\">test</th>\n",
       "      <th rowspan=\"6\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>78.105582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>74.768773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>66.687995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">multi</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>77.607454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <td>75.597322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <td>69.818424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>78.537758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>77.003916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>73.878313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>78.532902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>77.593455</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "trans_name                                bert-small-wordpiece-oscar-52000-10\n",
       "set  eval_unit variant prediction align                                      \n",
       "dev  morph     morph   gold       -                                 79.662546\n",
       "                       hybrid     -                                 78.164751\n",
       "                       yap        -                                 71.141128\n",
       "               multi   tokens     gold                              78.991742\n",
       "                                  hybrid                            78.010109\n",
       "                                  yap                               73.582826\n",
       "     token     morph   gold       tokens                            80.499798\n",
       "                       hybrid     tokens                            79.579169\n",
       "                       yap        tokens                            75.631056\n",
       "               multi   tokens     -                                 79.908810\n",
       "               single  tokens     -                                 80.778950\n",
       "test morph     morph   gold       -                                 78.105582\n",
       "                       hybrid     -                                 74.768773\n",
       "                       yap        -                                 66.687995\n",
       "               multi   tokens     gold                              77.607454\n",
       "                                  hybrid                            75.597322\n",
       "                                  yap                               69.818424\n",
       "     token     morph   gold       tokens                            78.537758\n",
       "                       hybrid     tokens                            77.003916\n",
       "                       yap        tokens                            73.878313\n",
       "               multi   tokens     -                                 78.532902\n",
       "               single  tokens     -                                 77.593455"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_scores = (all_df\n",
    "               .groupby(['set', 'eval_unit','variant', \n",
    "                         'prediction', 'align','trans_name'])\n",
    "               .f.mean().unstack().mul(100)[include_only]\n",
    "              )\n",
    "mean_scores"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb+align_morph_res_yap+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', \n",
    "                                                                                                                           'prediction', 'align', 'trans_name', \n",
    "                                                                                                                           'seed', 'p', 'r', 'f'])\n",
    "\n",
    "(at_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "(ne_df.groupby(['set', 'trans_name', 'eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df.to_csv(ALL_DF_PATH, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_scores.reset_index().to_csv('output/mean_results_alephbert.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "#seg_res_df.to_csv('output/all_seg_results_alephbert.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>trans_name</th>\n",
       "      <th>bert-small-wordpiece-oscar-52000-10</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_set</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dev</th>\n",
       "      <td>93.425476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>91.622853</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "trans_name  bert-small-wordpiece-oscar-52000-10\n",
       "pred_set                                       \n",
       "dev                                   93.425476\n",
       "test                                  91.622853"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(seg_res_df.groupby(['pred_set', 'trans_name'])\n",
    " .f_seg_pos.mean().unstack()[include_only]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>trans_name</th>\n",
       "      <th>bert-small-wordpiece-oscar-52000-10</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_set</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dev</th>\n",
       "      <td>97.889414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>97.715750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "trans_name  bert-small-wordpiece-oscar-52000-10\n",
       "pred_set                                       \n",
       "dev                                   97.889414\n",
       "test                                  97.715750"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(seg_res_df.groupby(['pred_set', 'trans_name'])\n",
    " .f_seg_only.mean().unstack()[include_only]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7950108459869848, 0.7864806866952789, 0.7907227615965479)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', '../NER/data/from_amit/morph_label_test.bmes', str_join_char='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.801255230125523, 0.7675350701402806, 0.7840327533265097)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', '../NER/data/from_amit/morph_label_dev.bmes', str_join_char='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8597826086956522, 0.8487124463519313, 0.8542116630669546)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', '/home/nlp/shaked571/NerBert/ft_ner_alefbert/seed1_BaseBert10e_maxlen150_batch4_single_random/test_predictions.txt', str_join_char='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}