From 4f75d5cf476cf091ca866975de709710961a822e Mon Sep 17 00:00:00 2001
From: severinsimmler <s.simmler@snapaddy.com>
Date: Tue, 24 May 2022 15:31:36 +0200
Subject: [PATCH] chore: cleanup

---
 chaine/crf.py                |   2 +-
 examples/notebooks/ner.ipynb | 658 ++++++++++++++++++++++++++++-------
 2 files changed, 527 insertions(+), 133 deletions(-)

diff --git a/chaine/crf.py b/chaine/crf.py
index 0eeaefd..cd3cb3a 100755
--- a/chaine/crf.py
+++ b/chaine/crf.py
@@ -220,7 +220,7 @@ def params(self) -> dict[str, Union[str, int, float, bool]]:
 class HyperparameterOptimizer:
     def __init__(
         self,
-        trials: int = 20,
+        trials: int = 10,
         seed: Optional[int] = None,
         metric: str = "f1",
         folds: int = 5,
diff --git a/examples/notebooks/ner.ipynb b/examples/notebooks/ner.ipynb
index d561a5e..8377f79 100755
--- a/examples/notebooks/ner.ipynb
+++ b/examples/notebooks/ner.ipynb
@@ -37,34 +37,6 @@
    "id": "6726588b-57a0-4996-9d4c-6a0856826f0b",
    "metadata": {},
    "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "46e78394db0a4ba69e53f62a858a8dd7",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading builder script:   0%|          | 0.00/2.58k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d95b887a32134aa19c25d6797d9ee489",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading metadata:   0%|          | 0.00/1.62k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
     {
      "name": "stderr",
      "output_type": "stream",
@@ -75,7 +47,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1b0e5b8adfe64fb09800b45435158fc2",
+       "model_id": "75c6ba99733b4b318360fef82e093eff",
        "version_major": 2,
        "version_minor": 0
       },
@@ -232,7 +204,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "d520c53c-f8be-48a1-8484-1fafe81ad9a2",
    "metadata": {},
    "outputs": [],
@@ -241,6 +213,59 @@
     "labels = preprocess_labels(dataset[\"train\"])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "ab99001e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'token.lower()': 'eu',\n",
+       " 'token[-3:]': 'EU',\n",
+       " 'token[-2:]': 'EU',\n",
+       " 'token.isupper()': True,\n",
+       " 'token.istitle()': False,\n",
+       " 'token.isdigit()': False,\n",
+       " 'pos_tag': 22,\n",
+       " 'BOS': True,\n",
+       " '+1:token.lower()': 'rejects',\n",
+       " '+1:token.istitle()': False,\n",
+       " '+1:token.isupper()': False,\n",
+       " '+1:pos_tag': 42}"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sentences[0][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "105b64d1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'B-ORG'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "labels[0][0]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "d957af6e-ed83-4d55-90de-522c94151814",
@@ -290,14 +315,14 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "         LOC       0.67      0.65      0.66      1668\n",
-      "        MISC       0.72      0.50      0.59       702\n",
-      "         ORG       0.67      0.34      0.45      1661\n",
-      "         PER       0.71      0.66      0.69      1617\n",
+      "         LOC       0.82      0.72      0.77      1668\n",
+      "        MISC       0.66      0.66      0.66       702\n",
+      "         ORG       0.70      0.59      0.64      1661\n",
+      "         PER       0.82      0.77      0.79      1617\n",
       "\n",
-      "   micro avg       0.69      0.54      0.61      5648\n",
-      "   macro avg       0.70      0.54      0.60      5648\n",
-      "weighted avg       0.69      0.54      0.60      5648\n",
+      "   micro avg       0.76      0.69      0.72      5648\n",
+      "   macro avg       0.75      0.69      0.72      5648\n",
+      "weighted avg       0.76      0.69      0.72      5648\n",
       "\n"
      ]
     }
@@ -320,8 +345,374 @@
    "cell_type": "code",
    "execution_count": 10,
    "id": "1569a1f1-212c-4fc2-b8d3-b601b5aea88c",
-   "metadata": {},
-   "outputs": [],
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2022-05-24 14:12:57,820] [INFO] Starting with arow (1/5)\n",
+      "[2022-05-24 14:12:57,821] [INFO] Baseline for arow\n",
+      "[2022-05-24 14:13:10,700] [INFO] Trial 1/10 for arow\n",
+      "[2022-05-24 14:13:35,112] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:13:35,113] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:13:35,115] [INFO] Trial 2/10 for arow\n",
+      "[2022-05-24 14:13:57,998] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:13:57,999] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:13:57,999] [INFO] Trial 3/10 for arow\n",
+      "[2022-05-24 14:14:22,100] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:14:22,101] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:14:22,101] [INFO] Trial 4/10 for arow\n",
+      "[2022-05-24 14:14:41,419] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:14:41,420] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:14:41,422] [INFO] Trial 5/10 for arow\n",
+      "[2022-05-24 14:15:02,888] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:02,889] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:02,890] [INFO] Trial 6/10 for arow\n",
+      "[2022-05-24 14:15:20,546] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:20,547] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:20,547] [INFO] Trial 7/10 for arow\n",
+      "[2022-05-24 14:15:39,751] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:39,752] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:39,753] [INFO] Trial 8/10 for arow\n",
+      "[2022-05-24 14:15:53,562] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:53,562] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:15:53,563] [INFO] Trial 9/10 for arow\n",
+      "[2022-05-24 14:16:14,962] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:16:14,963] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:16:14,964] [INFO] Trial 10/10 for arow\n",
+      "[2022-05-24 14:16:32,600] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:16:32,601] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:16:32,602] [INFO] Starting with ap (2/5)\n",
+      "[2022-05-24 14:16:32,602] [INFO] Baseline for ap\n",
+      "[2022-05-24 14:16:54,175] [INFO] Trial 1/10 for ap\n",
+      "[2022-05-24 14:17:17,903] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:17:17,905] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:17:17,906] [INFO] Trial 2/10 for ap\n",
+      "[2022-05-24 14:17:38,846] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:17:38,847] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:17:38,848] [INFO] Trial 3/10 for ap\n",
+      "[2022-05-24 14:18:03,125] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:18:03,125] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:18:03,126] [INFO] Trial 4/10 for ap\n",
+      "[2022-05-24 14:18:27,364] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:18:27,365] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:18:27,366] [INFO] Trial 5/10 for ap\n",
+      "[2022-05-24 14:18:53,926] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:18:53,927] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:18:53,928] [INFO] Trial 6/10 for ap\n",
+      "[2022-05-24 14:19:14,329] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:19:14,329] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:19:14,330] [INFO] Trial 7/10 for ap\n",
+      "[2022-05-24 14:19:35,188] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:19:35,188] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:19:35,189] [INFO] Trial 8/10 for ap\n",
+      "[2022-05-24 14:20:04,335] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:20:04,336] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:20:04,337] [INFO] Trial 9/10 for ap\n",
+      "[2022-05-24 14:20:25,747] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:20:25,747] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:20:25,748] [INFO] Trial 10/10 for ap\n",
+      "[2022-05-24 14:20:55,942] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:20:55,943] [INFO] Best optimized model: 0.8053851088265638\n",
+      "[2022-05-24 14:20:55,944] [INFO] Starting with lbfgs (3/5)\n",
+      "[2022-05-24 14:20:55,944] [INFO] Baseline for lbfgs\n",
+      "[2022-05-24 14:21:23,004] [INFO] Trial 1/10 for lbfgs\n",
+      "[2022-05-24 14:21:51,544] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:21:51,545] [INFO] Best optimized model: 0.8512321635171647\n",
+      "[2022-05-24 14:21:51,546] [INFO] Trial 2/10 for lbfgs\n",
+      "[2022-05-24 14:22:15,532] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:22:15,533] [INFO] Best optimized model: 0.8583669352872352\n",
+      "[2022-05-24 14:22:15,534] [INFO] Trial 3/10 for lbfgs\n",
+      "[2022-05-24 14:22:41,231] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:22:41,232] [INFO] Best optimized model: 0.8583669352872352\n",
+      "[2022-05-24 14:22:41,233] [INFO] Trial 4/10 for lbfgs\n",
+      "[2022-05-24 14:23:11,347] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:23:11,348] [INFO] Best optimized model: 0.8765343356353424\n",
+      "[2022-05-24 14:23:11,349] [INFO] Trial 5/10 for lbfgs\n",
+      "[2022-05-24 14:23:47,354] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:23:47,356] [INFO] Best optimized model: 0.8765343356353424\n",
+      "[2022-05-24 14:23:47,357] [INFO] Trial 6/10 for lbfgs\n",
+      "[2022-05-24 14:24:21,991] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:24:21,993] [INFO] Best optimized model: 0.8765343356353424\n",
+      "[2022-05-24 14:24:21,993] [INFO] Trial 7/10 for lbfgs\n",
+      "[2022-05-24 14:24:44,177] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:24:44,178] [INFO] Best optimized model: 0.8765343356353424\n",
+      "[2022-05-24 14:24:44,178] [INFO] Trial 8/10 for lbfgs\n",
+      "[2022-05-24 14:25:10,296] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:25:10,297] [INFO] Best optimized model: 0.8765343356353424\n",
+      "[2022-05-24 14:25:10,297] [INFO] Trial 9/10 for lbfgs\n",
+      "[2022-05-24 14:25:33,210] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:25:33,211] [INFO] Best optimized model: 0.8765343356353424\n",
+      "[2022-05-24 14:25:33,212] [INFO] Trial 10/10 for lbfgs\n",
+      "[2022-05-24 14:25:57,224] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:25:57,225] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:25:57,226] [INFO] Starting with l2sgd (4/5)\n",
+      "[2022-05-24 14:25:57,227] [INFO] Baseline for l2sgd\n",
+      "[2022-05-24 14:26:29,107] [INFO] Trial 1/10 for l2sgd\n",
+      "[2022-05-24 14:26:57,573] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:26:57,574] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:26:57,575] [INFO] Trial 2/10 for l2sgd\n",
+      "[2022-05-24 14:27:34,959] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:27:34,960] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:27:34,960] [INFO] Trial 3/10 for l2sgd\n",
+      "[2022-05-24 14:28:04,577] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:28:04,578] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:28:04,578] [INFO] Trial 4/10 for l2sgd\n",
+      "[2022-05-24 14:28:29,486] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:28:29,487] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:28:29,487] [INFO] Trial 5/10 for l2sgd\n",
+      "[2022-05-24 14:28:43,936] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:28:43,937] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:28:43,938] [INFO] Trial 6/10 for l2sgd\n",
+      "[2022-05-24 14:29:17,613] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:29:17,614] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:29:17,614] [INFO] Trial 7/10 for l2sgd\n",
+      "[2022-05-24 14:29:49,064] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:29:49,065] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:29:49,066] [INFO] Trial 8/10 for l2sgd\n",
+      "[2022-05-24 14:30:28,164] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:30:28,165] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:30:28,165] [INFO] Trial 9/10 for l2sgd\n",
+      "[2022-05-24 14:31:04,397] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:31:04,397] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:31:04,398] [INFO] Trial 10/10 for l2sgd\n",
+      "[2022-05-24 14:31:32,285] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:31:32,286] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:31:32,287] [INFO] Starting with pa (5/5)\n",
+      "[2022-05-24 14:31:32,289] [INFO] Baseline for pa\n",
+      "[2022-05-24 14:31:57,620] [INFO] Trial 1/10 for pa\n",
+      "[2022-05-24 14:32:24,894] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:32:24,895] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:32:24,895] [INFO] Trial 2/10 for pa\n",
+      "[2022-05-24 14:32:54,099] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:32:54,100] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:32:54,101] [INFO] Trial 3/10 for pa\n",
+      "[2022-05-24 14:33:17,758] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:33:17,759] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:33:17,760] [INFO] Trial 4/10 for pa\n",
+      "[2022-05-24 14:33:44,958] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:33:44,959] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:33:44,959] [INFO] Trial 5/10 for pa\n",
+      "[2022-05-24 14:34:08,806] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:34:08,807] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:34:08,808] [INFO] Trial 6/10 for pa\n",
+      "[2022-05-24 14:34:35,835] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:34:35,835] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:34:35,836] [INFO] Trial 7/10 for pa\n",
+      "[2022-05-24 14:35:02,833] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:35:02,833] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:35:02,834] [INFO] Trial 8/10 for pa\n",
+      "[2022-05-24 14:35:30,266] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:35:30,267] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:35:30,268] [INFO] Trial 9/10 for pa\n",
+      "[2022-05-24 14:35:57,715] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:35:57,716] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:35:57,717] [INFO] Trial 10/10 for pa\n",
+      "[2022-05-24 14:36:26,015] [INFO] Best baseline model: 0.8053851088265638\n",
+      "[2022-05-24 14:36:26,016] [INFO] Best optimized model: 0.8959067851139951\n",
+      "[2022-05-24 14:36:26,016] [INFO] Finished hyperparameter optimization\n",
+      "[2022-05-24 14:36:26,017] [INFO] Trained 55 models with different hyperparamters\n",
+      "[2022-05-24 14:36:26,018] [INFO] Loading data set\n",
+      "[2022-05-24 14:36:26,493] [INFO] Start training\n",
+      "[2022-05-24 14:36:26,494] [INFO] Processing training data\n",
+      "[2022-05-24 14:36:26,507] [INFO] Processed 10% of the training data\n",
+      "[2022-05-24 14:36:26,514] [INFO] Processed 20% of the training data\n",
+      "[2022-05-24 14:36:26,529] [INFO] Processed 30% of the training data\n",
+      "[2022-05-24 14:36:26,544] [INFO] Processed 40% of the training data\n",
+      "[2022-05-24 14:36:26,557] [INFO] Processed 50% of the training data\n",
+      "[2022-05-24 14:36:26,569] [INFO] Processed 60% of the training data\n",
+      "[2022-05-24 14:36:26,582] [INFO] Processed 70% of the training data\n",
+      "[2022-05-24 14:36:26,591] [INFO] Processed 80% of the training data\n",
+      "[2022-05-24 14:36:26,598] [INFO] Processed 90% of the training data\n",
+      "[2022-05-24 14:36:26,606] [INFO] Processed 100% of the training data\n",
+      "[2022-05-24 14:36:26,617] [INFO] Start training with L-BFGS\n",
+      "[2022-05-24 14:36:26,755] [INFO] Iteration 1, training loss: 66667.597031\n",
+      "[2022-05-24 14:36:26,793] [INFO] Iteration 2, training loss: 55623.925756\n",
+      "[2022-05-24 14:36:26,900] [INFO] Iteration 3, training loss: 41676.630126\n",
+      "[2022-05-24 14:36:26,936] [INFO] Iteration 4, training loss: 41402.868392\n",
+      "[2022-05-24 14:36:26,979] [INFO] Iteration 5, training loss: 39958.091519\n",
+      "[2022-05-24 14:36:27,019] [INFO] Iteration 6, training loss: 39633.105474\n",
+      "[2022-05-24 14:36:27,058] [INFO] Iteration 7, training loss: 39410.823374\n",
+      "[2022-05-24 14:36:27,096] [INFO] Iteration 8, training loss: 38754.652581\n",
+      "[2022-05-24 14:36:27,166] [INFO] Iteration 9, training loss: 38431.445766\n",
+      "[2022-05-24 14:36:27,241] [INFO] Iteration 10, training loss: 38137.852674\n",
+      "[2022-05-24 14:36:27,316] [INFO] Iteration 11, training loss: 37932.490149\n",
+      "[2022-05-24 14:36:27,393] [INFO] Iteration 12, training loss: 37656.066661\n",
+      "[2022-05-24 14:36:27,466] [INFO] Iteration 13, training loss: 37041.775136\n",
+      "[2022-05-24 14:36:27,541] [INFO] Iteration 14, training loss: 34564.163873\n",
+      "[2022-05-24 14:36:27,615] [INFO] Iteration 15, training loss: 29178.011268\n",
+      "[2022-05-24 14:36:27,689] [INFO] Iteration 16, training loss: 25701.461865\n",
+      "[2022-05-24 14:36:27,760] [INFO] Iteration 17, training loss: 25476.440977\n",
+      "[2022-05-24 14:36:27,848] [INFO] Iteration 18, training loss: 23889.822991\n",
+      "[2022-05-24 14:36:27,923] [INFO] Iteration 19, training loss: 23242.360239\n",
+      "[2022-05-24 14:36:27,998] [INFO] Iteration 20, training loss: 22419.544709\n",
+      "[2022-05-24 14:36:28,087] [INFO] Iteration 21, training loss: 21077.029722\n",
+      "[2022-05-24 14:36:28,182] [INFO] Iteration 22, training loss: 20253.875975\n",
+      "[2022-05-24 14:36:28,284] [INFO] Iteration 23, training loss: 19517.472648\n",
+      "[2022-05-24 14:36:28,359] [INFO] Iteration 24, training loss: 17391.050388\n",
+      "[2022-05-24 14:36:28,432] [INFO] Iteration 25, training loss: 16194.689023\n",
+      "[2022-05-24 14:36:28,506] [INFO] Iteration 26, training loss: 16041.706576\n",
+      "[2022-05-24 14:36:28,579] [INFO] Iteration 27, training loss: 14975.985721\n",
+      "[2022-05-24 14:36:28,661] [INFO] Iteration 28, training loss: 14545.272899\n",
+      "[2022-05-24 14:36:28,714] [INFO] Iteration 29, training loss: 14082.482185\n",
+      "[2022-05-24 14:36:28,751] [INFO] Iteration 30, training loss: 13145.371954\n",
+      "[2022-05-24 14:36:28,787] [INFO] Iteration 31, training loss: 12629.195146\n",
+      "[2022-05-24 14:36:28,827] [INFO] Iteration 32, training loss: 12032.039313\n",
+      "[2022-05-24 14:36:28,868] [INFO] Iteration 33, training loss: 11273.695638\n",
+      "[2022-05-24 14:36:28,908] [INFO] Iteration 34, training loss: 10490.230218\n",
+      "[2022-05-24 14:36:28,948] [INFO] Iteration 35, training loss: 10028.089928\n",
+      "[2022-05-24 14:36:28,986] [INFO] Iteration 36, training loss: 9620.020311\n",
+      "[2022-05-24 14:36:29,024] [INFO] Iteration 37, training loss: 9197.565547\n",
+      "[2022-05-24 14:36:29,060] [INFO] Iteration 38, training loss: 8835.211466\n",
+      "[2022-05-24 14:36:29,100] [INFO] Iteration 39, training loss: 8324.185003\n",
+      "[2022-05-24 14:36:29,138] [INFO] Iteration 40, training loss: 7813.055525\n",
+      "[2022-05-24 14:36:29,176] [INFO] Iteration 41, training loss: 7331.840266\n",
+      "[2022-05-24 14:36:29,216] [INFO] Iteration 42, training loss: 6723.739352\n",
+      "[2022-05-24 14:36:29,259] [INFO] Iteration 43, training loss: 6537.846199\n",
+      "[2022-05-24 14:36:29,295] [INFO] Iteration 44, training loss: 6297.711000\n",
+      "[2022-05-24 14:36:29,332] [INFO] Iteration 45, training loss: 6136.256732\n",
+      "[2022-05-24 14:36:29,368] [INFO] Iteration 46, training loss: 5932.489668\n",
+      "[2022-05-24 14:36:29,406] [INFO] Iteration 47, training loss: 5800.595526\n",
+      "[2022-05-24 14:36:29,442] [INFO] Iteration 48, training loss: 5610.988280\n",
+      "[2022-05-24 14:36:29,483] [INFO] Iteration 49, training loss: 5561.589990\n",
+      "[2022-05-24 14:36:29,520] [INFO] Iteration 50, training loss: 5407.647596\n",
+      "[2022-05-24 14:36:29,557] [INFO] Iteration 51, training loss: 5390.933736\n",
+      "[2022-05-24 14:36:29,593] [INFO] Iteration 52, training loss: 5245.036766\n",
+      "[2022-05-24 14:36:29,635] [INFO] Iteration 53, training loss: 5196.459345\n",
+      "[2022-05-24 14:36:29,673] [INFO] Iteration 54, training loss: 5067.404038\n",
+      "[2022-05-24 14:36:29,714] [INFO] Iteration 55, training loss: 5029.268875\n",
+      "[2022-05-24 14:36:29,761] [INFO] Iteration 56, training loss: 4961.855915\n",
+      "[2022-05-24 14:36:29,798] [INFO] Iteration 57, training loss: 4924.561740\n",
+      "[2022-05-24 14:36:29,835] [INFO] Iteration 58, training loss: 4857.772977\n",
+      "[2022-05-24 14:36:29,872] [INFO] Iteration 59, training loss: 4846.022914\n",
+      "[2022-05-24 14:36:29,920] [INFO] Iteration 60, training loss: 4808.945481\n",
+      "[2022-05-24 14:36:29,979] [INFO] Iteration 61, training loss: 4796.355064\n",
+      "[2022-05-24 14:36:30,041] [INFO] Iteration 62, training loss: 4761.780633\n",
+      "[2022-05-24 14:36:30,127] [INFO] Iteration 63, training loss: 4726.583580\n",
+      "[2022-05-24 14:36:30,169] [INFO] Iteration 64, training loss: 4710.835766\n",
+      "[2022-05-24 14:36:30,239] [INFO] Iteration 65, training loss: 4683.154076\n",
+      "[2022-05-24 14:36:30,335] [INFO] Iteration 66, training loss: 4667.099179\n",
+      "[2022-05-24 14:36:30,406] [INFO] Iteration 67, training loss: 4644.071842\n",
+      "[2022-05-24 14:36:30,484] [INFO] Iteration 68, training loss: 4625.609051\n",
+      "[2022-05-24 14:36:30,560] [INFO] Iteration 69, training loss: 4617.274602\n",
+      "[2022-05-24 14:36:30,633] [INFO] Iteration 70, training loss: 4592.817701\n",
+      "[2022-05-24 14:36:30,709] [INFO] Iteration 71, training loss: 4585.805803\n",
+      "[2022-05-24 14:36:30,785] [INFO] Iteration 72, training loss: 4569.307227\n",
+      "[2022-05-24 14:36:30,861] [INFO] Iteration 73, training loss: 4557.347450\n",
+      "[2022-05-24 14:36:30,945] [INFO] Iteration 74, training loss: 4540.827323\n",
+      "[2022-05-24 14:36:31,017] [INFO] Iteration 75, training loss: 4529.524746\n",
+      "[2022-05-24 14:36:31,090] [INFO] Iteration 76, training loss: 4512.530626\n",
+      "[2022-05-24 14:36:31,169] [INFO] Iteration 77, training loss: 4505.499193\n",
+      "[2022-05-24 14:36:31,243] [INFO] Iteration 78, training loss: 4495.679634\n",
+      "[2022-05-24 14:36:31,318] [INFO] Iteration 79, training loss: 4483.684821\n",
+      "[2022-05-24 14:36:31,395] [INFO] Iteration 80, training loss: 4468.690766\n",
+      "[2022-05-24 14:36:31,478] [INFO] Iteration 81, training loss: 4458.656264\n",
+      "[2022-05-24 14:36:31,567] [INFO] Iteration 82, training loss: 4447.938415\n",
+      "[2022-05-24 14:36:31,644] [INFO] Iteration 83, training loss: 4439.669589\n",
+      "[2022-05-24 14:36:31,722] [INFO] Iteration 84, training loss: 4429.247218\n",
+      "[2022-05-24 14:36:31,792] [INFO] Iteration 85, training loss: 4419.982076\n",
+      "[2022-05-24 14:36:31,829] [INFO] Iteration 86, training loss: 4411.021280\n",
+      "[2022-05-24 14:36:31,866] [INFO] Iteration 87, training loss: 4402.644481\n",
+      "[2022-05-24 14:36:31,904] [INFO] Iteration 88, training loss: 4399.168568\n",
+      "[2022-05-24 14:36:31,942] [INFO] Iteration 89, training loss: 4394.003830\n",
+      "[2022-05-24 14:36:31,980] [INFO] Iteration 90, training loss: 4388.713150\n",
+      "[2022-05-24 14:36:32,016] [INFO] Iteration 91, training loss: 4385.619733\n",
+      "[2022-05-24 14:36:32,053] [INFO] Iteration 92, training loss: 4381.598230\n",
+      "[2022-05-24 14:36:32,092] [INFO] Iteration 93, training loss: 4377.788408\n",
+      "[2022-05-24 14:36:32,129] [INFO] Iteration 94, training loss: 4373.437655\n",
+      "[2022-05-24 14:36:32,167] [INFO] Iteration 95, training loss: 4368.816208\n",
+      "[2022-05-24 14:36:32,205] [INFO] Iteration 96, training loss: 4365.251812\n",
+      "[2022-05-24 14:36:32,244] [INFO] Iteration 97, training loss: 4361.696093\n",
+      "[2022-05-24 14:36:32,280] [INFO] Iteration 98, training loss: 4357.857630\n",
+      "[2022-05-24 14:36:32,317] [INFO] Iteration 99, training loss: 4354.613360\n",
+      "[2022-05-24 14:36:32,357] [INFO] Iteration 100, training loss: 4350.888322\n",
+      "[2022-05-24 14:36:32,397] [INFO] Iteration 101, training loss: 4348.166740\n",
+      "[2022-05-24 14:36:32,438] [INFO] Iteration 102, training loss: 4344.833289\n",
+      "[2022-05-24 14:36:32,474] [INFO] Iteration 103, training loss: 4341.791561\n",
+      "[2022-05-24 14:36:32,520] [INFO] Iteration 104, training loss: 4338.324725\n",
+      "[2022-05-24 14:36:32,565] [INFO] Iteration 105, training loss: 4335.350730\n",
+      "[2022-05-24 14:36:32,604] [INFO] Iteration 106, training loss: 4332.347137\n",
+      "[2022-05-24 14:36:32,653] [INFO] Iteration 107, training loss: 4329.062501\n",
+      "[2022-05-24 14:36:32,696] [INFO] Iteration 108, training loss: 4326.420723\n",
+      "[2022-05-24 14:36:32,742] [INFO] Iteration 109, training loss: 4323.894801\n",
+      "[2022-05-24 14:36:32,782] [INFO] Iteration 110, training loss: 4320.632134\n",
+      "[2022-05-24 14:36:32,819] [INFO] Iteration 111, training loss: 4317.967978\n",
+      "[2022-05-24 14:36:32,857] [INFO] Iteration 112, training loss: 4315.757244\n",
+      "[2022-05-24 14:36:32,893] [INFO] Iteration 113, training loss: 4313.471763\n",
+      "[2022-05-24 14:36:32,932] [INFO] Iteration 114, training loss: 4311.927318\n",
+      "[2022-05-24 14:36:32,971] [INFO] Iteration 115, training loss: 4307.875027\n",
+      "[2022-05-24 14:36:33,008] [INFO] Iteration 116, training loss: 4305.993894\n",
+      "[2022-05-24 14:36:33,045] [INFO] Iteration 117, training loss: 4303.629664\n",
+      "[2022-05-24 14:36:33,083] [INFO] Iteration 118, training loss: 4302.472220\n",
+      "[2022-05-24 14:36:33,121] [INFO] Iteration 119, training loss: 4299.774125\n",
+      "[2022-05-24 14:36:33,179] [INFO] Iteration 120, training loss: 4298.337756\n",
+      "[2022-05-24 14:36:33,257] [INFO] Iteration 121, training loss: 4295.608496\n",
+      "[2022-05-24 14:36:33,327] [INFO] Iteration 122, training loss: 4294.142966\n",
+      "[2022-05-24 14:36:33,400] [INFO] Iteration 123, training loss: 4292.115642\n",
+      "[2022-05-24 14:36:33,491] [INFO] Iteration 124, training loss: 4291.371326\n",
+      "[2022-05-24 14:36:33,563] [INFO] Iteration 125, training loss: 4288.613849\n",
+      "[2022-05-24 14:36:33,637] [INFO] Iteration 126, training loss: 4287.839852\n",
+      "[2022-05-24 14:36:33,709] [INFO] Iteration 127, training loss: 4285.331459\n",
+      "[2022-05-24 14:36:33,782] [INFO] Iteration 128, training loss: 4284.892009\n",
+      "[2022-05-24 14:36:33,857] [INFO] Iteration 129, training loss: 4282.615233\n",
+      "[2022-05-24 14:36:33,934] [INFO] Iteration 130, training loss: 4282.102488\n",
+      "[2022-05-24 14:36:34,007] [INFO] Iteration 131, training loss: 4279.965314\n",
+      "[2022-05-24 14:36:34,081] [INFO] Iteration 132, training loss: 4279.103657\n",
+      "[2022-05-24 14:36:34,153] [INFO] Iteration 133, training loss: 4277.442945\n",
+      "[2022-05-24 14:36:34,228] [INFO] Iteration 134, training loss: 4276.815206\n",
+      "[2022-05-24 14:36:34,302] [INFO] Iteration 135, training loss: 4275.288222\n",
+      "[2022-05-24 14:36:34,375] [INFO] Iteration 136, training loss: 4274.658127\n",
+      "[2022-05-24 14:36:34,447] [INFO] Iteration 137, training loss: 4273.117486\n",
+      "[2022-05-24 14:36:34,520] [INFO] Iteration 138, training loss: 4272.456664\n",
+      "[2022-05-24 14:36:34,594] [INFO] Iteration 139, training loss: 4271.039188\n",
+      "[2022-05-24 14:36:34,667] [INFO] Iteration 140, training loss: 4270.628543\n",
+      "[2022-05-24 14:36:34,735] [INFO] Iteration 141, training loss: 4269.196799\n",
+      "[2022-05-24 14:36:34,771] [INFO] Iteration 142, training loss: 4268.917476\n",
+      "[2022-05-24 14:36:34,808] [INFO] Iteration 143, training loss: 4267.337330\n",
+      "[2022-05-24 14:36:34,852] [INFO] Iteration 144, training loss: 4267.307893\n",
+      "[2022-05-24 14:36:34,893] [INFO] Iteration 145, training loss: 4265.749655\n",
+      "[2022-05-24 14:36:34,968] [INFO] Iteration 146, training loss: 4264.680008\n",
+      "[2022-05-24 14:36:35,007] [INFO] Iteration 147, training loss: 4264.272858\n",
+      "[2022-05-24 14:36:35,046] [INFO] Iteration 148, training loss: 4262.504383\n",
+      "[2022-05-24 14:36:35,085] [INFO] Iteration 149, training loss: 4262.351065\n",
+      "[2022-05-24 14:36:35,125] [INFO] Iteration 150, training loss: 4261.240872\n",
+      "[2022-05-24 14:36:35,161] [INFO] Iteration 151, training loss: 4259.488339\n",
+      "[2022-05-24 14:36:35,198] [INFO] Iteration 152, training loss: 4258.201711\n",
+      "[2022-05-24 14:36:35,236] [INFO] Iteration 153, training loss: 4257.138645\n",
+      "[2022-05-24 14:36:35,272] [INFO] Iteration 154, training loss: 4255.885107\n",
+      "[2022-05-24 14:36:35,309] [INFO] Iteration 155, training loss: 4255.219267\n",
+      "[2022-05-24 14:36:35,344] [INFO] Iteration 156, training loss: 4254.131410\n",
+      "[2022-05-24 14:36:35,384] [INFO] Iteration 157, training loss: 4253.529838\n",
+      "[2022-05-24 14:36:35,420] [INFO] Iteration 158, training loss: 4252.574568\n",
+      "[2022-05-24 14:36:35,457] [INFO] Iteration 159, training loss: 4251.368121\n",
+      "[2022-05-24 14:36:35,492] [INFO] Iteration 160, training loss: 4250.650102\n",
+      "[2022-05-24 14:36:35,537] [INFO] Iteration 161, training loss: 4249.897824\n",
+      "[2022-05-24 14:36:35,573] [INFO] Iteration 162, training loss: 4249.316158\n",
+      "[2022-05-24 14:36:35,616] [INFO] Iteration 163, training loss: 4248.314410\n",
+      "[2022-05-24 14:36:35,653] [INFO] Iteration 164, training loss: 4247.794380\n",
+      "[2022-05-24 14:36:35,690] [INFO] Iteration 165, training loss: 4246.639875\n",
+      "[2022-05-24 14:36:35,725] [INFO] Iteration 166, training loss: 4246.142378\n",
+      "[2022-05-24 14:36:35,764] [INFO] Iteration 167, training loss: 4245.284106\n",
+      "[2022-05-24 14:36:35,800] [INFO] Iteration 168, training loss: 4244.794802\n",
+      "[2022-05-24 14:36:35,837] [INFO] Iteration 169, training loss: 4243.926213\n",
+      "[2022-05-24 14:36:35,874] [INFO] Iteration 170, training loss: 4243.502643\n",
+      "[2022-05-24 14:36:35,919] [INFO] Iteration 171, training loss: 4242.699617\n",
+      "[2022-05-24 14:36:35,963] [INFO] Iteration 172, training loss: 4242.252274\n",
+      "[2022-05-24 14:36:36,005] [INFO] Iteration 173, training loss: 4241.594073\n",
+      "[2022-05-24 14:36:36,048] [INFO] Iteration 174, training loss: 4241.395693\n",
+      "[2022-05-24 14:36:36,092] [INFO] Iteration 175, training loss: 4240.532660\n",
+      "[2022-05-24 14:36:36,136] [INFO] Iteration 176, training loss: 4240.510311\n",
+      "[2022-05-24 14:36:36,179] [INFO] Iteration 177, training loss: 4239.532711\n",
+      "[2022-05-24 14:36:36,294] [INFO] Iteration 178, training loss: 4238.947761\n",
+      "[2022-05-24 14:36:36,368] [INFO] Iteration 179, training loss: 4238.874127\n",
+      "[2022-05-24 14:36:36,370] [INFO] Terminated with the stopping criteria\n",
+      "[2022-05-24 14:36:36,371] [INFO] Saving model\n"
+     ]
+    }
+   ],
    "source": [
     "model = chaine.train(sentences, labels, verbose=0, optimize_hyperparameters=True)"
    ]
@@ -338,14 +729,14 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "         LOC       0.94      0.91      0.92      1668\n",
-      "        MISC       0.91      0.77      0.83       702\n",
-      "         ORG       0.90      0.88      0.89      1661\n",
-      "         PER       0.90      0.91      0.91      1617\n",
+      "         LOC       0.97      0.95      0.96      1668\n",
+      "        MISC       0.97      0.87      0.91       702\n",
+      "         ORG       0.94      0.93      0.94      1661\n",
+      "         PER       0.97      0.96      0.96      1617\n",
       "\n",
-      "   micro avg       0.92      0.88      0.90      5648\n",
-      "   macro avg       0.92      0.87      0.89      5648\n",
-      "weighted avg       0.92      0.88      0.90      5648\n",
+      "   micro avg       0.96      0.94      0.95      5648\n",
+      "   macro avg       0.96      0.93      0.94      5648\n",
+      "weighted avg       0.96      0.94      0.95      5648\n",
       "\n"
      ]
     }
@@ -401,61 +792,61 @@
        "      <th>12</th>\n",
        "      <td>B-PER</td>\n",
        "      <td>I-PER</td>\n",
-       "      <td>8.655663</td>\n",
+       "      <td>8.318353</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>30</th>\n",
        "      <td>B-ORG</td>\n",
        "      <td>I-ORG</td>\n",
-       "      <td>7.053397</td>\n",
+       "      <td>7.168033</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>33</th>\n",
-       "      <td>I-ORG</td>\n",
-       "      <td>I-ORG</td>\n",
-       "      <td>6.440869</td>\n",
+       "      <th>22</th>\n",
+       "      <td>B-MISC</td>\n",
+       "      <td>I-MISC</td>\n",
+       "      <td>6.609091</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>O</td>\n",
-       "      <td>O</td>\n",
-       "      <td>5.564141</td>\n",
+       "      <th>26</th>\n",
+       "      <td>I-MISC</td>\n",
+       "      <td>I-MISC</td>\n",
+       "      <td>6.177694</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>O</td>\n",
-       "      <td>B-MISC</td>\n",
-       "      <td>5.456652</td>\n",
+       "      <th>34</th>\n",
+       "      <td>I-ORG</td>\n",
+       "      <td>I-ORG</td>\n",
+       "      <td>6.171870</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>4</th>\n",
+       "      <th>0</th>\n",
        "      <td>O</td>\n",
-       "      <td>B-ORG</td>\n",
-       "      <td>5.394471</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>22</th>\n",
-       "      <td>B-MISC</td>\n",
-       "      <td>I-MISC</td>\n",
-       "      <td>5.257558</td>\n",
+       "      <td>O</td>\n",
+       "      <td>5.502018</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
        "      <td>B-LOC</td>\n",
        "      <td>I-LOC</td>\n",
-       "      <td>4.875593</td>\n",
+       "      <td>5.196033</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>O</td>\n",
-       "      <td>B-PER</td>\n",
-       "      <td>4.716164</td>\n",
+       "      <th>18</th>\n",
+       "      <td>I-LOC</td>\n",
+       "      <td>I-LOC</td>\n",
+       "      <td>4.680722</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1</th>\n",
+       "      <th>3</th>\n",
        "      <td>O</td>\n",
-       "      <td>B-LOC</td>\n",
-       "      <td>4.539984</td>\n",
+       "      <td>B-MISC</td>\n",
+       "      <td>4.323212</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>I-PER</td>\n",
+       "      <td>I-PER</td>\n",
+       "      <td>4.204643</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -463,16 +854,16 @@
       ],
       "text/plain": [
        "      from      to    weight\n",
-       "12   B-PER   I-PER  8.655663\n",
-       "30   B-ORG   I-ORG  7.053397\n",
-       "33   I-ORG   I-ORG  6.440869\n",
-       "0        O       O  5.564141\n",
-       "3        O  B-MISC  5.456652\n",
-       "4        O   B-ORG  5.394471\n",
-       "22  B-MISC  I-MISC  5.257558\n",
-       "8    B-LOC   I-LOC  4.875593\n",
-       "2        O   B-PER  4.716164\n",
-       "1        O   B-LOC  4.539984"
+       "12   B-PER   I-PER  8.318353\n",
+       "30   B-ORG   I-ORG  7.168033\n",
+       "22  B-MISC  I-MISC  6.609091\n",
+       "26  I-MISC  I-MISC  6.177694\n",
+       "34   I-ORG   I-ORG  6.171870\n",
+       "0        O       O  5.502018\n",
+       "8    B-LOC   I-LOC  5.196033\n",
+       "18   I-LOC   I-LOC  4.680722\n",
+       "3        O  B-MISC  4.323212\n",
+       "15   I-PER   I-PER  4.204643"
       ]
      },
      "execution_count": 12,
@@ -519,81 +910,81 @@
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <th>4385</th>\n",
+       "      <th>4353</th>\n",
        "      <td>token[-2:]:0M</td>\n",
        "      <td>O</td>\n",
-       "      <td>5.749702</td>\n",
+       "      <td>9.405752</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>4386</th>\n",
+       "      <th>4354</th>\n",
        "      <td>token[-2:]:5M</td>\n",
        "      <td>O</td>\n",
-       "      <td>5.541271</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>164</th>\n",
-       "      <td>EOS</td>\n",
-       "      <td>O</td>\n",
-       "      <td>5.192606</td>\n",
+       "      <td>8.817209</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>214</th>\n",
+       "      <th>216</th>\n",
        "      <td>+1:token.lower():1996-12-06</td>\n",
        "      <td>B-LOC</td>\n",
-       "      <td>4.757576</td>\n",
+       "      <td>5.713073</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>215</th>\n",
+       "      <th>3807</th>\n",
+       "      <td>token.lower():painewebber</td>\n",
+       "      <td>B-ORG</td>\n",
+       "      <td>5.619434</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>217</th>\n",
        "      <td>+1:token.lower():1996-12-06</td>\n",
        "      <td>I-LOC</td>\n",
-       "      <td>3.692241</td>\n",
+       "      <td>5.243976</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>3532</th>\n",
-       "      <td>+1:token.lower():1996-12-07</td>\n",
-       "      <td>B-LOC</td>\n",
-       "      <td>3.504000</td>\n",
+       "      <th>3704</th>\n",
+       "      <td>+1:token.lower():exxon</td>\n",
+       "      <td>O</td>\n",
+       "      <td>5.152197</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1658</th>\n",
-       "      <td>-1:token.lower():b</td>\n",
-       "      <td>B-PER</td>\n",
-       "      <td>3.502818</td>\n",
+       "      <th>1013</th>\n",
+       "      <td>token.lower():italy</td>\n",
+       "      <td>B-LOC</td>\n",
+       "      <td>5.052394</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>23</th>\n",
-       "      <td>token.isdigit()</td>\n",
+       "      <th>168</th>\n",
+       "      <td>EOS</td>\n",
        "      <td>O</td>\n",
-       "      <td>3.362199</td>\n",
+       "      <td>4.993690</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>36</th>\n",
-       "      <td>BOS</td>\n",
-       "      <td>O</td>\n",
-       "      <td>3.280833</td>\n",
+       "      <th>1605</th>\n",
+       "      <td>-1:token.lower():b</td>\n",
+       "      <td>B-PER</td>\n",
+       "      <td>4.895346</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1095</th>\n",
-       "      <td>-1:token.lower():at</td>\n",
-       "      <td>B-LOC</td>\n",
-       "      <td>3.198786</td>\n",
+       "      <th>3233</th>\n",
+       "      <td>token.lower():trans-atlantic</td>\n",
+       "      <td>B-MISC</td>\n",
+       "      <td>4.789043</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "                          feature  label    weight\n",
-       "4385                token[-2:]:0M      O  5.749702\n",
-       "4386                token[-2:]:5M      O  5.541271\n",
-       "164                           EOS      O  5.192606\n",
-       "214   +1:token.lower():1996-12-06  B-LOC  4.757576\n",
-       "215   +1:token.lower():1996-12-06  I-LOC  3.692241\n",
-       "3532  +1:token.lower():1996-12-07  B-LOC  3.504000\n",
-       "1658           -1:token.lower():b  B-PER  3.502818\n",
-       "23                token.isdigit()      O  3.362199\n",
-       "36                            BOS      O  3.280833\n",
-       "1095          -1:token.lower():at  B-LOC  3.198786"
+       "                           feature   label    weight\n",
+       "4353                 token[-2:]:0M       O  9.405752\n",
+       "4354                 token[-2:]:5M       O  8.817209\n",
+       "216    +1:token.lower():1996-12-06   B-LOC  5.713073\n",
+       "3807     token.lower():painewebber   B-ORG  5.619434\n",
+       "217    +1:token.lower():1996-12-06   I-LOC  5.243976\n",
+       "3704        +1:token.lower():exxon       O  5.152197\n",
+       "1013           token.lower():italy   B-LOC  5.052394\n",
+       "168                            EOS       O  4.993690\n",
+       "1605            -1:token.lower():b   B-PER  4.895346\n",
+       "3233  token.lower():trans-atlantic  B-MISC  4.789043"
       ]
      },
      "execution_count": 13,
@@ -608,6 +999,9 @@
   }
  ],
  "metadata": {
+  "interpreter": {
+   "hash": "b6305eb58051137afc5a5205478d9ee79d6169039fed1d5a046194240369b06c"
+  },
   "kernelspec": {
    "display_name": "Python 3 (ipykernel)",
    "language": "python",
@@ -623,7 +1017,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.3"
+   "version": "3.9.13"
   }
  },
  "nbformat": 4,