4_word_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YHYG3fot4KAT"
   },
   "source": [
    "# Create Custom Word Embeddings\n",
    "\n",
    "In this notebook we aim to create our custom word-embeddings which are generic/unsupervised and can be used in any of our NLP applications where we use a word embeddings.\n",
    "\n",
    "We majorly will be doing the following :\n",
    "\n",
    "  1. Install & Import Packages\n",
    "  2. Define our Text Pre-processing pipeline.\n",
    "  3. Creating Text Corpus\n",
    "  4. Train Word Embeddings\n",
    "  5. Convert & Save "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 27926,
     "status": "ok",
     "timestamp": 1578202924754,
     "user": {
      "displayName": "Amit Agarwal",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCNCK3WDEC63aFcnBVJrUB3M3cs4J2bImVRkNad8A=s64",
      "userId": "15805199744168269697"
     },
     "user_tz": -330
    },
    "id": "bGKB2JRvYnpx",
    "outputId": "f5960e63-c00f-4538-97aa-393db047e814"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "# Let's mount our G-Drive. \n",
    "\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nmHUJ69b5zBU"
   },
   "source": [
    "# 1. Install & Import Packages\n",
    "\n",
    "Make sure you restart the runtime once the pip installs has completed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 141353,
     "status": "ok",
     "timestamp": 1578203067898,
     "user": {
      "displayName": "Amit Agarwal",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCNCK3WDEC63aFcnBVJrUB3M3cs4J2bImVRkNad8A=s64",
      "userId": "15805199744168269697"
     },
     "user_tz": -330
    },
    "id": "tGgt0wshuY9g",
    "outputId": "9a9897dd-2b6a-4ee3-cf90-f76d61890af7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fasttext\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/61/2e01f1397ec533756c1d893c22d9d5ed3fce3a6e4af1976e0d86bb13ea97/fasttext-0.9.1.tar.gz (57kB)\n",
      "\u001b[K     |████████████████████████████████| 61kB 2.0MB/s \n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: pybind11>=2.2 in /usr/local/lib/python3.6/dist-packages (from fasttext) (2.4.3)\n",
      "Requirement already satisfied, skipping upgrade: setuptools>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from fasttext) (42.0.2)\n",
      "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from fasttext) (1.17.4)\n",
      "Building wheels for collected packages: fasttext\n",
      "  Building wheel for fasttext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for fasttext: filename=fasttext-0.9.1-cp36-cp36m-linux_x86_64.whl size=2384065 sha256=d4e85b70e9f7fdf878e13da282a573148197922f0aea470a1280c3cc258812bc\n",
      "  Stored in directory: /root/.cache/pip/wheels/9f/f0/04/caa82c912aee89ce76358ff954f3f0729b7577c8ff23a292e3\n",
      "Successfully built fasttext\n",
      "Installing collected packages: fasttext\n",
      "Successfully installed fasttext-0.9.1\n",
      "Collecting gensim\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/dd/112bd4258cee11e0baaaba064060eb156475a42362e59e3ff28e7ca2d29d/gensim-3.8.1-cp36-cp36m-manylinux1_x86_64.whl (24.2MB)\n",
      "\u001b[K     |████████████████████████████████| 24.2MB 42.1MB/s \n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.3.3)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.17.4)\n",
      "Requirement already satisfied, skipping upgrade: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.12.0)\n",
      "Requirement already satisfied, skipping upgrade: smart-open>=1.8.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.9.0)\n",
      "Requirement already satisfied, skipping upgrade: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.8.1->gensim) (1.10.40)\n",
      "Requirement already satisfied, skipping upgrade: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.8.1->gensim) (2.49.0)\n",
      "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.8.1->gensim) (2.21.0)\n",
      "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.8.1->gensim) (0.9.4)\n",
      "Requirement already satisfied, skipping upgrade: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.8.1->gensim) (0.2.1)\n",
      "Requirement already satisfied, skipping upgrade: botocore<1.14.0,>=1.13.40 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.8.1->gensim) (1.13.40)\n",
      "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim) (2.8)\n",
      "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim) (2019.11.28)\n",
      "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim) (3.0.4)\n",
      "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.8.1->gensim) (1.24.3)\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil<2.8.1,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.40->boto3->smart-open>=1.8.1->gensim) (2.6.1)\n",
      "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.40->boto3->smart-open>=1.8.1->gensim) (0.15.2)\n",
      "Installing collected packages: gensim\n",
      "  Found existing installation: gensim 3.6.0\n",
      "    Uninstalling gensim-3.6.0:\n",
      "      Successfully uninstalled gensim-3.6.0\n",
      "Successfully installed gensim-3.8.1\n",
      "Collecting tiny-tokenizer\n",
      "  Downloading https://files.pythonhosted.org/packages/8d/0f/aa52c227c5af69914be05723b3deaf221805a4ccbce87643194ef2cdde43/tiny_tokenizer-3.1.0.tar.gz\n",
      "Building wheels for collected packages: tiny-tokenizer\n",
      "  Building wheel for tiny-tokenizer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for tiny-tokenizer: filename=tiny_tokenizer-3.1.0-cp36-none-any.whl size=10550 sha256=6248d5e884acfc63fe084f7290df0d98b8e36ef62d28adde01ea970a2c667c5b\n",
      "  Stored in directory: /root/.cache/pip/wheels/d1/c8/36/334497a689fab90128232e86b5829b800dd271a3d5d5959c53\n",
      "Successfully built tiny-tokenizer\n",
      "Installing collected packages: tiny-tokenizer\n",
      "Successfully installed tiny-tokenizer-3.1.0\n",
      "Collecting flair\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/16/22/8fc8e5978ec05b710216735ca47415700e83f304dec7e4281d61cefb6831/flair-0.4.4-py3-none-any.whl (193kB)\n",
      "\u001b[K     |████████████████████████████████| 194kB 2.8MB/s \n",
      "\u001b[?25hCollecting sqlitedict>=1.6.0\n",
      "  Downloading https://files.pythonhosted.org/packages/0f/1c/c757b93147a219cf1e25cef7e1ad9b595b7f802159493c45ce116521caff/sqlitedict-1.6.0.tar.gz\n",
      "Collecting ipython==7.6.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/2c/c7d44277b599df35af734d8f4142d501192fdb7aef5d04daf882d7eccfbc/ipython-7.6.1-py3-none-any.whl (774kB)\n",
      "\u001b[K     |████████████████████████████████| 778kB 39.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from flair) (0.4.2)\n",
      "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from flair) (0.0)\n",
      "Collecting deprecated>=1.2.4\n",
      "  Downloading https://files.pythonhosted.org/packages/f6/89/62912e01f3cede11edcc0abf81298e3439d9c06c8dce644369380ed13f6d/Deprecated-1.2.7-py2.py3-none-any.whl\n",
      "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.6/dist-packages (from flair) (3.1.2)\n",
      "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from flair) (1.3.1)\n",
      "Requirement already satisfied: ipython-genutils==0.2.0 in /usr/local/lib/python3.6/dist-packages (from flair) (0.2.0)\n",
      "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from flair) (0.8.6)\n",
      "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from flair) (0.1.2)\n",
      "Collecting segtok>=1.5.7\n",
      "  Downloading https://files.pythonhosted.org/packages/1d/59/6ed78856ab99d2da04084b59e7da797972baa0efecb71546b16d48e49d9b/segtok-1.5.7.tar.gz\n",
      "Requirement already satisfied: gensim>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from flair) (3.8.1)\n",
      "Requirement already satisfied: pytest>=3.6.4 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.4)\n",
      "Collecting langdetect\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/59/59/4bc44158a767a6d66de18c4136c8aa90491d56cc951c10b74dd1e13213c9/langdetect-1.0.7.zip (998kB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 46.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from flair) (2019.12.9)\n",
      "Collecting bpemb>=0.2.9\n",
      "  Downloading https://files.pythonhosted.org/packages/bc/70/468a9652095b370f797ed37ff77e742b11565c6fd79eaeca5f2e50b164a7/bpemb-0.3.0-py3-none-any.whl\n",
      "Requirement already satisfied: tiny-tokenizer[all] in /usr/local/lib/python3.6/dist-packages (from flair) (3.1.0)\n",
      "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from flair) (3.10.0)\n",
      "Collecting transformers>=2.0.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/10/aeefced99c8a59d828a92cc11d213e2743212d3641c87c82d61b035a7d5c/transformers-2.3.0-py3-none-any.whl (447kB)\n",
      "\u001b[K     |████████████████████████████████| 450kB 45.0MB/s \n",
      "\u001b[?25hCollecting mpld3==0.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/95/a52d3a83d0a29ba0d6898f6727e9858fe7a43f6c2ce81a5fe7e05f0f4912/mpld3-0.3.tar.gz (788kB)\n",
      "\u001b[K     |████████████████████████████████| 798kB 43.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.6/dist-packages (from flair) (4.28.1)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.20 in /usr/local/lib/python3.6/dist-packages (from flair) (1.24.3)\n",
      "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (42.0.2)\n",
      "Requirement already satisfied: backcall in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.1.0)\n",
      "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.4.1)\n",
      "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (2.1.3)\n",
      "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.3.3)\n",
      "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.7.5)\n",
      "Collecting prompt-toolkit<2.1.0,>=2.0.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl (340kB)\n",
      "\u001b[K     |████████████████████████████████| 348kB 48.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.7.0)\n",
      "Requirement already satisfied: jedi>=0.10 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.15.1)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (1.12.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (1.17.4)\n",
      "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (4.3.0)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->flair) (0.21.3)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from deprecated>=1.2.4->flair) (1.11.2)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.6.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.4.5)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (1.1.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (0.16.0)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (1.3.3)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (2.4)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->flair) (1.9.0)\n",
      "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (0.7.1)\n",
      "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.8.0)\n",
      "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (8.0.2)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (19.3.0)\n",
      "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.3.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from bpemb>=0.2.9->flair) (2.21.0)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 52.3MB/s \n",
      "\u001b[?25hCollecting SudachiPy; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/04/c9/40bfb291a7995ad218451ef97083432f998b822e3ecbd9f586f593d2cfb6/SudachiPy-0.4.2-py3-none-any.whl (73kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 6.4MB/s \n",
      "\u001b[?25hCollecting kytea; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fe/bc/702d01a96d5d094bd9f3c2eb1d12153daf8edf7bf5d78b9a2dae1202df07/kytea-0.1.4-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n",
      "\u001b[K     |████████████████████████████████| 3.8MB 34.9MB/s \n",
      "\u001b[?25hCollecting SudachiDict-core@ https://object-storage.tyo2.conoha.io/v1/nc_2520839e1f9641b08211a5c85243124a/sudachi/SudachiDict_core-20190927.tar.gz ; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://object-storage.tyo2.conoha.io/v1/nc_2520839e1f9641b08211a5c85243124a/sudachi/SudachiDict_core-20190927.tar.gz (70.7MB)\n",
      "\u001b[K     |████████████████████████████████| 70.7MB 55kB/s \n",
      "\u001b[?25hCollecting natto-py; extra == \"all\"\n",
      "  Downloading https://files.pythonhosted.org/packages/f1/14/1d4258247a00b7b8a115563effb1d0bd30501d69580629d36593ce0af92d/natto-py-0.9.2.tar.gz\n",
      "Collecting janome; extra == \"all\"\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/79/f0/bd7f90806132d7d9d642d418bdc3e870cfdff5947254ea3cab27480983a7/Janome-0.3.10-py2.py3-none-any.whl (21.5MB)\n",
      "\u001b[K     |████████████████████████████████| 21.5MB 73.4MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1f/8e/ed5364a06a9ba720fddd9820155cc57300d28f5f43a6fd7b7e817177e642/sacremoses-0.0.35.tar.gz (859kB)\n",
      "\u001b[K     |████████████████████████████████| 860kB 54.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (1.10.40)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython==7.6.1->flair) (0.1.7)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython==7.6.1->flair) (0.6.0)\n",
      "Requirement already satisfied: parso>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from jedi>=0.10->ipython==7.6.1->flair) (0.5.2)\n",
      "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision->flair) (0.46)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->flair) (0.14.1)\n",
      "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.8.1->gensim>=3.4.0->flair) (2.49.0)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.2.9->flair) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.2.9->flair) (2019.11.28)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.2.9->flair) (3.0.4)\n",
      "Requirement already satisfied: sortedcontainers~=2.1.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (2.1.0)\n",
      "Collecting dartsclone~=0.6.0\n",
      "  Downloading https://files.pythonhosted.org/packages/7d/4d/45acbe9d0795d8ceef0fee1f9ac2dcbf27dca3a0578a023fcdc3fef6fd89/dartsclone-0.6.tar.gz\n",
      "Requirement already satisfied: cffi in /usr/local/lib/python3.6/dist-packages (from natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (1.13.2)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->flair) (7.0)\n",
      "Requirement already satisfied: botocore<1.14.0,>=1.13.40 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (1.13.40)\n",
      "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.2.1)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.9.4)\n",
      "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from dartsclone~=0.6.0->SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (0.29.14)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi->natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (2.19)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.40->boto3->transformers>=2.0.0->flair) (0.15.2)\n",
      "Building wheels for collected packages: sqlitedict, segtok, langdetect, mpld3, SudachiDict-core, natto-py, sacremoses, dartsclone\n",
      "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sqlitedict: filename=sqlitedict-1.6.0-cp36-none-any.whl size=14689 sha256=bf55d86daffeac9a30694661c4050b15779b49e7cc4b6560d1c618c41042812b\n",
      "  Stored in directory: /root/.cache/pip/wheels/bd/57/d3/907c3ee02d35e66f674ad0106e61f06eeeb98f6ee66a6cc3fe\n",
      "  Building wheel for segtok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for segtok: filename=segtok-1.5.7-cp36-none-any.whl size=23258 sha256=fa548298a437aae9ea332f10b0fb52bcdde3ee12b96545ae64861132118f606e\n",
      "  Stored in directory: /root/.cache/pip/wheels/15/ee/a8/6112173f1386d33eebedb3f73429cfa41a4c3084556bcee254\n",
      "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for langdetect: filename=langdetect-1.0.7-cp36-none-any.whl size=993460 sha256=f914b1c44b745b006ce6b077a07c3c9d0c784f6c49e606be813cd65bb74538cd\n",
      "  Stored in directory: /root/.cache/pip/wheels/ec/0c/a9/1647275e7ef5014e7b83ff30105180e332867d65e7617ddafe\n",
      "  Building wheel for mpld3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for mpld3: filename=mpld3-0.3-cp36-none-any.whl size=116679 sha256=69b59119984167892d5205729cebbda536432155097fa5d82f1332923c2d7f2c\n",
      "  Stored in directory: /root/.cache/pip/wheels/c0/47/fb/8a64f89aecfe0059830479308ad42d62e898a3e3cefdf6ba28\n",
      "  Building wheel for SudachiDict-core (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for SudachiDict-core: filename=SudachiDict_core-20190927-cp36-none-any.whl size=70878518 sha256=3d6ddbcfe63ee97c4676bb212a05503191caea88f495951d97f8527ba4408d56\n",
      "  Stored in directory: /root/.cache/pip/wheels/22/d8/6e/b107d7fef6e80915aa1e46db741b98a3da011f567526347ccc\n",
      "  Building wheel for natto-py (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for natto-py: filename=natto_py-0.9.2-cp36-none-any.whl size=45164 sha256=4fcd81b439ef41487db79c47539a8e63e695edf3802a549408cc4bc896a38c43\n",
      "  Stored in directory: /root/.cache/pip/wheels/ce/51/dd/67f87608b124a23eecf5c1fc3557cc0b7ffdeae33fe6ee89df\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.35-cp36-none-any.whl size=883999 sha256=bfa943a579f3ecffe4638b00d9b861330517c803af1019f7c8d98afc07a6af87\n",
      "  Stored in directory: /root/.cache/pip/wheels/63/2a/db/63e2909042c634ef551d0d9ac825b2b0b32dede4a6d87ddc94\n",
      "  Building wheel for dartsclone (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for dartsclone: filename=dartsclone-0.6-cp36-cp36m-linux_x86_64.whl size=413281 sha256=07e49cb5c7b69f275b35536f9610bdd06737f209ece4d02f8ccc0e8f78b1314d\n",
      "  Stored in directory: /root/.cache/pip/wheels/be/cd/70/fe43307bf7398243155108f4f5a258ef336923d65ec4af93cd\n",
      "Successfully built sqlitedict segtok langdetect mpld3 SudachiDict-core natto-py sacremoses dartsclone\n",
      "\u001b[31mERROR: jupyter-console 5.2.0 has requirement prompt-toolkit<2.0.0,>=1.0.0, but you'll have prompt-toolkit 2.0.10 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: google-colab 1.0.0 has requirement ipython~=5.5.0, but you'll have ipython 7.6.1 which is incompatible.\u001b[0m\n",
      "Installing collected packages: sqlitedict, prompt-toolkit, ipython, deprecated, segtok, langdetect, sentencepiece, bpemb, sacremoses, transformers, mpld3, flair, dartsclone, SudachiPy, kytea, SudachiDict-core, natto-py, janome\n",
      "  Found existing installation: prompt-toolkit 1.0.18\n",
      "    Uninstalling prompt-toolkit-1.0.18:\n",
      "      Successfully uninstalled prompt-toolkit-1.0.18\n",
      "  Found existing installation: ipython 5.5.0\n",
      "    Uninstalling ipython-5.5.0:\n",
      "      Successfully uninstalled ipython-5.5.0\n",
      "Successfully installed SudachiDict-core-20190927 SudachiPy-0.4.2 bpemb-0.3.0 dartsclone-0.6 deprecated-1.2.7 flair-0.4.4 ipython-7.6.1 janome-0.3.10 kytea-0.1.4 langdetect-1.0.7 mpld3-0.3 natto-py-0.9.2 prompt-toolkit-2.0.10 sacremoses-0.0.35 segtok-1.5.7 sentencepiece-0.1.85 sqlitedict-1.6.0 transformers-2.3.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.colab-display-data+json": {
       "pip_warning": {
        "packages": [
         "IPython",
         "prompt_toolkit"
        ]
       }
      }
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "!pip install -U fasttext\n",
    "!pip install -U gensim\n",
    "!pip install tiny-tokenizer\n",
    "!pip install flair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 63
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5854,
     "status": "ok",
     "timestamp": 1578203097032,
     "user": {
      "displayName": "Amit Agarwal",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCNCK3WDEC63aFcnBVJrUB3M3cs4J2bImVRkNad8A=s64",
      "userId": "15805199744168269697"
     },
     "user_tz": -330
    },
    "id": "9sZizSqp62NH",
    "outputId": "febfb3ac-ccbb-4df7-ee6b-a1a0bb86541a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p style=\"color: red;\">\n",
       "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
       "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
       "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
       "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Let's import our packages !\n",
    "\n",
    "import pandas as pd\n",
    "import fasttext\n",
    "import re\n",
    "import html\n",
    "from tqdm import tqdm \n",
    "from flair.embeddings import WordEmbeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lRhs4oEz6Se9"
   },
   "source": [
    "# 2. Text Pre-processing Pipeline\n",
    "\n",
    "In NLP with the structured & unstructured text, one constant that doen't change is text pre-processing. Based on the task & requirements we create a text-cleaning pipeline.\n",
    "\n",
    "Every try-except block can be written as a different modular function which can be invoked from preprocess_text() function. This serves as a pipeline of the series of text-cleaning that you might require for your dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2x_lpXigwzFC"
   },
   "outputs": [],
   "source": [
    "clean = re.compile('<.*?>')\n",
    "\n",
    "def preprocess_text(text) :\n",
    "  try :\n",
    "    # soup = BeautifulSoup(text, \"html.parser\")\n",
    "    # text = soup.get_text()\n",
    "    text=  re.sub(clean, '', text)\n",
    "    text = html.unescape(text)\n",
    "  except Exception as e:\n",
    "    print(\"Error in HTML Processing ...\")\n",
    "    print(text)\n",
    "    text = text\n",
    "    raise e\n",
    "  try :\n",
    "    # remove extra newlines (often might be present in really noisy text)\n",
    "    text = text.translate(text.maketrans(\"\\n\\t\\r\", \"   \"))\n",
    "  except :\n",
    "    print(\"Error in removing extra lines ...\")\n",
    "    print(text)\n",
    "    text = text\n",
    "\n",
    "  try :\n",
    "    # remove extra whitespace\n",
    "    text = re.sub(' +', ' ', text)\n",
    "    text = text.strip()\n",
    "  except :\n",
    "    print(\"Error in extra whitespace removal ...\")\n",
    "    print(text)\n",
    "    text = text\n",
    "\n",
    "  return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "g8l9m9_07fwg"
   },
   "source": [
    "# 3. Creating the Corpus\n",
    "\n",
    "To train a word-embedding we first need to create a single corpus which contains all of our text that we want the `Machines` to understand. This includes the following :\n",
    "\n",
    "1. Loading all the Documents (Text Files/Sentences/Documents etc) & pre-process\n",
    "2. *One Line Per Document* : Traning format\n",
    "3. Save to re-use.\n",
    "4. Train FastText Embedding\n",
    "5. Convert to Gensim re-usable & distributable format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 22285,
     "status": "ok",
     "timestamp": 1578203118513,
     "user": {
      "displayName": "Amit Agarwal",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCNCK3WDEC63aFcnBVJrUB3M3cs4J2bImVRkNad8A=s64",
      "userId": "15805199744168269697"
     },
     "user_tz": -330
    },
    "id": "GrYBqDyFeM9v",
    "outputId": "ab524d14-8af5-4338-a237-8916a360d5d5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>CreationMonth</th>\n",
       "      <th>CreationYear</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>120</td>\n",
       "      <td>83.0</td>\n",
       "      <td>2008-08-01 15:50:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>&lt;p&gt;Has anyone got experience creating &lt;strong&gt;...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[asp.net, sql]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>260</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2008-08-01 23:22:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49</td>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>&lt;p&gt;I have a little game written in C#. It uses...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[c#, .net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>330</td>\n",
       "      <td>63.0</td>\n",
       "      <td>2008-08-02 02:51:36</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29</td>\n",
       "      <td>Should I use nested classes in this case?</td>\n",
       "      <td>&lt;p&gt;I am working on a collection of classes use...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[c++]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>470</td>\n",
       "      <td>71.0</td>\n",
       "      <td>2008-08-02 15:11:47</td>\n",
       "      <td>2016-03-26T05:23:29Z</td>\n",
       "      <td>13</td>\n",
       "      <td>Homegrown consumption of web services</td>\n",
       "      <td>&lt;p&gt;I've been writing a few web services for a ...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[web-services, .net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>580</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2008-08-02 23:30:59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>Deploying SQL Server Databases from Test to Live</td>\n",
       "      <td>&lt;p&gt;I wonder how you guys manage deployment of ...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[sql-server]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Id  OwnerUserId  ... CreationYear                   Tag\n",
       "0  120         83.0  ...         2008        [asp.net, sql]\n",
       "1  260         91.0  ...         2008            [c#, .net]\n",
       "2  330         63.0  ...         2008                 [c++]\n",
       "3  470         71.0  ...         2008  [web-services, .net]\n",
       "4  580         91.0  ...         2008          [sql-server]\n",
       "\n",
       "[5 rows x 10 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the Base Path & Data Files\n",
    "path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/'\n",
    "data ='filtered_data/question_tag_text_mapping.pkl'\n",
    "\n",
    "# Load the structured file with all document Text\n",
    "question_tag = pd.read_pickle(path+data)\n",
    "question_tag.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 110840,
     "status": "ok",
     "timestamp": 1578203208289,
     "user": {
      "displayName": "Amit Agarwal",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCNCK3WDEC63aFcnBVJrUB3M3cs4J2bImVRkNad8A=s64",
      "userId": "15805199744168269697"
     },
     "user_tz": -330
    },
    "id": "DfCdRCLTw15j",
    "outputId": "634384ec-78a9-4b8f-ecd5-c46e77386c79"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>CreationMonth</th>\n",
       "      <th>CreationYear</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>120</td>\n",
       "      <td>83.0</td>\n",
       "      <td>2008-08-01 15:50:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>Has anyone got experience creating SQL-based A...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[asp.net, sql]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>260</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2008-08-01 23:22:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49</td>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>I have a little game written in C#. It uses a ...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[c#, .net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>330</td>\n",
       "      <td>63.0</td>\n",
       "      <td>2008-08-02 02:51:36</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29</td>\n",
       "      <td>Should I use nested classes in this case?</td>\n",
       "      <td>I am working on a collection of classes used f...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[c++]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>470</td>\n",
       "      <td>71.0</td>\n",
       "      <td>2008-08-02 15:11:47</td>\n",
       "      <td>2016-03-26T05:23:29Z</td>\n",
       "      <td>13</td>\n",
       "      <td>Homegrown consumption of web services</td>\n",
       "      <td>I've been writing a few web services for a .ne...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[web-services, .net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>580</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2008-08-02 23:30:59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>Deploying SQL Server Databases from Test to Live</td>\n",
       "      <td>I wonder how you guys manage deployment of a d...</td>\n",
       "      <td>8</td>\n",
       "      <td>2008</td>\n",
       "      <td>[sql-server]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Id  OwnerUserId  ... CreationYear                   Tag\n",
       "0  120         83.0  ...         2008        [asp.net, sql]\n",
       "1  260         91.0  ...         2008            [c#, .net]\n",
       "2  330         63.0  ...         2008                 [c++]\n",
       "3  470         71.0  ...         2008  [web-services, .net]\n",
       "4  580         91.0  ...         2008          [sql-server]\n",
       "\n",
       "[5 rows x 10 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Apply the pre-processing on the Question Body & Text\n",
    "\n",
    "question_tag['Title'] = question_tag['Title'].apply(preprocess_text)\n",
    "question_tag['Body'] = question_tag['Body'].apply(preprocess_text)\n",
    "question_tag.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_Dhz3KkFeNi-"
   },
   "outputs": [],
   "source": [
    "# Iterate & Create \"One Line Per Document : Traning format\"\n",
    "\n",
    "text_lines =  list()\n",
    "for index in tqdm(question_tag.index) :\n",
    "  title = question_tag.loc[index,'Title']\n",
    "  body =  question_tag.loc[index,'Body']\n",
    "\n",
    "  text =  title + '. ' + body\n",
    "  text_lines.append(text)\n",
    "\n",
    "# Save to re-use easily next-time\n",
    "with open(path+'/training_data/training_data.txt', 'w',encoding ='utf-8') as filehandle:\n",
    "    filehandle.writelines(\"%s\\n\" % sent for sent in text_lines)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bBy0myng-iEb"
   },
   "source": [
    "# 4. Train Word Embeddings\n",
    "\n",
    "Here we use FastText to train wor-embeddings as these are very fast compared to any other trainings available and serves the purpose to evaluate how valuable custom embeddings are to the task at hand.\n",
    "\n",
    "\n",
    "FastText open-sourced by Facebook, provides both command-line based library & python based. If possible use command-line based as it's more faster.\n",
    "\n",
    "The options & hyperparameters to be tune can be found in the documentation.\n",
    "\n",
    "\n",
    "**Link**  : https://github.com/facebookresearch/fastText/tree/master/python#train_unsupervised-parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TvwnOchDFmV2"
   },
   "outputs": [],
   "source": [
    "# Skipgram model :\n",
    "model = fasttext.train_unsupervised(path+'/training_data/training_data.txt', model='skipgram',dim=300,verbose=3)\n",
    "model.save_model(path+\"/training_data/training_data_skipgram_d300.bin\")\n",
    "print(\"Model Saved\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p76_G5hI31er"
   },
   "source": [
    "# 5. Convert & Save\n",
    "\n",
    "Generally you would want to convert the embedding to Gensim format as it's a widely used library and provides wrapper to most framework."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iFyTTcts34wj"
   },
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "\n",
    "# We use the .vec file which does not contain the Meta-Data of FastText word-embeddings\n",
    "model = KeyedVectors.load_word2vec_format('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/model.vec')\n",
    "\n",
    "# Save\n",
    "model.save('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7181,
     "status": "ok",
     "timestamp": 1578203280054,
     "user": {
      "displayName": "Amit Agarwal",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCNCK3WDEC63aFcnBVJrUB3M3cs4J2bImVRkNad8A=s64",
      "userId": "15805199744168269697"
     },
     "user_tz": -330
    },
    "id": "kaAOMqTo_k5y",
    "outputId": "674cfc63-87da-43bf-9381-e2bcae03629a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word Embedding Loaded Successfully in Flair\n"
     ]
    }
   ],
   "source": [
    "# Test that we are able to load it in for the down-stream tasks/code based we want.\n",
    "word_embeddings = [ WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model')]\n",
    "print(\"Word Embedding Loaded Successfully in Flair\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 6538,
     "status": "ok",
     "timestamp": 1578203483746,
     "user": {
      "displayName": "Amit Agarwal",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCNCK3WDEC63aFcnBVJrUB3M3cs4J2bImVRkNad8A=s64",
      "userId": "15805199744168269697"
     },
     "user_tz": -330
    },
    "id": "rr1xkJvs1iWQ",
    "outputId": "bf558f6f-0070-4d0e-bf0b-182267ca9f2c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word : Welcome\n",
      "tensor([-0.0477,  0.2607,  0.2033,  0.2048, -0.0809,  0.1777,  0.0807, -0.2186,\n",
      "         0.0905, -0.2447, -0.0885,  0.3721,  0.7040,  0.4241,  0.4265, -0.8608,\n",
      "         0.1301, -0.7587, -0.2028,  0.6566, -0.5186, -0.3107, -0.4054, -0.4756,\n",
      "         0.1132, -0.0198,  0.4574,  0.1947, -0.4173,  0.0510, -0.2822, -0.2527,\n",
      "         0.1500, -0.4198, -0.0433,  0.5972,  0.2721, -0.6502,  0.7049, -0.3756,\n",
      "         0.2631,  0.5738, -0.6479,  0.1652,  0.2238,  0.6156,  0.3448,  0.1264,\n",
      "        -0.0845,  0.2268, -0.3862,  0.6731,  0.7547,  0.3776, -0.0597, -0.0291,\n",
      "         0.0330,  0.1897,  0.1477, -0.2566,  0.4579, -0.1078,  0.0178, -0.5004,\n",
      "         0.1670,  0.4812,  0.0785,  0.3406, -0.3407, -0.3803,  0.0712,  0.2021,\n",
      "         0.4849,  0.5238,  0.1135,  0.4455, -0.0758,  0.2082, -0.1574,  0.5648,\n",
      "        -0.7159, -0.2169,  0.1408, -0.0399, -0.1633, -0.1153,  0.1199, -0.1713,\n",
      "         0.1180, -0.1937, -0.5096, -0.1051,  0.6814,  0.5651, -0.0935,  0.1203,\n",
      "        -0.0947, -0.3284,  0.1391,  0.0813,  0.6048,  0.1000,  0.1402,  0.3548,\n",
      "        -0.1649,  0.1968, -0.2948,  0.0599, -0.1155,  0.1784,  0.1908, -0.0869,\n",
      "        -0.0222,  0.1384, -0.2566,  0.0527, -0.1039,  0.0036,  0.0378, -0.1057,\n",
      "        -0.1941, -0.3144,  0.0380,  0.3552, -0.3119, -0.0611,  0.0601, -0.1305,\n",
      "         0.2254, -0.2399, -0.5827,  0.2432, -0.7743, -0.2801,  0.4681,  0.2877,\n",
      "         0.4603,  0.5209, -0.3538,  0.2424,  0.4651,  0.2446, -0.0604, -0.1114,\n",
      "        -0.2922,  0.1111,  0.1202, -0.5614,  0.4203,  0.1085, -0.5371, -0.1238,\n",
      "         0.2290, -0.1628,  0.7400, -0.1522, -0.1302, -0.0238,  0.4253, -0.3068,\n",
      "        -0.2043, -0.0613,  0.1968, -0.0627,  0.3542,  0.0955, -0.0962,  0.1723,\n",
      "        -0.5309,  0.2022, -0.0197, -0.2324, -0.0915, -0.0814, -0.1021, -0.1554,\n",
      "         0.8659, -0.4141,  0.8013, -0.2015, -0.6200, -0.2403,  0.0861, -0.0260,\n",
      "         0.6759, -0.7808,  0.2522, -0.1796,  0.7298, -0.2259,  0.4488, -0.4468,\n",
      "         0.2491,  0.2893,  0.0190, -0.0625, -0.3603,  0.4064, -0.3267, -0.0785,\n",
      "        -0.2645,  0.4354, -0.0096,  0.3311, -0.1990,  0.5262,  0.2386,  0.0338,\n",
      "         0.7150, -0.2158,  0.1287,  0.0694,  0.2785,  0.2883,  0.2819, -0.0183,\n",
      "         0.3470,  0.2169, -0.3125, -0.0083, -0.3984,  0.4667, -0.0934, -0.3095,\n",
      "         0.1747,  0.4949,  0.5290, -0.1844, -0.1328,  0.4990, -0.4137,  0.0142,\n",
      "         0.2231, -0.7409,  0.1878, -0.6512, -0.3607, -0.4580,  0.1743,  0.2627,\n",
      "        -0.4298,  0.1821, -0.4627,  0.3221, -0.6834,  0.5248,  0.0386, -0.0472,\n",
      "        -0.2425,  0.4906, -0.5452, -0.0665, -0.7728,  0.0350,  0.0031,  0.0249,\n",
      "        -0.0206, -0.1915,  0.1850, -0.2908,  0.1583, -0.4116, -0.4349,  0.3121,\n",
      "        -0.4534,  0.1179,  0.4158,  0.4703, -0.3740, -0.0942,  0.0206, -0.2515,\n",
      "        -0.3750, -0.5248,  0.2481,  0.4143, -0.0730,  0.4588,  0.2716,  0.0544,\n",
      "        -0.1402,  0.3310,  0.1597,  0.1506, -0.0339,  0.3757, -0.2522,  0.0631,\n",
      "         0.5123,  0.1778,  0.2557, -0.3883, -0.1859,  0.2727,  0.1589, -0.6136,\n",
      "        -0.1190,  0.3323, -0.4231, -0.8020])\n",
      "torch.Size([300])\n",
      "Word : to\n",
      "tensor([ 3.6339e-02, -2.4494e-02, -1.2114e-02, -1.2705e-01, -4.6956e-02,\n",
      "         3.6840e-02, -1.2733e-02,  2.1591e-02,  3.7397e-02, -1.1073e-01,\n",
      "         1.2404e-02, -6.8313e-02, -1.9869e-01,  1.6133e-02, -4.1168e-02,\n",
      "        -1.5978e-01,  9.2192e-04,  8.3535e-02, -5.4543e-02,  4.6919e-02,\n",
      "         3.2192e-02, -1.9524e-01,  1.3777e-01, -1.4319e-01, -6.1194e-02,\n",
      "        -7.2729e-02,  1.7402e-01,  2.3624e-01,  2.8061e-01,  2.2739e-01,\n",
      "        -1.1246e-01,  1.0598e-01, -5.3752e-02, -8.5361e-02, -8.3607e-02,\n",
      "         1.3170e-02,  2.1637e-01,  9.4864e-02, -2.9709e-03,  1.2273e-01,\n",
      "         3.9622e-04,  2.1609e-02,  2.4474e-01,  4.9231e-02, -6.6462e-02,\n",
      "        -4.4609e-02,  6.9039e-02, -1.3874e-01, -1.9272e-02, -3.5554e-01,\n",
      "         1.6994e-01,  4.7074e-02,  1.5899e-01, -1.0291e-01,  5.0377e-02,\n",
      "         3.5507e-02,  1.9964e-01, -3.9888e-02, -9.1357e-02,  5.2118e-02,\n",
      "         8.0902e-02, -9.1324e-02, -1.7544e-01,  6.6699e-02, -1.6353e-01,\n",
      "         2.7309e-01, -6.6885e-02,  7.1747e-02, -2.3031e-02, -8.7383e-03,\n",
      "        -6.1933e-03,  1.2225e-01,  5.2446e-02,  3.5992e-02, -5.5336e-02,\n",
      "        -3.1977e-01, -1.0469e-01, -7.6672e-02,  1.4887e-02,  5.1591e-02,\n",
      "        -1.0014e-01, -1.8650e-01, -1.6713e-02,  3.9944e-02, -2.4223e-02,\n",
      "        -2.4116e-01, -2.8577e-01,  2.5180e-02,  3.9042e-03,  8.5575e-02,\n",
      "        -1.9636e-01,  1.1042e-01,  7.3364e-02, -1.8316e-01, -2.1148e-02,\n",
      "        -9.4132e-02, -3.4958e-02,  9.7389e-03,  2.3408e-01, -2.7921e-01,\n",
      "         1.1024e-01, -7.3606e-02, -1.5106e-01, -7.1171e-04, -1.8282e-01,\n",
      "         3.2537e-02,  8.0269e-02, -1.9263e-01, -1.3819e-02,  5.6716e-02,\n",
      "         3.4858e-01,  2.3520e-02,  8.1346e-02, -1.3724e-01,  1.5058e-02,\n",
      "         8.7654e-02, -1.8557e-01, -8.4466e-02, -1.7620e-01,  7.2993e-02,\n",
      "         3.2901e-02,  1.5794e-01,  4.1096e-02,  2.6892e-01, -1.7185e-01,\n",
      "         1.0824e-01, -1.4709e-01, -3.1810e-03, -2.9899e-02, -1.7575e-01,\n",
      "        -2.6012e-01,  1.2588e-01,  1.4374e-01, -2.3272e-01,  1.1925e-01,\n",
      "         1.5979e-01, -2.3451e-01,  3.4719e-02,  9.0848e-02,  7.3157e-02,\n",
      "         4.0322e-02,  5.6931e-03, -9.8749e-02, -7.8369e-02,  1.3131e-01,\n",
      "         1.1319e-01,  8.1078e-02, -1.0865e-01,  1.0686e-01, -1.1393e-02,\n",
      "         9.8988e-02, -1.8462e-02, -1.2408e-01, -4.3529e-02, -3.7762e-03,\n",
      "         7.9885e-02,  1.4877e-01,  1.1809e-01,  4.8652e-01, -1.0640e-01,\n",
      "        -1.8633e-01,  1.0566e-01,  1.8228e-01, -1.4069e-01,  3.2112e-02,\n",
      "        -4.0267e-03,  7.8731e-02,  1.9791e-01, -2.2754e-01, -9.6517e-02,\n",
      "        -2.1130e-01,  1.1556e-01,  1.4593e-01,  2.6445e-02,  1.6149e-01,\n",
      "         1.4295e-01,  1.1540e-01, -1.9820e-01,  6.1401e-02, -2.6999e-01,\n",
      "        -1.9836e-01, -2.2361e-01, -1.2045e-01, -1.2053e-01,  7.9004e-02,\n",
      "         6.4592e-02,  8.0523e-02, -1.3980e-01,  2.8297e-01, -3.7510e-02,\n",
      "         1.0815e-01,  4.0558e-02,  4.6477e-02, -4.1432e-02,  6.6295e-02,\n",
      "        -1.4023e-01, -2.0442e-01, -9.7826e-02,  1.1204e-01,  8.1633e-02,\n",
      "         1.5653e-01, -4.2701e-03, -4.7276e-02,  1.2172e-01,  1.2070e-01,\n",
      "        -7.1879e-02, -7.6994e-02, -1.7037e-01, -8.1381e-02, -2.9198e-01,\n",
      "         4.7168e-03, -9.4306e-02,  1.0912e-01, -4.9978e-02,  4.3321e-02,\n",
      "        -8.1665e-02,  2.2771e-01,  3.2605e-02, -3.2101e-02,  5.3249e-02,\n",
      "        -1.3086e-01, -1.1705e-01,  1.6097e-01,  9.5395e-02, -3.7107e-02,\n",
      "        -1.9454e-01,  2.1949e-01, -7.3298e-02, -7.8769e-02,  9.0716e-03,\n",
      "        -1.7926e-01,  1.5139e-03, -8.2956e-02, -2.3480e-01, -3.3561e-02,\n",
      "         7.3072e-02, -1.8550e-01, -1.2536e-01,  1.0583e-01,  1.4059e-01,\n",
      "         1.8973e-01,  2.8990e-02, -1.3750e-01,  3.8123e-02, -2.5247e-02,\n",
      "        -3.3681e-02, -1.1256e-02,  5.9361e-02, -1.8321e-01, -1.1323e-01,\n",
      "        -1.1501e-01,  1.3758e-01, -2.3677e-01, -6.7796e-03,  3.9143e-02,\n",
      "        -2.1713e-01,  4.5885e-02, -2.1135e-01, -2.2025e-02, -2.0704e-01,\n",
      "        -5.8738e-02, -7.7714e-03, -3.2371e-02,  7.4073e-02, -5.7857e-03,\n",
      "        -9.6474e-02,  1.9055e-01, -1.9940e-01, -1.5870e-01,  2.1586e-01,\n",
      "         1.3211e-01, -1.2297e-01,  2.0725e-02,  1.0323e-01, -1.4639e-01,\n",
      "        -4.4907e-02, -9.6610e-02,  1.8466e-01,  1.0989e-01,  1.4788e-02,\n",
      "        -4.0181e-01, -2.8018e-02,  2.5785e-02,  6.9528e-02, -6.8117e-03,\n",
      "        -5.9360e-02, -1.8033e-01,  1.4572e-01,  6.5107e-02,  1.3278e-01,\n",
      "        -3.8762e-02, -1.1348e-01, -4.4674e-04, -2.5010e-01, -2.7483e-01,\n",
      "        -1.1358e-01, -8.5791e-03, -6.3143e-02, -2.7356e-01, -2.4504e-01])\n",
      "torch.Size([300])\n",
      "Word : ICDMAI\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])\n",
      "torch.Size([300])\n",
      "Word : 2020\n",
      "tensor([-6.3832e-01, -1.9210e-01, -5.8406e-01,  5.0551e-04, -4.4844e-01,\n",
      "        -5.6936e-01,  1.8351e-01, -2.7762e-02,  1.7576e-01, -2.3128e-01,\n",
      "         5.5308e-02,  2.0629e-01,  3.5434e-01, -8.7441e-01,  1.4771e-01,\n",
      "        -4.0790e-01, -3.5043e-01, -5.5714e-03,  5.9811e-01,  1.3928e-01,\n",
      "         6.2285e-01,  9.3456e-03,  4.1364e-02,  6.4162e-02, -3.4034e-01,\n",
      "         2.7925e-01, -4.7826e-01, -6.9867e-01, -3.6805e-01,  3.4335e-01,\n",
      "        -9.5720e-02, -3.8671e-01,  4.0257e-01,  2.8187e-01, -4.0245e-01,\n",
      "        -9.3132e-02,  8.1834e-02, -3.2275e-01, -5.1545e-02, -4.3386e-01,\n",
      "        -3.3424e-01, -3.4082e-02, -4.2068e-01,  4.4907e-02, -1.9710e-02,\n",
      "        -2.4904e-01,  2.9610e-01, -5.6753e-01,  3.4585e-01, -1.7640e-02,\n",
      "        -6.5118e-02,  4.2652e-01,  3.9818e-01,  4.4916e-01, -9.6959e-02,\n",
      "        -1.4142e-02,  3.4603e-01, -4.0277e-02, -2.5193e-01,  7.4403e-02,\n",
      "         5.2602e-02, -7.0803e-02, -1.3437e-02,  2.4193e-01,  1.7584e-01,\n",
      "         2.3492e-01, -5.7652e-01,  4.5163e-01, -4.4961e-01, -6.9063e-01,\n",
      "        -2.5798e-02, -7.5963e-02, -1.0182e-01, -1.0380e-01, -8.5851e-02,\n",
      "        -2.0400e-01,  1.7778e-01, -3.9171e-01,  1.8554e-01,  5.7752e-01,\n",
      "        -6.0216e-01, -1.0167e-02,  6.5794e-01,  5.5952e-03, -6.1826e-03,\n",
      "        -2.0427e-01, -2.6813e-01,  3.2257e-01,  7.9242e-02, -2.9145e-01,\n",
      "        -1.4937e-01,  3.3657e-01, -1.1333e+00,  5.0464e-02, -5.4116e-01,\n",
      "        -1.6894e-01, -2.2131e-01,  2.9998e-02,  5.4128e-01, -9.3132e-01,\n",
      "         1.1284e-01,  2.5626e-01,  7.8466e-02,  1.5884e-01, -1.9448e-01,\n",
      "         6.7930e-02,  1.6085e-01, -5.9318e-01, -4.3597e-01, -1.2595e-01,\n",
      "        -1.0939e-01, -3.3629e-01,  2.7279e-01,  2.9837e-01,  3.2631e-01,\n",
      "         4.4942e-02, -1.8282e-02, -1.9155e-01, -7.5872e-01,  9.3227e-02,\n",
      "         6.4257e-02, -3.2821e-01, -5.0941e-01, -2.0112e-01, -7.9618e-01,\n",
      "        -6.9795e-01, -6.7954e-02, -2.8998e-01,  1.1557e+00, -2.0922e-01,\n",
      "         2.4802e-01,  5.4030e-01, -1.6078e-02,  1.5201e-02, -4.0753e-01,\n",
      "        -3.6420e-01,  7.0776e-01,  6.4049e-01,  1.6587e-01,  6.4254e-01,\n",
      "         6.5424e-01, -7.2177e-01, -2.7456e-01,  1.1564e-01, -3.4462e-01,\n",
      "        -3.6522e-02, -4.7531e-02, -5.8457e-01,  1.4790e-01,  4.8415e-01,\n",
      "         5.7351e-01, -1.7009e-01,  1.9418e-01,  5.4367e-01,  9.4944e-02,\n",
      "        -2.6752e-01,  3.0033e-01, -1.2383e+00,  3.2258e-01, -2.1862e-01,\n",
      "         1.5433e-01, -2.5346e-01, -4.6482e-01, -1.9222e-01, -5.4318e-01,\n",
      "        -1.4270e-01, -1.3575e-01, -3.6685e-01, -7.4892e-02,  4.0256e-01,\n",
      "        -4.9060e-01,  2.5078e-01,  9.2086e-01,  1.3995e-01,  3.5400e-01,\n",
      "         1.2154e-02,  2.3849e-01,  1.0156e-01,  8.0635e-02, -5.0521e-01,\n",
      "        -3.3427e-01, -2.8553e-01, -4.6089e-01, -2.9576e-01,  5.4559e-02,\n",
      "        -3.8428e-01,  2.0325e-01, -6.4413e-02, -4.6290e-01, -4.1162e-01,\n",
      "         6.1319e-02,  1.9357e-01,  2.5716e-01,  7.3564e-01, -1.6924e-01,\n",
      "        -2.6720e-02,  2.9803e-01,  2.0321e-01,  3.1252e-01, -7.0571e-01,\n",
      "        -1.9916e-01, -9.6272e-01, -2.6336e-03,  3.7165e-01, -5.1018e-01,\n",
      "         1.7666e-01, -4.3593e-01, -5.2357e-01, -2.0698e-01, -3.4416e-01,\n",
      "        -3.1171e-01,  4.3532e-01,  1.0754e-01, -6.1839e-01,  7.1043e-01,\n",
      "         3.4818e-01,  5.2157e-01,  7.1192e-01,  1.5703e-01, -6.3793e-02,\n",
      "         3.7858e-01, -5.2433e-01,  1.9031e-01, -1.8325e-01, -1.2737e-01,\n",
      "        -2.3596e-01,  4.2190e-01,  5.2790e-01,  6.0442e-02, -1.6200e-01,\n",
      "        -1.6648e-02,  3.9547e-01, -2.0620e-02, -5.7150e-01, -5.6526e-02,\n",
      "         4.8915e-01,  3.8955e-01, -2.0638e-02,  6.8230e-02,  9.6628e-01,\n",
      "        -1.3771e-01,  1.6872e-01, -1.0750e+00,  5.6320e-01,  5.2810e-01,\n",
      "         1.8950e-02, -7.9869e-02,  1.9502e-01,  3.4897e-02,  1.6598e-01,\n",
      "        -1.6029e-01, -7.6039e-02, -6.7201e-02,  3.6286e-01, -1.0093e-01,\n",
      "        -3.3589e-01, -7.4335e-01,  1.5520e-01, -3.4121e-02, -8.9027e-01,\n",
      "         5.9047e-02,  3.1438e-01, -6.2919e-01,  4.4505e-01, -3.9279e-01,\n",
      "        -5.5026e-01,  4.4582e-01,  9.0196e-02, -3.3974e-01,  2.5195e-01,\n",
      "        -2.1641e-01, -6.8051e-01, -1.4900e-01, -5.2879e-01, -1.7298e-01,\n",
      "        -8.0913e-01,  3.4190e-01,  9.0294e-02,  6.0828e-01,  4.0602e-01,\n",
      "        -4.5306e-01, -7.6085e-01,  6.2211e-01, -5.0778e-01,  3.4282e-02,\n",
      "        -5.0618e-01, -2.3395e-01,  1.8536e-01, -5.9771e-01, -2.9528e-01,\n",
      "         3.2988e-01, -2.3367e-01, -3.8268e-01, -2.8757e-02, -3.1536e-01,\n",
      "         4.0688e-01, -2.2435e-01,  5.9086e-01, -1.7274e-01, -5.3829e-01])\n",
      "torch.Size([300])\n"
     ]
    }
   ],
   "source": [
    "# Looking at the embeddings\n",
    "from flair.embeddings import WordEmbeddings\n",
    "from flair.data import Sentence\n",
    "word_embeddings = WordEmbeddings('/content/drive/My Drive/ICDMAI_Tutorial/notebook/training_data/word_embedding/gensim_model')\n",
    "\n",
    "# some example sentence\n",
    "sentence = Sentence('Welcome to ICDMAI 2020')\n",
    "# sentence = Sentence(question_tag.head(1))\n",
    "\n",
    "# embed sentences\n",
    "word_embeddings.embed(sentence)\n",
    "\n",
    "# go through each token in sentence\n",
    "for token in sentence:\n",
    "    # print token string\n",
    "    print(\"Word : {}\".format(token.text))\n",
    "    # print embedding of this Token\n",
    "    print(token.embedding)\n",
    "\n",
    "    # print shape of embedding of this Token\n",
    "    print(token.embedding.shape)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "4_word_embedding.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}