main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FrHVAW_sKbrO"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import csv\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "np.random.seed(113)\n",
    "path_ukara = \"drive/My Drive/Ukara/\"\n",
    "path_nlp = \"drive/My Drive/a/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 129
    },
    "colab_type": "code",
    "id": "Ta4lq7xRK7rt",
    "outputId": "de732a0e-bf18-4231-a9a0-54289b326a2a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OCPUst8JKv_0"
   },
   "outputs": [],
   "source": [
    "data_A = pd.read_csv(path_ukara + \"data_train_A.csv\").values\n",
    "data_B = pd.read_csv(path_ukara + \"data_train_B.csv\").values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 109
    },
    "colab_type": "code",
    "id": "E2Y5I6FRLohS",
    "outputId": "5afeeaa7-4eee-4e47-a830-550ac7b372f4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['TRA1', 'intetraksi/beradaptasi terhadap lingkungan yang baru.',\n",
       "        1],\n",
       "       ['TRA2', 'seperti jatuhnya meteor tsunami gempa bumi', 0],\n",
       "       ['TRA3', 'hanya tuhan yang tahu tantangan nya itu apaan', 0]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_A[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7QxrgpL-LsNy"
   },
   "outputs": [],
   "source": [
    "xtrain_A, xtest_A, ytrain_A, ytest_A = train_test_split(data_A[:,:-1], data_A[:,-1],\n",
    "                                      test_size=0.2, \n",
    "                                      stratify=data_A[:,-1], \n",
    "                                      random_state=113)\n",
    "\n",
    "xtrain_B, xtest_B, ytrain_B, ytest_B = train_test_split(data_B[:,:-1], data_B[:,-1],\n",
    "                                      test_size=0.2, \n",
    "                                      stratify=data_B[:,-1], \n",
    "                                      random_state=113)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NlioKY4FzpZF"
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "rwUnaw0WTCbk",
    "outputId": "77f9b3dd-976f-4dfe-bc74-ccbeb7320c8c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['TRA226' 'negara munggin menjari donasi hidup baru'] 0\n"
     ]
    }
   ],
   "source": [
    "print(xtrain_A[0], ytrain_A[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "jwIpnescaGyJ",
    "outputId": "2b24a57d-b9e3-4f07-9f4e-938fc81fe054"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([0, 1], dtype=object), array([ 77, 191]))\n",
      "(array([0, 1], dtype=object), array([137, 168]))\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(data_A[:,-1], return_counts=True))\n",
    "print(np.unique(data_B[:,-1], return_counts=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qRfGi0Sbenq_"
   },
   "outputs": [],
   "source": [
    "def diff(first, second):\n",
    "    second = set(second)\n",
    "    return [item for item in first if item not in second]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WpU9mAfaMhOF"
   },
   "source": [
    "# Fasttext Part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 276
    },
    "colab_type": "code",
    "id": "U0dBD6LfMbOl",
    "outputId": "a88f6e4d-02db-446b-d5a0-e3d8bd2644c5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fasttext\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/61/2e01f1397ec533756c1d893c22d9d5ed3fce3a6e4af1976e0d86bb13ea97/fasttext-0.9.1.tar.gz (57kB)\n",
      "\u001b[K     |████████████████████████████████| 61kB 2.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: pybind11>=2.2 in /usr/local/lib/python3.6/dist-packages (from fasttext) (2.4.2)\n",
      "Requirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from fasttext) (41.2.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from fasttext) (1.16.5)\n",
      "Building wheels for collected packages: fasttext\n",
      "  Building wheel for fasttext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for fasttext: filename=fasttext-0.9.1-cp36-cp36m-linux_x86_64.whl size=2384810 sha256=8f8f730e115254945e09ef2636fed4dd2418c1e43ebefaf0de492c0d8c833f77\n",
      "  Stored in directory: /root/.cache/pip/wheels/9f/f0/04/caa82c912aee89ce76358ff954f3f0729b7577c8ff23a292e3\n",
      "Successfully built fasttext\n",
      "Installing collected packages: fasttext\n",
      "Successfully installed fasttext-0.9.1\n"
     ]
    }
   ],
   "source": [
    "!pip install fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 239
    },
    "colab_type": "code",
    "id": "guq49kx1M2VE",
    "outputId": "92407cce-f372-4a38-bfa1-d2fd1f674f90"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-10-06 15:08:31--  https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.id.300.bin.gz\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.20.22.166, 104.20.6.166, 2606:4700:10::6814:16a6, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.20.22.166|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 4507049071 (4.2G) [application/octet-stream]\n",
      "Saving to: ‘cc.id.300.bin.gz’\n",
      "\n",
      "cc.id.300.bin.gz    100%[===================>]   4.20G  29.6MB/s    in 2m 28s  \n",
      "\n",
      "2019-10-06 15:11:00 (29.0 MB/s) - ‘cc.id.300.bin.gz’ saved [4507049071/4507049071]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.id.300.bin.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uUi9O3ytM4so"
   },
   "outputs": [],
   "source": [
    "!gunzip cc.id.300.bin.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S0IQPsOCxDAx"
   },
   "outputs": [],
   "source": [
    "# !mv \"cc.id.300.bin\" \"drive/My Drive/a/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "NYiMCGC7M7U5",
    "outputId": "4e6488d6-4b9f-4020-e8dc-a3c67fe8c6b2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import fasttext\n",
    "model = fasttext.load_model(path_nlp + \"trained_fasttext_ukara.pk\")\n",
    "# print(model['mesin'][:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "zeOHuKsV_V3G",
    "outputId": "ed07b605-bf77-426b-ba34-312b6d0bb8b2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import fasttext\n",
    "model2 = fasttext.load_model(\"cc.id.300.bin\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2ux7aPk7LkeY"
   },
   "source": [
    "# PreProcess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 129
    },
    "colab_type": "code",
    "id": "A8Zwdi1pOjHa",
    "outputId": "6cd4a4e1-90bd-42e8-da87-1341e06128ef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting PySastrawi\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/61/84/b0a5454a040f81e81e6a95a5d5635f20ad43cc0c288f8b4966b339084962/PySastrawi-1.2.0-py2.py3-none-any.whl (210kB)\n",
      "\u001b[K     |████████████████████████████████| 215kB 2.8MB/s \n",
      "\u001b[?25hInstalling collected packages: PySastrawi\n",
      "Successfully installed PySastrawi-1.2.0\n"
     ]
    }
   ],
   "source": [
    "!pip install PySastrawi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LJLCW5k-Oeef"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import string\n",
    "from nltk.util import skipgrams\n",
    "from nltk import ngrams\n",
    "from collections import Counter\n",
    "import math\n",
    "\n",
    "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
    "factory = StemmerFactory()\n",
    "stemmer = factory.create_stemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5sk7yPrvTSc9"
   },
   "outputs": [],
   "source": [
    "lena = []\n",
    "for d in data_A[:,1]:\n",
    "    lena.append(len(str(d).split()))\n",
    "datalen_mean_A = np.mean(lena)\n",
    "datalen_std_A = np.std(lena)\n",
    "\n",
    "lenb = []\n",
    "for d in data_B[:,1]:\n",
    "    lenb.append(len(str(d).split()))\n",
    "datalen_mean_B = np.mean(lenb)\n",
    "datalen_std_B = np.std(lenb)\n",
    "\n",
    "stop_b = ['yang', 'lebih', 'untuk', 'dan']\n",
    "stop_a = ['yang', 'akan', 'mereka', 'dan']\n",
    "\n",
    "table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
    "def rmpun(text):\n",
    "    return text.translate(table)\n",
    "    \n",
    "def preprocess(data, tipe=\"A\"):\n",
    "    if type(data) != str:\n",
    "        return \" \"\n",
    "    if len(data) <1:\n",
    "        return \" \"\n",
    "    data = data.lower()\n",
    "    data = data.replace('/',' atau ')\n",
    "    data = data.replace('yg ','yang ')\n",
    "    data = rmpun(data)\n",
    "    data = \" \".join(data.split())\n",
    "    if tipe==\"A\":\n",
    "        for w in stop_a:\n",
    "            data = data.replace(w, '')\n",
    "    else:\n",
    "        for w in stop_b:\n",
    "            data = data.replace(w, '')\n",
    "    return data\n",
    "\n",
    "def preprocess_2(data, tipe=\"A\"):\n",
    "    data = data.lower()\n",
    "    data = data.replace('/',' atau ')\n",
    "    data = data.replace('\"','')\n",
    "    data = data.replace('\\'','')\n",
    "    data = data.replace('yg ','yang ')\n",
    "    data = data.replace('_',' ')\n",
    "    data = data.replace(',',' , ')\n",
    "    return data\n",
    "\n",
    "def vectorize(x, pre=True, tipe=\"A\"):\n",
    "    ans = []\n",
    "    for data in x:\n",
    "        if pre:\n",
    "            data = preprocess(str(data), tipe)\n",
    "        vect = model.get_sentence_vector(data)\n",
    "        ans.append(vect)\n",
    "    return np.asarray(ans)\n",
    "\n",
    "def vectorize2(x, pre=True, tipe=\"A\"):\n",
    "    ans = []\n",
    "    for data in x:\n",
    "        if pre:\n",
    "            data = preprocess_2(str(data), tipe)\n",
    "        vect = model2.get_sentence_vector(data)\n",
    "        ans.append(vect)\n",
    "    return np.asarray(ans)\n",
    "\n",
    "def find_len(x, tipe='A'):\n",
    "    ans = []\n",
    "    for data in x:\n",
    "        ans.append(len(str(data).split()))\n",
    "    ans = np.asarray(ans, dtype=\"float32\")\n",
    "    \n",
    "    if tipe==\"A\":\n",
    "        datalen_mean = datalen_mean_A\n",
    "        datalen_std = datalen_std_A\n",
    "    else:\n",
    "        datalen_mean = datalen_mean_B\n",
    "        datalen_std = datalen_std_B\n",
    "    \n",
    "    ans -= datalen_mean\n",
    "    return ans / datalen_std\n",
    "\n",
    "# 1, 58"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U4X4rmk3Lm2K"
   },
   "outputs": [],
   "source": [
    "# table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
    "# def rmpun(text):\n",
    "#     return text.translate(table)\n",
    "    \n",
    "# def preprocess_istri(data):\n",
    "#     if type(data) != str:\n",
    "#         return \" \"\n",
    "#     if len(data) <1:\n",
    "#         return \" \"\n",
    "#     data = data.lower()\n",
    "#     data = data.replace('/',' atau ')\n",
    "#     data = data.replace('yg ','yang ')\n",
    "#     data = rmpun(data)\n",
    "#     data = \" \".join(data.split())\n",
    "#     # data = stem(data)\n",
    "#     return data\n",
    "\n",
    "# word_b = \"\"\n",
    "# for data in data_A:\n",
    "#     word_b += preprocess_istri(data[1])\n",
    "# counter = Counter(word_b.split())\n",
    "# counter = list(reversed(counter.most_common()))\n",
    "# counter[:50]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VBPRxMypOcTO"
   },
   "source": [
    "# PPMI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P1tTSHXGOx87"
   },
   "outputs": [],
   "source": [
    "table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
    "def rmpun(text):\n",
    "    return text.translate(table)\n",
    "\n",
    "def stem(text):\n",
    "    text = stemmer.stem(text)\n",
    "    return text\n",
    "\n",
    "def preprocess_ppmi(data):\n",
    "    if type(data) != str:\n",
    "        return \" \"\n",
    "    if len(data) <1:\n",
    "        return \" \"\n",
    "    data = data.lower()\n",
    "    data = data.replace('/',' atau ')\n",
    "    data = data.replace('yg ','yang ')\n",
    "    data = rmpun(data)\n",
    "    data = \" \".join(data.split())\n",
    "    # data = stem(data)\n",
    "    return data\n",
    "\n",
    "def pmi(w1, w2, unifreq, bifreq):\n",
    "    pw1 = unifreq[w1] / float(sum(unifreq.values()))\n",
    "    pw2 = unifreq[w2] / float(sum(unifreq.values()))\n",
    "    pw1w2 = bifreq[\" \".join([w1, w2])] / float(sum(bifreq.values()))\n",
    "    try:\n",
    "        res = math.log(pw1w2/float(pw1*pw2),2)\n",
    "        if res < 0:\n",
    "            res = 0\n",
    "    except:\n",
    "        res = 0\n",
    "    return res\n",
    "\n",
    "def mkbigram(text):\n",
    "    gram2 = list(ngrams(text.split(), 2))\n",
    "    if len(gram2) == 0:\n",
    "        return [text]\n",
    "    return gram2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "pzYis9-eO2vq",
    "outputId": "a5afbe90-f527-48c3-cea3-ed21452fb94a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 92,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traina1ex = [\"Sebuah negara membeli pulau membeli tanah untuk memindahkan penduduknya di sana suatu hari.\",\n",
    "                \"Pemerintah berharap tidak menempatkan semua penduduk semua warga semua orang di sebidang tanah.\", \n",
    "                \"Tetapi jika itu diperlukan untuk menyelamatkan warga penduduk maka akan dilakukan.\",\n",
    "                \"Pulau tanah itu akan menjadi rumah baru bagi penduduk warga yang berpindah.\",\n",
    "                \"Relokasi warga penduduk karena perubahan iklim.\",\n",
    "                \"Sebuah negara sudah mulai memindahkan populasi penduduk dari daratan rendah ke sebuah pulau lain.\",\n",
    "                \"Tapi ini akan menjadi pertama kalinya seluruh negara harus pindah karena tanah yang dibangun tidak lagi ada.\",\n",
    "                \"Ini menimbulkan pertanyaan baru dan menakutkan: Jika suatu negara tidak lagi ada dalam bentuk fisik, dapatkah ia tetap ada sebagai entitas politik?\",\n",
    "                \"Bisakah suatu bangsa bangkit dan bergerak?\",\n",
    "                \"Jika negara itu berpindah kita tidak tahu apakah kebangsaan bangsa penduduknya bisa dilestarikan.\",\n",
    "                \"Negara harus tinggal, dilestarikan, jika memungkinkan.\",\n",
    "                \"Relokasi mungkin tidak dapat dihindari.\",\n",
    "                \"Meski dengan aktivis gerakan nol emisi tetapi negara itu masih tenggelam.\",\n",
    "                \"Secara historis, negara tidak dihancurkan secara fisik; mereka hanya menjadi negara lain,\",\n",
    "                \"tanah yang mereka tempati dikendalikan oleh orang lain.\",\n",
    "                \"Tetapi setidaknya, untuk eksis, suatu negara membutuhkan pemerintah, populasi, dan sebidang pemukiman di dalam wilayah yang baru.\",\n",
    "                \"untuk mempertahankan pengakuan internasional, negara kepulauan yang menghadapi kehancuran karena perubahan iklim harus memperkuat wilayah mereka\",\n",
    "                \"menjaga setidaknya beberapa struktur fisik di atas air dan menjaga sekelompok kecil penduduk, bahkan jika sebagian besar penduduk telah pindah.\",\n",
    "                \"Orang-orang dari negara kepulauan kecil ingin terus memiliki perwakilan politik dalam komunitas internasional, dan mereka memiliki kepentingan ekonomi untuk dilindungi - hak atas perikanan dan sumber daya alam di wilayah mereka\",\n",
    "                \"Sangat masuk akal bahwa dampak perubahan iklim akan membuat negara bangsa tertentu tidak dapat dihuni sebelum akhir abad ini.\",\n",
    "                \"Walaupun ini mungkin nasib sejumlah kecil negara bangsa yang paling rentan terhadap perubahan iklim\",\n",
    "                \"implikasinya terhadap evolusi kenegaraan dan hukum internasional.\",\n",
    "                \"Untuk menanggapi fenomena negara tak bertanah.\",\n",
    "                \"status yang memungkinkan keberlangsungan keberadaan negara berdaulat, memberikan semua hak dan manfaat kedaulatan di antara keluarga negara, untuk selamanya.\",\n",
    "                \"Dalam praktiknya hal ini membutuhkan pembentukan kerangka kerja pemerintah baru yang dapat menggunakan wewenang atas penduduk yang tersebar karena perpindahan\",\n",
    "                \"Prospek pendidikan dan pekerjaan di pulau terbatas, dan mayoritas anak muda yang keluarganya mampu berpindah ke negara lain.\",\n",
    "                \"Banyak orang berniat bermigrasi untuk merespons perubahan iklim.\",\n",
    "                \"sebagian besar generasi tua tidak ingin pindah karena mereka percaya akan kehilangan identitas, budaya, gaya hidup dan tradisi mereka.\",\n",
    "                \"Tapi pemerintah percaya bahwa generasi muda berniat untuk bermigrasi demi keselamatan generasi mendatang.\"]\n",
    "len(traina1ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "POQbpgw7QM6B"
   },
   "outputs": [],
   "source": [
    "def get_ppmi(x_lengkap):\n",
    "    traina1 = x_lengkap[x_lengkap[:,2] == 1, 1]\n",
    "    # print(\"===================\")\n",
    "    # print(traina1)\n",
    "    # print(\"===================\")\n",
    "    # cleansing\n",
    "    trainAclear = [preprocess_ppmi(text) for text in list(traina1)]\n",
    "    trainAclear.extend([preprocess_ppmi(text) for text in list(traina1ex)])\n",
    "\n",
    "    # combination skipgram\n",
    "    unigram = []\n",
    "    bigram = []\n",
    "    for sent in trainAclear:        \n",
    "        unigram.extend(sent.split())\n",
    "        bigram.extend([\" \".join(item) for item in list(ngrams(sent.split(), 2))])\n",
    "        \n",
    "    count = Counter(unigram)\n",
    "    countbigram = Counter(bigram)\n",
    "\n",
    "    # print(\"===================\")\n",
    "    # print(unigram)\n",
    "    # print(\"===================\")\n",
    "    # print(bigram)\n",
    "    # print(\"===================\")\n",
    "\n",
    "    # traina['CL RESPONSE'] = traina.RESPONSE.apply(preprocess)\n",
    "    PPMI = []\n",
    "    idx = 0\n",
    "    for sent in x_lengkap[:,1]:\n",
    "        sent = preprocess_ppmi(sent)\n",
    "        try:\n",
    "            big = mkbigram(sent)\n",
    "            ppmi = sum([pmi(item[0],item[1],count,countbigram) for item in big])/len(big)\n",
    "            PPMI.append(ppmi)\n",
    "        except:\n",
    "            PPMI.append(0)\n",
    "        # print(idx)\n",
    "        idx+=1\n",
    "\n",
    "    return PPMI, count, countbigram\n",
    "\n",
    "def get_ppmi_test(x_lengkap, count, countbigram):\n",
    "    PPMI = []\n",
    "    idx = 0\n",
    "    for sent in x_lengkap[:,1]:\n",
    "        sent = preprocess_ppmi(sent)\n",
    "        try:\n",
    "            big = mkbigram(sent)\n",
    "            ppmi = sum([pmi(item[0],item[1],count,countbigram) for item in big])/len(big)\n",
    "            PPMI.append(ppmi)\n",
    "        except:\n",
    "            PPMI.append(0)\n",
    "        # print(idx)\n",
    "        idx+=1\n",
    "    return PPMI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "W_nhtWdQO7iN"
   },
   "source": [
    "# Classifier Booster 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 166
    },
    "colab_type": "code",
    "id": "1cNtdS6lNBHP",
    "outputId": "c2209513-b21c-4c39-a264-fc1d8d564e28"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: xgboost in /usr/local/lib/python3.6/dist-packages (0.90)\n",
      "Requirement already satisfied: imblearn in /usr/local/lib/python3.6/dist-packages (0.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.16.5)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.3.1)\n",
      "Requirement already satisfied: imbalanced-learn in /usr/local/lib/python3.6/dist-packages (from imblearn) (0.4.3)\n",
      "Requirement already satisfied: scikit-learn>=0.20 in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (0.21.3)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.20->imbalanced-learn->imblearn) (0.13.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install xgboost imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "colab_type": "code",
    "id": "5rGCQuvRPKmn",
    "outputId": "80ebba95-6590-49e1-cba8-c884364d2a90"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/externals/six.py:31: DeprecationWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
      "  \"(https://pypi.org/project/six/).\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.svm import SVC\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iSMOn1Bzaafc"
   },
   "source": [
    "### DATA A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 551
    },
    "colab_type": "code",
    "id": "lDkZNxm00-m6",
    "outputId": "9d5157ad-383f-4c13-e651-68e9333e6aec",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(250,) 0.0001 800\n",
      "Fold 1/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M2 Accuracy pada test data: 0.8727272727272727\n",
      "   F1 pada test data: 0.9135802469135802\n",
      "Fold 2/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M2 Accuracy pada test data: 0.7592592592592593\n",
      "   F1 pada test data: 0.8354430379746836\n",
      "Fold 3/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M2 Accuracy pada test data: 0.7358490566037735\n",
      "   F1 pada test data: 0.8292682926829269\n",
      "Fold 4/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M2 Accuracy pada test data: 0.8867924528301887\n",
      "   F1 pada test data: 0.9268292682926829\n",
      "Fold 5/5\n",
      "M2 Accuracy pada test data: 0.9245283018867925\n",
      "   F1 pada test data: 0.9444444444444444\n",
      "AVERAGE F1 0.8899130580616637\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "# STACKING LV 0 DATA A\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "avg_f1 = 0\n",
    "\n",
    "# ACTIVATE FOR EXPLORING\n",
    "for p1 in [(250,)]: #  hiddens\n",
    "    for p2 in [0.0001]: # learning_rate\n",
    "        for p3 in [800]: # max_iter\n",
    "# for p1 in [75]: # n_estimator, hiddens\n",
    "#     for p2 in [ 0.001]: # learning_rate\n",
    "#         for p3 in [0.75]: # subsample, max_iter\n",
    "            avg_exp_f1 = 0\n",
    "            print()\n",
    "            print(p1, p2, p3)\n",
    "\n",
    "            np.random.seed(113)\n",
    "            sampler = SMOTE(sampling_strategy=0.5, random_state=113)\n",
    "\n",
    "            X = data_A[:,1].copy()\n",
    "            Y = np.load(\"A_Y.npy\")\n",
    "            skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=False)\n",
    "\n",
    "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
    "                print(\"Fold %d/5\" % (kf+1))\n",
    "\n",
    "                X_train = vectorize(data_A[id_train,1].copy(), pre=True)\n",
    "                Y_train = np.load(\"A_Y.npy\")[id_train]\n",
    "\n",
    "                X_test = vectorize(data_A[id_test,1].copy(), pre=True)\n",
    "                Y_test = np.load(\"A_Y.npy\")[id_test]\n",
    "\n",
    "                X_test = np.c_[X_test, find_len(data_A[id_test,1], tipe='A')]\n",
    "                X_train = np.c_[X_train, find_len(data_A[id_train,1], tipe='A')]\n",
    "\n",
    "                X_train, Y_train = sampler.fit_resample(X_train, Y_train)\n",
    "\n",
    "                # XGB\n",
    "                # clf_b_1 = XGBClassifier(random_state=113, n_estimators=p1, learning_rate=p2, subsample=p3)\n",
    "                # clf_b_1.fit(X_train, Y_train)\n",
    "\n",
    "                # # MLP \n",
    "                clf_b_2 = MLPClassifier(random_state=113, max_iter=p3, hidden_layer_sizes=p1, learning_rate_init=p2, solver='adam')\n",
    "                clf_b_2.fit(X_train, Y_train)\n",
    "\n",
    "                # ev1 = clf_b_1.predict(X_test)\n",
    "                ev2 = clf_b_2.predict(X_test)\n",
    "\n",
    "                avg_exp_f1 += f1_score(ev2, Y_test) # <<<< change this\n",
    "\n",
    "                # print(\"M1 Accuracy pada test data:\", accuracy_score(ev1, Y_test))\n",
    "                # print(\"   F1 pada test data:\", f1_score(ev1, Y_test))\n",
    "                print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
    "                print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
    "\n",
    "            print(\"AVERAGE F1\", avg_exp_f1/5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "dGkhEaSn05Mp",
    "outputId": "0348b86b-0a87-4258-b1a6-dbf2c0598916"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(45,) 3e-05 900\n",
      "Fold 1/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M1 Accuracy pada test data: 0.8363636363636363\n",
      "   F1 pada test data: 0.888888888888889\n",
      "M2 Accuracy pada test data: 0.8727272727272727\n",
      "   F1 pada test data: 0.9135802469135802\n",
      "==========================\n",
      "Accuracy pada test data: 0.8727272727272727\n",
      "F1 pada test data: 0.9135802469135802\n",
      "Fold 2/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M1 Accuracy pada test data: 0.7592592592592593\n",
      "   F1 pada test data: 0.8470588235294116\n",
      "M2 Accuracy pada test data: 0.7592592592592593\n",
      "   F1 pada test data: 0.8354430379746836\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (900) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.7592592592592593\n",
      "F1 pada test data: 0.8354430379746836\n",
      "Fold 3/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M1 Accuracy pada test data: 0.7547169811320755\n",
      "   F1 pada test data: 0.8470588235294116\n",
      "M2 Accuracy pada test data: 0.7358490566037735\n",
      "   F1 pada test data: 0.8292682926829269\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (900) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.7547169811320755\n",
      "F1 pada test data: 0.8433734939759038\n",
      "Fold 4/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M1 Accuracy pada test data: 0.8301886792452831\n",
      "   F1 pada test data: 0.891566265060241\n",
      "M2 Accuracy pada test data: 0.8867924528301887\n",
      "   F1 pada test data: 0.9268292682926829\n",
      "==========================\n",
      "Accuracy pada test data: 0.8679245283018868\n",
      "F1 pada test data: 0.9156626506024096\n",
      "Fold 5/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M1 Accuracy pada test data: 0.8301886792452831\n",
      "   F1 pada test data: 0.8831168831168831\n",
      "M2 Accuracy pada test data: 0.9245283018867925\n",
      "   F1 pada test data: 0.9444444444444444\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (900) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.9433962264150944\n",
      "F1 pada test data: 0.958904109589041\n",
      "\n",
      "Average F1 0.8933927078111237\n",
      "Average Lv0 F1-1 0.8715379368249673\n",
      "Average Lv0 F1-2 0.8899130580616637\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "# STACKING LV 1 DATA A\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "for p1 in [(45,)]: # n_estimator, hiddens 48, 0.00005 800\n",
    "    for p2 in [0.00003]: # learning_rate\n",
    "        for p3 in [900]: # subsample, max_iter\n",
    "\n",
    "            avg_f1 = 0\n",
    "            avg_m1 = 0\n",
    "            avg_m2 = 0\n",
    "            np.random.seed(113)\n",
    "            sampler = SMOTE(sampling_strategy=0.5, random_state=113)\n",
    "\n",
    "            print()\n",
    "            print(p1, p2, p3)\n",
    "\n",
    "            X = data_A[:,1].copy()\n",
    "            Y = np.load(\"A_Y.npy\")\n",
    "            skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=False)\n",
    "\n",
    "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
    "                print(\"Fold %d/5\" % (kf+1))\n",
    "\n",
    "                X_all = vectorize(data_A[:,1].copy(), pre=True)\n",
    "                Y = np.load(\"A_Y.npy\")\n",
    "\n",
    "                X_train = vectorize(data_A[id_train,1].copy(), pre=True)\n",
    "                Y_train = np.load(\"A_Y.npy\")[id_train]\n",
    "\n",
    "                X_test = vectorize(data_A[id_test,1].copy(), pre=True)\n",
    "                Y_test = np.load(\"A_Y.npy\")[id_test]\n",
    "\n",
    "                X_test = np.c_[X_test, find_len(data_A[id_test,1], tipe='A')]\n",
    "                X_train = np.c_[X_train, find_len(data_A[id_train,1], tipe='A')]\n",
    "                X_all = np.c_[X_all, find_len(data_A[:,1], tipe='A')]\n",
    "\n",
    "                X_train, Y_train = sampler.fit_resample(X_train, Y_train)\n",
    "\n",
    "                # # BASELINE\n",
    "                clf_a_1 = XGBClassifier(random_state=113, n_estimators=75, learning_rate=0.001, subsample=0.75)\n",
    "                clf_a_1.fit(X_train, Y_train)\n",
    "\n",
    "                # MLP / 14\n",
    "                clf_a_2 = MLPClassifier(random_state=113, max_iter=800, hidden_layer_sizes=(250,), learning_rate_init=0.0001, solver='adam')\n",
    "                clf_a_2.fit(X_train, Y_train)\n",
    "\n",
    "                A1 = np.log(clf_a_1.predict_proba(X_train))\n",
    "                A2 = np.log(clf_a_2.predict_proba(X_train))\n",
    "                out_A = np.c_[A1, A2]\n",
    "\n",
    "                # r1 = 0.05\n",
    "                # r2 = 0.35\n",
    "                # r3 = 0.25\n",
    "                # r4 = 0.35\n",
    "                # tot_B = B1*r1 + B2*r2 + B3*r3 + B4*r4\n",
    "\n",
    "                A1t = np.log(clf_a_1.predict_proba(X_test))\n",
    "                A2t = np.log(clf_a_2.predict_proba(X_test))\n",
    "                out_At = np.c_[A1t, A2t]\n",
    "                # tot_Bt = B1t*r1 + B2t*r2 + B3t*r3 + B4t*r4\n",
    "\n",
    "                ev1 = clf_a_1.predict(X_test)\n",
    "                ev2 = clf_a_2.predict(X_test)\n",
    "\n",
    "                print(\"==========================\")\n",
    "                print(\"M1 Accuracy pada test data:\", accuracy_score(ev1, Y_test))\n",
    "                print(\"   F1 pada test data:\", f1_score(ev1, Y_test))\n",
    "                print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
    "                print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
    "                print(\"==========================\")\n",
    "\n",
    "                meta_clf = MLPClassifier(\n",
    "                        random_state=113,\n",
    "                        max_iter=p3,\n",
    "                        hidden_layer_sizes=p1,\n",
    "                        learning_rate_init=p2,\n",
    "                        learning_rate = 'adaptive',\n",
    "                        solver='adam')\n",
    "\n",
    "                meta_clf.fit(out_A, Y_train)\n",
    "                prediction = meta_clf.predict(out_At).astype(int)\n",
    "                # prediction = np.argmax(tot_Bt, 1)\n",
    "\n",
    "                print(\"Accuracy pada test data:\", accuracy_score(prediction, Y_test))\n",
    "                print(\"F1 pada test data:\", f1_score(prediction, Y_test))\n",
    "                \n",
    "                avg_f1 += f1_score(prediction, Y_test)\n",
    "                avg_m1 += f1_score(ev1, Y_test)\n",
    "                avg_m2 += f1_score(ev2, Y_test)\n",
    "                            \n",
    "            print(\"\\nAverage F1\", avg_f1/5)\n",
    "            print(\"Average Lv0 F1-1\", avg_m1/5)\n",
    "            print(\"Average Lv0 F1-2\", avg_m2/5)\n",
    "\n",
    "            clf_a_1 = XGBClassifier(random_state=113, n_estimators=75, learning_rate=0.001, subsample=0.75)\n",
    "            clf_a_2 = MLPClassifier(random_state=113, max_iter=800, hidden_layer_sizes=(250,), learning_rate_init=0.0001, solver='adam')\n",
    "            clf_a_1.fit(X_all, Y)\n",
    "            clf_a_2.fit(X_all, Y)\n",
    "            A1 = np.log(clf_a_1.predict_proba(X_all))\n",
    "            A2 = np.log(clf_a_2.predict_proba(X_all))\n",
    "            out_A_all = np.c_[A1, A2]\n",
    "            meta_clf = MLPClassifier(\n",
    "                        random_state=113,\n",
    "                        max_iter=p3,\n",
    "                        hidden_layer_sizes=p1,\n",
    "                        learning_rate_init=p2,\n",
    "                        solver='adam')\n",
    "            meta_clf.fit(out_A_all, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LV 0\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "avg_exp_f1_model1 = 0\n",
    "avg_exp_f1_model2 = 0\n",
    "avg_f1 = 0\n",
    "np.random.seed(113)\n",
    "sampler = SMOTE(sampling_strategy=0.5, random_state=113)\n",
    "\n",
    "X = data_A[:,1].copy()\n",
    "X_all = vectorize(data_A[:,1].copy(), pre=True)\n",
    "Y = np.load(\"A_Y.npy\")\n",
    "skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=True)\n",
    "\n",
    "for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
    "    print(\"Fold %d/5\" % (kf+1))\n",
    "\n",
    "    X_train = vectorize(data_A[id_train,1].copy(), pre=True)\n",
    "    Y_train = np.load(\"A_Y.npy\")[id_train]\n",
    "\n",
    "    X_test = vectorize(data_A[id_test,1].copy(), pre=True)\n",
    "    Y_test = np.load(\"A_Y.npy\")[id_test]\n",
    "\n",
    "    X_test = np.c_[X_test, find_len(data_A[id_test,1], tipe='A')] # concat with len\n",
    "    X_train = np.c_[X_train, find_len(data_A[id_train,1], tipe='A')] # concat with len\n",
    "\n",
    "    X_train, Y_train = sampler.fit_resample(X_train, Y_train) # SMOTE\n",
    "\n",
    "    # XGB\n",
    "    clf_a_1 = XGBClassifier(random_state=113, n_estimators=75, learning_rate=0.001, subsample=0.75)\n",
    "    clf_a_1.fit(X_train, Y_train)\n",
    "\n",
    "    # MLP \n",
    "    clf_a_2 = MLPClassifier(random_state=113, max_iter=800, hidden_layer_sizes=(250,), learning_rate_init=0.0001, solver='adam')\n",
    "    clf_a_2.fit(X_train, Y_train)\n",
    "\n",
    "    ev1 = clf_a_1.predict(X_test)\n",
    "    ev2 = clf_a_2.predict(X_test)\n",
    "    \n",
    "    avg_exp_f1_model1 += f1_score(ev1, Y_test) \n",
    "    avg_exp_f1_model2 += f1_score(ev2, Y_test) \n",
    "\n",
    "    print(\"Model 1 Accuracy pada test data:\", accuracy_score(ev1, Y_test))\n",
    "    print(\"F1 pada test data:\", f1_score(ev1, Y_test))\n",
    "    print(\"Model 2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
    "    print(\"F1 pada test data:\", f1_score(ev2, Y_test))\n",
    "    print()\n",
    "\n",
    "    # META CLASSIFIER\n",
    "    meta_clf = MLPClassifier(random_state=113, max_iter=800, hidden_layer_sizes=48, learning_rate_init=0.00005, solver='adam')\n",
    "\n",
    "    # INPUT METACLF\n",
    "    A1_train = np.log(clf_a_1.predict_proba(X_train))\n",
    "    A2_train = np.log(clf_a_2.predict_proba(X_train))\n",
    "    out_A_train = np.c_[A1_train, A2_train]\n",
    "\n",
    "    A1_test = np.log(clf_a_1.predict_proba(X_test))\n",
    "    A2_test = np.log(clf_a_2.predict_proba(X_test))\n",
    "    out_A_test = np.c_[A1_test, A2_test]\n",
    "\n",
    "    meta_clf.fit(out_A_train, Y_train)\n",
    "    prediction = meta_clf.predict(out_A_test).astype(int)\n",
    "\n",
    "    print(\"META RESULT:\")\n",
    "    print(\"Accuracy pada test data:\", accuracy_score(prediction, Y_test))\n",
    "    print(\"F1 pada test data:\", f1_score(prediction, Y_test))\n",
    "    print()\n",
    "    \n",
    "    avg_f1 += f1_score(prediction, Y_test)\n",
    "                \n",
    "print(\"\\nAverage F1\", avg_f1/5)\n",
    "print(\"AVERAGE F1 MODEL 1\", avg_exp_f1_model1/5)\n",
    "print(\"AVERAGE F1 MODEL 2\", avg_exp_f1_model2/5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r-UDmpdOa7iC"
   },
   "source": [
    "### DATA B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Vf7eViPm5Ik6"
   },
   "outputs": [],
   "source": [
    "# avg_f1 = 0\n",
    "# X = vectorize2(data_B[:,1].copy(), pre=True)\n",
    "# # XE = np.load(\"B_X.npy\")\n",
    "# # X = vectorize(data_B[:,1].copy())\n",
    "# Y = np.load(\"B_Y.npy\")[:]\n",
    "\n",
    "# # X = np.c_[X, XE]\n",
    "# X = np.c_[X, find_len(data_B[:,1], tipe='B')]\n",
    "\n",
    "# skf = StratifiedKFold(n_splits=5, random_state=11, shuffle=True)\n",
    "\n",
    "# # hiddens = [(100,), (200,), (300,), (400,), (80,80),(120,120),(120,80)]\n",
    "# hiddens = [(200,100), (128,64), (300, 100)]\n",
    "# lris = [0.001, 0.0001, 0.005, 0.0005]\n",
    "# # solvers = ['adam', 'sgd']\n",
    "# max_iters = [400, 500, 600]\n",
    "# # lrs = ['adaptive', 'constant', 'invscaling']\n",
    "\n",
    "# hiddens = [(128,64)]\n",
    "# lris = [0.0001]\n",
    "# solvers = ['adam']\n",
    "# max_iters = [500]\n",
    "# lrs = ['adaptive']\n",
    "\n",
    "# # ssmpls = [0.7,  0.8]\n",
    "# # nestrs = [50, 75, 100, 125, 150, 175]\n",
    "# # lris = [0.001, 0.01, 0.002, 0.008, 0.005]\n",
    "# # mds = [3]\n",
    "# # cbt = [0.8, 0.9, 1]\n",
    "\n",
    "# # ssmpls = [0.8]\n",
    "# # nestrs = [200]\n",
    "# # mds = [3]\n",
    "# # lris = [0.01]\n",
    "# # cbt = [1]\n",
    "\n",
    "# # for cb in cbt:\n",
    "# #     for lri in lris:\n",
    "# #         for ss in ssmpls:\n",
    "# #             for ne in nestrs:\n",
    "# #                 for md in mds:\n",
    "# for h in hiddens:\n",
    "#     for lri in lris:\n",
    "#         for solver in solvers:\n",
    "#             for mi in max_iters:\n",
    "#                 for lr in lrs:\n",
    "# # for h in range(1):\n",
    "# #     for lri in range(1):\n",
    "# #         for c in [0.001, 0.01, 0.1, 0.3, 0.5, 0.7, 1.0]:\n",
    "# #             for gm in [0.01, 0.05, 0.1, 0.5, 1.0]:\n",
    "# #                 for lr in range(1):\n",
    "#                     print()\n",
    "#                     print(h, lri, solver, mi, lr)\n",
    "# #                     print(lri, ss, ne, md, cb)\n",
    "# #                     print(c, gm)\n",
    "                    \n",
    "#                     avg_f1 = 0\n",
    "            \n",
    "#                     # BASELINE\n",
    "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=200, max_depth=3, learning_rate=0.01, subsample=0.8)\n",
    "\n",
    "#                     # ATTEMPT 3\n",
    "# #                     clf2 = AdaBoostClassifier(random_state=113, n_estimators=50, learning_rate=0.05)\n",
    "    \n",
    "#                     # ATTEMPT 5\n",
    "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=150, gamma=5, max_depth=3, learning_rate=0.01, subsample=0.8)\n",
    "                    \n",
    "    \n",
    "#                     # ATTEMPT 12\n",
    "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=150, max_depth=3, learning_rate=0.0017, subsample=0.7, colsample_bytree=0.8)\n",
    "            \n",
    "#                     # Final attempt\n",
    "#                     clf2 = MLPClassifier(\n",
    "#                             random_state=113,\n",
    "#                             max_iter=mi,\n",
    "#                             hidden_layer_sizes=h,\n",
    "#                             learning_rate_init=lri,\n",
    "#                             learning_rate = lr,\n",
    "#                             solver=solver)\n",
    "    \n",
    "# #                     clf2 = SVC(C=c, gamma=gm, probability=True, class_weight=None, random_state=113)\n",
    "                    \n",
    "#                     done = True\n",
    "#                     for i, (train_index, test_index) in enumerate(skf.split(X, Y)):\n",
    "#                         done = False\n",
    "#                         x_train, y_train = X[train_index], Y[train_index]\n",
    "# #                         x_train, y_train = sampler.fit_resample(x_train, y_train)\n",
    "\n",
    "#                         x_test, y_test = X[test_index], Y[test_index]\n",
    "\n",
    "\n",
    "#                         clf2.fit(x_train, y_train)\n",
    "\n",
    "#                         prediction = clf2.predict(x_test).astype(int)\n",
    "\n",
    "#                         from sklearn.metrics import accuracy_score\n",
    "#                         from sklearn.metrics import precision_score\n",
    "#                         from sklearn.metrics import recall_score\n",
    "#                         from sklearn.metrics import f1_score\n",
    "\n",
    "# #                         print(\"Accuracy pada train data:\", accuracy_score(clf2.predict(x_train).astype(int), y_train))\n",
    "# #                         print(\"Accuracy pada test data:\", accuracy_score(prediction, y_test))\n",
    "# #                         print(\"Precision baseline pada test data:\", precision_score(prediction, y_test))\n",
    "# #                         print(\"Recall baseline pada test data:\", recall_score(prediction, y_test))\n",
    "#                         print(\"F1 baseline pada test data:\", f1_score(prediction, y_test))\n",
    "# #                         print()\n",
    "#                         avg_f1 += f1_score(prediction, y_test)\n",
    "\n",
    "# #                         if f1_score(prediction, y_test)<0.64 and i==2:\n",
    "# #                             break\n",
    "# #                         if f1_score(prediction, y_test)<0.84 and i==3:\n",
    "# #                             break\n",
    "#                         done = True\n",
    "                    \n",
    "#                     if done:\n",
    "#                         print(\"Average F1\", avg_f1/5)\n",
    "\n",
    "#                     # FINAL\n",
    "#                     clf2 = MLPClassifier(\n",
    "#                             random_state=113,\n",
    "#                             max_iter=mi,\n",
    "#                             hidden_layer_sizes=h,\n",
    "#                             learning_rate_init=lri,\n",
    "#                             learning_rate = lr,\n",
    "#                             solver=solver)\n",
    "\n",
    "#                     # BASELINE\n",
    "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=200, max_depth=3, learning_rate=0.01, subsample=0.8)\n",
    "                    \n",
    "#                     # 12\n",
    "#                     # clf2 = XGBClassifier(random_state=113, n_estimators=150, max_depth=3, learning_rate=0.0017, subsample=0.7, colsample_bytree=0.8)\n",
    "                    \n",
    "#                     clf2.fit(X, Y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 348
    },
    "colab_type": "code",
    "id": "93etZfyTcdLP",
    "outputId": "6abf915b-64b1-4ab7-989a-709a8c124065"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "70 0.02 0.9\n",
      "Fold 1/5\n",
      "M1 Accuracy pada test data: 0.6290322580645161\n",
      "   F1 pada test data: 0.7012987012987012\n",
      "Fold 2/5\n",
      "M1 Accuracy pada test data: 0.5806451612903226\n",
      "   F1 pada test data: 0.675\n",
      "Fold 3/5\n",
      "M1 Accuracy pada test data: 0.7213114754098361\n",
      "   F1 pada test data: 0.767123287671233\n",
      "Fold 4/5\n",
      "M1 Accuracy pada test data: 0.7\n",
      "   F1 pada test data: 0.7428571428571429\n",
      "Fold 5/5\n",
      "M1 Accuracy pada test data: 0.7833333333333333\n",
      "   F1 pada test data: 0.7936507936507938\n",
      "AVERAGE F1 0.7359859850955741\n"
     ]
    }
   ],
   "source": [
    "# STACKING LV 0\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "avg_f1 = 0\n",
    "\n",
    "# ACTIVATE FOR EXPLORING\n",
    "for p1 in [70]: # n_estimator, hiddens\n",
    "    for p2 in [0.02]: # learning_rate\n",
    "        for p3 in [0.9]: # subsample, max_iter\n",
    "# for p1 in [(128, 64), (500,), (600,)]: # n_estimator, hiddens\n",
    "#     for p2 in [0.0001, 0.001]: # learning_rate\n",
    "#         for p3 in [500, 300, 800]: # subsample, max_iter\n",
    "            avg_exp_f1 = 0\n",
    "            print()\n",
    "            print(p1, p2, p3)\n",
    "\n",
    "            np.random.seed(113)\n",
    "\n",
    "            X = data_B[:,1].copy()\n",
    "            XE = np.load(\"B_X.npy\")\n",
    "            Y = np.load(\"B_Y.npy\")\n",
    "#             X = np.c_[X, XE]\n",
    "            skf = StratifiedKFold(n_splits=5, random_state=11, shuffle=True)\n",
    "\n",
    "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
    "                print(\"Fold %d/5\" % (kf+1))\n",
    "\n",
    "                X_train = vectorize2(data_B[id_train,1].copy(), pre=True, tipe=\"B\")\n",
    "#                 X_train = XE[id_train]\n",
    "                Y_train = np.load(\"B_Y.npy\")[id_train]\n",
    "\n",
    "                X_test = vectorize2(data_B[id_test,1].copy(), pre=True, tipe=\"B\")\n",
    "#                 X_test = XE[id_test]\n",
    "                Y_test = np.load(\"B_Y.npy\")[id_test]\n",
    "\n",
    "                X_test = np.c_[X_test, XE[id_test], find_len(data_B[id_test,1], tipe='B')]\n",
    "                X_train = np.c_[X_train, XE[id_train],find_len(data_B[id_train,1], tipe='B')]\n",
    "\n",
    "                # # XGB\n",
    "                clf_b_1 = XGBClassifier(random_state=113, n_estimators=p1,\n",
    "                                        learning_rate=p2, subsample=p3,\n",
    "                                        gamma=0, colsample_bytree=0.5)\n",
    "                clf_b_1.fit(X_train, Y_train)\n",
    "\n",
    "#                 # # MLP\n",
    "#                 clf_b_2 = MLPClassifier(random_state=113, max_iter=p3, hidden_layer_sizes=p1, learning_rate_init=p2, solver='adam')\n",
    "#                 clf_b_2.fit(X_train, Y_train)\n",
    "\n",
    "                ev1 = clf_b_1.predict(X_test)\n",
    "#                 ev2 = clf_b_2.predict(X_test)\n",
    "\n",
    "                avg_exp_f1 += f1_score(ev1, Y_test) # <<<< change this\n",
    "\n",
    "                print(\"M1 Accuracy pada test data:\", accuracy_score(ev1, Y_test))\n",
    "                print(\"   F1 pada test data:\", f1_score(ev1, Y_test))\n",
    "#                 print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
    "#                 print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
    "\n",
    "            print(\"AVERAGE F1\", avg_exp_f1/5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "mGif9g-q3O0Q",
    "outputId": "4586367e-6059-404f-816e-b44cfc2fe046"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(32,) 3e-05 1000\n",
      "Fold 1/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M2 Accuracy pada test data: 0.6290322580645161\n",
      "   F1 pada test data: 0.6666666666666666\n",
      "M3 Accuracy pada test data: 0.6612903225806451\n",
      "   F1 pada test data: 0.6956521739130436\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.6612903225806451\n",
      "F1 pada test data: 0.7042253521126761\n",
      "Fold 2/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M2 Accuracy pada test data: 0.5967741935483871\n",
      "   F1 pada test data: 0.6987951807228915\n",
      "M3 Accuracy pada test data: 0.6290322580645161\n",
      "   F1 pada test data: 0.6933333333333332\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.6451612903225806\n",
      "F1 pada test data: 0.725\n",
      "Fold 3/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M2 Accuracy pada test data: 0.7868852459016393\n",
      "   F1 pada test data: 0.8219178082191781\n",
      "M3 Accuracy pada test data: 0.7868852459016393\n",
      "   F1 pada test data: 0.8059701492537314\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.7540983606557377\n",
      "F1 pada test data: 0.782608695652174\n",
      "Fold 4/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M2 Accuracy pada test data: 0.7833333333333333\n",
      "   F1 pada test data: 0.8169014084507042\n",
      "M3 Accuracy pada test data: 0.7666666666666667\n",
      "   F1 pada test data: 0.8108108108108109\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.7666666666666667\n",
      "F1 pada test data: 0.8157894736842105\n",
      "Fold 5/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================\n",
      "M2 Accuracy pada test data: 0.6833333333333333\n",
      "   F1 pada test data: 0.7164179104477613\n",
      "M3 Accuracy pada test data: 0.7666666666666667\n",
      "   F1 pada test data: 0.7741935483870968\n",
      "==========================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.75\n",
      "F1 pada test data: 0.7761194029850745\n",
      "\n",
      "Average F1 0.7607485848868271\n",
      "Average Lv0 F1-2 0.7441397949014403\n",
      "Average Lv0 F1-3 0.7559920031396032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "# STACKING LV 1 FILM B\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "for p1 in [(32,)]: # n_estimator, hiddens\n",
    "    for p2 in [0.00003]: # learning_rate\n",
    "        for p3 in [1000]: # subsample, max_iter\n",
    "\n",
    "            avg_f1 = 0\n",
    "            avg_m1 = 0\n",
    "            avg_m2 = 0\n",
    "            avg_m3 = 0\n",
    "            avg_m4 = 0\n",
    "            np.random.seed(113)\n",
    "            print()\n",
    "            print(p1, p2, p3)\n",
    "\n",
    "            X = data_B[:,1].copy()\n",
    "            Y = np.load(\"B_Y.npy\")\n",
    "            skf = StratifiedKFold(n_splits=5, random_state=11, shuffle=True)\n",
    "\n",
    "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
    "                print(\"Fold %d/5\" % (kf+1))\n",
    "\n",
    "                # id_test = np.random.choice(len(data_B), 39)\n",
    "                # id_train = diff(range(len(data_B)), id_test)\n",
    "\n",
    "                X_all = vectorize2(data_B[:,1].copy(), pre=True)\n",
    "                Y = np.load(\"B_Y.npy\")\n",
    "\n",
    "                X_train = vectorize2(data_B[id_train,1].copy(), pre=True)\n",
    "                Y_train = np.load(\"B_Y.npy\")[id_train]\n",
    "\n",
    "                X_test = vectorize2(data_B[id_test,1].copy(), pre=True)\n",
    "                Y_test = np.load(\"B_Y.npy\")[id_test]\n",
    "\n",
    "                X_test = np.c_[X_test, find_len(data_B[id_test,1], tipe='B')]\n",
    "                X_train = np.c_[X_train, find_len(data_B[id_train,1], tipe='B')]\n",
    "                X_all = np.c_[X_all, find_len(data_B[:,1], tipe='B')]\n",
    "\n",
    "                # # BASELINE\n",
    "                # 12\n",
    "                clf_b_2 = XGBClassifier(random_state=113, n_estimators=100, learning_rate=0.01, subsample=0.8, colsample_bytree=0.8)\n",
    "                clf_b_2.fit(X_train, Y_train)\n",
    "\n",
    "                # MLP / 14\n",
    "                clf_b_3 = MLPClassifier(random_state=113, max_iter=500, hidden_layer_sizes=(128,64), learning_rate_init=0.0001, solver='adam')\n",
    "                clf_b_3.fit(X_train, Y_train)\n",
    "\n",
    "                B2 = np.log(clf_b_2.predict_proba(X_train))\n",
    "                B3 = np.log(clf_b_3.predict_proba(X_train))\n",
    "                out_B = np.c_[B2, B3]\n",
    "\n",
    "                B2t = np.log(clf_b_2.predict_proba(X_test))\n",
    "                B3t = np.log(clf_b_3.predict_proba(X_test))\n",
    "                out_Bt = np.c_[B2t, B3t]\n",
    "\n",
    "                ev2 = clf_b_2.predict(X_test)\n",
    "                ev3 = clf_b_3.predict(X_test)\n",
    "\n",
    "                print(\"==========================\")\n",
    "                print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
    "                print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
    "                print(\"M3 Accuracy pada test data:\", accuracy_score(ev3, Y_test))\n",
    "                print(\"   F1 pada test data:\", f1_score(ev3, Y_test))\n",
    "                print(\"==========================\")\n",
    "\n",
    "                meta_clf2 = MLPClassifier(\n",
    "                        random_state=113,\n",
    "                        max_iter=p3,\n",
    "                        hidden_layer_sizes=p1,\n",
    "                        learning_rate_init=p2,\n",
    "                        learning_rate = 'adaptive',\n",
    "                        solver='adam')\n",
    "\n",
    "                meta_clf2.fit(out_B, Y_train)\n",
    "\n",
    "                prediction = meta_clf2.predict(out_Bt).astype(int)\n",
    "\n",
    "                print(\"Accuracy pada test data:\", accuracy_score(prediction, Y_test))\n",
    "                print(\"F1 pada test data:\", f1_score(prediction, Y_test))\n",
    "                \n",
    "                avg_f1 += f1_score(prediction, Y_test)\n",
    "                avg_m2 += f1_score(ev2, Y_test)\n",
    "                avg_m3 += f1_score(ev3, Y_test)\n",
    "\n",
    "                            \n",
    "            print(\"\\nAverage F1\", avg_f1/5)\n",
    "            print(\"Average Lv0 F1-2\", avg_m2/5)\n",
    "            print(\"Average Lv0 F1-3\", avg_m3/5)\n",
    "\n",
    "            clf_b_2 = XGBClassifier(random_state=113, n_estimators=100, learning_rate=0.01, subsample=0.8, colsample_bytree=0.8)\n",
    "            clf_b_3 = MLPClassifier(random_state=113, max_iter=500, hidden_layer_sizes=(128,64), learning_rate_init=0.0001, solver='adam')\n",
    "            clf_b_2.fit(X_all, Y)\n",
    "            clf_b_3.fit(X_all, Y)\n",
    "            B2 = np.log(clf_b_2.predict_proba(X_all))\n",
    "            B3 = np.log(clf_b_3.predict_proba(X_all))\n",
    "            out_B_all = np.c_[B2, B3]\n",
    "            meta_clf2 = MLPClassifier(\n",
    "                        random_state=113,\n",
    "                        max_iter=p3,\n",
    "                        hidden_layer_sizes=p1,\n",
    "                        learning_rate_init=p2,\n",
    "                        solver='adam')\n",
    "            meta_clf2.fit(out_B_all, Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mzNIrkhuPWun"
   },
   "source": [
    "# Classifier LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "351bImnsPVo1"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "\n",
    "def seed():\n",
    "    torch.manual_seed(113)\n",
    "    torch.cuda.manual_seed(113)\n",
    "    torch.cuda.manual_seed_all(113)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "\n",
    "    np.random.seed(113)\n",
    "\n",
    "    import random\n",
    "    random.seed(113)\n",
    "\n",
    "def _init_fn(worker_id):\n",
    "    np.random.seed(int(seed))\n",
    "    \n",
    "seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JECKBpP5SRuB"
   },
   "outputs": [],
   "source": [
    "len_seq = 14\n",
    "def batcher(sent):\n",
    "    ret = []\n",
    "    counter = 0\n",
    "    null = np.zeros(150)\n",
    "    for word in sent.split():\n",
    "        if word in [\"./,;!?()\"]:\n",
    "            continue\n",
    "        word = word.lower()\n",
    "        ret.append(model[word])\n",
    "    while(len(ret) < len_seq):\n",
    "        ret.insert(0, null)\n",
    "    while(len(ret) > len_seq):\n",
    "        ret.pop(-1)  \n",
    "    return np.asarray(ret)\n",
    "\n",
    "def batcher2(sent):\n",
    "    ret = []\n",
    "    counter = 0\n",
    "    null = np.zeros(300)\n",
    "    for word in sent.split():\n",
    "        if word in [\"./,;!?()\"]:\n",
    "            continue\n",
    "        ret.append(model2[word])\n",
    "    while(len(ret) < len_seq):\n",
    "        ret.insert(0, null)\n",
    "    while(len(ret) > len_seq):\n",
    "        ret.pop(-1)  \n",
    "    return np.asarray(ret)\n",
    "\n",
    "def data_batcher(x):\n",
    "    res = []\n",
    "    for i in x:\n",
    "        res.append(batcher(i[1]))\n",
    "    return np.asarray(res) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "15DIxkgUTqCz"
   },
   "outputs": [],
   "source": [
    "# xwer_A = data_batcher(xtrain_A)\n",
    "# xwer_B = data_batcher(xtrain_B)\n",
    "# xwet_A = data_batcher(xtest_A)\n",
    "# xwet_B = data_batcher(xtest_B)\n",
    "\n",
    "# yr_A = np.asarray(ytrain_A).astype(int)\n",
    "# yt_A = np.asarray(ytest_A).astype(int)\n",
    "# yr_B = np.asarray(ytrain_B).astype(int)\n",
    "# yt_B = np.asarray(ytest_A).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "57FC6Sba88mI"
   },
   "outputs": [],
   "source": [
    "class LSTMModel(nn.Module):   \n",
    "\n",
    "    def __init__(self, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):\n",
    "        super(LSTMModel, self).__init__()\n",
    "\n",
    "        self.output_size = output_size\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_dim = hidden_dim\n",
    "        \n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, \n",
    "                            dropout=drop_prob, batch_first=True)\n",
    "        \n",
    "        self.dropout = nn.Dropout(0.5)\n",
    "        \n",
    "        self.fc = nn.Linear(hidden_dim, output_size)\n",
    "\n",
    "    def forward(self, x, hidden):\n",
    "        \n",
    "        batch_size = x.size(0)\n",
    "        seq_len = x.size(1)\n",
    "\n",
    "        lstm_out, hidden = self.lstm(x, hidden)\n",
    "#         print(lstm_out.shape)\n",
    "        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)\n",
    "#         print(lstm_out.shape)\n",
    "        out = self.dropout(lstm_out)\n",
    "        out = self.fc(out)\n",
    "        out = F.relu(out)\n",
    "        \n",
    "        out = out.view(batch_size, seq_len, -1)\n",
    "#         print(out.shape)\n",
    "        out = out[:, -1]\n",
    "#         print(out.shape)\n",
    "        return F.log_softmax(out), hidden\n",
    "    \n",
    "    \n",
    "    def init_hidden(self, batch_size):\n",
    "        weight = next(self.parameters()).data\n",
    "        \n",
    "        hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),\n",
    "                  weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())\n",
    "        \n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W8Hudww27Trd"
   },
   "outputs": [],
   "source": [
    "\n",
    "# def reset_data():\n",
    "# dataloaders\n",
    "batch_size = 42\n",
    "\n",
    "# make sure the SHUFFLE your training data\n",
    "train_loader = []\n",
    "valid_loader = []\n",
    "test_size = [0.2, 0.15, 0.1, 0.2, 0.15]\n",
    "rand_i = [113, 1117, 11, 3, 317]\n",
    "\n",
    "for i in range(5):\n",
    "    xtrain_A, xtest_A, ytrain_A, ytest_A = train_test_split(data_A[:,:-1], data_A[:,-1],\n",
    "                                      test_size=test_size[i], \n",
    "                                      stratify=data_A[:,-1], \n",
    "                                      random_state=rand_i[i])\n",
    "\n",
    "    xtrain_B, xtest_B, ytrain_B, ytest_B = train_test_split(data_B[:,:-1], data_B[:,-1],\n",
    "                                          test_size=test_size[i], \n",
    "                                          stratify=data_B[:,-1], \n",
    "                                          random_state=rand_i[i])\n",
    "\n",
    "    xwer_A = data_batcher(xtrain_A)\n",
    "    xwer_B = data_batcher(xtrain_B)\n",
    "    xwet_A = data_batcher(xtest_A)\n",
    "    xwet_B = data_batcher(xtest_B)\n",
    "\n",
    "    yr_A = np.asarray(ytrain_A).astype(int)\n",
    "    yt_A = np.asarray(ytest_A).astype(int)\n",
    "    yr_B = np.asarray(ytrain_B).astype(int)\n",
    "    yt_B = np.asarray(ytest_A).astype(int)\n",
    "\n",
    "    train_data = TensorDataset(torch.from_numpy(xwer_A), torch.from_numpy(yr_A))\n",
    "    valid_data = TensorDataset(torch.from_numpy(xwet_A), torch.from_numpy(yt_A))\n",
    "\n",
    "    train_loader.append(DataLoader(train_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))\n",
    "    valid_loader.append(DataLoader(valid_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))\n",
    "        \n",
    "# reset_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 570
    },
    "colab_type": "code",
    "id": "hTJtWuhf-dCS",
    "outputId": "8bce2efe-a7c2-42e3-ac96-ff1394ba6e8a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:54: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy pada test data: 0.7777777777777778\n",
      "Precision baseline pada test data: 0.868421052631579\n",
      "Recall baseline pada test data: 0.825\n",
      "F1 baseline pada test data: 0.8461538461538461\n",
      "\n",
      "Accuracy pada test data: 0.8048780487804879\n",
      "Precision baseline pada test data: 0.9655172413793104\n",
      "Recall baseline pada test data: 0.8\n",
      "F1 baseline pada test data: 0.8750000000000001\n",
      "\n",
      "Accuracy pada test data: 0.7777777777777778\n",
      "Precision baseline pada test data: 0.8421052631578947\n",
      "Recall baseline pada test data: 0.8421052631578947\n",
      "F1 baseline pada test data: 0.8421052631578947\n",
      "\n",
      "Accuracy pada test data: 0.7592592592592593\n",
      "Precision baseline pada test data: 0.9473684210526315\n",
      "Recall baseline pada test data: 0.7659574468085106\n",
      "F1 baseline pada test data: 0.8470588235294116\n",
      "\n",
      "Accuracy pada test data: 0.8292682926829268\n",
      "Precision baseline pada test data: 0.9655172413793104\n",
      "Recall baseline pada test data: 0.8235294117647058\n",
      "F1 baseline pada test data: 0.888888888888889\n",
      "\n",
      "0.8598413643460084\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "seed()\n",
    "\n",
    "output_size = 2\n",
    "embedding_dim = 150\n",
    "hidden_dim = 256\n",
    "n_layers = 1\n",
    "lrs= 0.00007\n",
    "epochs = 45\n",
    "counter = 0\n",
    "print_every = 500\n",
    "clip=5 # gradient clipping\n",
    "\n",
    "# for epoch in epochs:\n",
    "#     for lr in lrs:\n",
    "#         for hidden_dim in hidden_dims:\n",
    "#             for n_layer in n_layers:\n",
    "                \n",
    "#                 print()\n",
    "#                 print(epoch, lr, hidden_dim, n_layer)\n",
    "\n",
    "avg_f1 = 0\n",
    "\n",
    "for i in range(5):\n",
    "\n",
    "    clf = LSTMModel(output_size, embedding_dim, hidden_dim, n_layers)\n",
    "    criterion = nn.NLLLoss()\n",
    "    optimizer = torch.optim.Adam(clf.parameters(), lr=lrs)\n",
    "\n",
    "    clf.cuda()\n",
    "    clf.train()\n",
    "\n",
    "    for e in range(epochs):\n",
    "\n",
    "        for _, (inputs, labels) in enumerate(train_loader[i]):\n",
    "\n",
    "            counter += 1\n",
    "\n",
    "            inputs, labels = inputs.cuda(), labels.cuda()\n",
    "            inputs = inputs.float()\n",
    "\n",
    "            h = clf.init_hidden(inputs.size(0))\n",
    "            h = tuple([each.data for each in h])\n",
    "\n",
    "            clf.zero_grad()\n",
    "            output, h = clf(inputs, h)\n",
    "\n",
    "            loss = criterion(output.squeeze(), labels)\n",
    "            loss.backward()\n",
    "\n",
    "            # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
    "#             nn.utils.clip_grad_norm_(clf.parameters(), clip)\n",
    "            optimizer.step()\n",
    "\n",
    "        clf.eval()\n",
    "        val_losses = []\n",
    "        acc = torch.Tensor([]).cuda()\n",
    "        for inputs, labels in valid_loader[i]:\n",
    "\n",
    "            val_h = clf.init_hidden(inputs.size(0))\n",
    "            val_h = tuple([each.data for each in val_h])\n",
    "\n",
    "            inputs, labels = inputs.cuda(), labels.cuda()\n",
    "            inputs = inputs.float()\n",
    "\n",
    "            output, val_h = clf(inputs, val_h)\n",
    "    #         print(labels)\n",
    "    #         print(torch.argmax(output.squeeze(), dim=1))\n",
    "            pred = torch.argmax(output.squeeze(), dim=1)==labels\n",
    "            acc = torch.cat((acc, pred.float()),0)\n",
    "            val_loss = criterion(output.squeeze(), labels)\n",
    "\n",
    "            val_losses.append(val_loss.item())\n",
    "\n",
    "        clf.train()\n",
    "#                         if e%print_every == 0:\n",
    "#                             print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
    "#                                   \"Loss: {:.6f}...\".format(loss.item()),\n",
    "#                                   \"Val Loss: {:.6f}\".format(np.mean(val_losses)),\n",
    "#                                   \"Val Acc: {:.6f}\".format(torch.mean(acc)))\n",
    "\n",
    "\n",
    "    clf.eval()\n",
    "    preds = torch.Tensor([]).cuda()\n",
    "    labels_all = torch.Tensor([]).cuda()\n",
    "    for inputs, labels in valid_loader[i]:\n",
    "\n",
    "        val_h = clf.init_hidden(inputs.size(0))\n",
    "        val_h = tuple([each.data for each in val_h])\n",
    "\n",
    "        inputs, labels = inputs.cuda(), labels.cuda()\n",
    "        inputs = inputs.float()\n",
    "\n",
    "        output, val_h = clf(inputs, val_h)\n",
    "    #     print(output.shape)\n",
    "        pred = torch.argmax(output.squeeze(), dim=1)\n",
    "        preds = torch.cat((preds, pred.float()),0)\n",
    "        labels_all = torch.cat((labels_all, labels.float()),0)\n",
    "\n",
    "    preds = preds.detach().cpu().numpy()\n",
    "    labels_all = labels_all.cpu().numpy()\n",
    "#                     print(preds)\n",
    "\n",
    "    print(\"Accuracy pada test data:\", accuracy_score(preds, labels_all))\n",
    "    print(\"Accuracy pada test data:\", accuracy_score(preds, labels_all))\n",
    "    print(\"Precision baseline pada test data:\", precision_score(preds, labels_all))\n",
    "    print(\"Recall baseline pada test data:\", recall_score(preds, labels_all))\n",
    "    print(\"F1 baseline pada test data:\", f1_score(preds, labels_all))\n",
    "    print()\n",
    "    avg_f1 += f1_score(preds, labels_all)\n",
    "\n",
    "\n",
    "print(avg_f1/5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_iCdiAaHgLc4"
   },
   "outputs": [],
   "source": [
    "# dataloaders\n",
    "batch_size = 42\n",
    "\n",
    "# make sure the SHUFFLE your training data\n",
    "train_loader = []\n",
    "valid_loader = []\n",
    "test_size = [0.2, 0.15, 0.1, 0.2, 0.15]\n",
    "rand_i = [113, 1117, 11, 3, 317]\n",
    "\n",
    "for i in range(5):\n",
    "    xtrain_B, xtest_B, ytrain_B, ytest_B = train_test_split(data_B[:,:-1], data_B[:,-1],\n",
    "                                          test_size=test_size[i], \n",
    "                                          stratify=data_B[:,-1], \n",
    "                                          random_state=rand_i[i])\n",
    "    \n",
    "    xwer_B = data_batcher(xtrain_B)\n",
    "    xwet_B = data_batcher(xtest_B)\n",
    "\n",
    "    yr_B = np.asarray(ytrain_B).astype(int)\n",
    "    yt_B = np.asarray(ytest_A).astype(int)\n",
    "    \n",
    "    train_data = TensorDataset(torch.from_numpy(xwer_B), torch.from_numpy(yr_B))\n",
    "    valid_data = TensorDataset(torch.from_numpy(xwet_B), torch.from_numpy(yt_B))\n",
    "    \n",
    "    train_loader.append(DataLoader(train_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))\n",
    "    valid_loader.append(DataLoader(valid_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pgniphDhgFHe"
   },
   "outputs": [],
   "source": [
    "seed()\n",
    "\n",
    "output_size = 2\n",
    "embedding_dim = 300\n",
    "hidden_dim = 500\n",
    "n_layers = 1\n",
    "lr=0.00007\n",
    "epochs = 75 # 3-4 is approx where I noticed the validation loss stop decreasing\n",
    "counter = 0\n",
    "print_every = 30\n",
    "clip=5 # gradient clipping\n",
    "\n",
    "avg_f1 = 0\n",
    "\n",
    "for i in range(5):\n",
    "    \n",
    "    clf = LSTMModel(output_size, embedding_dim, hidden_dim, n_layers)\n",
    "    criterion = nn.NLLLoss()\n",
    "    optimizer = torch.optim.Adam(clf.parameters(), lr=lr)\n",
    "    \n",
    "    clf.cuda()\n",
    "    clf.train()\n",
    "    \n",
    "    for e in range(epochs):\n",
    "\n",
    "        for inputs, labels in train_loader[i]:\n",
    "\n",
    "            counter += 1\n",
    "\n",
    "            inputs, labels = inputs.cuda(), labels.cuda()\n",
    "            inputs = inputs.float()\n",
    "\n",
    "            h = clf.init_hidden(inputs.size(0))\n",
    "            h = tuple([each.data for each in h])\n",
    "\n",
    "            clf.zero_grad()\n",
    "            output, h = clf(inputs, h)\n",
    "\n",
    "            loss = criterion(output.squeeze(), labels)\n",
    "            loss.backward()\n",
    "\n",
    "            # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
    "            nn.utils.clip_grad_norm_(clf.parameters(), clip)\n",
    "            optimizer.step()\n",
    "\n",
    "        clf.eval()\n",
    "        val_losses = []\n",
    "        acc = torch.Tensor([]).cuda()\n",
    "        for inputs, labels in valid_loader[i]:\n",
    "\n",
    "            val_h = clf.init_hidden(inputs.size(0))\n",
    "            val_h = tuple([each.data for each in val_h])\n",
    "\n",
    "            inputs, labels = inputs.cuda(), labels.cuda()\n",
    "            inputs = inputs.float()\n",
    "\n",
    "            output, val_h = clf(inputs, val_h)\n",
    "    #         print(labels)\n",
    "    #         print(torch.argmax(output.squeeze(), dim=1))\n",
    "            pred = torch.argmax(output.squeeze(), dim=1)==labels\n",
    "            acc = torch.cat((acc, pred.float()),0)\n",
    "            val_loss = criterion(output.squeeze(), labels)\n",
    "\n",
    "            val_losses.append(val_loss.item())\n",
    "\n",
    "        clf.train()\n",
    "        if e%print_every == 0:\n",
    "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
    "                  \"Loss: {:.6f}...\".format(loss.item()),\n",
    "                  \"Val Loss: {:.6f}\".format(np.mean(val_losses)),\n",
    "                  \"Val Acc: {:.6f}\".format(torch.mean(acc)))\n",
    "\n",
    "\n",
    "    clf.eval()\n",
    "    preds = torch.Tensor([]).cuda()\n",
    "    labels_all = torch.Tensor([]).cuda()\n",
    "    for inputs, labels in valid_loader[i]:\n",
    "\n",
    "        val_h = clf.init_hidden(inputs.size(0))\n",
    "        val_h = tuple([each.data for each in val_h])\n",
    "\n",
    "        inputs, labels = inputs.cuda(), labels.cuda()\n",
    "        inputs = inputs.float()\n",
    "\n",
    "        output, val_h = clf(inputs, val_h)\n",
    "    #     print(output.shape)\n",
    "        pred = torch.argmax(output.squeeze(), dim=1)\n",
    "        preds = torch.cat((preds, pred.float()),0)\n",
    "        labels_all = torch.cat((labels_all, labels.float()),0)\n",
    "\n",
    "    preds = preds.detach().cpu().numpy()\n",
    "    labels_all = labels_all.cpu().numpy()\n",
    "    print(preds)\n",
    "    from sklearn.metrics import accuracy_score\n",
    "    from sklearn.metrics import precision_score\n",
    "    from sklearn.metrics import recall_score\n",
    "    from sklearn.metrics import f1_score\n",
    "\n",
    "    print(\"Accuracy pada test data:\", accuracy_score(preds, labels_all))\n",
    "    print(\"Precision baseline pada test data:\", precision_score(preds, labels_all))\n",
    "    print(\"Recall baseline pada test data:\", recall_score(preds, labels_all))\n",
    "    print(\"F1 baseline pada test data:\", f1_score(preds, labels_all))\n",
    "    print()\n",
    "    avg_f1 += f1_score(preds, labels_all)\n",
    "    \n",
    "    \n",
    "print(avg_f1/5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TmeYjiPQevx9"
   },
   "source": [
    "# Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Oro3tPO8ew7B"
   },
   "outputs": [],
   "source": [
    "data_dev_A_df = pd.read_csv(\"drive/My Drive/Ukara/data_dev_A.csv\")\n",
    "data_dev_B_df = pd.read_csv(\"drive/My Drive/Ukara/data_dev_B.csv\")\n",
    "data_dev_A = data_dev_A_df.values\n",
    "data_dev_B = data_dev_B_df.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LK_Le69DBehz"
   },
   "outputs": [],
   "source": [
    "data_tes_A_df = pd.read_csv(\"drive/My Drive/Ukara/data_test_A.csv\")\n",
    "data_tes_B_df = pd.read_csv(\"drive/My Drive/Ukara/data_test_B.csv\")\n",
    "data_tes_A = data_tes_A_df.values\n",
    "data_tes_B = data_tes_B_df.values\n",
    "\n",
    "# ppmi_test_a = pd.read_csv(path_ukara + 'data_test_A_ppmi.csv')['PPMI'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 92
    },
    "colab_type": "code",
    "id": "nkELjk6P8Iq5",
    "outputId": "78354be0-4ae6-4286-d445-d8b585b99c9b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['TSA1',\n",
       "       'beradaptasi dengan lingkunagn baru, lahan dan harta mereka yang dulu akan hilang, tertinggalnya teknologi karena bencana lingkunagn tersebut.'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_tes_A[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3lLRLPUMfA93"
   },
   "outputs": [],
   "source": [
    "# DEV\n",
    "# xfd_A = vectorize(data_dev_A[:,1], pre=True)\n",
    "# xfd_B = vectorize2(data_dev_B[:,1], pre=True)\n",
    "\n",
    "# xfd_A = np.c_[xfd_A, find_len(data_dev_A[:,1], tipe='A'), ppmi_test_a]\n",
    "# xfd_B = np.c_[xfd_B, find_len(data_dev_B[:,1], tipe='B')]\n",
    "\n",
    "# predict_A = clf.predict(xfd_A)\n",
    "# predict_B = clf2.predict(xfd_B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "d2e_BxiOiyIG"
   },
   "outputs": [],
   "source": [
    "xft_A = vectorize(data_tes_A[:,1], pre=True)\n",
    "xft_B = vectorize2(data_tes_B[:,1], pre=True)\n",
    "\n",
    "# STACKING DATA\n",
    "xft_A = np.c_[xft_A, find_len(data_tes_A[:,1], tipe='A')]\n",
    "xft_B = np.c_[xft_B, find_len(data_tes_B[:,1], tipe='B')]\n",
    "\n",
    "A1 = np.log(clf_a_1.predict_proba(xft_A))\n",
    "A2 = np.log(clf_a_2.predict_proba(xft_A))\n",
    "B1 = np.log(clf_b_2.predict_proba(xft_B))\n",
    "B2 = np.log(clf_b_3.predict_proba(xft_B))\n",
    "# r1 = 0.05\n",
    "# r2 = 0.35\n",
    "# r3 = 0.25\n",
    "# r4 = 0.35\n",
    "# tot_B = B1*r1 + B2*r2 + B3*r3 + B4*r4\n",
    "\n",
    "out_A = np.c_[A1, A2]\n",
    "out_B = np.c_[B1, B2]\n",
    "\n",
    "predict_tes_A = meta_clf.predict(out_A)\n",
    "predict_tes_B = meta_clf2.predict(out_B)\n",
    "# predict_tes_B = np.argmax(tot_B, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9D2OC3QWi_fL"
   },
   "outputs": [],
   "source": [
    "# # PREDICT NON_STACKING\n",
    "xft_A = vectorize(data_tes_A[:,1].copy(), pre=True)\n",
    "xft_B = vectorize2(data_tes_B[:,1].copy(), pre=True)\n",
    "xft_A = np.c_[xft_A, find_len(data_tes_A[:,1], tipe='A'), ppmi_test_a]\n",
    "xft_B = np.c_[xft_B, find_len(data_tes_B[:,1], tipe='B')]\n",
    "\n",
    "predict_tes_A = clf.predict(xft_A)\n",
    "predict_tes_B = clf2.predict(xft_B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 164
    },
    "colab_type": "code",
    "id": "8WGdYk3VOxJa",
    "outputId": "06b3bb1a-ff96-4f4f-9f18-18bdb4dfcc4c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TSA369 0\n",
      "TSA373 0\n",
      "TSA390 0\n",
      "TSA465 0\n",
      "TSA536 1\n",
      "TSA715 0\n",
      "TSB592 0\n",
      "TSB751 0\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(data_tes_A)):\n",
    "    if len(str(data_tes_A[i,1]))<=5:\n",
    "        print(data_tes_A[i,0], predict_tes_A[i])\n",
    "        predict_tes_A[i]=0\n",
    "\n",
    "for i in range(len(data_tes_B)):\n",
    "    if len(str(data_tes_B[i,1]))<=5:\n",
    "        print(data_tes_B[i,0], predict_tes_B[i])\n",
    "        predict_tes_B[i]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9aONw_r2fJuy"
   },
   "outputs": [],
   "source": [
    "# from pandas import DataFrame\n",
    "\n",
    "# prediksi_data_A = {'RES_ID': data_dev_A_df['RES_ID'],\n",
    "#                 'LABEL': predict_A\n",
    "#                 }\n",
    "\n",
    "# prediksi_data_B = {'RES_ID': data_dev_B_df['RES_ID'],\n",
    "#                 'LABEL': predict_B\n",
    "#                 }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-Y5fRteSgMMJ"
   },
   "outputs": [],
   "source": [
    "# df_data_A = DataFrame(prediksi_data_A, columns= ['RES_ID', 'LABEL'])\n",
    "# df_data_B = DataFrame(prediksi_data_B, columns= ['RES_ID', 'LABEL'])\n",
    "# frame = [df_data_A, df_data_B]\n",
    "# answer = pd.concat(frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "71XOS2W-gdR0"
   },
   "outputs": [],
   "source": [
    "# answer.to_json('ANS_BEST_DEV.json', orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kNtwR3nbENdt"
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "\n",
    "prediksi_data_A = {'RES_ID': data_tes_A_df['RES_ID'],\n",
    "                'LABEL': predict_tes_A\n",
    "                }\n",
    "\n",
    "prediksi_data_B = {'RES_ID': data_tes_B_df['RES_ID'],\n",
    "                'LABEL': predict_tes_B\n",
    "                }\n",
    "\n",
    "df_data_A = DataFrame(prediksi_data_A, columns= ['RES_ID', 'LABEL'])\n",
    "df_data_B = DataFrame(prediksi_data_B, columns= ['RES_ID', 'LABEL'])\n",
    "frame = [df_data_A, df_data_B]\n",
    "answer = pd.concat(frame)\n",
    "\n",
    "answer.to_json('ANS_STACKING_ANEWPRE_BFINAL.json', orient='records')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PSW33Ebmyn90"
   },
   "source": [
    "# TRIAL SECTION"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "GT.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}