FE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ep-PxDVknP0V"
   },
   "source": [
    "# TF-IDF Corpus Tweet TOP 1 Fitur Full"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "unbeNuLcnP0V"
   },
   "source": [
    "## Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hDngzYc_qgFB",
    "outputId": "64650f0f-9f3e-4bdd-9561-4934676fbaee"
   },
   "outputs": [],
   "source": [
    "!pip install git+https://github.com/laxmimerit/preprocess_kgptalkie.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "k_EF8oSZnP0W",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import array\n",
    "import pandas as pd\n",
    "\n",
    "from IPython.display import Audio\n",
    "\n",
    "#import preprocess_kgptalkie as ps\n",
    "\n",
    "from ast import literal_eval\n",
    "\n",
    "import string \n",
    "import re\n",
    "from nltk.tokenize import word_tokenize \n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "# import Sastrawi package\n",
    "#from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
    "#import swifter\n",
    "\n",
    "from IPython.display import Audio\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "vmwTEaACnP0Y"
   },
   "outputs": [],
   "source": [
    "def Average(lst):\n",
    "    return sum(lst) / len(lst)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QYWJoF1lnP0Y"
   },
   "source": [
    "# Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "id": "-vRRyr2nnP0Z",
    "outputId": "ae6847c0-b279-485e-dff1-d20ff38e3bcc",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>LabelFinal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kejat jatim tunjuk jpu tangan perkara tragedi ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dikysoemarno cuekin aja tum kalo yg teriak car...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>belum nyata sikap yg dri tuntut mula smgt lipa...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>listyosigitp berapa gas air mata yang kalian t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bodo susup usut km harun mas kanjuruhan fs fak...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25320</th>\n",
       "      <td>putri candrawathi tak tahan juga ferdy masih k...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25321</th>\n",
       "      <td>kapolri jenderal listyo sigit prabowo alas put...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25322</th>\n",
       "      <td>dalih manusia amp rekomendasi komnas perempuan...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25323</th>\n",
       "      <td>lalu tahan mako brimob putri candrawathi belum...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25324</th>\n",
       "      <td>kapolri jenderal listyo sigit prabowo angkat b...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25325 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Text  LabelFinal\n",
       "0      kejat jatim tunjuk jpu tangan perkara tragedi ...           0\n",
       "1      dikysoemarno cuekin aja tum kalo yg teriak car...           0\n",
       "2      belum nyata sikap yg dri tuntut mula smgt lipa...           0\n",
       "3      listyosigitp berapa gas air mata yang kalian t...           0\n",
       "4      bodo susup usut km harun mas kanjuruhan fs fak...           0\n",
       "...                                                  ...         ...\n",
       "25320  putri candrawathi tak tahan juga ferdy masih k...           1\n",
       "25321  kapolri jenderal listyo sigit prabowo alas put...           1\n",
       "25322  dalih manusia amp rekomendasi komnas perempuan...           1\n",
       "25323  lalu tahan mako brimob putri candrawathi belum...           1\n",
       "25324  kapolri jenderal listyo sigit prabowo angkat b...           1\n",
       "\n",
       "[25325 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweetberita = pd.read_excel(\"dataset_FINAL_afterPreprocessing.xlsx\", usecols=['Text', 'LabelFinal'])\n",
    "df_tweetberita"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 458
    },
    "id": "ftHBXiNTrdYb",
    "outputId": "7df263ba-9cdf-4515-8e1b-efa7492c3371"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Error loading punkt: <urlopen error [Errno 11001]\n",
      "[nltk_data]     getaddrinfo failed>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>LabelFinal</th>\n",
       "      <th>TokenizedText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kejat jatim tunjuk jpu tangan perkara tragedi ...</td>\n",
       "      <td>0</td>\n",
       "      <td>[kejat, jatim, tunjuk, jpu, tangan, perkara, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dikysoemarno cuekin aja tum kalo yg teriak car...</td>\n",
       "      <td>0</td>\n",
       "      <td>[dikysoemarno, cuekin, aja, tum, kalo, yg, ter...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>belum nyata sikap yg dri tuntut mula smgt lipa...</td>\n",
       "      <td>0</td>\n",
       "      <td>[belum, nyata, sikap, yg, dri, tuntut, mula, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>listyosigitp berapa gas air mata yang kalian t...</td>\n",
       "      <td>0</td>\n",
       "      <td>[listyosigitp, berapa, gas, air, mata, yang, k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bodo susup usut km harun mas kanjuruhan fs fak...</td>\n",
       "      <td>0</td>\n",
       "      <td>[bodo, susup, usut, km, harun, mas, kanjuruhan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25320</th>\n",
       "      <td>putri candrawathi tak tahan juga ferdy masih k...</td>\n",
       "      <td>1</td>\n",
       "      <td>[putri, candrawathi, tak, tahan, juga, ferdy, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25321</th>\n",
       "      <td>kapolri jenderal listyo sigit prabowo alas put...</td>\n",
       "      <td>1</td>\n",
       "      <td>[kapolri, jenderal, listyo, sigit, prabowo, al...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25322</th>\n",
       "      <td>dalih manusia amp rekomendasi komnas perempuan...</td>\n",
       "      <td>1</td>\n",
       "      <td>[dalih, manusia, amp, rekomendasi, komnas, per...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25323</th>\n",
       "      <td>lalu tahan mako brimob putri candrawathi belum...</td>\n",
       "      <td>1</td>\n",
       "      <td>[lalu, tahan, mako, brimob, putri, candrawathi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25324</th>\n",
       "      <td>kapolri jenderal listyo sigit prabowo angkat b...</td>\n",
       "      <td>1</td>\n",
       "      <td>[kapolri, jenderal, listyo, sigit, prabowo, an...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25325 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Text  LabelFinal  \\\n",
       "0      kejat jatim tunjuk jpu tangan perkara tragedi ...           0   \n",
       "1      dikysoemarno cuekin aja tum kalo yg teriak car...           0   \n",
       "2      belum nyata sikap yg dri tuntut mula smgt lipa...           0   \n",
       "3      listyosigitp berapa gas air mata yang kalian t...           0   \n",
       "4      bodo susup usut km harun mas kanjuruhan fs fak...           0   \n",
       "...                                                  ...         ...   \n",
       "25320  putri candrawathi tak tahan juga ferdy masih k...           1   \n",
       "25321  kapolri jenderal listyo sigit prabowo alas put...           1   \n",
       "25322  dalih manusia amp rekomendasi komnas perempuan...           1   \n",
       "25323  lalu tahan mako brimob putri candrawathi belum...           1   \n",
       "25324  kapolri jenderal listyo sigit prabowo angkat b...           1   \n",
       "\n",
       "                                           TokenizedText  \n",
       "0      [kejat, jatim, tunjuk, jpu, tangan, perkara, t...  \n",
       "1      [dikysoemarno, cuekin, aja, tum, kalo, yg, ter...  \n",
       "2      [belum, nyata, sikap, yg, dri, tuntut, mula, s...  \n",
       "3      [listyosigitp, berapa, gas, air, mata, yang, k...  \n",
       "4      [bodo, susup, usut, km, harun, mas, kanjuruhan...  \n",
       "...                                                  ...  \n",
       "25320  [putri, candrawathi, tak, tahan, juga, ferdy, ...  \n",
       "25321  [kapolri, jenderal, listyo, sigit, prabowo, al...  \n",
       "25322  [dalih, manusia, amp, rekomendasi, komnas, per...  \n",
       "25323  [lalu, tahan, mako, brimob, putri, candrawathi...  \n",
       "25324  [kapolri, jenderal, listyo, sigit, prabowo, an...  \n",
       "\n",
       "[25325 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "\n",
    "df_tweetberita['TokenizedText'] = df_tweetberita['Text']\n",
    "df_tweetberita[\"TokenizedText\"] = df_tweetberita.apply(lambda row: nltk.word_tokenize(row['TokenizedText']), axis=1)\n",
    "data = list(df_tweetberita['TokenizedText'])\n",
    "df_tweetberita['TokenizedText'] = data\n",
    "df_tweetberita"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MmFVZNZynP0a",
    "tags": []
   },
   "source": [
    "#### TF-IDF Vectorizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "y7ZIQISnnP0a",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "C:\\Users\\HP\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 44s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aamiin</th>\n",
       "      <th>abai</th>\n",
       "      <th>abang</th>\n",
       "      <th>abdi</th>\n",
       "      <th>abdul</th>\n",
       "      <th>abis</th>\n",
       "      <th>abu</th>\n",
       "      <th>abu waras</th>\n",
       "      <th>abur</th>\n",
       "      <th>abuse</th>\n",
       "      <th>...</th>\n",
       "      <th>zhico</th>\n",
       "      <th>zina</th>\n",
       "      <th>zoelfick</th>\n",
       "      <th>zon</th>\n",
       "      <th>zrn</th>\n",
       "      <th>zulfiarrahman</th>\n",
       "      <th>zulkiflilubis</th>\n",
       "      <th>zumhana</th>\n",
       "      <th>zusen</th>\n",
       "      <th>zusen tahan</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25320</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25321</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25322</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25323</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25324</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25325 rows × 10000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       aamiin  abai  abang  abdi  abdul  abis  abu  abu waras  abur  abuse  \\\n",
       "0         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "1         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "2         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "3         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "4         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "...       ...   ...    ...   ...    ...   ...  ...        ...   ...    ...   \n",
       "25320     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25321     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25322     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25323     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25324     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "\n",
       "       ...  zhico  zina  zoelfick  zon  zrn  zulfiarrahman  zulkiflilubis  \\\n",
       "0      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "1      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "2      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "3      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "4      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "...    ...    ...   ...       ...  ...  ...            ...            ...   \n",
       "25320  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25321  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25322  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25323  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25324  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "\n",
       "       zumhana  zusen  zusen tahan  \n",
       "0          0.0    0.0          0.0  \n",
       "1          0.0    0.0          0.0  \n",
       "2          0.0    0.0          0.0  \n",
       "3          0.0    0.0          0.0  \n",
       "4          0.0    0.0          0.0  \n",
       "...        ...    ...          ...  \n",
       "25320      0.0    0.0          0.0  \n",
       "25321      0.0    0.0          0.0  \n",
       "25322      0.0    0.0          0.0  \n",
       "25323      0.0    0.0          0.0  \n",
       "25324      0.0    0.0          0.0  \n",
       "\n",
       "[25325 rows x 10000 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "\n",
    "#define tf-idf\n",
    "tf_idf_vec = TfidfVectorizer(ngram_range=(1,2), max_features=10000)\n",
    "\n",
    "#transform\n",
    "tf_idf_data = tf_idf_vec.fit_transform(df_tweetberita['Text'])\n",
    "\n",
    "#create dataframe\n",
    "tf_idf_df  = pd.DataFrame(tf_idf_data.toarray(),columns=tf_idf_vec.get_feature_names())\n",
    "features_v = tf_idf_vec.get_feature_names()\n",
    "vector     = tf_idf_df.values.tolist()\n",
    "df_tweetberita['TFIDF_vec'] = vector\n",
    "tf_idf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "YXqiyLPtnP0b",
    "outputId": "dee1b179-4727-47aa-8295-ed576420948b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aamiin</th>\n",
       "      <th>abai</th>\n",
       "      <th>abang</th>\n",
       "      <th>abdi</th>\n",
       "      <th>abdul</th>\n",
       "      <th>abis</th>\n",
       "      <th>abu</th>\n",
       "      <th>abu waras</th>\n",
       "      <th>abur</th>\n",
       "      <th>abuse</th>\n",
       "      <th>...</th>\n",
       "      <th>zhico</th>\n",
       "      <th>zina</th>\n",
       "      <th>zoelfick</th>\n",
       "      <th>zon</th>\n",
       "      <th>zrn</th>\n",
       "      <th>zulfiarrahman</th>\n",
       "      <th>zulkiflilubis</th>\n",
       "      <th>zumhana</th>\n",
       "      <th>zusen</th>\n",
       "      <th>zusen tahan</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25320</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25321</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25322</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25323</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25324</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25325 rows × 10000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       aamiin  abai  abang  abdi  abdul  abis  abu  abu waras  abur  abuse  \\\n",
       "0         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "1         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "2         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "3         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "4         0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "...       ...   ...    ...   ...    ...   ...  ...        ...   ...    ...   \n",
       "25320     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25321     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25322     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25323     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "25324     0.0   0.0    0.0   0.0    0.0   0.0  0.0        0.0   0.0    0.0   \n",
       "\n",
       "       ...  zhico  zina  zoelfick  zon  zrn  zulfiarrahman  zulkiflilubis  \\\n",
       "0      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "1      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "2      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "3      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "4      ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "...    ...    ...   ...       ...  ...  ...            ...            ...   \n",
       "25320  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25321  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25322  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25323  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "25324  ...    0.0   0.0       0.0  0.0  0.0            0.0            0.0   \n",
       "\n",
       "       zumhana  zusen  zusen tahan  \n",
       "0          0.0    0.0          0.0  \n",
       "1          0.0    0.0          0.0  \n",
       "2          0.0    0.0          0.0  \n",
       "3          0.0    0.0          0.0  \n",
       "4          0.0    0.0          0.0  \n",
       "...        ...    ...          ...  \n",
       "25320      0.0    0.0          0.0  \n",
       "25321      0.0    0.0          0.0  \n",
       "25322      0.0    0.0          0.0  \n",
       "25323      0.0    0.0          0.0  \n",
       "25324      0.0    0.0          0.0  \n",
       "\n",
       "[25325 rows x 10000 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_idf_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jeujyV2HnP0b"
   },
   "source": [
    "#### Load Corpus Tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "Cipf_ICWnP0c",
    "outputId": "cb938bcd-9f52-49c3-dbbc-dc3cec464b3d",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Words</th>\n",
       "      <th>Rank 1</th>\n",
       "      <th>Rank 2</th>\n",
       "      <th>Rank 3</th>\n",
       "      <th>Rank 4</th>\n",
       "      <th>Rank 5</th>\n",
       "      <th>Rank 6</th>\n",
       "      <th>Rank 7</th>\n",
       "      <th>Rank 8</th>\n",
       "      <th>Rank 9</th>\n",
       "      <th>Rank 10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kejat</td>\n",
       "      <td>dki</td>\n",
       "      <td>kajati</td>\n",
       "      <td>jatim</td>\n",
       "      <td>aspidsus</td>\n",
       "      <td>dahlan</td>\n",
       "      <td>jaksa</td>\n",
       "      <td>iskan</td>\n",
       "      <td>gubernur</td>\n",
       "      <td>pemprov</td>\n",
       "      <td>diy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jatim</td>\n",
       "      <td>kejat</td>\n",
       "      <td>timur</td>\n",
       "      <td>kaltim</td>\n",
       "      <td>jawa</td>\n",
       "      <td>jabar</td>\n",
       "      <td>polda</td>\n",
       "      <td>jateng</td>\n",
       "      <td>minarti</td>\n",
       "      <td>sepejaten</td>\n",
       "      <td>madura</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tunjuk</td>\n",
       "      <td>langsung</td>\n",
       "      <td>kapolri</td>\n",
       "      <td>bukti</td>\n",
       "      <td>bijak</td>\n",
       "      <td>mesti</td>\n",
       "      <td>timbang</td>\n",
       "      <td>alat</td>\n",
       "      <td>proses</td>\n",
       "      <td>presiden</td>\n",
       "      <td>apa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jpu</td>\n",
       "      <td>tuntut</td>\n",
       "      <td>dakwa</td>\n",
       "      <td>jaksa</td>\n",
       "      <td>pulung</td>\n",
       "      <td>eksepsi</td>\n",
       "      <td>haerudin</td>\n",
       "      <td>sidang</td>\n",
       "      <td>fitroh</td>\n",
       "      <td>majelis</td>\n",
       "      <td>irene</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tangan</td>\n",
       "      <td>ott</td>\n",
       "      <td>operasi</td>\n",
       "      <td>kasuskasus</td>\n",
       "      <td>perkara</td>\n",
       "      <td>lambai</td>\n",
       "      <td>serius</td>\n",
       "      <td>tanda</td>\n",
       "      <td>libat</td>\n",
       "      <td>tangkap</td>\n",
       "      <td>kait</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Words    Rank 1   Rank 2      Rank 3    Rank 4   Rank 5    Rank 6  Rank 7  \\\n",
       "0   kejat       dki   kajati       jatim  aspidsus   dahlan     jaksa   iskan   \n",
       "1   jatim     kejat    timur      kaltim      jawa    jabar     polda  jateng   \n",
       "2  tunjuk  langsung  kapolri       bukti     bijak    mesti   timbang    alat   \n",
       "3     jpu    tuntut    dakwa       jaksa    pulung  eksepsi  haerudin  sidang   \n",
       "4  tangan       ott  operasi  kasuskasus   perkara   lambai    serius   tanda   \n",
       "\n",
       "     Rank 8     Rank 9 Rank 10  \n",
       "0  gubernur    pemprov     diy  \n",
       "1   minarti  sepejaten  madura  \n",
       "2    proses   presiden     apa  \n",
       "3    fitroh    majelis   irene  \n",
       "4     libat    tangkap    kait  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_corpus = pd.read_csv(\"similaritytweetberita.csv\")\n",
    "df_corpus.drop('Unnamed: 0', axis=1, inplace=True)\n",
    "df_corpus.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "nVvEcDeBnP0c"
   },
   "outputs": [],
   "source": [
    "df_corpus['combined'] = df_corpus.apply(lambda x: list([x['Rank 1']]),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "aY5RfngBnP0c",
    "outputId": "9b00a17b-0392-46d9-dd85-0cb8953d2103"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['dki']\n",
      "['dki']\n"
     ]
    }
   ],
   "source": [
    "print(df_corpus['combined'][0])\n",
    "print(df_corpus['combined'][0][0:10])\n",
    "d = dict([([Words,combined]) for Words, combined in zip(df_corpus.Words, df_corpus.combined)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pePrIv4pnP0d"
   },
   "source": [
    "## Feature Expansion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vt3J3FZwnP0d",
    "tags": []
   },
   "source": [
    "<img src=\"../images/Feature Expansion.PNG\" width=\"40%\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "HdsN2LUJnP0d",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def checkWords(i, j):\n",
    "    try:\n",
    "        # pengecekan if one word in W appears in tweet T\n",
    "        a = [t for t in d[features_v[j]] if t in [i]]\n",
    "        return a\n",
    "    except:\n",
    "        return []\n",
    "    \n",
    "def weightCheck(w, i, j):\n",
    "    try:\n",
    "        if w != []:\n",
    "            return vector[i][features_v.index(w[0])]\n",
    "        else:\n",
    "            return vector[i][j]\n",
    "    except:\n",
    "        return vector[i][j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "v2J8yHBFnP0e"
   },
   "outputs": [],
   "source": [
    "pd.set_option(\"display.max_columns\", None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rb7VMORnnP0e",
    "outputId": "e182607b-bd15-44da-fc21-3e5d2af11f8e",
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "============ FEATURE EXPANSION IS PROCESSING ============\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "fe_vec = []\n",
    "print()\n",
    "print('============ FEATURE EXPANSION IS PROCESSING ============')\n",
    "for i in range(len(vector)):\n",
    "    v = [weightCheck(checkWords(i,j), i, j) if (vector[i][j] == 0.0) else vector[i][j] for j in range(len(vector[i]))]\n",
    "    fe_vec.append(v)\n",
    "df_tweetberita['TFIDF_vec_fe'] = fe_vec\n",
    "fe_TFIDF_df = pd.DataFrame(fe_vec, columns=features_v)\n",
    "fe_TFIDF_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fe_TFIDF_df.to_excel(\"FETFIDF_top1-tweetberita.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "2RoE2Be9nP0e",
    "outputId": "b99cab3b-942a-4727-e539-0c10696ff2c1",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BERHASIL!\n",
      "len\n",
      "10000 10000\n",
      "same? False\n"
     ]
    }
   ],
   "source": [
    "lis1 = list(df_tweetberita['TFIDF_vec'])\n",
    "lis2 = list(df_tweetberita['TFIDF_vec_fe'])\n",
    "if lis1 != lis2:\n",
    "    print('BERHASIL!')\n",
    "    \n",
    "print('len')\n",
    "print(len(fe_vec[0]), len(vector[0]))\n",
    "print('same?', (fe_vec[0] == vector[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "tBom1ZdKnP0f",
    "outputId": "bd4d7bbc-f0d3-444b-8b68-2f31360094c8",
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======= CHECK KEBENARAN =======\n",
      "tweet:\n",
      "   ['kejat', 'jatim', 'tunjuk', 'jpu', 'tangan', 'perkara', 'tragedi', 'kanjuruhan']\n",
      "\n",
      "expansion vector @idx 1810 covid\n",
      "  corpus idx - 334\n",
      "0.0 0.17765272338113253\n",
      "expansion vector @idx 1857 dakwa\n",
      "  corpus idx - 6299\n",
      "0.0 0.2982215547938298\n",
      "expansion vector @idx 4494 langsung\n",
      "  corpus idx - 115\n",
      "0.0 0.3389707534945348\n",
      "expansion vector @idx 4871 malang\n",
      "  corpus idx - 161\n",
      "0.0 0.17765272338113253\n",
      "expansion vector @idx 6498 pokok\n",
      "  corpus idx - 728\n",
      "0.0 0.24839702295156185\n",
      "expansion vector @idx 9394 tuntut\n",
      "  corpus idx - 36\n",
      "0.0 0.2982215547938298\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print('======= CHECK KEBENARAN =======')\n",
    "print(\"tweet:\")\n",
    "print(\"  \", data[0])\n",
    "print()\n",
    "for i in range(len(fe_vec[0])):\n",
    "    if vector[0][i] != fe_vec[0][i]:\n",
    "        print('expansion vector @idx', i, features_v[i])\n",
    "        print('  corpus idx -', df_corpus[df_corpus[\"Words\"]==features_v[i]].index.values[0])\n",
    "        print(vector[0][i], fe_vec[0][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "6PyjfiISnP0f",
    "outputId": "fd6b8c5b-bd41-4e5b-ba85-5a2176e53560"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Words</th>\n",
       "      <th>Rank 1</th>\n",
       "      <th>Rank 2</th>\n",
       "      <th>Rank 3</th>\n",
       "      <th>Rank 4</th>\n",
       "      <th>Rank 5</th>\n",
       "      <th>Rank 6</th>\n",
       "      <th>Rank 7</th>\n",
       "      <th>Rank 8</th>\n",
       "      <th>Rank 9</th>\n",
       "      <th>Rank 10</th>\n",
       "      <th>combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>marah</td>\n",
       "      <td>picu</td>\n",
       "      <td>kesal</td>\n",
       "      <td>takut</td>\n",
       "      <td>jasmerah</td>\n",
       "      <td>heran</td>\n",
       "      <td>khawatir</td>\n",
       "      <td>bingung</td>\n",
       "      <td>jelek</td>\n",
       "      <td>gelisah</td>\n",
       "      <td>emosi</td>\n",
       "      <td>[picu]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Words Rank 1 Rank 2 Rank 3    Rank 4 Rank 5    Rank 6   Rank 7 Rank 8  \\\n",
       "682  marah   picu  kesal  takut  jasmerah  heran  khawatir  bingung  jelek   \n",
       "\n",
       "      Rank 9 Rank 10 combined  \n",
       "682  gelisah   emosi   [picu]  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_corpus.iloc[[682]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}