docs/1. dataset.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "TLUbCDzYEII0"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import torch\n",
        "import re\n",
        "import nltk\n",
        "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
        "import torch\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nq0GoLGzEII3",
        "outputId": "6f8b9bb0-76d8-4c39-91f7-96097747b4a8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "device"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AoJsRDJ4EII8"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O11kYRVzEIJA"
      },
      "source": [
        "**data cleansing** "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "dGWCFZFFEIJD"
      },
      "outputs": [],
      "source": [
        "def remove_numeric(text):\n",
        "    return re.sub(r'\\d+', '', text)\n",
        "\n",
        "\n",
        "# define the regular expression pattern to match the label and text\n",
        "pattern = r\"^(\\d)\\t(.+)$\"\n",
        "\n",
        "# create an empty list to store the data\n",
        "data = []\n",
        "\n",
        "# open the text file and iterate through each line\n",
        "with open(\"VideoCommentsThreatCorpus.txt\", \"r\") as file:\n",
        "    for line in file:\n",
        "        # remove any leading/trailing white spaces\n",
        "        line = line.strip()\n",
        "        # check if the line matches the pattern\n",
        "        match = re.match(pattern, line)\n",
        "        if match:\n",
        "            # extract the label and text from the matched line\n",
        "            label = int(match.group(1))\n",
        "            text = match.group(2)\n",
        "            # append the label and text to the data list\n",
        "            data.append([text, label])\n",
        "\n",
        "# create a Pandas dataframe from the data list\n",
        "\n",
        "df = pd.DataFrame(data, columns=[\"text\", \"label\"])\n",
        "\n",
        "df[\"text\"] = df[\"text\"].apply(lambda x: remove_numeric(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "O5uPdRhtEIJH"
      },
      "outputs": [],
      "source": [
        "df = df.drop_duplicates()\n",
        "\n",
        "def remove_special_characters(text):\n",
        "    pattern = r'[^a-zA-Z0-9\\s]'\n",
        "    text = re.sub(pattern, '', text)\n",
        "    return text\n",
        "\n",
        "df['text'] = df['text'].apply(remove_special_characters)\n",
        "\n",
        "df['text'] = df['text'].apply(lambda x: x.lower())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D82wPPwAEIJJ"
      },
      "source": [
        "**determine max_sequence_length of your data**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 448
        },
        "id": "7nkYbNcQEIJK",
        "outputId": "9be5f5aa-ff1d-473e-db64-788f4237c30c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: >"
            ]
          },
          "metadata": {},
          "execution_count": 11
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxc0lEQVR4nO3de3SU9Z3H8U8SkoEIkwg0mWQJMQUrREAQBKZaFiUkYJZq5eypikIrwoENrpAuQipigGJYLCIqwrpe4h6hCj1iFVAyhAKi4ZYlcrOpF1ysMmFXCsN1GJJn/9iTZ51ySSbMkPmF9+ucOWae5zu/5/d8J8DH5zITY1mWJQAAAIPENvcEAAAAQkWAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYp1VzTyBS6urq9O2336pdu3aKiYlp7ukAAIBGsCxLx48fV3p6umJjL36cpcUGmG+//VYZGRnNPQ0AANAEX3/9tTp16nTR9S02wLRr107S/zXA6XSGbdxAIKCysjLl5uYqPj4+bOOC3kYSvY0cehsZ9DVyor23Pp9PGRkZ9r/jF9NiA0z9aSOn0xn2AJOYmCin0xmVb7zJ6G3k0NvIobeRQV8jx5TeNnT5BxfxAgAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABinVXNP4Gp23fQ1DdZ8NS//CswEAACzcAQGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA43EYd5bjVGgCA83EEBgAAGIcAAwAAjBNSgFmyZIl69eolp9Mpp9Mpt9ut999/314/ePBgxcTEBD0mTJgQNMbBgweVn5+vxMREpaSkaOrUqTp37lxQzcaNG3XzzTfL4XCoa9euKi0tbfoeAgCAFieka2A6deqkefPm6frrr5dlWXr99dd11113adeuXbrxxhslSePGjdPs2bPt1yQmJto/19bWKj8/Xy6XSx9//LEOHTqk0aNHKz4+Xk899ZQk6cCBA8rPz9eECRO0bNkylZeX6+GHH1ZaWpry8vLCsc8AAMBwIQWYESNGBD2fO3eulixZoq1bt9oBJjExUS6X64KvLysr0/79+7V+/Xqlpqaqd+/emjNnjqZNm6bi4mIlJCRo6dKlysrK0oIFCyRJ3bt315YtW7Rw4UICDAAAkHQZdyHV1tZq5cqVOnnypNxut7182bJleuONN+RyuTRixAg98cQT9lGYiooK9ezZU6mpqXZ9Xl6eJk6cqH379qlPnz6qqKhQTk5O0Lby8vI0efLkS87H7/fL7/fbz30+nyQpEAgoEAg0dTfPUz9WOMZ0xFmXPYYUnrlEg3D2FsHobeTQ28igr5ET7b1t7LxCDjB79uyR2+3WmTNn1LZtW61atUrZ2dmSpPvvv1+ZmZlKT0/X7t27NW3aNFVXV+vtt9+WJHm93qDwIsl+7vV6L1nj8/l0+vRptWnT5oLzKikp0axZs85bXlZWFnQaK1w8Hs9ljzG/fxgmImnt2rXhGShKhKO3uDB6Gzn0NjLoa+REa29PnTrVqLqQA8wNN9ygqqoqHTt2TL///e81ZswYbdq0SdnZ2Ro/frxd17NnT6WlpWnIkCH64osv1KVLl1A3FZKioiIVFhbaz30+nzIyMpSbmyun0xm27QQCAXk8Hg0dOlTx8fGXNVaP4nVhmdPe4pZxai2cvUUwehs59DYy6GvkRHtv68+gNCTkAJOQkKCuXbtKkvr27asdO3Zo0aJF+rd/+7fzagcMGCBJ+vzzz9WlSxe5XC5t3749qKampkaS7OtmXC6Xvez7NU6n86JHXyTJ4XDI4XCctzw+Pj4ib1A4xvXXxoRtLi1JpN4z0NtIoreRQV8jJ1p729g5XfbnwNTV1QVde/J9VVVVkqS0tDRJktvt1p49e3T48GG7xuPxyOl02qeh3G63ysvLg8bxeDxB19kAAICrW0hHYIqKijR8+HB17txZx48f1/Lly7Vx40atW7dOX3zxhZYvX64777xTHTp00O7duzVlyhQNGjRIvXr1kiTl5uYqOztbDz74oObPny+v16sZM2aooKDAPnoyYcIEvfDCC3rsscf00EMPacOGDVqxYoXWrGn4I/UBAMDVIaQAc/jwYY0ePVqHDh1SUlKSevXqpXXr1mno0KH6+uuvtX79ej377LM6efKkMjIyNHLkSM2YMcN+fVxcnFavXq2JEyfK7Xbrmmuu0ZgxY4I+NyYrK0tr1qzRlClTtGjRInXq1Ekvv/wyt1ADAABbSAHmlVdeuei6jIwMbdq0qcExMjMzG7xrZvDgwdq1a1coUwMAAFcRvgsJAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGKdVc0+gpbpu+prmngIAAC0WR2AAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOK2aewK4fNdNX9NgzVfz8q/ATAAAuDJCOgKzZMkS9erVS06nU06nU263W++//769/syZMyooKFCHDh3Utm1bjRw5UjU1NUFjHDx4UPn5+UpMTFRKSoqmTp2qc+fOBdVs3LhRN998sxwOh7p27arS0tKm7yEAAGhxQgownTp10rx581RZWamdO3fqjjvu0F133aV9+/ZJkqZMmaL33ntPK1eu1KZNm/Ttt9/qnnvusV9fW1ur/Px8nT17Vh9//LFef/11lZaWaubMmXbNgQMHlJ+fr9tvv11VVVWaPHmyHn74Ya1bty5MuwwAAEwX0imkESNGBD2fO3eulixZoq1bt6pTp0565ZVXtHz5ct1xxx2SpNdee03du3fX1q1bNXDgQJWVlWn//v1av369UlNT1bt3b82ZM0fTpk1TcXGxEhIStHTpUmVlZWnBggWSpO7du2vLli1auHCh8vLywrTbAADAZE2+Bqa2tlYrV67UyZMn5Xa7VVlZqUAgoJycHLumW7du6ty5syoqKjRw4EBVVFSoZ8+eSk1NtWvy8vI0ceJE7du3T3369FFFRUXQGPU1kydPvuR8/H6//H6//dzn80mSAoGAAoFAU3fzPPVjNTSmI84K2zbDIZw9iJTG9haho7eRQ28jg75GTrT3trHzCjnA7NmzR263W2fOnFHbtm21atUqZWdnq6qqSgkJCUpOTg6qT01NldfrlSR5vd6g8FK/vn7dpWp8Pp9Onz6tNm3aXHBeJSUlmjVr1nnLy8rKlJiYGOpuNsjj8Vxy/fz+Yd/kZVm7dm1zT6HRGuotmo7eRg69jQz6GjnR2ttTp041qi7kAHPDDTeoqqpKx44d0+9//3uNGTNGmzZtCnmC4VZUVKTCwkL7uc/nU0ZGhnJzc+V0OsO2nUAgII/Ho6FDhyo+Pv6idT2Ko+uanb3F0X/6rbG9RejobeTQ28igr5ET7b2tP4PSkJADTEJCgrp27SpJ6tu3r3bs2KFFixbp5z//uc6ePaujR48GHYWpqamRy+WSJLlcLm3fvj1ovPq7lL5f87d3LtXU1MjpdF706IskORwOORyO85bHx8dH5A1qaFx/bUzYt3k5ovGX9GIi9Z6B3kYSvY0M+ho50drbxs7psj/Irq6uTn6/X3379lV8fLzKy8vtddXV1Tp48KDcbrckye12a8+ePTp8+LBd4/F45HQ6lZ2dbdd8f4z6mvoxAAAAQjoCU1RUpOHDh6tz5846fvy4li9fro0bN2rdunVKSkrS2LFjVVhYqPbt28vpdOqRRx6R2+3WwIEDJUm5ubnKzs7Wgw8+qPnz58vr9WrGjBkqKCiwj55MmDBBL7zwgh577DE99NBD2rBhg1asWKE1axr+sDYAAHB1CCnAHD58WKNHj9ahQ4eUlJSkXr16ad26dRo6dKgkaeHChYqNjdXIkSPl9/uVl5enF1980X59XFycVq9erYkTJ8rtduuaa67RmDFjNHv2bLsmKytLa9as0ZQpU7Ro0SJ16tRJL7/8MrdQAwAAW0gB5pVXXrnk+tatW2vx4sVavHjxRWsyMzMbvCNm8ODB2rVrVyhTAwAAVxG+zBEAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOO0au4J4Mq4bvqaRtV9NS8/wjMBAODycQQGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADBOSAGmpKREt9xyi9q1a6eUlBTdfffdqq6uDqoZPHiwYmJigh4TJkwIqjl48KDy8/OVmJiolJQUTZ06VefOnQuq2bhxo26++WY5HA517dpVpaWlTdtDAADQ4oQUYDZt2qSCggJt3bpVHo9HgUBAubm5OnnyZFDduHHjdOjQIfsxf/58e11tba3y8/N19uxZffzxx3r99ddVWlqqmTNn2jUHDhxQfn6+br/9dlVVVWny5Ml6+OGHtW7dusvcXQAA0BKE9FUCH3zwQdDz0tJSpaSkqLKyUoMGDbKXJyYmyuVyXXCMsrIy7d+/X+vXr1dqaqp69+6tOXPmaNq0aSouLlZCQoKWLl2qrKwsLViwQJLUvXt3bdmyRQsXLlReXl6o+wgAAFqYy7oG5tixY5Kk9u3bBy1ftmyZOnbsqB49eqioqEinTp2y11VUVKhnz55KTU21l+Xl5cnn82nfvn12TU5OTtCYeXl5qqiouJzpAgCAFqLJX+ZYV1enyZMn69Zbb1WPHj3s5ffff78yMzOVnp6u3bt3a9q0aaqurtbbb78tSfJ6vUHhRZL93Ov1XrLG5/Pp9OnTatOmzXnz8fv98vv99nOfzydJCgQCCgQCTd3N89SP1dCYjjgrbNu8ksLZq6Zuuznn0FLR28iht5FBXyMn2nvb2Hk1OcAUFBRo79692rJlS9Dy8ePH2z/37NlTaWlpGjJkiL744gt16dKlqZtrUElJiWbNmnXe8rKyMiUmJoZ9ex6P55Lr5/cP+yaviLVr1zb3FBrsLZqO3kYOvY0M+ho50drb75+1uZQmBZhJkyZp9erV2rx5szp16nTJ2gEDBkiSPv/8c3Xp0kUul0vbt28PqqmpqZEk+7oZl8tlL/t+jdPpvODRF0kqKipSYWGh/dzn8ykjI0O5ublyOp2h7eAlBAIBeTweDR06VPHx8Ret61Fs5gXHe4ub7xqjxvYWoaO3kUNvI4O+Rk6097b+DEpDQgowlmXpkUce0apVq7Rx40ZlZWU1+JqqqipJUlpamiTJ7XZr7ty5Onz4sFJSUiT9Xwp0Op3Kzs62a/72SIDH45Hb7b7odhwOhxwOx3nL4+PjI/IGNTSuvzYm7Nu8EqLhlzlS7xnobSTR28igr5ETrb1t7JxCuoi3oKBAb7zxhpYvX6527drJ6/XK6/Xq9OnTkqQvvvhCc+bMUWVlpb766iu9++67Gj16tAYNGqRevXpJknJzc5Wdna0HH3xQn3zyidatW6cZM2aooKDADiATJkzQl19+qccee0x/+tOf9OKLL2rFihWaMmVKKNMFAAAtVEgBZsmSJTp27JgGDx6stLQ0+/HWW29JkhISErR+/Xrl5uaqW7du+tWvfqWRI0fqvffes8eIi4vT6tWrFRcXJ7fbrQceeECjR4/W7Nmz7ZqsrCytWbNGHo9HN910kxYsWKCXX36ZW6gBAICkJpxCupSMjAxt2rSpwXEyMzMbvFh08ODB2rVrVyjTAwAAVwm+CwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOK2aewKILtdNX9NgzVfz8q/ATAAAuDiOwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIwTUoApKSnRLbfconbt2iklJUV33323qqurg2rOnDmjgoICdejQQW3bttXIkSNVU1MTVHPw4EHl5+crMTFRKSkpmjp1qs6dOxdUs3HjRt18881yOBzq2rWrSktLm7aHAACgxQkpwGzatEkFBQXaunWrPB6PAoGAcnNzdfLkSbtmypQpeu+997Ry5Upt2rRJ3377re655x57fW1trfLz83X27Fl9/PHHev3111VaWqqZM2faNQcOHFB+fr5uv/12VVVVafLkyXr44Ye1bt26MOwyAAAwXatQij/44IOg56WlpUpJSVFlZaUGDRqkY8eO6ZVXXtHy5ct1xx13SJJee+01de/eXVu3btXAgQNVVlam/fv3a/369UpNTVXv3r01Z84cTZs2TcXFxUpISNDSpUuVlZWlBQsWSJK6d++uLVu2aOHChcrLywvTrgMAAFOFFGD+1rFjxyRJ7du3lyRVVlYqEAgoJyfHrunWrZs6d+6siooKDRw4UBUVFerZs6dSU1Ptmry8PE2cOFH79u1Tnz59VFFRETRGfc3kyZMvOhe/3y+/328/9/l8kqRAIKBAIHA5uxmkfqyGxnTEWWHbZrQJZz8vNG6kxr+a0dvIobeRQV8jJ9p729h5NTnA1NXVafLkybr11lvVo0cPSZLX61VCQoKSk5ODalNTU+X1eu2a74eX+vX16y5V4/P5dPr0abVp0+a8+ZSUlGjWrFnnLS8rK1NiYmLTdvISPB7PJdfP7x/2TUaNtWvXRnT8hnqLpqO3kUNvI4O+Rk609vbUqVONqmtygCkoKNDevXu1ZcuWpg4RVkVFRSosLLSf+3w+ZWRkKDc3V06nM2zbCQQC8ng8Gjp0qOLj4y9a16O45V6vs7c4MqfxGttbhI7eRg69jQz6GjnR3tv6MygNaVKAmTRpklavXq3NmzerU6dO9nKXy6WzZ8/q6NGjQUdhampq5HK57Jrt27cHjVd/l9L3a/72zqWamho5nc4LHn2RJIfDIYfDcd7y+Pj4iLxBDY3rr40J+zajRaR/4SP1noHeRhK9jQz6GjnR2tvGzimku5Asy9KkSZO0atUqbdiwQVlZWUHr+/btq/j4eJWXl9vLqqurdfDgQbndbkmS2+3Wnj17dPjwYbvG4/HI6XQqOzvbrvn+GPU19WMAAICrW0hHYAoKCrR8+XL94Q9/ULt27exrVpKSktSmTRslJSVp7NixKiwsVPv27eV0OvXII4/I7XZr4MCBkqTc3FxlZ2frwQcf1Pz58+X1ejVjxgwVFBTYR1AmTJigF154QY899pgeeughbdiwQStWrNCaNWvCvPsAAMBEIR2BWbJkiY4dO6bBgwcrLS3Nfrz11lt2zcKFC/UP//APGjlypAYNGiSXy6W3337bXh8XF6fVq1crLi5ObrdbDzzwgEaPHq3Zs2fbNVlZWVqzZo08Ho9uuukmLViwQC+//DK3UAMAAEkhHoGxrIZvDW7durUWL16sxYsXX7QmMzOzwTtZBg8erF27doUyPQAAcJXgu5AAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABjnsr7MEVen66Y3/Hk8X83LvwIzAQBcrTgCAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwTsgBZvPmzRoxYoTS09MVExOjd955J2j9L37xC8XExAQ9hg0bFlRz5MgRjRo1Sk6nU8nJyRo7dqxOnDgRVLN792795Cc/UevWrZWRkaH58+eHvncAAKBFCjnAnDx5UjfddJMWL1580Zphw4bp0KFD9uN3v/td0PpRo0Zp37598ng8Wr16tTZv3qzx48fb630+n3Jzc5WZmanKyko9/fTTKi4u1ksvvRTqdAEAQAvUKtQXDB8+XMOHD79kjcPhkMvluuC6Tz/9VB988IF27Nihfv36SZKef/553Xnnnfrtb3+r9PR0LVu2TGfPntWrr76qhIQE3XjjjaqqqtIzzzwTFHQAAMDVKeQA0xgbN25USkqKrr32Wt1xxx36zW9+ow4dOkiSKioqlJycbIcXScrJyVFsbKy2bdumn/3sZ6qoqNCgQYOUkJBg1+Tl5elf//Vf9de//lXXXnvtedv0+/3y+/32c5/PJ0kKBAIKBAJh27f6sRoa0xFnhW2bJmpKzxvbW4SO3kYOvY0M+ho50d7bxs4r7AFm2LBhuueee5SVlaUvvvhCv/71rzV8+HBVVFQoLi5OXq9XKSkpwZNo1Urt27eX1+uVJHm9XmVlZQXVpKam2usuFGBKSko0a9as85aXlZUpMTExXLtn83g8l1w/v3/YN2mUtWvXNvm1DfUWTUdvI4feRgZ9jZxo7e2pU6caVRf2AHPvvffaP/fs2VO9evVSly5dtHHjRg0ZMiTcm7MVFRWpsLDQfu7z+ZSRkaHc3Fw5nc6wbScQCMjj8Wjo0KGKj4+/aF2P4nVh26aJ9hbnhfyaxvYWoaO3kUNvI4O+Rk6097b+DEpDInIK6ft++MMfqmPHjvr88881ZMgQuVwuHT58OKjm3LlzOnLkiH3djMvlUk1NTVBN/fOLXVvjcDjkcDjOWx4fHx+RN6ihcf21MWHfpkkup+eRes9AbyOJ3kYGfY2caO1tY+cU8c+B+ctf/qLvvvtOaWlpkiS3262jR4+qsrLSrtmwYYPq6uo0YMAAu2bz5s1B58E8Ho9uuOGGC54+AgAAV5eQA8yJEydUVVWlqqoqSdKBAwdUVVWlgwcP6sSJE5o6daq2bt2qr776SuXl5brrrrvUtWtX5eX93ymF7t27a9iwYRo3bpy2b9+ujz76SJMmTdK9996r9PR0SdL999+vhIQEjR07Vvv27dNbb72lRYsWBZ0iAgAAV6+QA8zOnTvVp08f9enTR5JUWFioPn36aObMmYqLi9Pu3bv105/+VD/60Y80duxY9e3bVx9++GHQ6Z1ly5apW7duGjJkiO68807ddtttQZ/xkpSUpLKyMh04cEB9+/bVr371K82cOZNbqAEAgKQmXAMzePBgWdbFbxFet67hi1fbt2+v5cuXX7KmV69e+vDDD0OdHgAAuArwXUgAAMA4BBgAAGAcAgwAADAOAQYAABgn4h9kh6vTddPXNFjz1bz8KzATAEBLxBEYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxmnV3BPA1eu66WuCnjviLM3vL/UoXid/bYwk6at5+c0xNQBAlOMIDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYJ+QAs3nzZo0YMULp6emKiYnRO++8E7TesizNnDlTaWlpatOmjXJycvTZZ58F1Rw5ckSjRo2S0+lUcnKyxo4dqxMnTgTV7N69Wz/5yU/UunVrZWRkaP78+aHvHQAAaJFCDjAnT57UTTfdpMWLF19w/fz58/Xcc89p6dKl2rZtm6655hrl5eXpzJkzds2oUaO0b98+eTwerV69Wps3b9b48ePt9T6fT7m5ucrMzFRlZaWefvppFRcX66WXXmrCLgIAgJYm5C9zHD58uIYPH37BdZZl6dlnn9WMGTN01113SZL+4z/+Q6mpqXrnnXd077336tNPP9UHH3ygHTt2qF+/fpKk559/Xnfeead++9vfKj09XcuWLdPZs2f16quvKiEhQTfeeKOqqqr0zDPPBAUdAABwdQrrNTAHDhyQ1+tVTk6OvSwpKUkDBgxQRUWFJKmiokLJycl2eJGknJwcxcbGatu2bXbNoEGDlJCQYNfk5eWpurpaf/3rX8M5ZQAAYKCQj8BcitfrlSSlpqYGLU9NTbXXeb1epaSkBE+iVSu1b98+qCYrK+u8MerXXXvttedt2+/3y+/32899Pp8kKRAIKBAIXM5uBakfq6ExHXFW2LZ5tXDEWkH/lRruMxqnsb+3CB29jQz6GjnR3tvGziusAaY5lZSUaNasWectLysrU2JiYti35/F4Lrl+fv+wb/KqMadfnf3z2rVrm3EmLU9Dv7doOnobGfQ1cqK1t6dOnWpUXVgDjMvlkiTV1NQoLS3NXl5TU6PevXvbNYcPHw563blz53TkyBH79S6XSzU1NUE19c/ra/5WUVGRCgsL7ec+n08ZGRnKzc2V0+m8vB37nkAgII/Ho6FDhyo+Pv6idT2K14Vtm1cLR6ylOf3q9MTOWPnrYiRJe4vzmnlWLUNjf28ROnobGfQ1cqK9t/VnUBoS1gCTlZUll8ul8vJyO7D4fD5t27ZNEydOlCS53W4dPXpUlZWV6tu3ryRpw4YNqqur04ABA+yaxx9/XIFAwG6ux+PRDTfccMHTR5LkcDjkcDjOWx4fHx+RN6ihcf21MWHf5tXCXxdj9y8a/3CZLFJ/HkBvI4W+Rk609raxcwr5It4TJ06oqqpKVVVVkv7vwt2qqiodPHhQMTExmjx5sn7zm9/o3Xff1Z49ezR69Gilp6fr7rvvliR1795dw4YN07hx47R9+3Z99NFHmjRpku69916lp6dLku6//34lJCRo7Nix2rdvn9566y0tWrQo6AgLAAC4eoV8BGbnzp26/fbb7ef1oWLMmDEqLS3VY489ppMnT2r8+PE6evSobrvtNn3wwQdq3bq1/Zply5Zp0qRJGjJkiGJjYzVy5Eg999xz9vqkpCSVlZWpoKBAffv2VceOHTVz5kxuoQYAAJKaEGAGDx4sy7r4HTYxMTGaPXu2Zs+efdGa9u3ba/ny5ZfcTq9evfThhx+GOj20MNdNX9NgzVfz8q/ATAAA0YTvQgIAAMYhwAAAAOMQYAAAgHEIMAAAwDgt5pN4gUvhYmAAaFk4AgMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4/BdSDBeY77nCADQsnAEBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHHCHmCKi4sVExMT9OjWrZu9/syZMyooKFCHDh3Utm1bjRw5UjU1NUFjHDx4UPn5+UpMTFRKSoqmTp2qc+fOhXuqAADAUK0iMeiNN96o9evX//9GWv3/ZqZMmaI1a9Zo5cqVSkpK0qRJk3TPPffoo48+kiTV1tYqPz9fLpdLH3/8sQ4dOqTRo0crPj5eTz31VCSmCwAADBORANOqVSu5XK7zlh87dkyvvPKKli9frjvuuEOS9Nprr6l79+7aunWrBg4cqLKyMu3fv1/r169XamqqevfurTlz5mjatGkqLi5WQkJCJKYMAAAMEpEA89lnnyk9PV2tW7eW2+1WSUmJOnfurMrKSgUCAeXk5Ni13bp1U+fOnVVRUaGBAweqoqJCPXv2VGpqql2Tl5eniRMnat++ferTp88Ft+n3++X3++3nPp9PkhQIBBQIBMK2b/VjNTSmI84K2zavFo5YK+i/V1o4f0+iTWN/bxE6ehsZ9DVyor23jZ1X2APMgAEDVFpaqhtuuEGHDh3SrFmz9JOf/ER79+6V1+tVQkKCkpOTg16Tmpoqr9crSfJ6vUHhpX59/bqLKSkp0axZs85bXlZWpsTExMvcq/N5PJ5Lrp/fP+ybvGrM6VfXLNtdu3Zts2z3Smro9xZNR28jg75GTrT29tSpU42qC3uAGT58uP1zr169NGDAAGVmZmrFihVq06ZNuDdnKyoqUmFhof3c5/MpIyNDubm5cjqdYdtOIBCQx+PR0KFDFR8ff9G6HsXrwrbNq4Uj1tKcfnV6Ymes/HUxV3z7e4vzrvg2r5TG/t4idPQ2Muhr5ER7b+vPoDQkIqeQvi85OVk/+tGP9Pnnn2vo0KE6e/asjh49GnQUpqamxr5mxuVyafv27UFj1N+ldKHrauo5HA45HI7zlsfHx0fkDWpoXH/tlf8HuKXw18U0S/+i8Q9yuEXqzwPobaTQ18iJ1t42dk4R/xyYEydO6IsvvlBaWpr69u2r+Ph4lZeX2+urq6t18OBBud1uSZLb7daePXt0+PBhu8bj8cjpdCo7OzvS0wUAAAYI+xGYf/mXf9GIESOUmZmpb7/9Vk8++aTi4uJ03333KSkpSWPHjlVhYaHat28vp9OpRx55RG63WwMHDpQk5ebmKjs7Ww8++KDmz58vr9erGTNmqKCg4IJHWAAAwNUn7AHmL3/5i+677z599913+sEPfqDbbrtNW7du1Q9+8ANJ0sKFCxUbG6uRI0fK7/crLy9PL774ov36uLg4rV69WhMnTpTb7dY111yjMWPGaPbs2eGeKhDkuulrGqz5al7+FZgJAKAhYQ8wb7755iXXt27dWosXL9bixYsvWpOZmXlV3BECAACahu9CAgAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwTsS/SgBoSfisGACIDhyBAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDrdRA2HGrdYAEHkcgQEAAMYhwAAAAOMQYAAAgHEIMAAAwDhcxAs0Ay70BYDLwxEYAABgHAIMAAAwDqeQgCjFaSYAuDiOwAAAAOMQYAAAgHE4hdREPYrXyV8b09zTAADgqsQRGAAAYBwCDAAAMA4BBgAAGIdrYACDcas1gKsVAQZo4a6bvkaOOEvz+1/84nNCDgDTEGAANApHewBEE66BAQAAxiHAAAAA43AKCUCjTg+FaxxOMwEIB47AAAAA43AEBoCRONoDXN0IMACuKIIHgHCI6gCzePFiPf300/J6vbrpppv0/PPPq3///s09LQARFq5rcgC0XFEbYN566y0VFhZq6dKlGjBggJ599lnl5eWpurpaKSkpzT09AAa4EkGo/kMCAVxZURtgnnnmGY0bN06//OUvJUlLly7VmjVr9Oqrr2r69OnNPDsACHaxTzmOhMacYou2U3XRNh+YLyoDzNmzZ1VZWamioiJ7WWxsrHJyclRRUXHB1/j9fvn9fvv5sWPHJElHjhxRIBAI29wCgYBOnTqlVoFY1dZdmb+srhat6iydOlVHbyOA3kZOc/S267+saLCmMX+5N2acbUVDGqwZUFIe9vk4Yi3N6FOn3o+/LX+IfW3MnBujMfsVrv6Ea86NUf/v2Hfffaf4+Pgrtt3GOn78uCTJsqxLF1pR6JtvvrEkWR9//HHQ8qlTp1r9+/e/4GuefPJJSxIPHjx48ODBowU8vv7660tmhag8AtMURUVFKiwstJ/X1dXpyJEj6tChg2Jiwvd/RT6fTxkZGfr666/ldDrDNi7obSTR28iht5FBXyMn2ntrWZaOHz+u9PT0S9ZFZYDp2LGj4uLiVFNTE7S8pqZGLpfrgq9xOBxyOBxBy5KTkyM1RTmdzqh841sCehs59DZy6G1k0NfIiebeJiUlNVgTlZ/Em5CQoL59+6q8/P/PG9bV1am8vFxut7sZZwYAAKJBVB6BkaTCwkKNGTNG/fr1U//+/fXss8/q5MmT9l1JAADg6hW1AebnP/+5/vu//1szZ86U1+tV79699cEHHyg1NbVZ5+VwOPTkk0+ed7oKl4/eRg69jRx6Gxn0NXJaSm9jLKuh+5QAAACiS1ReAwMAAHApBBgAAGAcAgwAADAOAQYAABiHABOixYsX67rrrlPr1q01YMAAbd++vbmnZJSSkhLdcsstateunVJSUnT33Xeruro6qObMmTMqKChQhw4d1LZtW40cOfK8DzVEw+bNm6eYmBhNnjzZXkZvm+6bb77RAw88oA4dOqhNmzbq2bOndu7caa+3LEszZ85UWlqa2rRpo5ycHH322WfNOOPoV1tbqyeeeEJZWVlq06aNunTpojlz5gR9Bw59bZzNmzdrxIgRSk9PV0xMjN55552g9Y3p45EjRzRq1Cg5nU4lJydr7NixOnHixBXcixBd/jcXXT3efPNNKyEhwXr11Vetffv2WePGjbOSk5Otmpqa5p6aMfLy8qzXXnvN2rt3r1VVVWXdeeedVufOna0TJ07YNRMmTLAyMjKs8vJya+fOndbAgQOtH//4x804a/Ns377duu6666xevXpZjz76qL2c3jbNkSNHrMzMTOsXv/iFtW3bNuvLL7+01q1bZ33++ed2zbx586ykpCTrnXfesT755BPrpz/9qZWVlWWdPn26GWce3ebOnWt16NDBWr16tXXgwAFr5cqVVtu2ba1FixbZNfS1cdauXWs9/vjj1ttvv21JslatWhW0vjF9HDZsmHXTTTdZW7dutT788EOra9eu1n333XeF96TxCDAh6N+/v1VQUGA/r62ttdLT062SkpJmnJXZDh8+bEmyNm3aZFmWZR09etSKj4+3Vq5cadd8+umnliSroqKiuaZplOPHj1vXX3+95fF4rL//+7+3Awy9bbpp06ZZt91220XX19XVWS6Xy3r66aftZUePHrUcDof1u9/97kpM0Uj5+fnWQw89FLTsnnvusUaNGmVZFn1tqr8NMI3p4/79+y1J1o4dO+ya999/34qJibG++eabKzb3UHAKqZHOnj2ryspK5eTk2MtiY2OVk5OjioqKZpyZ2Y4dOyZJat++vSSpsrJSgUAgqM/dunVT586d6XMjFRQUKD8/P6iHEr29HO+++6769eunf/zHf1RKSor69Omjf//3f7fXHzhwQF6vN6i3SUlJGjBgAL29hB//+McqLy/Xn//8Z0nSJ598oi1btmj48OGS6Gu4NKaPFRUVSk5OVr9+/eyanJwcxcbGatu2bVd8zo0RtZ/EG23+53/+R7W1ted9EnBqaqr+9Kc/NdOszFZXV6fJkyfr1ltvVY8ePSRJXq9XCQkJ530RZ2pqqrxebzPM0ixvvvmm/vM//1M7duw4bx29bbovv/xSS5YsUWFhoX79619rx44d+ud//mclJCRozJgxdv8u9PcDvb246dOny+fzqVu3boqLi1Ntba3mzp2rUaNGSRJ9DZPG9NHr9SolJSVofatWrdS+ffuo7TUBBs2moKBAe/fu1ZYtW5p7Ki3C119/rUcffVQej0etW7du7um0KHV1derXr5+eeuopSVKfPn20d+9eLV26VGPGjGnm2ZlrxYoVWrZsmZYvX64bb7xRVVVVmjx5stLT0+krGsQppEbq2LGj4uLizrtjo6amRi6Xq5lmZa5JkyZp9erV+uMf/6hOnTrZy10ul86ePaujR48G1dPnhlVWVurw4cO6+eab1apVK7Vq1UqbNm3Sc889p1atWik1NZXeNlFaWpqys7ODlnXv3l0HDx6UJLt//P0QmqlTp2r69Om699571bNnTz344IOaMmWKSkpKJNHXcGlMH10ulw4fPhy0/ty5czpy5EjU9poA00gJCQnq27evysvL7WV1dXUqLy+X2+1uxpmZxbIsTZo0SatWrdKGDRuUlZUVtL5v376Kj48P6nN1dbUOHjxInxswZMgQ7dmzR1VVVfajX79+GjVqlP0zvW2aW2+99bzb/f/85z8rMzNTkpSVlSWXyxXUW5/Pp23bttHbSzh16pRiY4P/GYqLi1NdXZ0k+houjemj2+3W0aNHVVlZadds2LBBdXV1GjBgwBWfc6M091XEJnnzzTcth8NhlZaWWvv377fGjx9vJScnW16vt7mnZoyJEydaSUlJ1saNG61Dhw7Zj1OnTtk1EyZMsDp37mxt2LDB2rlzp+V2uy23292MszbX9+9Csix621Tbt2+3WrVqZc2dO9f67LPPrGXLllmJiYnWG2+8YdfMmzfPSk5Otv7whz9Yu3fvtu666y5u923AmDFjrL/7u7+zb6N+++23rY4dO1qPPfaYXUNfG+f48ePWrl27rF27dlmSrGeeecbatWuX9V//9V+WZTWuj8OGDbP69Oljbdu2zdqyZYt1/fXXcxt1S/L8889bnTt3thISEqz+/ftbW7dube4pGUXSBR+vvfaaXXP69Gnrn/7pn6xrr73WSkxMtH72s59Zhw4dar5JG+xvAwy9bbr33nvP6tGjh+VwOKxu3bpZL730UtD6uro664knnrBSU1Mth8NhDRkyxKqurm6m2ZrB5/NZjz76qNW5c2erdevW1g9/+EPr8ccft/x+v11DXxvnj3/84wX/bh0zZoxlWY3r43fffWfdd999Vtu2bS2n02n98pe/tI4fP94Me9M4MZb1vY88BAAAMADXwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgnP8F9mb89LGtrp4AAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "seq_len = [len(row.split()) for row in df[\"text\"]]\n",
        "pd.Series(seq_len).hist(bins=50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "hho2SaTIEIJM"
      },
      "outputs": [],
      "source": [
        "max_length = 40\n",
        "\n",
        "indexes = []\n",
        "\n",
        "for i, row in df.iterrows():\n",
        "\n",
        "    if len(row[\"text\"].split()) > max_length:\n",
        "\n",
        "        indexes.append(i)\n",
        "\n",
        "df.drop(indexes, inplace=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hlLq-9YsEIJO"
      },
      "source": [
        "stemming and lemmatization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8wqZwjT_EIJR",
        "outputId": "28066271-2e13-4066-fec5-cd2c2675be7e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
          ]
        }
      ],
      "source": [
        "nltk.download(\"punkt\")\n",
        "nltk.download(\"wordnet\")\n",
        "\n",
        "# Instantiate the stemmer and lemmatizer\n",
        "stemmer = PorterStemmer()\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "# Define a function to apply stemming and lemmatization to a sentence\n",
        "def stem_and_lemmatize(sentence):\n",
        "    # Tokenize the sentence into words\n",
        "    words = nltk.word_tokenize(sentence.lower())\n",
        "    # Apply stemming to each word\n",
        "    stemmed_words = [stemmer.stem(word) for word in words]\n",
        "    # Apply lemmatization to each word\n",
        "    lemmatized_words = [lemmatizer.lemmatize(word) for word in stemmed_words]\n",
        "    # Join the lemmatized words back into a sentence\n",
        "    lemmatized_sentence = \" \".join(lemmatized_words)\n",
        "    return lemmatized_sentence\n",
        "\n",
        "\n",
        "# Apply the stem_and_lemmatize function to your text data\n",
        "df[\"text\"] = df[\"text\"].apply(stem_and_lemmatize)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6s0g-2CFEIJT"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f8Lurb3WEIJV"
      },
      "source": [
        "**statisticall information of dataset**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "Dv0OZXGsEIJW"
      },
      "outputs": [],
      "source": [
        "threaten_comment    = df[df[\"label\"] == 1]\n",
        "un_threaten_comment = df[df[\"label\"] == 0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A1eHtqFbEIJX",
        "outputId": "3c67625a-4741-4b30-e0fb-1e156cca09e8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "size of the dataset : 27425 \n",
            "size of threaten comments : 1302 \n",
            "size of un_threaten comments : 26123 \n"
          ]
        }
      ],
      "source": [
        "print(f\"size of the dataset : {df.shape[0]} \")\n",
        "print(f\"size of threaten comments : {threaten_comment.shape[0] } \")\n",
        "print(f\"size of un_threaten comments : {un_threaten_comment.shape[0] } \")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "W8Q27t_JEIJZ",
        "outputId": "f996a298-40f1-40a4-b1f7-a32ad60a2f1d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "threaten comments\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                 text  label\n",
              "2   but it okbecaus europ will soon rebel against ...      1\n",
              "13   all it mean is there will be more of you to kill      1\n",
              "35  i just hope when multicultur prove to be a rea...      1\n",
              "70                                     death to islam      1\n",
              "75               the quran can go burn itself in hell      1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-597640dd-f310-446f-8667-1f2f68e1e050\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>but it okbecaus europ will soon rebel against ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>all it mean is there will be more of you to kill</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>i just hope when multicultur prove to be a rea...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>death to islam</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75</th>\n",
              "      <td>the quran can go burn itself in hell</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-597640dd-f310-446f-8667-1f2f68e1e050')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-597640dd-f310-446f-8667-1f2f68e1e050 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-597640dd-f310-446f-8667-1f2f68e1e050');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "print(\"threaten comments\")\n",
        "threaten_comment.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "Jx7ZErr0EIJa",
        "outputId": "19d388e3-57a3-4ef1-90bf-ab1fef37daa4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "un-threaten comments\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  label\n",
              "0  it becaus european do not want to chang their ...      0\n",
              "1  the muslim there do not want to assimil proper...      0\n",
              "3  i forse a big civil war in europ in the futur ...      0\n",
              "4    islam a simpl humanitarian and attract religion      0\n",
              "5                                    welcom to islam      0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-665750f1-32cd-4e15-bad7-f14643eebf26\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>it becaus european do not want to chang their ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>the muslim there do not want to assimil proper...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>i forse a big civil war in europ in the futur ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>islam a simpl humanitarian and attract religion</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>welcom to islam</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-665750f1-32cd-4e15-bad7-f14643eebf26')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-665750f1-32cd-4e15-bad7-f14643eebf26 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-665750f1-32cd-4e15-bad7-f14643eebf26');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "print(\"un-threaten comments\")\n",
        "un_threaten_comment.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0o0q6FKNEIJd"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KBPtmbe1EIJe"
      },
      "source": [
        "**sampling methods**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "biORU_7rEIJf"
      },
      "source": [
        "Stratified Sampling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "Rj7QYzJkEIJf"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "# Stratified sampling\n",
        "X = df[\"text\"]\n",
        "y = df[\"label\"]\n",
        "train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.2, stratify=y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "laRemoGIEIJg",
        "outputId": "2a7c1336-a709-4116-dc8f-b13f6d24a11e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20898, 1042)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "train_y.value_counts()[0] , train_y.value_counts()[1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e3WE2jONEIJh",
        "outputId": "0afd3aa8-4561-4cff-c46f-63fa9de83a96"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5225, 260)"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "test_y.value_counts()[0] , test_y.value_counts()[1]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "__a7jBtiEIJk"
      },
      "source": [
        "balanced sampling : Sample as much minority data as majority data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "hZ8mrzkJEIJl"
      },
      "outputs": [],
      "source": [
        "balanced_unthreaten = un_threaten_comment.sample(n = threaten_comment.shape[0] , random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "Aj6bJxx0EIJm",
        "outputId": "e275c4be-9f46-4500-c00f-6e3f816cda96"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label\n",
              "1603   your fuck countri becaus you want to see the e...      0\n",
              "12947  imagin yourself be born in an european countri...      0\n",
              "11977  hahah you guysi wouldnt be surpris if there wa...      0\n",
              "17109  vitray of evameyrovitch research and director ...      0\n",
              "17231            so devil can be kill just by ray of sun      0\n",
              "...                                                  ...    ...\n",
              "16556  second of all the arab did creat algebra look ...      0\n",
              "16752  im not pull for thi man at all im not readi to...      0\n",
              "3380                           i find youll think it doe      0\n",
              "3356                but the law is not on your side mate      0\n",
              "21117  those who support american diver nonwhit immig...      0\n",
              "\n",
              "[1302 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6727f1ee-6243-44d6-80ff-90ede71ba0b6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1603</th>\n",
              "      <td>your fuck countri becaus you want to see the e...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12947</th>\n",
              "      <td>imagin yourself be born in an european countri...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11977</th>\n",
              "      <td>hahah you guysi wouldnt be surpris if there wa...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17109</th>\n",
              "      <td>vitray of evameyrovitch research and director ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17231</th>\n",
              "      <td>so devil can be kill just by ray of sun</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16556</th>\n",
              "      <td>second of all the arab did creat algebra look ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16752</th>\n",
              "      <td>im not pull for thi man at all im not readi to...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3380</th>\n",
              "      <td>i find youll think it doe</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3356</th>\n",
              "      <td>but the law is not on your side mate</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21117</th>\n",
              "      <td>those who support american diver nonwhit immig...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1302 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6727f1ee-6243-44d6-80ff-90ede71ba0b6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6727f1ee-6243-44d6-80ff-90ede71ba0b6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6727f1ee-6243-44d6-80ff-90ede71ba0b6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ],
      "source": [
        "balanced_unthreaten"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ffGDKN5nEIJn"
      },
      "source": [
        "**we will use the balanced sampling method**\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TRKiwNIEEIJo"
      },
      "source": [
        "train test split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "J8jcC-nTEIJp",
        "outputId": "2a7f4a38-9021-4aff-8080-e9f8eba2f9c0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label\n",
              "1603   your fuck countri becaus you want to see the e...      0\n",
              "12947  imagin yourself be born in an european countri...      0\n",
              "11977  hahah you guysi wouldnt be surpris if there wa...      0\n",
              "17109  vitray of evameyrovitch research and director ...      0\n",
              "17231            so devil can be kill just by ray of sun      0\n",
              "...                                                  ...    ...\n",
              "28602  if that look like happen all your devil shrine...      1\n",
              "28609  we must creat a new hitler that will destroy e...      1\n",
              "28616                                      kill them all      1\n",
              "28617              in poland and ukrain they kill muslim      1\n",
              "28638  yeah we are all monstersim gon na kill u right...      1\n",
              "\n",
              "[2604 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9f23b9f4-143f-4bda-a4f8-9db956a7aca1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1603</th>\n",
              "      <td>your fuck countri becaus you want to see the e...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12947</th>\n",
              "      <td>imagin yourself be born in an european countri...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11977</th>\n",
              "      <td>hahah you guysi wouldnt be surpris if there wa...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17109</th>\n",
              "      <td>vitray of evameyrovitch research and director ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17231</th>\n",
              "      <td>so devil can be kill just by ray of sun</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28602</th>\n",
              "      <td>if that look like happen all your devil shrine...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28609</th>\n",
              "      <td>we must creat a new hitler that will destroy e...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28616</th>\n",
              "      <td>kill them all</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28617</th>\n",
              "      <td>in poland and ukrain they kill muslim</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28638</th>\n",
              "      <td>yeah we are all monstersim gon na kill u right...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2604 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9f23b9f4-143f-4bda-a4f8-9db956a7aca1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9f23b9f4-143f-4bda-a4f8-9db956a7aca1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9f23b9f4-143f-4bda-a4f8-9db956a7aca1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "dataset = pd.concat([balanced_unthreaten, threaten_comment])\n",
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "98IukEeyEIJq"
      },
      "outputs": [],
      "source": [
        "X = dataset[\"text\"]\n",
        "y = dataset[\"label\"]\n",
        "train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.2, stratify=y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dSG2hDyfEIJr",
        "outputId": "65641c96-a395-4596-d17b-af2a7d23d93b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "value count of train_y \n",
            "0    1042\n",
            "1    1041\n",
            "Name: label, dtype: int64 \n",
            "value count of test_y \n",
            "1    261\n",
            "0    260\n",
            "Name: label, dtype: int64 \n"
          ]
        }
      ],
      "source": [
        "print(f\"value count of train_y \\n{train_y.value_counts()} \")\n",
        "print(f\"value count of test_y \\n{test_y.value_counts()} \")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uGw3tyicEIJs"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "HzZgtMSpEIJt"
      },
      "outputs": [],
      "source": [
        "train_df = pd.concat([train_X, train_y], axis=1)\n",
        "test_df  = pd.concat([test_X, test_y], axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "oLRVS4aHEIJ7",
        "outputId": "5be363c5-a7a7-437d-a2a1-2e20fdab61b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train_data.shape  : (2083, 2)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label\n",
              "13464      muslim should be deport from all western land      1\n",
              "2538   go ahead if you ever onc think about carri out...      1\n",
              "27042                                      kill them all      1\n",
              "2114      now what ha thi to do with the muslim of today      0\n",
              "5704       wrotegood for you read more of the holi quran      0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b618f97a-1f75-4112-a3b8-c3aff4dcf121\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>13464</th>\n",
              "      <td>muslim should be deport from all western land</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2538</th>\n",
              "      <td>go ahead if you ever onc think about carri out...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27042</th>\n",
              "      <td>kill them all</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2114</th>\n",
              "      <td>now what ha thi to do with the muslim of today</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5704</th>\n",
              "      <td>wrotegood for you read more of the holi quran</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b618f97a-1f75-4112-a3b8-c3aff4dcf121')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b618f97a-1f75-4112-a3b8-c3aff4dcf121 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b618f97a-1f75-4112-a3b8-c3aff4dcf121');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ],
      "source": [
        "print(f\"train_data.shape  : {train_df.shape}\")\n",
        "\n",
        "train_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "SvU9o_SqEIJ9",
        "outputId": "74894ce6-d8db-4084-96ba-636222cb38d8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "test_data.shape  : (521, 2)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label\n",
              "15368                   i pray for your gener to die out      1\n",
              "26273                            hehe polskojzyczni jaki      0\n",
              "17620  plea ani one want to pick up the debat il crus...      1\n",
              "21721                                       like a snake      0\n",
              "11880  tun tun lmao special fuck you to all american too      0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-65de91ab-3c55-4149-85ab-18a0f262dae1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>15368</th>\n",
              "      <td>i pray for your gener to die out</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26273</th>\n",
              "      <td>hehe polskojzyczni jaki</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17620</th>\n",
              "      <td>plea ani one want to pick up the debat il crus...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21721</th>\n",
              "      <td>like a snake</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11880</th>\n",
              "      <td>tun tun lmao special fuck you to all american too</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-65de91ab-3c55-4149-85ab-18a0f262dae1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-65de91ab-3c55-4149-85ab-18a0f262dae1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-65de91ab-3c55-4149-85ab-18a0f262dae1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "print(f\"test_data.shape  : {test_df.shape}\")\n",
        "\n",
        "test_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_oIRyw9SEIJ-"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dY1-sjuyEIJ_"
      },
      "source": [
        "get the tf-idf matrix of dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aJc6V-TiEIKA",
        "outputId": "214db36b-15d8-4b32-e0c4-b5ed4702da4d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<2083x2205 sparse matrix of type '<class 'numpy.float64'>'\n",
              "\twith 27868 stored elements in Compressed Sparse Row format>"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ],
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "vectorizer = TfidfVectorizer(\n",
        "    strip_accents='unicode',\n",
        "    ngram_range=(1, 2),\n",
        "    min_df=0.001,\n",
        "    max_df=0.8)\n",
        "\n",
        "X_train_tf = vectorizer.fit_transform(train_df[\"text\"])\n",
        "X_test_tf = vectorizer.transform(test_df[\"text\"])\n",
        "\n",
        "X_train_tf"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**dimension reduction using svd on tf_idf matrix**"
      ],
      "metadata": {
        "id": "EUP4qZwGcFxn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import TruncatedSVD"
      ],
      "metadata": {
        "id": "eNw3rp_GcJ65"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "truncated_svd = TruncatedSVD(n_components=200, random_state=0)\n",
        "X_train_reduced = truncated_svd.fit_transform(X_train)\n",
        "X_test_reduced = truncated_svd.transform(X_test)\n",
        "X_train_reduced.shape, X_test_reduced.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9p6Yy5N4cXQv",
        "outputId": "3c9c2b16-0177-425c-e886-08b69b826390"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((2083, 200), (521, 200))"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "truncated_svd.explained_variance_ratio_.sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2e-o-r5vce1p",
        "outputId": "83fb92a7-7c33-4b78-948e-fdd6ce0fe089"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.49293074027072803"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.metrics import classification_report\n",
        "\n"
      ],
      "metadata": {
        "id": "RxGeCllgdaJi"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "clf = MultinomialNB()\n",
        "clf.fit(X_train, train_y)\n",
        "\n",
        "# y_pred_train = clf.predict(X_train)\n",
        "y_pred_test = clf.predict(X_test)\n",
        "print(classification_report(test_y, y_pred_test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IaPs1KuJdm95",
        "outputId": "d7d2be1e-e771-460d-ef5a-997edd120934"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.87      0.88       260\n",
            "           1       0.88      0.90      0.89       261\n",
            "\n",
            "    accuracy                           0.88       521\n",
            "   macro avg       0.89      0.88      0.88       521\n",
            "weighted avg       0.89      0.88      0.88       521\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PRAK2rYzEIKB"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J6uxCZWxEIKC"
      },
      "source": [
        "**get the word representations using pretrained bert model**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4iweAi0cEIKD"
      },
      "outputs": [],
      "source": [
        "! pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "bLJL8N12EIKD"
      },
      "outputs": [],
      "source": [
        "from transformers import BertTokenizer, BertModel\n",
        "\n",
        "# Load pre-trained BERT tokenizer and model\n",
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "model = BertModel.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "# Define a function to get the CLS representation of a text using BERT\n",
        "def get_cls_representation(text):\n",
        "\n",
        "    tokens = tokenizer.batch_encode_plus(\n",
        "        text,\n",
        "        add_special_tokens=True,\n",
        "        truncation=True,\n",
        "        padding=\"max_length\",\n",
        "        max_length=max_length + 2,\n",
        "        return_tensors=\"pt\",\n",
        "        return_attention_mask=True,\n",
        "    )\n",
        "\n",
        "    input_ids = tokens[\"input_ids\"]\n",
        "    attention_mask = tokens[\"attention_mask\"]\n",
        "\n",
        "    outputs = model(input_ids, attention_mask=attention_mask)\n",
        "    cls_representation = outputs[0][:, 0, :]\n",
        "\n",
        "    return cls_representation.detach().numpy()\n",
        "\n",
        "\n",
        "train_text_representations = [\n",
        "    get_cls_representation(train_df[\"text\"][i : i + 64].tolist())\n",
        "    for i in range(0, train_df.shape[0], 64)\n",
        "]\n",
        "\n",
        "test_text_representations = [\n",
        "    get_cls_representation(test_df[\"text\"][i : i + 64].tolist())\n",
        "    for i in range(0, test_df.shape[0], 64)\n",
        "]\n",
        "\n",
        "train_rep = np.concatenate(train_text_representations).reshape((train_df.shape[0], 768))\n",
        "test_rep = np.concatenate(test_text_representations).reshape((test_df.shape[0], 768))\n",
        "\n",
        "train_df[\"text_represantion\"] = train_rep.tolist()\n",
        "\n",
        "test_df[\"text_represantion\"] = test_rep.tolist()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Bj2CdUldV_fZ",
        "outputId": "7e366a45-eb93-4767-a999-8f8b814c40e1"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label  \\\n",
              "13464      muslim should be deport from all western land      1   \n",
              "2538   go ahead if you ever onc think about carri out...      1   \n",
              "27042                                      kill them all      1   \n",
              "2114      now what ha thi to do with the muslim of today      0   \n",
              "5704       wrotegood for you read more of the holi quran      0   \n",
              "\n",
              "                                       text_represantion  \n",
              "13464  [-0.5980727672576904, 0.4174318015575409, -0.4...  \n",
              "2538   [0.06734813749790192, 0.14853157103061676, -0....  \n",
              "27042  [0.07035636156797409, 0.5827975273132324, -0.3...  \n",
              "2114   [-0.03710252419114113, 0.1834038496017456, -0....  \n",
              "5704   [-0.15349437296390533, 0.1581261307001114, 0.0...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9c636fc0-0bb7-455b-9edf-4117fb8b105d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>text_represantion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>13464</th>\n",
              "      <td>muslim should be deport from all western land</td>\n",
              "      <td>1</td>\n",
              "      <td>[-0.5980727672576904, 0.4174318015575409, -0.4...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2538</th>\n",
              "      <td>go ahead if you ever onc think about carri out...</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.06734813749790192, 0.14853157103061676, -0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27042</th>\n",
              "      <td>kill them all</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.07035636156797409, 0.5827975273132324, -0.3...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2114</th>\n",
              "      <td>now what ha thi to do with the muslim of today</td>\n",
              "      <td>0</td>\n",
              "      <td>[-0.03710252419114113, 0.1834038496017456, -0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5704</th>\n",
              "      <td>wrotegood for you read more of the holi quran</td>\n",
              "      <td>0</td>\n",
              "      <td>[-0.15349437296390533, 0.1581261307001114, 0.0...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9c636fc0-0bb7-455b-9edf-4117fb8b105d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9c636fc0-0bb7-455b-9edf-4117fb8b105d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9c636fc0-0bb7-455b-9edf-4117fb8b105d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "DTXylCCoWBq-",
        "outputId": "6704496c-42a8-4fdc-c017-34383eb6d5ee"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    text  label  \\\n",
              "15368                   i pray for your gener to die out      1   \n",
              "26273                            hehe polskojzyczni jaki      0   \n",
              "17620  plea ani one want to pick up the debat il crus...      1   \n",
              "21721                                       like a snake      0   \n",
              "11880  tun tun lmao special fuck you to all american too      0   \n",
              "\n",
              "                                       text_represantion  \n",
              "15368  [0.14766113460063934, 0.236196368932724, 0.045...  \n",
              "26273  [-0.4059525430202484, 0.21916669607162476, -0....  \n",
              "17620  [-0.4428701102733612, 0.08135899901390076, 0.0...  \n",
              "21721  [-0.2700501084327698, 0.1964525580406189, -0.1...  \n",
              "11880  [-0.15307843685150146, 0.14068268239498138, 0....  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-db869452-98cc-40fe-ad27-7bc19f395d82\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>text_represantion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>15368</th>\n",
              "      <td>i pray for your gener to die out</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.14766113460063934, 0.236196368932724, 0.045...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26273</th>\n",
              "      <td>hehe polskojzyczni jaki</td>\n",
              "      <td>0</td>\n",
              "      <td>[-0.4059525430202484, 0.21916669607162476, -0....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17620</th>\n",
              "      <td>plea ani one want to pick up the debat il crus...</td>\n",
              "      <td>1</td>\n",
              "      <td>[-0.4428701102733612, 0.08135899901390076, 0.0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21721</th>\n",
              "      <td>like a snake</td>\n",
              "      <td>0</td>\n",
              "      <td>[-0.2700501084327698, 0.1964525580406189, -0.1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11880</th>\n",
              "      <td>tun tun lmao special fuck you to all american too</td>\n",
              "      <td>0</td>\n",
              "      <td>[-0.15307843685150146, 0.14068268239498138, 0....</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-db869452-98cc-40fe-ad27-7bc19f395d82')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-db869452-98cc-40fe-ad27-7bc19f395d82 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-db869452-98cc-40fe-ad27-7bc19f395d82');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.9.12 ('myenv')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "598dc2fa64422c55e7d42274a4ea170b07e6b0df7ea783722f3eedf52eed3650"
      }
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}