data_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3955f75f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6365,
     "status": "ok",
     "timestamp": 1678951361939,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "3955f75f",
    "outputId": "b57a39ee-c51a-4b71-e2c8-6478c238fff5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lingua-language-detector==1.1.0 in /Users/new/opt/anaconda3/lib/python3.9/site-packages (1.1.0)\r\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.23.2 in /Users/new/opt/anaconda3/lib/python3.9/site-packages (from lingua-language-detector==1.1.0) (1.24.2)\r\n",
      "Requirement already satisfied: regex<2023.0.0,>=2022.8.17 in /Users/new/opt/anaconda3/lib/python3.9/site-packages (from lingua-language-detector==1.1.0) (2022.10.31)\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/new/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /Users/new/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to /Users/new/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/new/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package maxent_ne_chunker to\n",
      "[nltk_data]     /Users/new/nltk_data...\n",
      "[nltk_data]   Package maxent_ne_chunker is already up-to-date!\n",
      "[nltk_data] Downloading package words to /Users/new/nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from utils.analysis import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "73663428",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "executionInfo": {
     "elapsed": 746,
     "status": "ok",
     "timestamp": 1678951396015,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "73663428",
    "outputId": "f2e8ae37-adaa-481e-e580-70854232fa67"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ah Ah.. Wait o.. No be you Tinubu dey insult s...</td>\n",
       "      <td>Wait... Is it not you Tinubu has been insultin...</td>\n",
       "      <td>Hasn't Tinubu been insulting you since Monday?...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ah, o ti o! I don forget say I suppose call my...</td>\n",
       "      <td>Oh no! I forgot that I was supposed to call my...</td>\n",
       "      <td>Oh no! I forgot to call my mother today.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>An ode to my lover Oh! I’ve met women before D...</td>\n",
       "      <td>An ode to my lover Oh! I've met women before D...</td>\n",
       "      <td>An ode to my lover! I've met women before Dúpé...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Tweets  \\\n",
       "0     “buju na my guy but je kin monetize e o baby!”   \n",
       "1         Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "2  Ah Ah.. Wait o.. No be you Tinubu dey insult s...   \n",
       "3  Ah, o ti o! I don forget say I suppose call my...   \n",
       "4  An ode to my lover Oh! I’ve met women before D...   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "2  Wait... Is it not you Tinubu has been insultin...   \n",
       "3  Oh no! I forgot that I was supposed to call my...   \n",
       "4  An ode to my lover Oh! I've met women before D...   \n",
       "\n",
       "                                             Summary  \n",
       "0     \"Buju na my guy but let me monetize you baby!\"  \n",
       "1                                I twisted my ankle.  \n",
       "2  Hasn't Tinubu been insulting you since Monday?...  \n",
       "3           Oh no! I forgot to call my mother today.  \n",
       "4  An ode to my lover! I've met women before Dúpé...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_loc = \"Data/twitter_data.csv\"\n",
    "twitter_data = pd.read_csv(file_loc)\n",
    "twitter_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1953f9b9",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 94,
     "status": "ok",
     "timestamp": 1678951399544,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "1953f9b9",
    "outputId": "5a3328af-8c8e-4b0d-a848-c9a96d6c696f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "Zau_Cml7OycZ",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "executionInfo": {
     "elapsed": 277,
     "status": "ok",
     "timestamp": 1678951402651,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "Zau_Cml7OycZ",
    "outputId": "8379d2e3-ed89-4d12-8766-ad6594732c4d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  \n",
       "0   [\"buju na my guy but let me monetize you baby!\"]  \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenize sentence\n",
    "twitter_data['tokenized_tweet_sent']= twitter_data['Tweets'].apply(lambda x: tokenize_sent(x))\n",
    "twitter_data['tokenized_eng_sent']= twitter_data['Eng_source'].apply(lambda x: tokenize_sent(x))\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4a6e6d87",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1678951418673,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "4a6e6d87",
    "outputId": "6c2234a9-e71f-4aee-c4ea-e5702166f03b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  \n",
       "0                 1               1  \n",
       "1                 2               2  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new variable containing the number of sentences in the summary, tweet, and english text\n",
    "twitter_data['sum_sent_count'] = twitter_data['Summary'].apply(lambda x : count_sents(x))\n",
    "twitter_data['tweet_sent_count'] = twitter_data['Tweets'].apply(lambda x : count_sents(x))\n",
    "twitter_data['eng_sent_count'] = twitter_data['Eng_source'].apply(lambda x : count_sents(x))\n",
    "\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3cfc6277",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 280
    },
    "executionInfo": {
     "elapsed": 121,
     "status": "ok",
     "timestamp": 1678951430188,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "3cfc6277",
    "outputId": "c5eb1bef-8d95-4f14-d82d-21601e82efaf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>eng_char_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  sum_char_count  tweet_char_count  \\\n",
       "0                 1               1              37                36   \n",
       "1                 2               2              16                34   \n",
       "\n",
       "   eng_char_count  \n",
       "0              37  \n",
       "1              43  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new variable containing the number of characters in the summary, tweet, and english text\n",
    "twitter_data['sum_char_count'] = twitter_data['Summary'].apply(lambda x : count_chars(x))\n",
    "twitter_data['tweet_char_count'] = twitter_data['Tweets'].apply(lambda x : count_chars(x))\n",
    "twitter_data['eng_char_count'] = twitter_data['Eng_source'].apply(lambda x : count_chars(x))\n",
    "\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "82a188c5",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 280
    },
    "executionInfo": {
     "elapsed": 152,
     "status": "ok",
     "timestamp": 1678951434535,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "82a188c5",
    "outputId": "dff5fe6a-bee2-47d1-deb4-a2b1c81c67e6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>eng_char_count</th>\n",
       "      <th>sum_punc_count</th>\n",
       "      <th>tweet_punc_count</th>\n",
       "      <th>eng_punc_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "      <td>37</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  sum_char_count  tweet_char_count  \\\n",
       "0                 1               1              37                36   \n",
       "1                 2               2              16                34   \n",
       "\n",
       "   eng_char_count  sum_punc_count  tweet_punc_count  eng_punc_count  \n",
       "0              37               3                 1               3  \n",
       "1              43               1                 2               4  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new variable containing the number of punctuations in the summary, tweet, and english text\n",
    "twitter_data['sum_punc_count'] = twitter_data['Summary'].apply(lambda x : count_punc(x))\n",
    "twitter_data['tweet_punc_count'] = twitter_data['Tweets'].apply(lambda x : count_punc(x))\n",
    "twitter_data['eng_punc_count'] = twitter_data['Eng_source'].apply(lambda x : count_punc(x))\n",
    "\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2407e7d0",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 280
    },
    "executionInfo": {
     "elapsed": 160,
     "status": "ok",
     "timestamp": 1678951444876,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "2407e7d0",
    "outputId": "f277a44f-6955-4734-a197-55419e74c9eb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>eng_char_count</th>\n",
       "      <th>sum_punc_count</th>\n",
       "      <th>tweet_punc_count</th>\n",
       "      <th>eng_punc_count</th>\n",
       "      <th>tweet_without_punc</th>\n",
       "      <th>eng_without_punc</th>\n",
       "      <th>sum_without_punc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "      <td>37</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>“buju na my guy but je kin monetize e o baby”</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>Aanu ara mi n semi I’ve twisted my ankle</td>\n",
       "      <td>Feeling sorry for myself I’ve twisted my ankle</td>\n",
       "      <td>I twisted my ankle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  sum_char_count  tweet_char_count  \\\n",
       "0                 1               1              37                36   \n",
       "1                 2               2              16                34   \n",
       "\n",
       "   eng_char_count  sum_punc_count  tweet_punc_count  eng_punc_count  \\\n",
       "0              37               3                 1               3   \n",
       "1              43               1                 2               4   \n",
       "\n",
       "                              tweet_without_punc  \\\n",
       "0  “buju na my guy but je kin monetize e o baby”   \n",
       "1       Aanu ara mi n semi I’ve twisted my ankle   \n",
       "\n",
       "                                 eng_without_punc  \\\n",
       "0     Buju na my guy but let me monetize you baby   \n",
       "1  Feeling sorry for myself I’ve twisted my ankle   \n",
       "\n",
       "                              sum_without_punc  \n",
       "0  Buju na my guy but let me monetize you baby  \n",
       "1                           I twisted my ankle  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# storing the puntuation free text\n",
    "twitter_data['tweet_without_punc'] = twitter_data['Tweets'].apply(lambda x: remove_punctuation(x))\n",
    "twitter_data['eng_without_punc'] = twitter_data['Eng_source'].apply(lambda x: remove_punctuation(x))\n",
    "twitter_data['sum_without_punc'] = twitter_data['Summary'].apply(lambda x: remove_punctuation(x))\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b476ed91",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 280
    },
    "executionInfo": {
     "elapsed": 157,
     "status": "ok",
     "timestamp": 1678951484621,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "b476ed91",
    "outputId": "9c6ff650-0d3b-4c6d-b4eb-c2dd24bddb44"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>eng_char_count</th>\n",
       "      <th>sum_punc_count</th>\n",
       "      <th>tweet_punc_count</th>\n",
       "      <th>eng_punc_count</th>\n",
       "      <th>tweet_without_punc</th>\n",
       "      <th>eng_without_punc</th>\n",
       "      <th>sum_without_punc</th>\n",
       "      <th>tokenized_tweet</th>\n",
       "      <th>tokenized_eng</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "      <td>37</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>“buju na my guy but je kin monetize e o baby”</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "      <td>[“, buju, na, my, guy, but, je, kin, monetize,...</td>\n",
       "      <td>[buju, na, my, guy, but, let, me, monetize, yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>Aanu ara mi n semi I’ve twisted my ankle</td>\n",
       "      <td>Feeling sorry for myself I’ve twisted my ankle</td>\n",
       "      <td>I twisted my ankle</td>\n",
       "      <td>[aanu, ara, mi, n, semi, i, ’, ve, twisted, my...</td>\n",
       "      <td>[feeling, sorry, for, myself, i, ’, ve, twiste...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  sum_char_count  tweet_char_count  \\\n",
       "0                 1               1              37                36   \n",
       "1                 2               2              16                34   \n",
       "\n",
       "   eng_char_count  sum_punc_count  tweet_punc_count  eng_punc_count  \\\n",
       "0              37               3                 1               3   \n",
       "1              43               1                 2               4   \n",
       "\n",
       "                              tweet_without_punc  \\\n",
       "0  “buju na my guy but je kin monetize e o baby”   \n",
       "1       Aanu ara mi n semi I’ve twisted my ankle   \n",
       "\n",
       "                                 eng_without_punc  \\\n",
       "0     Buju na my guy but let me monetize you baby   \n",
       "1  Feeling sorry for myself I’ve twisted my ankle   \n",
       "\n",
       "                              sum_without_punc  \\\n",
       "0  Buju na my guy but let me monetize you baby   \n",
       "1                           I twisted my ankle   \n",
       "\n",
       "                                     tokenized_tweet  \\\n",
       "0  [“, buju, na, my, guy, but, je, kin, monetize,...   \n",
       "1  [aanu, ara, mi, n, semi, i, ’, ve, twisted, my...   \n",
       "\n",
       "                                       tokenized_eng  \n",
       "0  [buju, na, my, guy, but, let, me, monetize, yo...  \n",
       "1  [feeling, sorry, for, myself, i, ’, ve, twiste...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenize text\n",
    "twitter_data['tokenized_tweet']= twitter_data['tweet_without_punc'].apply(lambda x: tokenize_text(x))\n",
    "twitter_data['tokenized_eng']= twitter_data['eng_without_punc'].apply(lambda x: tokenize_text(x))\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d21baa95",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 310
    },
    "executionInfo": {
     "elapsed": 181,
     "status": "ok",
     "timestamp": 1678951491011,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "d21baa95",
    "outputId": "c93c6423-9d12-4105-9515-288a2d8764cd"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>...</th>\n",
       "      <th>tweet_punc_count</th>\n",
       "      <th>eng_punc_count</th>\n",
       "      <th>tweet_without_punc</th>\n",
       "      <th>eng_without_punc</th>\n",
       "      <th>sum_without_punc</th>\n",
       "      <th>tokenized_tweet</th>\n",
       "      <th>tokenized_eng</th>\n",
       "      <th>sum_word_count</th>\n",
       "      <th>tweet_word_count</th>\n",
       "      <th>eng_word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>“buju na my guy but je kin monetize e o baby”</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "      <td>[“, buju, na, my, guy, but, je, kin, monetize,...</td>\n",
       "      <td>[buju, na, my, guy, but, let, me, monetize, yo...</td>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>Aanu ara mi n semi I’ve twisted my ankle</td>\n",
       "      <td>Feeling sorry for myself I’ve twisted my ankle</td>\n",
       "      <td>I twisted my ankle</td>\n",
       "      <td>[aanu, ara, mi, n, semi, i, ’, ve, twisted, my...</td>\n",
       "      <td>[feeling, sorry, for, myself, i, ’, ve, twiste...</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  sum_char_count  tweet_char_count  ...  \\\n",
       "0                 1               1              37                36  ...   \n",
       "1                 2               2              16                34  ...   \n",
       "\n",
       "   tweet_punc_count  eng_punc_count  \\\n",
       "0                 1               3   \n",
       "1                 2               4   \n",
       "\n",
       "                              tweet_without_punc  \\\n",
       "0  “buju na my guy but je kin monetize e o baby”   \n",
       "1       Aanu ara mi n semi I’ve twisted my ankle   \n",
       "\n",
       "                                 eng_without_punc  \\\n",
       "0     Buju na my guy but let me monetize you baby   \n",
       "1  Feeling sorry for myself I’ve twisted my ankle   \n",
       "\n",
       "                              sum_without_punc  \\\n",
       "0  Buju na my guy but let me monetize you baby   \n",
       "1                           I twisted my ankle   \n",
       "\n",
       "                                     tokenized_tweet  \\\n",
       "0  [“, buju, na, my, guy, but, je, kin, monetize,...   \n",
       "1  [aanu, ara, mi, n, semi, i, ’, ve, twisted, my...   \n",
       "\n",
       "                                       tokenized_eng sum_word_count  \\\n",
       "0  [buju, na, my, guy, but, let, me, monetize, yo...             10   \n",
       "1  [feeling, sorry, for, myself, i, ’, ve, twiste...              4   \n",
       "\n",
       "  tweet_word_count  eng_word_count  \n",
       "0               13              10  \n",
       "1               11              10  \n",
       "\n",
       "[2 rows x 22 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new variable containing the number of words in the summary, tweet, and english text\n",
    "twitter_data['sum_word_count'] = twitter_data['sum_without_punc'].apply(lambda x : count_words(x))\n",
    "twitter_data['tweet_word_count'] = twitter_data['tweet_without_punc'].apply(lambda x : count_words(x))\n",
    "twitter_data['eng_word_count'] = twitter_data['eng_without_punc'].apply(lambda x : count_words(x))\n",
    "\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ca7bd5a9",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 310
    },
    "executionInfo": {
     "elapsed": 400,
     "status": "ok",
     "timestamp": 1678951501106,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "ca7bd5a9",
    "outputId": "a44eb597-8292-4ea2-f087-70a7dc0aae09"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>...</th>\n",
       "      <th>eng_without_punc</th>\n",
       "      <th>sum_without_punc</th>\n",
       "      <th>tokenized_tweet</th>\n",
       "      <th>tokenized_eng</th>\n",
       "      <th>sum_word_count</th>\n",
       "      <th>tweet_word_count</th>\n",
       "      <th>eng_word_count</th>\n",
       "      <th>sum_word_density</th>\n",
       "      <th>tweet_word_density</th>\n",
       "      <th>eng_word_density</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "      <td>Buju na my guy but let me monetize you baby</td>\n",
       "      <td>[“, buju, na, my, guy, but, je, kin, monetize,...</td>\n",
       "      <td>[buju, na, my, guy, but, let, me, monetize, yo...</td>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.351351</td>\n",
       "      <td>0.263158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>...</td>\n",
       "      <td>Feeling sorry for myself I’ve twisted my ankle</td>\n",
       "      <td>I twisted my ankle</td>\n",
       "      <td>[aanu, ara, mi, n, semi, i, ’, ve, twisted, my...</td>\n",
       "      <td>[feeling, sorry, for, myself, i, ’, ve, twiste...</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.314286</td>\n",
       "      <td>0.227273</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  sum_char_count  tweet_char_count  ...  \\\n",
       "0                 1               1              37                36  ...   \n",
       "1                 2               2              16                34  ...   \n",
       "\n",
       "                                 eng_without_punc  \\\n",
       "0     Buju na my guy but let me monetize you baby   \n",
       "1  Feeling sorry for myself I’ve twisted my ankle   \n",
       "\n",
       "                              sum_without_punc  \\\n",
       "0  Buju na my guy but let me monetize you baby   \n",
       "1                           I twisted my ankle   \n",
       "\n",
       "                                     tokenized_tweet  \\\n",
       "0  [“, buju, na, my, guy, but, je, kin, monetize,...   \n",
       "1  [aanu, ara, mi, n, semi, i, ’, ve, twisted, my...   \n",
       "\n",
       "                                       tokenized_eng sum_word_count  \\\n",
       "0  [buju, na, my, guy, but, let, me, monetize, yo...             10   \n",
       "1  [feeling, sorry, for, myself, i, ’, ve, twiste...              4   \n",
       "\n",
       "  tweet_word_count eng_word_count sum_word_density tweet_word_density  \\\n",
       "0               13             10         0.263158           0.351351   \n",
       "1               11             10         0.235294           0.314286   \n",
       "\n",
       "   eng_word_density  \n",
       "0          0.263158  \n",
       "1          0.227273  \n",
       "\n",
       "[2 rows x 25 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new variable containing the density of words in the summary, tweet, and english text\n",
    "twitter_data['sum_word_density'] = word_density(twitter_data, 'sum_word_count', 'sum_char_count')\n",
    "twitter_data['tweet_word_density'] = word_density(twitter_data, 'tweet_word_count', 'tweet_char_count')\n",
    "twitter_data['eng_word_density'] = word_density(twitter_data, 'eng_word_count', 'eng_char_count')\n",
    "\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5832a842",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 310
    },
    "executionInfo": {
     "elapsed": 168,
     "status": "ok",
     "timestamp": 1678951516196,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "5832a842",
    "outputId": "cbbd0f1b-3d18-42c2-8519-6664d2a588d8"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweets</th>\n",
       "      <th>Eng_source</th>\n",
       "      <th>Summary</th>\n",
       "      <th>tokenized_tweet_sent</th>\n",
       "      <th>tokenized_eng_sent</th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>...</th>\n",
       "      <th>tokenized_eng</th>\n",
       "      <th>sum_word_count</th>\n",
       "      <th>tweet_word_count</th>\n",
       "      <th>eng_word_count</th>\n",
       "      <th>sum_word_density</th>\n",
       "      <th>tweet_word_density</th>\n",
       "      <th>eng_word_density</th>\n",
       "      <th>sum_sent_density</th>\n",
       "      <th>tweet_sent_density</th>\n",
       "      <th>eng_sent_density</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“buju na my guy but je kin monetize e o baby!”</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>\"Buju na my guy but let me monetize you baby!\"</td>\n",
       "      <td>[“buju na my guy but je kin monetize e o baby!”]</td>\n",
       "      <td>[\"buju na my guy but let me monetize you baby!\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>[buju, na, my, guy, but, let, me, monetize, yo...</td>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.351351</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aanu ara mi n semi! I’ve twisted my ankle.</td>\n",
       "      <td>Feeling sorry for.. myself! I’ve twisted my an...</td>\n",
       "      <td>I twisted my ankle.</td>\n",
       "      <td>[aanu ara mi n semi!, i’ve twisted my ankle.]</td>\n",
       "      <td>[feeling sorry for.. myself!, i’ve twisted my ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>...</td>\n",
       "      <td>[feeling, sorry, for, myself, i, ’, ve, twiste...</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.314286</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.181818</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Tweets  \\\n",
       "0  “buju na my guy but je kin monetize e o baby!”   \n",
       "1      Aanu ara mi n semi! I’ve twisted my ankle.   \n",
       "\n",
       "                                          Eng_source  \\\n",
       "0     \"Buju na my guy but let me monetize you baby!\"   \n",
       "1  Feeling sorry for.. myself! I’ve twisted my an...   \n",
       "\n",
       "                                          Summary  \\\n",
       "0  \"Buju na my guy but let me monetize you baby!\"   \n",
       "1                             I twisted my ankle.   \n",
       "\n",
       "                               tokenized_tweet_sent  \\\n",
       "0  [“buju na my guy but je kin monetize e o baby!”]   \n",
       "1     [aanu ara mi n semi!, i’ve twisted my ankle.]   \n",
       "\n",
       "                                  tokenized_eng_sent  sum_sent_count  \\\n",
       "0   [\"buju na my guy but let me monetize you baby!\"]               1   \n",
       "1  [feeling sorry for.. myself!, i’ve twisted my ...               1   \n",
       "\n",
       "   tweet_sent_count  eng_sent_count  sum_char_count  tweet_char_count  ...  \\\n",
       "0                 1               1              37                36  ...   \n",
       "1                 2               2              16                34  ...   \n",
       "\n",
       "                                       tokenized_eng  sum_word_count  \\\n",
       "0  [buju, na, my, guy, but, let, me, monetize, yo...              10   \n",
       "1  [feeling, sorry, for, myself, i, ’, ve, twiste...               4   \n",
       "\n",
       "   tweet_word_count  eng_word_count sum_word_density tweet_word_density  \\\n",
       "0                13              10         0.263158           0.351351   \n",
       "1                11              10         0.235294           0.314286   \n",
       "\n",
       "  eng_word_density sum_sent_density tweet_sent_density  eng_sent_density  \n",
       "0         0.263158         0.090909           0.071429          0.090909  \n",
       "1         0.227273         0.200000           0.166667          0.181818  \n",
       "\n",
       "[2 rows x 28 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new variable containing the density of sentences in the summary, tweet, and english text\n",
    "twitter_data['sum_sent_density'] = sent_density(twitter_data, 'sum_sent_count', 'sum_word_count')\n",
    "twitter_data['tweet_sent_density'] = sent_density(twitter_data, 'tweet_sent_count', 'tweet_word_count')\n",
    "twitter_data['eng_sent_density'] = sent_density(twitter_data, 'eng_sent_count', 'eng_word_count')\n",
    "\n",
    "twitter_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "657ca39e",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 110,
     "status": "ok",
     "timestamp": 1678951524276,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "657ca39e",
    "outputId": "8e4ea5f7-753c-4f6c-fe3a-3251eb66c117"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sum of values in numerical columns\n",
      "----------------------------------\n",
      "sum_sent_count         128.000000\n",
      "tweet_sent_count       195.000000\n",
      "eng_sent_count         194.000000\n",
      "sum_char_count        5569.000000\n",
      "tweet_char_count      8112.000000\n",
      "eng_char_count        8845.000000\n",
      "sum_punc_count         227.000000\n",
      "tweet_punc_count       376.000000\n",
      "eng_punc_count         417.000000\n",
      "sum_word_count        1283.000000\n",
      "tweet_word_count      2071.000000\n",
      "eng_word_count        2051.000000\n",
      "sum_word_density        22.855251\n",
      "tweet_word_density      26.135824\n",
      "eng_word_density        23.095553\n",
      "sum_sent_density        10.453667\n",
      "tweet_sent_density      10.366935\n",
      "eng_sent_density        10.398464\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# get the sum of values of numeric columns\n",
    "data_sum(twitter_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b7edf81b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 136,
     "status": "ok",
     "timestamp": 1678951565587,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "b7edf81b",
    "outputId": "aea5bf4b-46f3-448d-ebd1-0b3bd72e5403"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average of values in numerical columns\n",
      "--------------------------------------\n",
      "sum_sent_count         1.280000\n",
      "tweet_sent_count       1.950000\n",
      "eng_sent_count         1.940000\n",
      "sum_char_count        55.690000\n",
      "tweet_char_count      81.120000\n",
      "eng_char_count        88.450000\n",
      "sum_punc_count         2.270000\n",
      "tweet_punc_count       3.760000\n",
      "eng_punc_count         4.170000\n",
      "sum_word_count        12.830000\n",
      "tweet_word_count      20.710000\n",
      "eng_word_count        20.510000\n",
      "sum_word_density       0.228553\n",
      "tweet_word_density     0.261358\n",
      "eng_word_density       0.230956\n",
      "sum_sent_density       0.104537\n",
      "tweet_sent_density     0.103669\n",
      "eng_sent_density       0.103985\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# get the average of values of numeric columns\n",
    "data_average(twitter_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "023b7441",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sum_sent_count</th>\n",
       "      <th>tweet_sent_count</th>\n",
       "      <th>eng_sent_count</th>\n",
       "      <th>sum_char_count</th>\n",
       "      <th>tweet_char_count</th>\n",
       "      <th>eng_char_count</th>\n",
       "      <th>sum_punc_count</th>\n",
       "      <th>tweet_punc_count</th>\n",
       "      <th>eng_punc_count</th>\n",
       "      <th>sum_word_count</th>\n",
       "      <th>tweet_word_count</th>\n",
       "      <th>eng_word_count</th>\n",
       "      <th>sum_word_density</th>\n",
       "      <th>tweet_word_density</th>\n",
       "      <th>eng_word_density</th>\n",
       "      <th>sum_sent_density</th>\n",
       "      <th>tweet_sent_density</th>\n",
       "      <th>eng_sent_density</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.280000</td>\n",
       "      <td>1.950000</td>\n",
       "      <td>1.940000</td>\n",
       "      <td>55.690000</td>\n",
       "      <td>81.120000</td>\n",
       "      <td>88.450000</td>\n",
       "      <td>2.270000</td>\n",
       "      <td>3.760000</td>\n",
       "      <td>4.170000</td>\n",
       "      <td>12.830000</td>\n",
       "      <td>20.710000</td>\n",
       "      <td>20.510000</td>\n",
       "      <td>0.228553</td>\n",
       "      <td>0.261358</td>\n",
       "      <td>0.230956</td>\n",
       "      <td>0.104537</td>\n",
       "      <td>0.103669</td>\n",
       "      <td>0.103985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.533333</td>\n",
       "      <td>1.018763</td>\n",
       "      <td>1.032991</td>\n",
       "      <td>26.632818</td>\n",
       "      <td>53.527918</td>\n",
       "      <td>56.747247</td>\n",
       "      <td>1.270131</td>\n",
       "      <td>2.763836</td>\n",
       "      <td>2.839316</td>\n",
       "      <td>6.103576</td>\n",
       "      <td>12.779636</td>\n",
       "      <td>13.196644</td>\n",
       "      <td>0.033179</td>\n",
       "      <td>0.042917</td>\n",
       "      <td>0.029194</td>\n",
       "      <td>0.050396</td>\n",
       "      <td>0.056574</td>\n",
       "      <td>0.056914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.098361</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.033898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>37.750000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>11.750000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.207837</td>\n",
       "      <td>0.237977</td>\n",
       "      <td>0.213346</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.068966</td>\n",
       "      <td>0.071429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>50.500000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>73.500000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>16.500000</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.258621</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.094494</td>\n",
       "      <td>0.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>65.250000</td>\n",
       "      <td>112.750000</td>\n",
       "      <td>125.500000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>26.250000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.288863</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.119485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>132.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>285.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>0.394737</td>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       sum_sent_count  tweet_sent_count  eng_sent_count  sum_char_count  \\\n",
       "count      100.000000        100.000000      100.000000      100.000000   \n",
       "mean         1.280000          1.950000        1.940000       55.690000   \n",
       "std          0.533333          1.018763        1.032991       26.632818   \n",
       "min          1.000000          1.000000        1.000000       10.000000   \n",
       "25%          1.000000          1.000000        1.000000       36.000000   \n",
       "50%          1.000000          2.000000        2.000000       50.500000   \n",
       "75%          1.000000          2.000000        2.250000       65.250000   \n",
       "max          3.000000          5.000000        5.000000      132.000000   \n",
       "\n",
       "       tweet_char_count  eng_char_count  sum_punc_count  tweet_punc_count  \\\n",
       "count        100.000000      100.000000      100.000000        100.000000   \n",
       "mean          81.120000       88.450000        2.270000          3.760000   \n",
       "std           53.527918       56.747247        1.270131          2.763836   \n",
       "min           10.000000       11.000000        0.000000          0.000000   \n",
       "25%           37.750000       43.000000        1.000000          2.000000   \n",
       "50%           69.000000       73.500000        2.000000          3.000000   \n",
       "75%          112.750000      125.500000        3.000000          5.000000   \n",
       "max          255.000000      285.000000        7.000000         15.000000   \n",
       "\n",
       "       eng_punc_count  sum_word_count  tweet_word_count  eng_word_count  \\\n",
       "count      100.000000      100.000000        100.000000      100.000000   \n",
       "mean         4.170000       12.830000         20.710000       20.510000   \n",
       "std          2.839316        6.103576         12.779636       13.196644   \n",
       "min          0.000000        2.000000          3.000000        3.000000   \n",
       "25%          2.000000        9.000000         11.750000       10.000000   \n",
       "50%          4.000000       11.000000         18.000000       16.500000   \n",
       "75%          5.000000       15.000000         28.000000       26.250000   \n",
       "max         16.000000       35.000000         64.000000       66.000000   \n",
       "\n",
       "       sum_word_density  tweet_word_density  eng_word_density  \\\n",
       "count        100.000000          100.000000        100.000000   \n",
       "mean           0.228553            0.261358          0.230956   \n",
       "std            0.033179            0.042917          0.029194   \n",
       "min            0.125000            0.098361          0.125000   \n",
       "25%            0.207837            0.237977          0.213346   \n",
       "50%            0.227273            0.258621          0.230769   \n",
       "75%            0.250000            0.288863          0.250000   \n",
       "max            0.312500            0.394737          0.307692   \n",
       "\n",
       "       sum_sent_density  tweet_sent_density  eng_sent_density  \n",
       "count        100.000000          100.000000        100.000000  \n",
       "mean           0.104537            0.103669          0.103985  \n",
       "std            0.050396            0.056574          0.056914  \n",
       "min            0.033333            0.034483          0.033898  \n",
       "25%            0.071429            0.068966          0.071429  \n",
       "50%            0.090909            0.094494          0.090909  \n",
       "75%            0.125000            0.125000          0.119485  \n",
       "max            0.333333            0.500000          0.500000  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cddf23ae",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4464,
     "status": "ok",
     "timestamp": 1678951574107,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "cddf23ae",
    "outputId": "ae7e4ec4-006e-4093-d5fb-c5b39606a0a9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Yoruba words:  647\n",
      "Number of English words:  1424\n",
      "Percentage of Yoruba words: 31.24%\n",
      "Percentage of English words: 68.76%\n"
     ]
    }
   ],
   "source": [
    "# language count in tweets using lingua library\n",
    "lang_count(twitter_data, 'tokenized_tweet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1fb52a7d",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1678951578196,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "1fb52a7d",
    "outputId": "565daad5-3872-467a-ebd7-c39bee20b197"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Yoruba words:  779\n",
      "Number of Eng words:  1292\n",
      "Percentage of Yoruba words: 37.61%\n",
      "Percentage of English words: 62.39%\n"
     ]
    }
   ],
   "source": [
    "# language count in tweets by comparing with english text\n",
    "lang_count_comp(twitter_data, 'tokenized_tweet', 'tokenized_eng')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "LTpRUsWoKeb7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 894,
     "status": "ok",
     "timestamp": 1678951627977,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "LTpRUsWoKeb7",
    "outputId": "09a7c660-c82a-479c-fda9-32c5cac2f356"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique words in Code-switched tweets = 867\n",
      "Number of unique words in English tweets = 774\n"
     ]
    }
   ],
   "source": [
    "# get unique words\n",
    "get_unique_words(twitter_data, 'tokenized_tweet', 'Code-switched tweets')\n",
    "get_unique_words(twitter_data, 'tokenized_eng', 'English tweets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "mc9Btwy3N1P2",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1678951633909,
     "user": {
      "displayName": "Glory Odeyemi",
      "userId": "04491820538436953732"
     },
     "user_tz": 240
    },
    "id": "mc9Btwy3N1P2",
    "outputId": "be8d170b-c90b-407e-cfc9-766ffa2efad2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique sentences in Code-switched tweets = 194\n",
      "Number of unique sentences in English tweets = 193\n"
     ]
    }
   ],
   "source": [
    "# get unique sentences\n",
    "get_unique_sents(twitter_data, 'tokenized_tweet_sent', 'Code-switched tweets')\n",
    "get_unique_sents(twitter_data, 'tokenized_eng_sent', 'English tweets')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}