Scrape + Clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "6c10c873-4f08-48e9-9077-3bc5e62e3e52",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "fcab51b8-197f-4e44-8c45-38076b1538d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.boxofficemojo.com/year/2017/?grossesOption=totalGrosses&sort=rank&sortDir=asc'\n",
    "#https://www.boxofficemojo.com/year/2018/?grossesOption=totalGrosses&sort=rank&sortDir=asc\n",
    "#https://www.boxofficemojo.com/year/2019/?grossesOption=totalGrosses&sort=rank&sortDir=asc\n",
    "#https://www.boxofficemojo.com/year/2020/?grossesOption=totalGrosses&sort=rank&sortDir=asc\n",
    "#https://www.boxofficemojo.com/year/2021/?grossesOption=totalGrosses&sort=rank&sortDir=asc\n",
    "#https://www.boxofficemojo.com/year/2022/?grossesOption=totalGrosses&sort=rank&sortDir=asc\n",
    "response = requests.get(url)    \n",
    "page = response.text\n",
    "soup = BeautifulSoup(page,\"lxml\")\n",
    "table = soup.find('table')\n",
    "rows = [row for row in table.find_all('tr')]  # tr tag is for rows\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "82f9a6b7-7a6f-4757-be56-2fd7b06be25f",
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = {}\n",
    "\n",
    "for row in rows[1:]:\n",
    "    items = row.find_all('td') #td finds the target cell\n",
    "    link = items[1].find('a') #a finds the anchor\n",
    "    title, url = link.text, link['href'] #href gives the url\n",
    "    movies[title] = [url]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "08c6bf46-4a8b-440d-9251-2100265ca0ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Link_stub</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Star Wars: Episode VIII - The Last Jedi</td>\n",
       "      <td>/release/rl2708702721/?ref_=bo_yld_table_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Beauty and the Beast</td>\n",
       "      <td>/release/rl222594561/?ref_=bo_yld_table_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Wonder Woman</td>\n",
       "      <td>/release/rl578455041/?ref_=bo_yld_table_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Jumanji: Welcome to the Jungle</td>\n",
       "      <td>/release/rl3095234049/?ref_=bo_yld_table_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Guardians of the Galaxy Vol. 2</td>\n",
       "      <td>/release/rl2976089601/?ref_=bo_yld_table_5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     index  \\\n",
       "0  Star Wars: Episode VIII - The Last Jedi   \n",
       "1                     Beauty and the Beast   \n",
       "2                             Wonder Woman   \n",
       "3           Jumanji: Welcome to the Jungle   \n",
       "4           Guardians of the Galaxy Vol. 2   \n",
       "\n",
       "                                    Link_stub  \n",
       "0  /release/rl2708702721/?ref_=bo_yld_table_1  \n",
       "1   /release/rl222594561/?ref_=bo_yld_table_2  \n",
       "2   /release/rl578455041/?ref_=bo_yld_table_3  \n",
       "3  /release/rl3095234049/?ref_=bo_yld_table_4  \n",
       "4  /release/rl2976089601/?ref_=bo_yld_table_5  "
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_links = pd.DataFrame(movies).T #transpose\n",
    "movie_links.columns = ['Link_stub']\n",
    "movie_links.reset_index().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "a99c59f1-8769-4f16-8338-052194b9ba49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 199 entries, Star Wars: Episode VIII - The Last Jedi to Free Fire\n",
      "Data columns (total 1 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   Link_stub  199 non-null    object\n",
      "dtypes: object(1)\n",
      "memory usage: 3.1+ KB\n"
     ]
    }
   ],
   "source": [
    "movie_links.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "bed2d21c-29bb-413e-82b5-4ab60f32c1c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_value(soup, field_name):\n",
    "    \n",
    "    obj = soup.find(text=re.compile(field_name))\n",
    "    \n",
    "    if not obj: \n",
    "        return None\n",
    "    \n",
    "    # this works for most of the values\n",
    "    next_element = obj.findNext()\n",
    "    \n",
    "    if next_element:\n",
    "        return next_element.text \n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "id": "1be93e17-e7fb-40e6-92e1-887fd06c19be",
   "metadata": {},
   "outputs": [],
   "source": [
    "def runtime_to_min(runtimestring):\n",
    "    try:    \n",
    "        runtime = runtimestring.split()\n",
    "        minutes = int(runtime[0])*60 + int(runtime[2])\n",
    "        return minutes\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "id": "2ccf6e27-7334-4b1f-9dd2-edfc54537eb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def movie_revenue(cash):\n",
    "    revenue = []\n",
    "    try:\n",
    "        money = cash.find(class_='mojo-performance-summary-table').find_all('span', class_='money')[0].text\n",
    "        revenue.append(money)\n",
    "    except:\n",
    "        revenue.append('None')\n",
    "    \n",
    "    try:\n",
    "        money = cash.find(class_='mojo-performance-summary-table').find_all('span', class_='money')[1].text\n",
    "        revenue.append(money)\n",
    "    except:\n",
    "        revenue.append('None')\n",
    "    \n",
    "    try:\n",
    "        money = cash.find(class_='mojo-performance-summary-table').find_all('span', class_='money')[2].text\n",
    "        revenue.append(money)\n",
    "    except:\n",
    "        revenue.append('None')\n",
    "    return revenue\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "id": "23d38a6b-9df8-44ef-8bd2-0d15549a7d7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def money_to_int(moneystring):\n",
    "    try:\n",
    "        moneystring = moneystring.replace('$', '').replace(',', '')\n",
    "        return int(moneystring)\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "id": "e2654716-b95d-4fd9-832b-5198a3e446a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def theatre_number(places_string):\n",
    "    try:\n",
    "        places = places_string.split()\n",
    "        number = places[0].replace(',','')\n",
    "        return number\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "id": "c9b52081-7b24-4de2-a4a4-d3f844b49c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_genre(category):\n",
    "    try:\n",
    "        category = category.split()\n",
    "        return category\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "id": "134dd0a1-ba1f-4d97-b8be-178490a17a92",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_movie_dict(link):\n",
    "    \n",
    "    base_url = 'https://www.boxofficemojo.com'\n",
    "    \n",
    "    #Create full url to scrape\n",
    "    url = base_url + link\n",
    "    \n",
    "    #Request HTML and parse\n",
    "    response = requests.get(url)\n",
    "    page = response.text\n",
    "    soup = BeautifulSoup(page,\"lxml\")\n",
    "    \n",
    "    headers = ['Title', 'Distributor','Opening','Release_date','In_Release',\n",
    "               'Rating','Run_time','Genre','Theatres','Domestic_gross',\n",
    "              'Internation_gross','Worldwide_gross']\n",
    "    \n",
    "    #Get title\n",
    "    title_string = soup.find('title').text.split('-')[0].strip()\n",
    "\n",
    "    #Get Distributor\n",
    "    distributor = get_movie_value(soup,'Distributor')\n",
    "    \n",
    "    #Get opening day box office gross\n",
    "    opening = soup.find(class_='money').text.replace('$','').replace(',','')\n",
    "    \n",
    "    #Get release date\n",
    "    release_date = get_movie_value(soup,'Release Date')\n",
    "    \n",
    "    #Get days since release date\n",
    "    Days_since_release = get_movie_value(soup,'In Release')\n",
    "    \n",
    "    #Get rating\n",
    "    rating = get_movie_value(soup,'MPAA')\n",
    "    \n",
    "    #Get run time\n",
    "    run_time = get_movie_value(soup,'Running') \n",
    "    minutes = runtime_to_min(run_time)\n",
    "    \n",
    "    #Get genre\n",
    "    genre = get_movie_value(soup,'Genres')\n",
    "    genre = clean_genre(genre)\n",
    "    \n",
    "    #Get release theatres\n",
    "    theatre = get_movie_value(soup,'Widest Release')\n",
    "    theatre = theatre_number(theatre) \n",
    "    \n",
    "    #Get revenue gross\n",
    "    domestic = movie_revenue(soup)[0]\n",
    "    domestic = money_to_int(domestic)\n",
    "    international = movie_revenue(soup)[1]\n",
    "    international = money_to_int(international)\n",
    "    worldwide = movie_revenue(soup)[2]\n",
    "    worldwide = money_to_int(worldwide)\n",
    "    \n",
    "    #Create movie dictionary and return\n",
    "    movie_dict = dict(zip(headers, [title_string, distributor, opening, release_date,\n",
    "                                    Days_since_release, rating, minutes, genre, theatre,\n",
    "                                    domestic, international, worldwide]))\n",
    "\n",
    "    return movie_dict\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "id": "60ef0b33-86ac-4413-ad47-06b6590d1acb",
   "metadata": {},
   "outputs": [],
   "source": [
    "movies_pages = []\n",
    "\n",
    "for link in movie_links.Link_stub:\n",
    "    movies_pages.append(get_movie_dict(link))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 516,
   "id": "0d306164-ab8f-47db-814d-583926d17516",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_2022 = pd.DataFrame(movies_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "id": "ef62856c-5461-48a4-ae02-258d64753d70",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_2021 = pd.DataFrame(movies_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "id": "049176fc-1a62-4d59-998d-66b92785191c",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_2020 = pd.DataFrame(movies_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "765b7598-bdea-4974-954f-95aa6fcf3078",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_2019 = pd.DataFrame(movies_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "ccc05215-3e8c-45f5-8ab6-fd6367dcd0fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_2018 = pd.DataFrame(movies_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "id": "8f4f63b2-8925-4ed4-a05e-4a633a462cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_2017 = pd.DataFrame(movies_pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 494,
   "id": "8eb312ff-8c0d-4f41-8940-d86fd11fe590",
   "metadata": {},
   "outputs": [],
   "source": [
    "combine = [movie_2022, movie_2021, movie_2020, movie_2019, movie_2018, movie_2017]\n",
    "movie_df = pd.concat(combine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 495,
   "id": "6fe737de-1b9b-40af-beba-ede6cb532ae2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1063 entries, 0 to 198\n",
      "Data columns (total 12 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   Title              1063 non-null   object \n",
      " 1   Distributor        1063 non-null   object \n",
      " 2   Opening            1063 non-null   object \n",
      " 3   Release_date       1063 non-null   object \n",
      " 4   In_Release         1063 non-null   object \n",
      " 5   Rating             892 non-null    object \n",
      " 6   Run_time           1034 non-null   float64\n",
      " 7   Genre              1059 non-null   object \n",
      " 8   Theatres           1050 non-null   object \n",
      " 9   Domestic_gross     1063 non-null   int64  \n",
      " 10  Internation_gross  1063 non-null   int64  \n",
      " 11  Worldwide_gross    944 non-null    float64\n",
      "dtypes: float64(2), int64(2), object(8)\n",
      "memory usage: 108.0+ KB\n"
     ]
    }
   ],
   "source": [
    "movie_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "id": "b2526aee-fdc6-4744-af59-822eea2b513f",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df.Run_time.fillna('0',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 498,
   "id": "6f31fba0-9a6f-4921-a5d8-bfce3dccf680",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df.Genre.fillna('None',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 499,
   "id": "9ec50f4c-f49a-44f3-898b-919b76832680",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df.Theatres.fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 500,
   "id": "91cf32d2-5e7a-40b6-8fb3-1be9872bd7ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df.Worldwide_gross.fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "id": "4825207a-564f-44d9-b87b-1ce3e539915c",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df.Rating.fillna('None',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "id": "63c7a424-3065-4373-bd3d-6d1d07aa236b",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_df['Theatres'] = pd.to_numeric(movie_df['Theatres'])\n",
    "movie_df['Run_time'] = pd.to_numeric(movie_df['Run_time'])\n",
    "movie_df['Opening'] = pd.to_numeric(movie_df['Opening'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "id": "80737283-57ab-4704-bd51-eb4763e42ba8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(movie_df)\n",
    "df.to_csv('movie_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686854af-e99a-4aa7-a31b-acced3903996",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}