FlightScraper python bot for kayak.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import sleep, strftime\n",
    "from random import randint\n",
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import smtplib\n",
    "from email.mime.multipart import MIMEMultipart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "chromedriver_path = 'C:/{YOUR PATH HERE}/chromedriver_win32/chromedriver.exe' # Change this to your own chromedriver path!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome(executable_path=chromedriver_path)\n",
    "sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "kayak = 'https://www.kayak.com/flights/LIS-SIN/2019-07-29-flexible/2019-08-15-flexible?sort=bestflight_a'\n",
    "driver.get(kayak)\n",
    "sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Closing the popup\n",
    "\n",
    "# xp_popup_close = '//button[contains(@id,\"dialog-close\") and contains(@class,\"Button-No-Standard-Style close \")]'\n",
    "# driver.find_elements_by_xpath(xp_popup_close)[5].click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load more results to maximize the scraping\n",
    "\n",
    "def load_more():\n",
    "    try:\n",
    "        more_results = '//a[@class = \"moreButton\"]'\n",
    "        driver.find_element_by_xpath(more_results).click()\n",
    "        print('sleeping.....')\n",
    "        sleep(randint(25,35))\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def start_kayak(city_from, city_to, date_start, date_end):\n",
    "    \"\"\"City codes - it's the IATA codes!\n",
    "    Date format -  YYYY-MM-DD\"\"\"\n",
    "    \n",
    "    kayak = ('https://www.kayak.com/flights/' + city_from + '-' + city_to +\n",
    "             '/' + date_start + '-flexible/' + date_end + '-flexible?sort=bestflight_a')\n",
    "    driver.get(kayak)\n",
    "    sleep(randint(8,10))\n",
    "    \n",
    "    # sometimes a popup shows up, so we can use a try statement to check it and close\n",
    "    try:\n",
    "        xp_popup_close = '//button[contains(@id,\"dialog-close\") and contains(@class,\"Button-No-Standard-Style close \")]'\n",
    "        driver.find_elements_by_xpath(xp_popup_close)[5].click()\n",
    "    except Exception as e:\n",
    "        pass\n",
    "    sleep(randint(60,95))\n",
    "    print('loading more.....')\n",
    "    \n",
    "#     load_more()\n",
    "    \n",
    "    print('starting first scrape.....')\n",
    "    df_flights_best = page_scrape()\n",
    "    df_flights_best['sort'] = 'best'\n",
    "    sleep(randint(60,80))\n",
    "    \n",
    "    # Let's also get the lowest prices from the matrix on top\n",
    "    matrix = driver.find_elements_by_xpath('//*[contains(@id,\"FlexMatrixCell\")]')\n",
    "    matrix_prices = [price.text.replace('$','') for price in matrix]\n",
    "    matrix_prices = list(map(int, matrix_prices))\n",
    "    matrix_min = min(matrix_prices)\n",
    "    matrix_avg = sum(matrix_prices)/len(matrix_prices)\n",
    "    \n",
    "    print('switching to cheapest results.....')\n",
    "    cheap_results = '//a[@data-code = \"price\"]'\n",
    "    driver.find_element_by_xpath(cheap_results).click()\n",
    "    sleep(randint(60,90))\n",
    "    print('loading more.....')\n",
    "    \n",
    "#     load_more()\n",
    "    \n",
    "    print('starting second scrape.....')\n",
    "    df_flights_cheap = page_scrape()\n",
    "    df_flights_cheap['sort'] = 'cheap'\n",
    "    sleep(randint(60,80))\n",
    "    \n",
    "    print('switching to quickest results.....')\n",
    "    quick_results = '//a[@data-code = \"duration\"]'\n",
    "    driver.find_element_by_xpath(quick_results).click()  \n",
    "    sleep(randint(60,90))\n",
    "    print('loading more.....')\n",
    "    \n",
    "#     load_more()\n",
    "    \n",
    "    print('starting third scrape.....')\n",
    "    df_flights_fast = page_scrape()\n",
    "    df_flights_fast['sort'] = 'fast'\n",
    "    sleep(randint(60,80))\n",
    "    \n",
    "    # saving a new dataframe as an excel file. the name is custom made to your cities and dates\n",
    "    final_df = df_flights_cheap.append(df_flights_best).append(df_flights_fast)\n",
    "    final_df.to_excel('search_backups//{}_flights_{}-{}_from_{}_to_{}.xlsx'.format(strftime(\"%Y%m%d-%H%M\"),\n",
    "                                                                                   city_from, city_to, \n",
    "                                                                                   date_start, date_end), index=False)\n",
    "    print('saved df.....')\n",
    "    \n",
    "    # We can keep track of what they predict and how it actually turns out!\n",
    "    xp_loading = '//div[contains(@id,\"advice\")]'\n",
    "    loading = driver.find_element_by_xpath(xp_loading).text\n",
    "    xp_prediction = '//span[@class=\"info-text\"]'\n",
    "    prediction = driver.find_element_by_xpath(xp_prediction).text\n",
    "    print(loading+'\\n'+prediction)\n",
    "    \n",
    "    # sometimes we get this string in the loading variable, which will conflict with the email we send later\n",
    "    # just change it to \"Not Sure\" if it happens\n",
    "    weird = '¯\\\\_(ツ)_/¯'\n",
    "    if loading == weird:\n",
    "        loading = 'Not sure'\n",
    "    \n",
    "    username = 'YOUREMAIL@hotmail.com'\n",
    "    password = 'YOUR PASSWORD'\n",
    "\n",
    "    server = smtplib.SMTP('smtp.outlook.com', 587)\n",
    "    server.ehlo()\n",
    "    server.starttls()\n",
    "    server.login(username, password)\n",
    "    msg = ('Subject: Flight Scraper\\n\\n\\\n",
    "Cheapest Flight: {}\\nAverage Price: {}\\n\\nRecommendation: {}\\n\\nEnd of message'.format(matrix_min, matrix_avg, (loading+'\\n'+prediction)))\n",
    "    message = MIMEMultipart()\n",
    "    message['From'] = 'YOUREMAIL@hotmail.com'\n",
    "    message['to'] = 'YOUROTHEREMAIL@domain.com'\n",
    "    server.sendmail('YOUREMAIL@hotmail.com', 'YOUROTHEREMAIL@domain.com', msg)\n",
    "    print('sent email.....')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def page_scrape():\n",
    "    \"\"\"This function takes care of the scraping part\"\"\"\n",
    "    \n",
    "    xp_sections = '//*[@class=\"section duration\"]'\n",
    "    sections = driver.find_elements_by_xpath(xp_sections)\n",
    "    sections_list = [value.text for value in sections]\n",
    "    section_a_list = sections_list[::2] # This is to separate the two flights\n",
    "    section_b_list = sections_list[1::2] # This is to separate the two flights\n",
    "    \n",
    "    # if you run into a reCaptcha, you might want to do something about it\n",
    "    # you will know there's a problem if the lists above are empty\n",
    "    # this if statement lets you exit the bot or do something else\n",
    "    # you can add a sleep here, to let you solve the captcha and continue scraping\n",
    "    # i'm using a SystemExit because i want to test everything from the start\n",
    "    if section_a_list == []:\n",
    "        raise SystemExit\n",
    "    \n",
    "    # I'll use the letter A for the outbound flight and B for the inbound\n",
    "    a_duration = []\n",
    "    a_section_names = []\n",
    "    for n in section_a_list:\n",
    "        # Separate the time from the cities\n",
    "        a_section_names.append(''.join(n.split()[2:5]))\n",
    "        a_duration.append(''.join(n.split()[0:2]))\n",
    "    b_duration = []\n",
    "    b_section_names = []\n",
    "    for n in section_b_list:\n",
    "        # Separate the time from the cities\n",
    "        b_section_names.append(''.join(n.split()[2:5]))\n",
    "        b_duration.append(''.join(n.split()[0:2]))\n",
    "\n",
    "    xp_dates = '//div[@class=\"section date\"]'\n",
    "    dates = driver.find_elements_by_xpath(xp_dates)\n",
    "    dates_list = [value.text for value in dates]\n",
    "    a_date_list = dates_list[::2]\n",
    "    b_date_list = dates_list[1::2]\n",
    "    # Separating the weekday from the day\n",
    "    a_day = [value.split()[0] for value in a_date_list]\n",
    "    a_weekday = [value.split()[1] for value in a_date_list]\n",
    "    b_day = [value.split()[0] for value in b_date_list]\n",
    "    b_weekday = [value.split()[1] for value in b_date_list]\n",
    "    \n",
    "    # getting the prices\n",
    "    xp_prices = '//a[@class=\"booking-link\"]/span[@class=\"price option-text\"]'\n",
    "    prices = driver.find_elements_by_xpath(xp_prices)\n",
    "    prices_list = [price.text.replace('$','') for price in prices if price.text != '']\n",
    "    prices_list = list(map(int, prices_list))\n",
    "\n",
    "    # the stops are a big list with one leg on the even index and second leg on odd index\n",
    "    xp_stops = '//div[@class=\"section stops\"]/div[1]'\n",
    "    stops = driver.find_elements_by_xpath(xp_stops)\n",
    "    stops_list = [stop.text[0].replace('n','0') for stop in stops]\n",
    "    a_stop_list = stops_list[::2]\n",
    "    b_stop_list = stops_list[1::2]\n",
    "\n",
    "    xp_stops_cities = '//div[@class=\"section stops\"]/div[2]'\n",
    "    stops_cities = driver.find_elements_by_xpath(xp_stops_cities)\n",
    "    stops_cities_list = [stop.text for stop in stops_cities]\n",
    "    a_stop_name_list = stops_cities_list[::2]\n",
    "    b_stop_name_list = stops_cities_list[1::2]\n",
    "    \n",
    "    # this part gets me the airline company and the departure and arrival times, for both legs\n",
    "    xp_schedule = '//div[@class=\"section times\"]'\n",
    "    schedules = driver.find_elements_by_xpath(xp_schedule)\n",
    "    hours_list = []\n",
    "    carrier_list = []\n",
    "    for schedule in schedules:\n",
    "        hours_list.append(schedule.text.split('\\n')[0])\n",
    "        carrier_list.append(schedule.text.split('\\n')[1])\n",
    "    # split the hours and carriers, between a and b legs\n",
    "    a_hours = hours_list[::2]\n",
    "    a_carrier = carrier_list[::2]\n",
    "    b_hours = hours_list[1::2]\n",
    "    b_carrier = carrier_list[1::2]\n",
    "\n",
    "    \n",
    "    cols = (['Out Day', 'Out Time', 'Out Weekday', 'Out Airline', 'Out Cities', 'Out Duration', 'Out Stops', 'Out Stop Cities',\n",
    "            'Return Day', 'Return Time', 'Return Weekday', 'Return Airline', 'Return Cities', 'Return Duration', 'Return Stops', 'Return Stop Cities',\n",
    "            'Price'])\n",
    "\n",
    "    flights_df = pd.DataFrame({'Out Day': a_day,\n",
    "                               'Out Weekday': a_weekday,\n",
    "                               'Out Duration': a_duration,\n",
    "                               'Out Cities': a_section_names,\n",
    "                               'Return Day': b_day,\n",
    "                               'Return Weekday': b_weekday,\n",
    "                               'Return Duration': b_duration,\n",
    "                               'Return Cities': b_section_names,\n",
    "                               'Out Stops': a_stop_list,\n",
    "                               'Out Stop Cities': a_stop_name_list,\n",
    "                               'Return Stops': b_stop_list,\n",
    "                               'Return Stop Cities': b_stop_name_list,\n",
    "                               'Out Time': a_hours,\n",
    "                               'Out Airline': a_carrier,\n",
    "                               'Return Time': b_hours,\n",
    "                               'Return Airline': b_carrier,                           \n",
    "                               'Price': prices_list})[cols]\n",
    "    \n",
    "    flights_df['timestamp'] = strftime(\"%Y%m%d-%H%M\") # so we can know when it was scraped\n",
    "    return flights_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading more.....\n",
      "starting first scrape.....\n",
      "switching to cheapest results.....\n",
      "loading more.....\n",
      "starting second scrape.....\n",
      "switching to quickest results.....\n",
      "loading more.....\n",
      "starting third scrape.....\n",
      "saved df.....\n",
      "¯\\_(ツ)_/¯\n",
      "We're still gathering data for this route\n",
      "sent email.....\n",
      "iteration 0 was complete @ 20190502-2339\n",
      "sleep finished.....\n",
      "loading more.....\n",
      "starting first scrape.....\n",
      "switching to cheapest results.....\n",
      "loading more.....\n",
      "starting second scrape.....\n",
      "switching to quickest results.....\n",
      "loading more.....\n",
      "starting third scrape.....\n",
      "saved df.....\n",
      "¯\\_(ツ)_/¯\n",
      "We're still gathering data for this route\n",
      "sent email.....\n",
      "iteration 1 was complete @ 20190503-0046\n",
      "sleep finished.....\n"
     ]
    }
   ],
   "source": [
    "city_from = input('From which city? ')\n",
    "city_to = input('Where to? ')\n",
    "date_start = input('Search around which departure date? Please use YYYY-MM-DD format only ')\n",
    "date_end = input('Return when? Please use YYYY-MM-DD format only ')\n",
    "\n",
    "# city_from = 'LIS'\n",
    "# city_to = 'SIN'\n",
    "# date_start = '2019-08-21'\n",
    "# date_end = '2019-09-07'\n",
    "\n",
    "for n in range(0,5):\n",
    "    start_kayak(city_from, city_to, date_start, date_end)\n",
    "    print('iteration {} was complete @ {}'.format(n, strftime(\"%Y%m%d-%H%M\")))\n",
    "    \n",
    "    # Wait 4 hours\n",
    "    sleep(60*60*4)\n",
    "    print('sleep finished.....')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Bonus: save a screenshot!\n",
    "driver.save_screenshot('pythonscraping.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.quit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}