HW1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_jan = pd.read_parquet('C:/Users/monal/Downloads/yellow_tripdata_2022-01.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>tip_amount</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>airport_fee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2022-01-01 00:35:40</td>\n",
       "      <td>2022-01-01 00:53:29</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.80</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>142</td>\n",
       "      <td>236</td>\n",
       "      <td>1</td>\n",
       "      <td>14.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.65</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>21.95</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2022-01-01 00:33:43</td>\n",
       "      <td>2022-01-01 00:42:07</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.10</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>236</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>13.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2022-01-01 00:53:21</td>\n",
       "      <td>2022-01-01 01:02:19</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.97</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>166</td>\n",
       "      <td>166</td>\n",
       "      <td>1</td>\n",
       "      <td>7.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.76</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>10.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2022-01-01 00:25:21</td>\n",
       "      <td>2022-01-01 00:35:23</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.09</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>114</td>\n",
       "      <td>68</td>\n",
       "      <td>2</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>11.80</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>2022-01-01 00:36:48</td>\n",
       "      <td>2022-01-01 01:14:20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.30</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>68</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>23.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>30.30</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "0         1  2022-01-01 00:35:40   2022-01-01 00:53:29              2.0   \n",
       "1         1  2022-01-01 00:33:43   2022-01-01 00:42:07              1.0   \n",
       "2         2  2022-01-01 00:53:21   2022-01-01 01:02:19              1.0   \n",
       "3         2  2022-01-01 00:25:21   2022-01-01 00:35:23              1.0   \n",
       "4         2  2022-01-01 00:36:48   2022-01-01 01:14:20              1.0   \n",
       "\n",
       "   trip_distance  RatecodeID store_and_fwd_flag  PULocationID  DOLocationID  \\\n",
       "0           3.80         1.0                  N           142           236   \n",
       "1           2.10         1.0                  N           236            42   \n",
       "2           0.97         1.0                  N           166           166   \n",
       "3           1.09         1.0                  N           114            68   \n",
       "4           4.30         1.0                  N            68           163   \n",
       "\n",
       "   payment_type  fare_amount  extra  mta_tax  tip_amount  tolls_amount  \\\n",
       "0             1         14.5    3.0      0.5        3.65           0.0   \n",
       "1             1          8.0    0.5      0.5        4.00           0.0   \n",
       "2             1          7.5    0.5      0.5        1.76           0.0   \n",
       "3             2          8.0    0.5      0.5        0.00           0.0   \n",
       "4             1         23.5    0.5      0.5        3.00           0.0   \n",
       "\n",
       "   improvement_surcharge  total_amount  congestion_surcharge  airport_fee  \n",
       "0                    0.3         21.95                   2.5          0.0  \n",
       "1                    0.3         13.30                   0.0          0.0  \n",
       "2                    0.3         10.56                   0.0          0.0  \n",
       "3                    0.3         11.80                   2.5          0.0  \n",
       "4                    0.3         30.30                   2.5          0.0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_jan.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_feb = pd.read_parquet('C:/Users/monal/Downloads/yellow_tripdata_2022-02.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>tip_amount</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>airport_fee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2022-02-01 00:06:58</td>\n",
       "      <td>2022-02-01 00:19:24</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.40</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>252</td>\n",
       "      <td>1</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.75</td>\n",
       "      <td>0.5</td>\n",
       "      <td>3.90</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>23.45</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2022-02-01 00:38:22</td>\n",
       "      <td>2022-02-01 00:55:55</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.40</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>41</td>\n",
       "      <td>2</td>\n",
       "      <td>21.0</td>\n",
       "      <td>1.75</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6.55</td>\n",
       "      <td>0.3</td>\n",
       "      <td>30.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2022-02-01 00:03:20</td>\n",
       "      <td>2022-02-01 00:26:59</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>200</td>\n",
       "      <td>2</td>\n",
       "      <td>35.5</td>\n",
       "      <td>1.75</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6.55</td>\n",
       "      <td>0.3</td>\n",
       "      <td>44.60</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2022-02-01 00:08:00</td>\n",
       "      <td>2022-02-01 00:28:05</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.88</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>239</td>\n",
       "      <td>200</td>\n",
       "      <td>2</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>34.80</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>2022-02-01 00:06:48</td>\n",
       "      <td>2022-02-01 00:33:07</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.16</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>138</td>\n",
       "      <td>125</td>\n",
       "      <td>1</td>\n",
       "      <td>35.5</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.5</td>\n",
       "      <td>8.11</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.3</td>\n",
       "      <td>48.66</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1.25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "0         1  2022-02-01 00:06:58   2022-02-01 00:19:24              1.0   \n",
       "1         1  2022-02-01 00:38:22   2022-02-01 00:55:55              1.0   \n",
       "2         1  2022-02-01 00:03:20   2022-02-01 00:26:59              1.0   \n",
       "3         2  2022-02-01 00:08:00   2022-02-01 00:28:05              1.0   \n",
       "4         2  2022-02-01 00:06:48   2022-02-01 00:33:07              1.0   \n",
       "\n",
       "   trip_distance  RatecodeID store_and_fwd_flag  PULocationID  DOLocationID  \\\n",
       "0           5.40         1.0                  N           138           252   \n",
       "1           6.40         1.0                  N           138            41   \n",
       "2          12.50         1.0                  N           138           200   \n",
       "3           9.88         1.0                  N           239           200   \n",
       "4          12.16         1.0                  N           138           125   \n",
       "\n",
       "   payment_type  fare_amount  extra  mta_tax  tip_amount  tolls_amount  \\\n",
       "0             1         17.0   1.75      0.5        3.90          0.00   \n",
       "1             2         21.0   1.75      0.5        0.00          6.55   \n",
       "2             2         35.5   1.75      0.5        0.00          6.55   \n",
       "3             2         28.0   0.50      0.5        0.00          3.00   \n",
       "4             1         35.5   0.50      0.5        8.11          0.00   \n",
       "\n",
       "   improvement_surcharge  total_amount  congestion_surcharge  airport_fee  \n",
       "0                    0.3         23.45                   0.0         1.25  \n",
       "1                    0.3         30.10                   0.0         1.25  \n",
       "2                    0.3         44.60                   0.0         1.25  \n",
       "3                    0.3         34.80                   2.5         0.00  \n",
       "4                    0.3         48.66                   2.5         1.25  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_feb.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Number of columns in Jan 2022 Yellow Taxi Trip data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_jan.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Standard deviation of the trips duration in Jan 2022 Yellow Taxi Trip data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46.43333333333333"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df_jan.tpep_dropoff_datetime - df_jan.tpep_pickup_datetime).std().seconds / 60"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropping outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2463931"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_jan.tpep_dropoff_datetime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_duration = df_jan.tpep_dropoff_datetime - df_jan.tpep_pickup_datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_duration = df_duration.dt.total_seconds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_duration_feb = df_feb.tpep_dropoff_datetime - df_feb.tpep_pickup_datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_duration_feb = df_duration_feb.dt.total_seconds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with_outliers = len(df_duration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_limit = 1 * 60\n",
    "upper_limit = 60 * 60\n",
    "\n",
    "df_duration_outliers_Removed = [i for i in df_duration if i >= lower_limit and i <= upper_limit]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliers_rem = len(df_duration_outliers_Removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9827547930522406"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outliers_rem / with_outliers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dimensionality after OHE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "257\n",
      "261\n"
     ]
    }
   ],
   "source": [
    "print(len(df_jan['PULocationID'].unique()))\n",
    "print(len(df_jan['DOLocationID'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = df_jan[['PULocationID', 'DOLocationID']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df_feb = df_feb[['PULocationID', 'DOLocationID']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PULocationID', 'DOLocationID'], dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "518\n"
     ]
    }
   ],
   "source": [
    "one_hot_encoded_data = pd.get_dummies(final_df, columns = ['PULocationID', 'DOLocationID'])\n",
    "print(len(one_hot_encoded_data.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "516\n"
     ]
    }
   ],
   "source": [
    "one_hot_encoded_data_feb = pd.get_dummies(final_df_feb, columns = ['PULocationID', 'DOLocationID'])\n",
    "print(len(one_hot_encoded_data_feb.columns))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Turning the dataframe into a list of dictionaries\n",
    "having memory issue so i have selected till 70639 rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "one_hot_encoded_data = one_hot_encoded_data[0:70639].to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "one_hot_encoded_data_feb = one_hot_encoded_data_feb[0:70639].to_dict('records')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit a dictionary vectorizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_vectorize = DictVectorizer(sparse=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get a feature matrix from it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = dict_vectorize.fit_transform(one_hot_encoded_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val = dict_vectorize.transform(one_hot_encoded_data_feb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70639, 518)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70639,)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train = df_duration.values[0:70639] / 60\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val = df_duration_feb.values[0:70639] / 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17.816666666666666"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12.433333333333334"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_val[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Training a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lr.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "mse_train = mean_squared_error(y_train, y_pred, squared=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.134281288015298\n"
     ]
    }
   ],
   "source": [
    "RMSE_train = math.sqrt(mse_train)\n",
    "print(RMSE_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_val = lr.predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "mse_val = mean_squared_error(y_val, y_pred_val, squared=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1117418393736.596\n"
     ]
    }
   ],
   "source": [
    "print(mse_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "RMSE_val = math.sqrt(mse_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1057080.1264504958\n"
     ]
    }
   ],
   "source": [
    "print(RMSE_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12.433333333333334 17.982421875\n"
     ]
    }
   ],
   "source": [
    "print(y_val[0], y_pred_val[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}