MisfitDemoNotebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b80dce38",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "%config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebd0d050",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import (datasets, dummy, ensemble,\n",
    "                     linear_model, metrics,\n",
    "                     model_selection as skms,\n",
    "                     naive_bayes, neighbors, tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a568038d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import (make_learning_curve, make_complexity_curve, \n",
    "                   rms_error, rmse,\n",
    "                   manage_ames_nans, manage_ames_ordinal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "646a17b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca55ef5b",
   "metadata": {},
   "source": [
    "# Part 1:  Fit-Predict-Evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1edf1c66",
   "metadata": {},
   "source": [
    "# Simple sklearn (in-sample only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47beaa0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "cah_df = pd.read_csv('datasets/cali.csv.gz')\n",
    "cah_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81e63527",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cah_df.MedHouseVal.plot(kind='hist');\n",
    "# sns.pairplot(cah_df) # slow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8edc43e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# explain why dropping\n",
    "housing_ftrs = cah_df.drop(columns=['Latitude', 'Longitude', 'MedHouseVal'])\n",
    "housing_tgt  = cah_df[['MedHouseVal']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa1b11fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "knn   = neighbors.KNeighborsRegressor(n_neighbors=3)\n",
    "fit   = knn.fit(housing_ftrs, housing_tgt)\n",
    "preds = knn.predict(housing_ftrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7135b6e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# or a \"one-liner\" (broken up for readability)\n",
    "preds = (neighbors.KNeighborsRegressor(n_neighbors=3)\n",
    "                  .fit(housing_ftrs, housing_tgt)\n",
    "                  .predict(housing_ftrs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9954fe7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ??rms_error to show source code\n",
    "rms_error(housing_tgt, preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7cce672",
   "metadata": {},
   "source": [
    "# Simple sklearn (train-test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e517cacf",
   "metadata": {},
   "outputs": [],
   "source": [
    "housing_ftrs = cah_df.drop(columns=['Latitude', 'Longitude', 'MedHouseVal'])\n",
    "housing_tgt  = cah_df[['MedHouseVal']]\n",
    "\n",
    "(train_ftrs, test_ftrs,\n",
    " train_tgt,  test_tgt) = skms.train_test_split(housing_ftrs, \n",
    "                                               housing_tgt, \n",
    "                                               test_size=.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5d0bbf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# here we separate about predictions b/c we want to use it twice\n",
    "knr = (neighbors.KNeighborsRegressor(n_neighbors=3)\n",
    "                .fit(train_ftrs, train_tgt))\n",
    "\n",
    "# predict for train and then for test\n",
    "train_preds = knr.predict(train_ftrs)\n",
    "test_preds = knr.predict(test_ftrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c3d5d40",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_rmse = rms_error(train_tgt, train_preds)\n",
    "print('train set rmse: {:0.4f}'.format(train_rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "938dfcbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_rmse = rms_error(test_tgt, test_preds)\n",
    "print('test set rmse: {:0.4f}'.format(test_rmse))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ca20c1d",
   "metadata": {},
   "source": [
    "# Part 2:  Comparing Models on TTS"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b112aba",
   "metadata": {},
   "source": [
    "# Baseline `Predict-the-Mean` Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8450fdea",
   "metadata": {},
   "outputs": [],
   "source": [
    "base = (dummy.DummyRegressor(strategy='mean') # default\n",
    "            .fit(train_ftrs, train_tgt))\n",
    "\n",
    "train_preds = base.predict(train_ftrs)\n",
    "test_preds  = base.predict(test_ftrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4edf3bd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_rmse = rms_error(train_tgt, train_preds)\n",
    "print('in-sample train rmse: {:0.4f}'.format(train_rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf45ae88",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_rmse = rms_error(test_tgt, test_preds)\n",
    "print('test rmse: {:0.4f}'.format(test_rmse))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a77c829",
   "metadata": {},
   "source": [
    "# Two Different Nearest Neighbors Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ecb4738",
   "metadata": {},
   "outputs": [],
   "source": [
    "knr = (neighbors.KNeighborsRegressor(n_neighbors=3)\n",
    "                .fit(train_ftrs, train_tgt))\n",
    "\n",
    "train_preds = knr.predict(train_ftrs)\n",
    "test_preds = knr.predict(test_ftrs)\n",
    "\n",
    "train_rmse = rms_error(train_tgt, train_preds)\n",
    "test_rmse = rms_error(test_tgt, test_preds)\n",
    "\n",
    "print('kNN(k={:2d}) train/test rmse: {:0.4f} {:0.4f}'.format(3, \n",
    "                                                             train_rmse, \n",
    "                                                             test_rmse))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "420e1baf",
   "metadata": {},
   "outputs": [],
   "source": [
    "knr = (neighbors.KNeighborsRegressor(n_neighbors=10)\n",
    "                .fit(train_ftrs, train_tgt))\n",
    "\n",
    "train_preds = knr.predict(train_ftrs)\n",
    "test_preds = knr.predict(test_ftrs)\n",
    "\n",
    "train_rmse = rms_error(train_tgt, train_preds)\n",
    "test_rmse = rms_error(test_tgt, test_preds)\n",
    "\n",
    "print('kNN(k={:2d}) train/test rmse: {:0.4f} {:0.4f}'.format(10, \n",
    "                                                             train_rmse, \n",
    "                                                             test_rmse))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a3c4200",
   "metadata": {},
   "source": [
    "# Two Decision Tree Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77ab9709",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtr = (tree.DecisionTreeRegressor(max_depth=1)\n",
    "           .fit(train_ftrs, train_tgt))\n",
    "\n",
    "train_preds = dtr.predict(train_ftrs)\n",
    "test_preds  = dtr.predict(test_ftrs)\n",
    "\n",
    "train_rmse = rms_error(train_tgt, train_preds)\n",
    "test_rmse  = rms_error(test_tgt,  test_preds)\n",
    "\n",
    "print('DT-R(depth={:1d}) train/test rmse: {:0.4f} {:0.4f}'.format(1, \n",
    "                                                                  train_rmse, \n",
    "                                                                  test_rmse))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c62e905f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtr = (tree.DecisionTreeRegressor(max_depth=3)\n",
    "           .fit(train_ftrs, train_tgt))\n",
    "\n",
    "train_preds = dtr.predict(train_ftrs)\n",
    "test_preds  = dtr.predict(test_ftrs)\n",
    "\n",
    "train_rmse = rms_error(train_tgt, train_preds)\n",
    "test_rmse  = rms_error(test_tgt,  test_preds)\n",
    "\n",
    "print('DT-R(depth={:1d}) train/test rmse: {:0.4f} {:0.4f}'.format(3, \n",
    "                                                                  train_rmse, \n",
    "                                                                  test_rmse))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f11fb9c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# other model possibilities:\n",
    "# https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "167133ad",
   "metadata": {},
   "source": [
    "# Part 3:  Cross-Validation and Model Choice"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f97c164",
   "metadata": {},
   "source": [
    "### Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "925a30cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "seq = np.array(list('abcdefghi'))\n",
    "print(seq)\n",
    "print()\n",
    "\n",
    "# warning:  here we are shuffling.  see sk docs for more details.\n",
    "# https://scikit-learn.org/stable/common_pitfalls.html#randomness\n",
    "three_fold = KFold(n_splits=3, shuffle=True)\n",
    "for train, test in three_fold.split(seq):\n",
    "    print('train:', seq[train])\n",
    "    print('\\ttest:', seq[test])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3764218b",
   "metadata": {},
   "source": [
    "### Evaluate with Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81bbf958",
   "metadata": {},
   "outputs": [],
   "source": [
    "# default cv=None means do 5-fold CV\n",
    "# train->test->score five times\n",
    "skms.cross_val_score(neighbors.KNeighborsRegressor(n_neighbors=3), \n",
    "                     housing_ftrs, \n",
    "                     housing_tgt, \n",
    "                     scoring=rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf7a1079",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [dummy.DummyRegressor(strategy='mean'),\n",
    "          neighbors.KNeighborsRegressor(n_neighbors=3),\n",
    "          neighbors.KNeighborsRegressor(n_neighbors=10),\n",
    "          tree.DecisionTreeRegressor(max_depth=1),\n",
    "          tree.DecisionTreeRegressor(max_depth=3)]\n",
    "\n",
    "for model in models:\n",
    "    cvs = skms.cross_val_score(model, housing_ftrs, housing_tgt, scoring=rmse)\n",
    "    print(model, \n",
    "          # cvs, \n",
    "          'mean ~ std: {:.3f} ~ {:.3f}'.format(cvs.mean(), \n",
    "                                               cvs.std()), \n",
    "          sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bffb274",
   "metadata": {},
   "source": [
    "### A Good k for Nearest Neighbors (Complexity Curves)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27e255cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1,1)\n",
    "knn = neighbors.KNeighborsRegressor\n",
    "\n",
    "n_neighbors = range(11)\n",
    "make_complexity_curve(knn(), \"KNN\", \n",
    "                      'n_neighbors', n_neighbors, \n",
    "                      housing_ftrs, housing_tgt, \n",
    "                      ax=ax);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c8072f4",
   "metadata": {},
   "source": [
    "### Does More Data Help?  (Learning Curves)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b9bebb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# learning curves tell us what happens as we use more and more data to train\n",
    "# notes:  1. the data is CV-split first and then 2. fractions of that data are used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcc94eab",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1,1)\n",
    "knn = neighbors.KNeighborsRegressor\n",
    "make_learning_curve(knn(n_neighbors=5), \"KNR(5)\", \n",
    "                    housing_ftrs, housing_tgt, \n",
    "                    ax=ax);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4df2a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "# and for decision trees\n",
    "fig, axes = plt.subplots(2,1)\n",
    "dtr = tree.DecisionTreeRegressor\n",
    "\n",
    "max_depth = range(1,11)\n",
    "make_complexity_curve(dtr(), \"DT-R\", 'max_depth', max_depth, \n",
    "                      housing_ftrs, housing_tgt, ax=axes[0])\n",
    "\n",
    "make_learning_curve(dtr(max_depth=2), \"DT-R(6)\", housing_ftrs, housing_tgt, ax=axes[1])\n",
    "fig.tight_layout();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "456c20ef",
   "metadata": {},
   "source": [
    "# Part 4:  Improving our Fits"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46546fb1",
   "metadata": {},
   "source": [
    "### We'll Be Doing This a Lot:  Function-ify!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "306480b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_two_graphs(model, hyper_name, \n",
    "                  hyper_values, hyper_focus):\n",
    "    ' produce two plots for given model, hypers, on housing_ftrs/tgt'\n",
    "    fig, axes = plt.subplots(1,2, figsize=(12,3), sharey=True)\n",
    "    name = model.__name__\n",
    "    args = {hyper_name:hyper_focus}\n",
    "    \n",
    "    make_complexity_curve(model(), name, \n",
    "                          hyper_name, hyper_values, \n",
    "                          housing_ftrs, housing_tgt, ax=axes[0])\n",
    "    label = \"{}({})\".format(name, hyper_focus)\n",
    "    make_learning_curve(model(**args), label, \n",
    "                        housing_ftrs, housing_tgt, ax=axes[1])\n",
    "    fig.tight_layout()\n",
    "    \n",
    "    cvs = skms.cross_val_score(model(**args), \n",
    "                           housing_ftrs, housing_tgt, \n",
    "                           cv=5, scoring=rmse)\n",
    "    print('mean CV(5) RMSE for {} {:0.3f}'.format(label, cvs.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d91463b6",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "do_two_graphs(neighbors.KNeighborsRegressor, 'n_neighbors', range(11), 5)\n",
    "do_two_graphs(tree.DecisionTreeRegressor, 'max_depth', range(1,11), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fd7b0ba",
   "metadata": {},
   "source": [
    "### Now to Improve!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b316662d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FIXME/TODO:  \n",
    "# so, how to handle these modifications with Cali if i don't want to muck with different sets (!?!)\n",
    "\n",
    "# not going to use different feature sets\n",
    "# add notes for students to try with:\n",
    "# (1) more features\n",
    "# (2) more examples\n",
    "# (3) both (more features and more examples!)\n",
    "\n",
    "# i definitely want to demo selecting features\n",
    "# (using feature importances)\n",
    "\n",
    "# ask students to train on all data in a csv then evluate on a \n",
    "# separate HOT file\n",
    "\n",
    "\n",
    "# so, i'm going to demo necessary code on Cali and students will have that for example code.\n",
    "#     students can:\n",
    "#     (1) rerun/experiment with code on Cali\n",
    "#     (2) modify code (\"directly\") for Ames\n",
    "#     (3) modify code for Ames and apply additional techniques \n",
    "#         (other learners, preprocessing, etc.)\n",
    "\n",
    "# so my presentations will be:\n",
    "# (1) conceptual\n",
    "# (2) Cali code\n",
    "# (3) [after they work on exercises]\n",
    "#     (some) amounts of Ames code. \n",
    "#     (which should be quick as I have it layed out in ForwardProgress)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53e8049a",
   "metadata": {},
   "source": [
    "### Let's Be Selective about our Features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e47d1eeb",
   "metadata": {},
   "source": [
    "#### Feature Importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "187ac203",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff8e3186",
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = RandomForestRegressor()\n",
    "forest.fit(housing_ftrs, housing_tgt)\n",
    "fis = forest.feature_importances_\n",
    "fis = pd.Series(fis, index=housing_ftrs.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fda9f7dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "imp_df = (pd.DataFrame({'ftr_imp':fis})\n",
    "            .sort_values(by='ftr_imp', ascending=False))\n",
    "print(len(imp_df))\n",
    "imp_df.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cacf45e3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# imp_df.plot.bar();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10ec6f77",
   "metadata": {},
   "outputs": [],
   "source": [
    "hfi = imp_df[:2].index\n",
    "hfi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feebaac5",
   "metadata": {},
   "source": [
    "#### Using \"good\" Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e973b133",
   "metadata": {},
   "outputs": [],
   "source": [
    "housing_ftrs = cah_df[hfi]\n",
    "housing_tgt  = cah_df[['MedHouseVal']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f9b8dfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "do_two_graphs(neighbors.KNeighborsRegressor, 'n_neighbors', range(11), 5)\n",
    "do_two_graphs(tree.DecisionTreeRegressor, 'max_depth', range(1,11), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f215c34",
   "metadata": {},
   "source": [
    "# Bonus: And Some Ensemble Learners"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5949e828",
   "metadata": {},
   "source": [
    "##### Boosted Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69ac0058",
   "metadata": {},
   "outputs": [],
   "source": [
    "# boosting reduces bias (allows more complexity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75c8a2d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "gbr = ensemble.GradientBoostingRegressor\n",
    "\n",
    "n_estimators = np.arange(1,10) * 10\n",
    "gs = skms.GridSearchCV(gbr(), {'n_estimators':n_estimators}, scoring=rmse, cv=5)\n",
    "gs.fit(housing_ftrs, housing_tgt)\n",
    "\n",
    "param_df = pd.DataFrame.from_records(gs.cv_results_['params'])\n",
    "param_df['mean_rmse'] = gs.cv_results_['mean_test_score']\n",
    "param_df.sort_values('mean_rmse').head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2e17965",
   "metadata": {},
   "outputs": [],
   "source": [
    "cvs = skms.cross_val_score(gbr(n_estimators=90), \n",
    "                           housing_ftrs, housing_tgt, \n",
    "                           cv=5, scoring=rmse)\n",
    "print('mean 5-fold CV RMSE:', cvs.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a0a36e1",
   "metadata": {},
   "source": [
    "##### Bagged Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01a33e88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# RFR (bagging reduces bias and variance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2288da00",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1,1, figsize=(6,3))\n",
    "rfr = ensemble.RandomForestRegressor # default = 100 estimators\n",
    "make_learning_curve(rfr(), \"RFR(default)\", housing_ftrs, housing_tgt, ax=ax)\n",
    "\n",
    "cvs = skms.cross_val_score(rfr(), \n",
    "                           housing_ftrs, housing_tgt, \n",
    "                           cv=5, scoring=rmse)\n",
    "print('mean 5-fold CV RMSE:', cvs.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a09ca17",
   "metadata": {},
   "source": [
    "# Next Steps"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dace3b3",
   "metadata": {},
   "source": [
    "  * A Book:\n",
    "    * Machine Learning with Python for Everyone (by me!)\n",
    "    * https://www.pearson.com/store/p/machine-learning-with-python-for-everyone/P200000009467/9780134845623\n",
    "  * Videos:\n",
    "    * Machine Learning in Python for Everyone (Video Collection) (by me!)\n",
    "    * https://learning.oreilly.com/videos/machine-learning-in/9780138092818/\n",
    "  * sklearn docs:\n",
    "    * https://scikit-learn.org/stable/user_guide.html"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}