ch07.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3ecef393-857c-4d5e-a340-27e5d5076f07",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chapter 7: Data Cleaning and Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1493784c-3391-4980-bf4c-34fc17ef1c4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import Series, DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a25d9a98-71da-4028-b094-2a117ff6ca0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "float_data = pd.Series([1.2, -3.5, np.nan, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b325a775-8070-42b0-a36b-af2dc1a21bb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.2\n",
       "1   -3.5\n",
       "2    NaN\n",
       "3    0.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data # NaN is a sentinel value: when present, it indicates a missing (or null) value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6b4d27c1-a733-46a3-aa74-4a5c9fe990e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2     True\n",
       "3    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "77a71e2c-160a-4222-8dea-052071aec25a",
   "metadata": {},
   "outputs": [],
   "source": [
    "string_data = pd.Series([\"aardark\", np.nan, None, \"avocado\"]) # NA refers to Missing data, which stands for not available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "4d01c52c-326d-4b3f-8774-1501ef488f33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    aardark\n",
       "1        NaN\n",
       "2       None\n",
       "3    avocado\n",
       "dtype: object"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_data # The built-in Python None value is also treated as NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "21a554a5-4f7a-48dd-a435-81565b58887b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1     True\n",
       "2     True\n",
       "3    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_data.isna() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "48ed058e-5070-4f39-b44c-3a7c2a1bd0f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "float_data = pd.Series([1, 2, None], dtype='float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "574f92f5-6228-4b5f-8eb8-05eddeb76ff9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    2.0\n",
       "2    NaN\n",
       "dtype: float64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f6403352-3bb3-4e68-820d-dcc5cbe12451",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float_data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a3aed95a-7e66-46f6-804b-1aea3f736c84",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.Series([1, np.nan, 3.5, np.nan, 7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6b68cd1c-d342-4a5b-a3f3-e0b5e7b80d86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    3.5\n",
       "3    NaN\n",
       "4    7.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "de3f9cd7-89c8-422d-8941-cebd2e56cd7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "2    3.5\n",
       "4    7.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "cb4ade28-bc01-4347-a205-ee37bba1480c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "2    3.5\n",
       "4    7.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is the same thing as doing:\n",
    "data[data.notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "69a4a3bb-236b-4fe1-ab05-0142d8a258d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame([[1., 6.5, 3.], [1., np.nan, np.nan],\n",
    "                     [np.nan, np.nan, np.nan], [np.nan, 6.5, 3.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "fbed82f6-5c6f-4f47-a316-90ee068210e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0\n",
       "1  1.0  NaN  NaN\n",
       "2  NaN  NaN  NaN\n",
       "3  NaN  6.5  3.0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e151da16-601a-40c8-8fe0-d997073db1b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "87b7c17d-053d-43be-aed0-b15f1961eacd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0\n",
       "1  1.0  NaN  NaN\n",
       "3  NaN  6.5  3.0"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(how=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "730bfd9d-db92-44e1-9366-7d943ec04b0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# These functions return new objects by default and do not modify the contents of the original object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ef87c014-1f9b-4d47-929b-368e58bba8f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[4] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d1162e9d-ed8e-4cb8-bb11-a3bbe9741098",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2   4\n",
       "0  1.0  6.5  3.0 NaN\n",
       "1  1.0  NaN  NaN NaN\n",
       "2  NaN  NaN  NaN NaN\n",
       "3  NaN  6.5  3.0 NaN"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "6bb36ac4-52b4-42d1-9f84-957d02a32a5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2\n",
       "0  1.0  6.5  3.0\n",
       "1  1.0  NaN  NaN\n",
       "2  NaN  NaN  NaN\n",
       "3  NaN  6.5  3.0"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(axis=\"columns\", how=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "ec724bc8-a0a7-4f4b-b9ad-a7a1a45abd59",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.random.standard_normal((7, 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "dd789cf8-2f95-4576-9944-0f981830c8c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.633379</td>\n",
       "      <td>-0.188013</td>\n",
       "      <td>-0.101596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.621413</td>\n",
       "      <td>0.934805</td>\n",
       "      <td>-1.436407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.245828</td>\n",
       "      <td>-0.394157</td>\n",
       "      <td>0.313533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.015369</td>\n",
       "      <td>1.679599</td>\n",
       "      <td>0.870040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.499889</td>\n",
       "      <td>-0.516297</td>\n",
       "      <td>0.242709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.078585</td>\n",
       "      <td>0.226359</td>\n",
       "      <td>0.654416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.309446</td>\n",
       "      <td>-0.705697</td>\n",
       "      <td>-1.287513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  0.633379 -0.188013 -0.101596\n",
       "1 -0.621413  0.934805 -1.436407\n",
       "2  2.245828 -0.394157  0.313533\n",
       "3 -1.015369  1.679599  0.870040\n",
       "4 -0.499889 -0.516297  0.242709\n",
       "5 -1.078585  0.226359  0.654416\n",
       "6  0.309446 -0.705697 -1.287513"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "20c672a7-6d77-4054-aa46-e39301fe958b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[:4, 1] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "fe8ba012-288e-4d6c-8c10-c5a031c31717",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.633379</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.101596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.621413</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.436407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.245828</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.313533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.015369</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.870040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.499889</td>\n",
       "      <td>-0.516297</td>\n",
       "      <td>0.242709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.078585</td>\n",
       "      <td>0.226359</td>\n",
       "      <td>0.654416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.309446</td>\n",
       "      <td>-0.705697</td>\n",
       "      <td>-1.287513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  0.633379       NaN -0.101596\n",
       "1 -0.621413       NaN -1.436407\n",
       "2  2.245828       NaN  0.313533\n",
       "3 -1.015369       NaN  0.870040\n",
       "4 -0.499889 -0.516297  0.242709\n",
       "5 -1.078585  0.226359  0.654416\n",
       "6  0.309446 -0.705697 -1.287513"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "a8d546eb-0b65-4ead-8470-79b520c66909",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[:2, 2] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "15318ad2-6065-4c24-85cc-067fc84343d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.633379</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.621413</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.245828</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.313533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.015369</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.870040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.499889</td>\n",
       "      <td>-0.516297</td>\n",
       "      <td>0.242709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.078585</td>\n",
       "      <td>0.226359</td>\n",
       "      <td>0.654416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.309446</td>\n",
       "      <td>-0.705697</td>\n",
       "      <td>-1.287513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  0.633379       NaN       NaN\n",
       "1 -0.621413       NaN       NaN\n",
       "2  2.245828       NaN  0.313533\n",
       "3 -1.015369       NaN  0.870040\n",
       "4 -0.499889 -0.516297  0.242709\n",
       "5 -1.078585  0.226359  0.654416\n",
       "6  0.309446 -0.705697 -1.287513"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "ecc76cfd-6c6e-46aa-9373-881759b2800d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.499889</td>\n",
       "      <td>-0.516297</td>\n",
       "      <td>0.242709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.078585</td>\n",
       "      <td>0.226359</td>\n",
       "      <td>0.654416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.309446</td>\n",
       "      <td>-0.705697</td>\n",
       "      <td>-1.287513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "4 -0.499889 -0.516297  0.242709\n",
       "5 -1.078585  0.226359  0.654416\n",
       "6  0.309446 -0.705697 -1.287513"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "0706adcc-2d12-447d-93da-aac146d27b13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.245828</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.313533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.015369</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.870040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.499889</td>\n",
       "      <td>-0.516297</td>\n",
       "      <td>0.242709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.078585</td>\n",
       "      <td>0.226359</td>\n",
       "      <td>0.654416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.309446</td>\n",
       "      <td>-0.705697</td>\n",
       "      <td>-1.287513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "2  2.245828       NaN  0.313533\n",
       "3 -1.015369       NaN  0.870040\n",
       "4 -0.499889 -0.516297  0.242709\n",
       "5 -1.078585  0.226359  0.654416\n",
       "6  0.309446 -0.705697 -1.287513"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna(thresh=2) # Keep only rows containing at most a certain number of missing observations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "c88c2acc-9bc9-4fa8-ba46-b0b7563fbef2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.633379</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.621413</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.245828</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.313533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.015369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.870040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.499889</td>\n",
       "      <td>-0.516297</td>\n",
       "      <td>0.242709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.078585</td>\n",
       "      <td>0.226359</td>\n",
       "      <td>0.654416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.309446</td>\n",
       "      <td>-0.705697</td>\n",
       "      <td>-1.287513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  0.633379  0.000000  0.000000\n",
       "1 -0.621413  0.000000  0.000000\n",
       "2  2.245828  0.000000  0.313533\n",
       "3 -1.015369  0.000000  0.870040\n",
       "4 -0.499889 -0.516297  0.242709\n",
       "5 -1.078585  0.226359  0.654416\n",
       "6  0.309446 -0.705697 -1.287513"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "9b90310e-67b2-4b5d-a9ff-653ff20aaa48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.633379</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.621413</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.245828</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.313533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.015369</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.870040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.499889</td>\n",
       "      <td>-0.516297</td>\n",
       "      <td>0.242709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.078585</td>\n",
       "      <td>0.226359</td>\n",
       "      <td>0.654416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.309446</td>\n",
       "      <td>-0.705697</td>\n",
       "      <td>-1.287513</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0  0.633379  0.500000  0.000000\n",
       "1 -0.621413  0.500000  0.000000\n",
       "2  2.245828  0.500000  0.313533\n",
       "3 -1.015369  0.500000  0.870040\n",
       "4 -0.499889 -0.516297  0.242709\n",
       "5 -1.078585  0.226359  0.654416\n",
       "6  0.309446 -0.705697 -1.287513"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna({1: 0.5, 2: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "a977ee96-4023-4a4f-b7db-b059e7483d67",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.random.standard_normal((6, 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "1c5c59a0-e6c1-448f-868a-6d70d6a97b2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.708622</td>\n",
       "      <td>-0.358532</td>\n",
       "      <td>0.012731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.163665</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>0.690906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.565765</td>\n",
       "      <td>0.061619</td>\n",
       "      <td>-0.085516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.278933</td>\n",
       "      <td>-0.066694</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.265632</td>\n",
       "      <td>0.747721</td>\n",
       "      <td>0.333549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.911526</td>\n",
       "      <td>0.256441</td>\n",
       "      <td>0.034631</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -0.708622 -0.358532  0.012731\n",
       "1 -0.163665  0.138102  0.690906\n",
       "2 -0.565765  0.061619 -0.085516\n",
       "3  0.278933 -0.066694 -0.136896\n",
       "4  1.265632  0.747721  0.333549\n",
       "5 -0.911526  0.256441  0.034631"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "823adb12-c6f4-4492-9283-0af57ce15232",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[2:, 1] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "3b4a5b8e-8a37-41d1-9d28-4dc78ba792c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.708622</td>\n",
       "      <td>-0.358532</td>\n",
       "      <td>0.012731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.163665</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>0.690906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.565765</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.085516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.278933</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.265632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.333549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.911526</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.034631</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -0.708622 -0.358532  0.012731\n",
       "1 -0.163665  0.138102  0.690906\n",
       "2 -0.565765       NaN -0.085516\n",
       "3  0.278933       NaN -0.136896\n",
       "4  1.265632       NaN  0.333549\n",
       "5 -0.911526       NaN  0.034631"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "bc2bf77b-8e95-4f6a-8763-a552ea59041e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.iloc[4:, 2] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "8e21b86a-475c-4da9-b5a1-d1efcf64aacf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.708622</td>\n",
       "      <td>-0.358532</td>\n",
       "      <td>0.012731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.163665</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>0.690906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.565765</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.085516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.278933</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.265632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.911526</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -0.708622 -0.358532  0.012731\n",
       "1 -0.163665  0.138102  0.690906\n",
       "2 -0.565765       NaN -0.085516\n",
       "3  0.278933       NaN -0.136896\n",
       "4  1.265632       NaN       NaN\n",
       "5 -0.911526       NaN       NaN"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "8f4b1e13-571e-407d-9b91-fdda74d6237d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\TEMP\\ipykernel_13520\\2535960854.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n",
      "  df.fillna(method=\"ffill\") # forward fill\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.708622</td>\n",
       "      <td>-0.358532</td>\n",
       "      <td>0.012731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.163665</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>0.690906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.565765</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>-0.085516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.278933</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.265632</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.911526</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -0.708622 -0.358532  0.012731\n",
       "1 -0.163665  0.138102  0.690906\n",
       "2 -0.565765  0.138102 -0.085516\n",
       "3  0.278933  0.138102 -0.136896\n",
       "4  1.265632  0.138102 -0.136896\n",
       "5 -0.911526  0.138102 -0.136896"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(method=\"ffill\") # forward fill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "5ee40860-b9f5-4e25-b1e3-26dc4e90d409",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\TEMP\\ipykernel_13520\\1627181726.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n",
      "  df.fillna(method=\"ffill\", limit=2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.708622</td>\n",
       "      <td>-0.358532</td>\n",
       "      <td>0.012731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.163665</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>0.690906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.565765</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>-0.085516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.278933</td>\n",
       "      <td>0.138102</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.265632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.911526</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.136896</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2\n",
       "0 -0.708622 -0.358532  0.012731\n",
       "1 -0.163665  0.138102  0.690906\n",
       "2 -0.565765  0.138102 -0.085516\n",
       "3  0.278933  0.138102 -0.136896\n",
       "4  1.265632       NaN -0.136896\n",
       "5 -0.911526       NaN -0.136896"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(method=\"ffill\", limit=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "6d470cd6-d278-4bf5-81ec-cbbdbf52e846",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.Series([1., np.nan, 3.5, np.nan, 7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "671993c2-2387-4903-afd5-f83a3df4d745",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    3.5\n",
       "3    NaN\n",
       "4    7.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "bf9571b4-1a8a-49a8-b47b-46a7e577ed2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.000000\n",
       "1    3.833333\n",
       "2    3.500000\n",
       "3    3.833333\n",
       "4    7.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.fillna(data.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "792192ff-9f12-4b53-93ec-38823295c144",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame({\"k1\": [\"one\", \"two\"] * 3 + [\"two\"],\n",
    "                     \"k2\": [1, 1, 2, 3, 3, 4, 4]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "852716b0-9675-499d-b1b9-a73b4a55a161",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2\n",
       "0  one   1\n",
       "1  two   1\n",
       "2  one   2\n",
       "3  two   3\n",
       "4  one   3\n",
       "5  two   4\n",
       "6  two   4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ca2f94b4-4a55-4c21-b01c-36d593120cba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3    False\n",
       "4    False\n",
       "5    False\n",
       "6     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9296d0ab-c86f-40f0-bdef-7e4ab427c23c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2\n",
       "0  one   1\n",
       "1  two   1\n",
       "2  one   2\n",
       "3  two   3\n",
       "4  one   3\n",
       "5  two   4"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "751bcfaa-2dad-40b0-adce-713419f76b9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"v1\"] = range(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e84d1852-d765-4ef1-8a35-45cbec6ce8eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th>v1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2  v1\n",
       "0  one   1   0\n",
       "1  two   1   1\n",
       "2  one   2   2\n",
       "3  two   3   3\n",
       "4  one   3   4\n",
       "5  two   4   5\n",
       "6  two   4   6"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ec2e14e8-7a90-464b-a546-b74e329f46cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th>v1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2  v1\n",
       "0  one   1   0\n",
       "1  two   1   1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates(subset=[\"k1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "120db2e6-45f8-4fc7-aed4-21971c0350ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k1</th>\n",
       "      <th>k2</th>\n",
       "      <th>v1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>two</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>one</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>two</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>one</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>two</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k1  k2  v1\n",
       "0  one   1   0\n",
       "1  two   1   1\n",
       "2  one   2   2\n",
       "3  two   3   3\n",
       "4  one   3   4\n",
       "6  two   4   6"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# \"duplicated\" and \"drop_duplicates\" by default keep the first observed value combi|nation. \n",
    "# Passing keep=\"last\" will return the last one\n",
    "data.drop_duplicates([\"k1\", \"k2\"], keep=\"last\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7a52e357-33cd-4040-a1ee-17fa4aede18d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame({\"food\": [\"bacon\", \"pulled pork\", \"bacon\",\n",
    "                              \"pastrami\", \"corned beef\", \"bacon\",\n",
    "                              \"pastrami\", \"honey ham\", \"nova lox\"],\n",
    "                     \"ounces\": [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "13c1ec77-3308-4742-b5af-870fab6d1552",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>food</th>\n",
       "      <th>ounces</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bacon</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pulled pork</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bacon</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>corned beef</td>\n",
       "      <td>7.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bacon</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>honey ham</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>nova lox</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          food  ounces\n",
       "0        bacon     4.0\n",
       "1  pulled pork     3.0\n",
       "2        bacon    12.0\n",
       "3     pastrami     6.0\n",
       "4  corned beef     7.5\n",
       "5        bacon     8.0\n",
       "6     pastrami     3.0\n",
       "7    honey ham     5.0\n",
       "8     nova lox     6.0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "39e366fa-eaf1-4088-885c-8c6991af30ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "meat_to_animal = {\n",
    "  \"bacon\": \"pig\",\n",
    "  \"pulled pork\": \"pig\",\n",
    "  \"pastrami\": \"cow\",\n",
    "  \"corned beef\": \"cow\",\n",
    "  \"honey ham\": \"pig\",\n",
    "  \"nova lox\": \"salmon\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d3d51235-c604-48fa-9c39-f5daf3f941a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"animal\"] = data[\"food\"].map(meat_to_animal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8d2628c3-b354-4147-b0cf-e4d979183319",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>food</th>\n",
       "      <th>ounces</th>\n",
       "      <th>animal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bacon</td>\n",
       "      <td>4.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pulled pork</td>\n",
       "      <td>3.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bacon</td>\n",
       "      <td>12.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>6.0</td>\n",
       "      <td>cow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>corned beef</td>\n",
       "      <td>7.5</td>\n",
       "      <td>cow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bacon</td>\n",
       "      <td>8.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pastrami</td>\n",
       "      <td>3.0</td>\n",
       "      <td>cow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>honey ham</td>\n",
       "      <td>5.0</td>\n",
       "      <td>pig</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>nova lox</td>\n",
       "      <td>6.0</td>\n",
       "      <td>salmon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          food  ounces  animal\n",
       "0        bacon     4.0     pig\n",
       "1  pulled pork     3.0     pig\n",
       "2        bacon    12.0     pig\n",
       "3     pastrami     6.0     cow\n",
       "4  corned beef     7.5     cow\n",
       "5        bacon     8.0     pig\n",
       "6     pastrami     3.0     cow\n",
       "7    honey ham     5.0     pig\n",
       "8     nova lox     6.0  salmon"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c0ff35e9-bc51-429a-8754-265fdfd5416b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_animal(x):\n",
    "    return meat_to_animal[x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a6658992-2aee-40a4-9cd3-ffa6eefeb9a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       pig\n",
       "1       pig\n",
       "2       pig\n",
       "3       cow\n",
       "4       cow\n",
       "5       pig\n",
       "6       cow\n",
       "7       pig\n",
       "8    salmon\n",
       "Name: food, dtype: object"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"food\"].map(get_animal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f5694d5d-7a9e-4ea4-acd9-6f4819ebf271",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.Series([1., -999., 2., -999., -1000., 3.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dabee927-61d8-435d-a671-7559e2f93a08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1.0\n",
       "1    -999.0\n",
       "2       2.0\n",
       "3    -999.0\n",
       "4   -1000.0\n",
       "5       3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2eb66df9-b322-431e-a0a8-83495fd34c16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1.0\n",
       "1       NaN\n",
       "2       2.0\n",
       "3       NaN\n",
       "4   -1000.0\n",
       "5       3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace(-999, np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "07c515d3-44ae-463e-a4ed-c3d3dbe3f420",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    2.0\n",
       "3    NaN\n",
       "4    NaN\n",
       "5    3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace([-999, -1000], np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0d4a1cf7-1ec4-4d9a-9e40-ac00a5542009",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    2.0\n",
       "3    NaN\n",
       "4    0.0\n",
       "5    3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace([-999, -1000], [np.nan, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "38978d8d-5727-4e4e-bd83-b0c5213909c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    NaN\n",
       "2    2.0\n",
       "3    NaN\n",
       "4    0.0\n",
       "5    3.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace({-999: np.nan, -1000: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2e0e8b7a-8cf5-4774-a670-db41254a14ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(np.arange(12).reshape((3, 4)),\n",
    "                    index=[\"OhiO\", \"Colorado\", \"New York\"],\n",
    "                    columns=[\"one\", \"two\", \"three\", \"four\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "80c7a0ad-c4f8-4a90-86dc-e86044ba57fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>OhiO</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          one  two  three  four\n",
       "OhiO        0    1      2     3\n",
       "Colorado    4    5      6     7\n",
       "New York    8    9     10    11"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1ad5faba-9544-4240-bc04-4fd887f8ec83",
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(x):\n",
    "    return x[:4].upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "85579f4d-818c-4899-9d02-890257fdcc76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['OHIO', 'COLO', 'NEW '], dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.index.map(transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ae37fdfc-6133-4d3c-bde8-5413e803d2db",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.index = data.index.map(transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b2722f59-4ce2-4657-8e54-c22ffc0891f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>OHIO</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COLO</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NEW</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      one  two  three  four\n",
       "OHIO    0    1      2     3\n",
       "COLO    4    5      6     7\n",
       "NEW     8    9     10    11"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "66309d91-b0aa-430c-af3d-00c146dc848e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ONE</th>\n",
       "      <th>TWO</th>\n",
       "      <th>THREE</th>\n",
       "      <th>FOUR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Ohio</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colo</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ONE  TWO  THREE  FOUR\n",
       "Ohio    0    1      2     3\n",
       "Colo    4    5      6     7\n",
       "New     8    9     10    11"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.rename(index=str.title, columns=str.upper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f603b8c0-037d-455e-a7dd-931a2de09853",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>three</th>\n",
       "      <th>four</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>OHIO</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COLO</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NEW</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      one  two  three  four\n",
       "OHIO    0    1      2     3\n",
       "COLO    4    5      6     7\n",
       "NEW     8    9     10    11"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data # create a transformed version of a dataset without modifying the original."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ab7a8d27-bb87-435d-a4d9-40eb7f439592",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>one</th>\n",
       "      <th>two</th>\n",
       "      <th>peekaboo</th>\n",
       "      <th>four</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>INDIANA</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>COLO</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NEW</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         one  two  peekaboo  four\n",
       "INDIANA    0    1         2     3\n",
       "COLO       4    5         6     7\n",
       "NEW        8    9        10    11"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.rename(index={\"OHIO\": \"INDIANA\"},\n",
    "            columns={\"three\": \"peekaboo\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c5474c7d-30da-4041-8124-4a33747989b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a2b6ed91-1f79-4c5a-91a8-10d047f84e1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [18, 25, 35, 60, 100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "884421e5-af5e-403e-9435-b03dc7d3a94f",
   "metadata": {},
   "outputs": [],
   "source": [
    "age_categories = pd.cut(ages, bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3205545e-f58d-431e-9065-eb7d3087359f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]\n",
       "Length: 12\n",
       "Categories (4, interval[int64, right]): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7e30ad67-db14-4657-9211-d5d60748ecd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 1, 0, 0, 2, 1, 3, 2, 2, 1], dtype=int8)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Outputs an array that represents the category each data point belongs to. \n",
    "# Each number corresponds to the position of the interval in the category list.\n",
    "age_categories.codes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "bc81418f-b782-4ebc-9a98-bec1b1053ef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IntervalIndex([(18, 25], (25, 35], (35, 60], (60, 100]], dtype='interval[int64, right]')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_categories.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c30067b2-c646-4b53-9c8e-fe456e273a70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Interval(18, 25, closed='right')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_categories.categories[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c3793c3f-d200-4b86-81d6-b60f81f534c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Locke\\AppData\\Local\\Temp\\ipykernel_12616\\3010498523.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(age_categories)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(18, 25]     5\n",
       "(25, 35]     3\n",
       "(35, 60]     3\n",
       "(60, 100]    1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(age_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "742cc9a8-2482-4dc8-bcda-2e73c284de99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18, 25]     5\n",
       "(25, 35]     3\n",
       "(35, 60]     3\n",
       "(60, 100]    1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(age_categories).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "0a753595-bbb7-45d4-abcf-08180fc79d48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[18, 25), [18, 25), [25, 35), [25, 35), [18, 25), ..., [25, 35), [60, 100), [35, 60), [35, 60), [25, 35)]\n",
       "Length: 12\n",
       "Categories (4, interval[int64, left]): [[18, 25) < [25, 35) < [35, 60) < [60, 100)]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.cut(ages, bins, right=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "443301bc-2b0b-44f4-9cff-15222c536840",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_names = [\"Youth\", \"YoungAdult\", \"MiddleAged\", \"Senior\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "936d0a52-4225-47cf-bb21-a183ac52dff1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Youth', 'Youth', 'Youth', 'YoungAdult', 'Youth', ..., 'YoungAdult', 'Senior', 'MiddleAged', 'MiddleAged', 'YoungAdult']\n",
       "Length: 12\n",
       "Categories (4, object): ['Youth' < 'YoungAdult' < 'MiddleAged' < 'Senior']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.cut(ages, bins, labels=group_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "da5c3885-9230-4acf-8b66-d3988339ba2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.random.uniform(size = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "34260be3-1726-4eaa-9854-41382de9eabe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.14183762, 0.02509081, 0.25507064, 0.90535758, 0.98466371,\n",
       "       0.26935295, 0.56856877, 0.91573819, 0.10462058, 0.93272929,\n",
       "       0.58342359, 0.48397811, 0.15581581, 0.71203824, 0.62331612,\n",
       "       0.24932962, 0.42964752, 0.48854163, 0.21731694, 0.40892649])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f16800cb-9770-4db4-b071-2b3511aab189",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.024, 0.26], (0.024, 0.26], (0.024, 0.26], (0.74, 0.98], (0.74, 0.98], ..., (0.024, 0.26], (0.26, 0.5], (0.26, 0.5], (0.024, 0.26], (0.26, 0.5]]\n",
       "Length: 20\n",
       "Categories (4, interval[float64, right]): [(0.024, 0.26] < (0.26, 0.5] < (0.5, 0.74] < (0.74, 0.98]]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.cut(data, 4, precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d1469afe-2178-434b-a2cd-d09a0cd419b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.random.standard_normal(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "78792748-373c-4bbb-87e9-5b8752ed7c72",
   "metadata": {},
   "outputs": [],
   "source": [
    "quartiles = pd.qcut(data, 4, precision=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "065aaa86-f86b-4c99-a552-47910260823a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.68, 3.39], (-0.031, 0.68], (0.68, 3.39], (-0.71, -0.031], (-0.71, -0.031], ..., (-0.031, 0.68], (-0.031, 0.68], (-0.71, -0.031], (-0.031, 0.68], (-2.78, -0.71]]\n",
       "Length: 1000\n",
       "Categories (4, interval[float64, right]): [(-2.78, -0.71] < (-0.71, -0.031] < (-0.031, 0.68] < (0.68, 3.39]]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quartiles # obtain roughly equally sized bins."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "7832ea61-5489-46cb-a8d1-be9cf1bc2f0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-2.78, -0.71]     250\n",
       "(-0.71, -0.031]    250\n",
       "(-0.031, 0.68]     250\n",
       "(0.68, 3.39]       250\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(quartiles).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a8442a5c-9d62-46c8-94c2-991c4b35fb4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-2.7729999999999997, -1.314]    100\n",
       "(-1.314, -0.0306]                400\n",
       "(-0.0306, 1.37]                  400\n",
       "(1.37, 3.388]                    100\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.qcut(data, [0, 0.1, 0.5, 0.9, 1.]).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8635fe8b-e28b-4d8e-b33b-41742677b38c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(np.random.standard_normal((1000, 4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "27afd1a4-9b5f-4879-898b-999df6373ec4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.039895</td>\n",
       "      <td>-0.001904</td>\n",
       "      <td>0.020121</td>\n",
       "      <td>0.071512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.037253</td>\n",
       "      <td>0.959715</td>\n",
       "      <td>1.021091</td>\n",
       "      <td>0.963054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.307885</td>\n",
       "      <td>-3.188160</td>\n",
       "      <td>-4.106648</td>\n",
       "      <td>-2.671398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.619790</td>\n",
       "      <td>-0.678995</td>\n",
       "      <td>-0.671932</td>\n",
       "      <td>-0.597959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.057015</td>\n",
       "      <td>-0.047932</td>\n",
       "      <td>-0.013736</td>\n",
       "      <td>0.062578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.752186</td>\n",
       "      <td>0.664939</td>\n",
       "      <td>0.686131</td>\n",
       "      <td>0.677515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.208430</td>\n",
       "      <td>3.226905</td>\n",
       "      <td>3.256160</td>\n",
       "      <td>3.991072</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean      0.039895    -0.001904     0.020121     0.071512\n",
       "std       1.037253     0.959715     1.021091     0.963054\n",
       "min      -3.307885    -3.188160    -4.106648    -2.671398\n",
       "25%      -0.619790    -0.678995    -0.671932    -0.597959\n",
       "50%       0.057015    -0.047932    -0.013736     0.062578\n",
       "75%       0.752186     0.664939     0.686131     0.677515\n",
       "max       3.208430     3.226905     3.256160     3.991072"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1367c636-caec-409b-9d5c-ce4325ca2f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "col = data[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f55dc7b6-59c5-4603-92bc-0e3ed6fd8a92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "216    3.256160\n",
       "400   -4.106648\n",
       "481    3.079395\n",
       "634    3.034615\n",
       "739   -3.152273\n",
       "753   -3.059182\n",
       "Name: 2, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col[col.abs() > 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "605432a1-d7af-47c1-b229-ead67ec735c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>1.585706</td>\n",
       "      <td>-3.188160</td>\n",
       "      <td>0.233903</td>\n",
       "      <td>0.346104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>3.141658</td>\n",
       "      <td>-0.097743</td>\n",
       "      <td>-1.524578</td>\n",
       "      <td>-0.708210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>-3.130391</td>\n",
       "      <td>1.137169</td>\n",
       "      <td>1.598413</td>\n",
       "      <td>0.590192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>3.120847</td>\n",
       "      <td>1.842470</td>\n",
       "      <td>-0.763319</td>\n",
       "      <td>-0.181725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>0.948868</td>\n",
       "      <td>-1.277980</td>\n",
       "      <td>3.256160</td>\n",
       "      <td>2.101211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>-3.183775</td>\n",
       "      <td>0.439288</td>\n",
       "      <td>-0.000760</td>\n",
       "      <td>0.333492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>0.317315</td>\n",
       "      <td>0.474916</td>\n",
       "      <td>0.910558</td>\n",
       "      <td>3.991072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>-0.485232</td>\n",
       "      <td>-0.114088</td>\n",
       "      <td>-4.106648</td>\n",
       "      <td>-1.114838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>-1.571596</td>\n",
       "      <td>0.470205</td>\n",
       "      <td>3.079395</td>\n",
       "      <td>0.810130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>621</th>\n",
       "      <td>-3.093881</td>\n",
       "      <td>0.608228</td>\n",
       "      <td>-1.187669</td>\n",
       "      <td>-1.308063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>-1.149771</td>\n",
       "      <td>-0.841001</td>\n",
       "      <td>3.034615</td>\n",
       "      <td>0.904384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>739</th>\n",
       "      <td>0.174434</td>\n",
       "      <td>-1.180628</td>\n",
       "      <td>-3.152273</td>\n",
       "      <td>1.257155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>753</th>\n",
       "      <td>-2.658944</td>\n",
       "      <td>-0.716147</td>\n",
       "      <td>-3.059182</td>\n",
       "      <td>1.168877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>759</th>\n",
       "      <td>0.406406</td>\n",
       "      <td>3.226905</td>\n",
       "      <td>0.011426</td>\n",
       "      <td>-0.360705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>794</th>\n",
       "      <td>-3.307885</td>\n",
       "      <td>-1.253889</td>\n",
       "      <td>0.047877</td>\n",
       "      <td>-1.925208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>3.208430</td>\n",
       "      <td>1.047657</td>\n",
       "      <td>-0.725456</td>\n",
       "      <td>-0.553024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3\n",
       "110  1.585706 -3.188160  0.233903  0.346104\n",
       "132  3.141658 -0.097743 -1.524578 -0.708210\n",
       "209 -3.130391  1.137169  1.598413  0.590192\n",
       "213  3.120847  1.842470 -0.763319 -0.181725\n",
       "216  0.948868 -1.277980  3.256160  2.101211\n",
       "288 -3.183775  0.439288 -0.000760  0.333492\n",
       "311  0.317315  0.474916  0.910558  3.991072\n",
       "400 -0.485232 -0.114088 -4.106648 -1.114838\n",
       "481 -1.571596  0.470205  3.079395  0.810130\n",
       "621 -3.093881  0.608228 -1.187669 -1.308063\n",
       "634 -1.149771 -0.841001  3.034615  0.904384\n",
       "739  0.174434 -1.180628 -3.152273  1.257155\n",
       "753 -2.658944 -0.716147 -3.059182  1.168877\n",
       "759  0.406406  3.226905  0.011426 -0.360705\n",
       "794 -3.307885 -1.253889  0.047877 -1.925208\n",
       "819  3.208430  1.047657 -0.725456 -0.553024"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use the any method on a Boolean DataFrame to \n",
    "# select all rows having a value exceeding 3 or –3.\n",
    "data[(data.abs() > 3).any(axis=\"columns\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e73cf0e6-9ad9-40c8-bc1c-7456baab0c30",
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.sign(): \n",
    "# It returns -1 for negative values, 1 for positive values, and 0 for any zeros.\n",
    "data[data.abs() > 3] = np.sign(data) * 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8b708e03-78f1-46fd-808a-032217d3bf23",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.040140</td>\n",
       "      <td>-0.001942</td>\n",
       "      <td>0.021069</td>\n",
       "      <td>0.070521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.033689</td>\n",
       "      <td>0.958370</td>\n",
       "      <td>1.015442</td>\n",
       "      <td>0.959519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.000000</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>-2.671398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.619790</td>\n",
       "      <td>-0.678995</td>\n",
       "      <td>-0.671932</td>\n",
       "      <td>-0.597959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.057015</td>\n",
       "      <td>-0.047932</td>\n",
       "      <td>-0.013736</td>\n",
       "      <td>0.062578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.752186</td>\n",
       "      <td>0.664939</td>\n",
       "      <td>0.686131</td>\n",
       "      <td>0.677515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean      0.040140    -0.001942     0.021069     0.070521\n",
       "std       1.033689     0.958370     1.015442     0.959519\n",
       "min      -3.000000    -3.000000    -3.000000    -2.671398\n",
       "25%      -0.619790    -0.678995    -0.671932    -0.597959\n",
       "50%       0.057015    -0.047932    -0.013736     0.062578\n",
       "75%       0.752186     0.664939     0.686131     0.677515\n",
       "max       3.000000     3.000000     3.000000     3.000000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "637aaf5f-29dc-4894-b39b-63d471471329",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3\n",
       "0 -1.0 -1.0  1.0 -1.0\n",
       "1  1.0  1.0  1.0 -1.0\n",
       "2  1.0  1.0  1.0 -1.0\n",
       "3 -1.0  1.0  1.0 -1.0\n",
       "4 -1.0 -1.0  1.0  1.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sign(data).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dca50537-258f-45bf-acca-989fbf3e73f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.arange(5 * 7).reshape((5, 7)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "57a9d05e-9d48-4e00-a654-07d1f58a6a97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "0   0   1   2   3   4   5   6\n",
       "1   7   8   9  10  11  12  13\n",
       "2  14  15  16  17  18  19  20\n",
       "3  21  22  23  24  25  26  27\n",
       "4  28  29  30  31  32  33  34"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "68e5ba1f-24b4-441b-9855-dd7958aea2fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "sampler = np.random.permutation(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "820fa258-255e-4513-a422-5c8e99059e6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 2, 4, 1, 0])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "766513af-ff39-47d0-a208-ee5b8baa162e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "3  21  22  23  24  25  26  27\n",
       "2  14  15  16  17  18  19  20\n",
       "4  28  29  30  31  32  33  34\n",
       "1   7   8   9  10  11  12  13\n",
       "0   0   1   2   3   4   5   6"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.take(sampler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9ea7a7ba-c6ee-478c-842b-1d2c5359c04c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "3  21  22  23  24  25  26  27\n",
       "2  14  15  16  17  18  19  20\n",
       "4  28  29  30  31  32  33  34\n",
       "1   7   8   9  10  11  12  13\n",
       "0   0   1   2   3   4   5   6"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[sampler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "aefb8969-aaa8-4a74-a957-365e642dfbad",
   "metadata": {},
   "outputs": [],
   "source": [
    "column_sampler = np.random.permutation(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bab11358-ab07-4953-be14-6ad7c5311668",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 5, 3, 1, 4, 6, 2])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "354595f6-90c2-4f39-b496-412d8dfb80c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>5</th>\n",
       "      <th>3</th>\n",
       "      <th>1</th>\n",
       "      <th>4</th>\n",
       "      <th>6</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>18</td>\n",
       "      <td>20</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>26</td>\n",
       "      <td>24</td>\n",
       "      <td>22</td>\n",
       "      <td>25</td>\n",
       "      <td>27</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>33</td>\n",
       "      <td>31</td>\n",
       "      <td>29</td>\n",
       "      <td>32</td>\n",
       "      <td>34</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   5   3   1   4   6   2\n",
       "0   0   5   3   1   4   6   2\n",
       "1   7  12  10   8  11  13   9\n",
       "2  14  19  17  15  18  20  16\n",
       "3  21  26  24  22  25  27  23\n",
       "4  28  33  31  29  32  34  30"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.take(column_sampler, axis=\"columns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "68ccbd84-5510-43b3-a420-45aea271e301",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "4  28  29  30  31  32  33  34\n",
       "3  21  22  23  24  25  26  27\n",
       "1   7   8   9  10  11  12  13"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(n=3) # select a random subset without replacement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6400766b-81e7-45a1-b90c-c716cb8627e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "choices = pd.Series([5, 7, -1, 6, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "86f6973d-38a6-4493-9196-60a4a6fe68ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    6\n",
       "3    6\n",
       "1    7\n",
       "3    6\n",
       "1    7\n",
       "1    7\n",
       "1    7\n",
       "1    7\n",
       "4    4\n",
       "3    6\n",
       "dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "choices.sample(n=10, replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9fbedaf2-909b-4fad-8764-9006fbb26f6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\"key\": [\"b\", \"b\", \"a\", \"c\", \"a\", \"b\"],\n",
    "                   \"data1\": range(6)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "52c5426b-9134-47d9-85ed-3405b2e5bb8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>data1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  key  data1\n",
       "0   b      0\n",
       "1   b      1\n",
       "2   a      2\n",
       "3   c      3\n",
       "4   a      4\n",
       "5   b      5"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d8638241-25c3-4034-bdd2-c9ab3c5ba215",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     a    b    c\n",
       "0  0.0  1.0  0.0\n",
       "1  0.0  1.0  0.0\n",
       "2  1.0  0.0  0.0\n",
       "3  0.0  0.0  1.0\n",
       "4  1.0  0.0  0.0\n",
       "5  0.0  1.0  0.0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(df[\"key\"], dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "37ea966e-33b7-49c8-afd7-e7b20cd02d46",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies = pd.get_dummies(df[\"key\"], prefix=\"key\", dtype=float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ff52f571-678e-42a1-87ea-d4113c2bee60",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_with_dummy = df[[\"data1\"]].join(dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "51162b4c-a296-4e86-ac58-ef343443a68f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data1</th>\n",
       "      <th>key_a</th>\n",
       "      <th>key_b</th>\n",
       "      <th>key_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   data1  key_a  key_b  key_c\n",
       "0      0    0.0    1.0    0.0\n",
       "1      1    0.0    1.0    0.0\n",
       "2      2    1.0    0.0    0.0\n",
       "3      3    0.0    0.0    1.0\n",
       "4      4    1.0    0.0    0.0\n",
       "5      5    0.0    1.0    0.0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_dummy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "234d9d29-ba97-4e1d-81f1-e33ca80a08e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnames = [\"movie_id\", \"title\", \"genres\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "87b2ca2b-2314-4663-bea3-7fc1ed30f155",
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = pd.read_table(\"datasets/movielens/movies.dat\", sep=\"::\",\n",
    "                       header=None, names=mnames, engine=\"python\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ae830182-8a0e-4de4-835a-7b8c08a67cd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>Heat (1995)</td>\n",
       "      <td>Action|Crime|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>Sabrina (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>Tom and Huck (1995)</td>\n",
       "      <td>Adventure|Children's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Sudden Death (1995)</td>\n",
       "      <td>Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>Action|Adventure|Thriller</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movie_id                               title                        genres\n",
       "0         1                    Toy Story (1995)   Animation|Children's|Comedy\n",
       "1         2                      Jumanji (1995)  Adventure|Children's|Fantasy\n",
       "2         3             Grumpier Old Men (1995)                Comedy|Romance\n",
       "3         4            Waiting to Exhale (1995)                  Comedy|Drama\n",
       "4         5  Father of the Bride Part II (1995)                        Comedy\n",
       "5         6                         Heat (1995)         Action|Crime|Thriller\n",
       "6         7                      Sabrina (1995)                Comedy|Romance\n",
       "7         8                 Tom and Huck (1995)          Adventure|Children's\n",
       "8         9                 Sudden Death (1995)                        Action\n",
       "9        10                    GoldenEye (1995)     Action|Adventure|Thriller"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "939bc3f9-6cd5-4022-9957-7e717ee00bee",
   "metadata": {},
   "outputs": [],
   "source": [
    "dummies = movies[\"genres\"].str.get_dummies(\"|\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f42e7c72-00fb-4062-9f58-e302eb19c885",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Action</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Children's</th>\n",
       "      <th>Comedy</th>\n",
       "      <th>Crime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Action  Adventure  Animation  Children's  Comedy  Crime\n",
       "0       0          0          1           1       1      0\n",
       "1       0          1          0           1       0      0\n",
       "2       0          0          0           0       1      0\n",
       "3       0          0          0           0       1      0\n",
       "4       0          0          0           0       1      0\n",
       "5       1          0          0           0       0      1\n",
       "6       0          0          0           0       1      0\n",
       "7       0          1          0           1       0      0\n",
       "8       1          0          0           0       0      0\n",
       "9       1          1          0           0       0      0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies.iloc[:10, :6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "86d499f0-f623-4cf1-b0be-37178f32f41e",
   "metadata": {},
   "outputs": [],
   "source": [
    "movies_windic = movies.join(dummies.add_prefix(\"Genre_\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "82a8e87a-2159-4ec7-9f28-d2216969f970",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>Genre_Action</th>\n",
       "      <th>Genre_Adventure</th>\n",
       "      <th>Genre_Animation</th>\n",
       "      <th>Genre_Children's</th>\n",
       "      <th>Genre_Comedy</th>\n",
       "      <th>Genre_Crime</th>\n",
       "      <th>Genre_Documentary</th>\n",
       "      <th>...</th>\n",
       "      <th>Genre_Fantasy</th>\n",
       "      <th>Genre_Film-Noir</th>\n",
       "      <th>Genre_Horror</th>\n",
       "      <th>Genre_Musical</th>\n",
       "      <th>Genre_Mystery</th>\n",
       "      <th>Genre_Romance</th>\n",
       "      <th>Genre_Sci-Fi</th>\n",
       "      <th>Genre_Thriller</th>\n",
       "      <th>Genre_War</th>\n",
       "      <th>Genre_Western</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>Heat (1995)</td>\n",
       "      <td>Action|Crime|Thriller</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>Sabrina (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>Tom and Huck (1995)</td>\n",
       "      <td>Adventure|Children's</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Sudden Death (1995)</td>\n",
       "      <td>Action</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>Action|Adventure|Thriller</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   movie_id                               title                        genres  \\\n",
       "0         1                    Toy Story (1995)   Animation|Children's|Comedy   \n",
       "1         2                      Jumanji (1995)  Adventure|Children's|Fantasy   \n",
       "2         3             Grumpier Old Men (1995)                Comedy|Romance   \n",
       "3         4            Waiting to Exhale (1995)                  Comedy|Drama   \n",
       "4         5  Father of the Bride Part II (1995)                        Comedy   \n",
       "5         6                         Heat (1995)         Action|Crime|Thriller   \n",
       "6         7                      Sabrina (1995)                Comedy|Romance   \n",
       "7         8                 Tom and Huck (1995)          Adventure|Children's   \n",
       "8         9                 Sudden Death (1995)                        Action   \n",
       "9        10                    GoldenEye (1995)     Action|Adventure|Thriller   \n",
       "\n",
       "   Genre_Action  Genre_Adventure  Genre_Animation  Genre_Children's  \\\n",
       "0             0                0                1                 1   \n",
       "1             0                1                0                 1   \n",
       "2             0                0                0                 0   \n",
       "3             0                0                0                 0   \n",
       "4             0                0                0                 0   \n",
       "5             1                0                0                 0   \n",
       "6             0                0                0                 0   \n",
       "7             0                1                0                 1   \n",
       "8             1                0                0                 0   \n",
       "9             1                1                0                 0   \n",
       "\n",
       "   Genre_Comedy  Genre_Crime  Genre_Documentary  ...  Genre_Fantasy  \\\n",
       "0             1            0                  0  ...              0   \n",
       "1             0            0                  0  ...              1   \n",
       "2             1            0                  0  ...              0   \n",
       "3             1            0                  0  ...              0   \n",
       "4             1            0                  0  ...              0   \n",
       "5             0            1                  0  ...              0   \n",
       "6             1            0                  0  ...              0   \n",
       "7             0            0                  0  ...              0   \n",
       "8             0            0                  0  ...              0   \n",
       "9             0            0                  0  ...              0   \n",
       "\n",
       "   Genre_Film-Noir  Genre_Horror  Genre_Musical  Genre_Mystery  Genre_Romance  \\\n",
       "0                0             0              0              0              0   \n",
       "1                0             0              0              0              0   \n",
       "2                0             0              0              0              1   \n",
       "3                0             0              0              0              0   \n",
       "4                0             0              0              0              0   \n",
       "5                0             0              0              0              0   \n",
       "6                0             0              0              0              1   \n",
       "7                0             0              0              0              0   \n",
       "8                0             0              0              0              0   \n",
       "9                0             0              0              0              0   \n",
       "\n",
       "   Genre_Sci-Fi  Genre_Thriller  Genre_War  Genre_Western  \n",
       "0             0               0          0              0  \n",
       "1             0               0          0              0  \n",
       "2             0               0          0              0  \n",
       "3             0               0          0              0  \n",
       "4             0               0          0              0  \n",
       "5             0               1          0              0  \n",
       "6             0               0          0              0  \n",
       "7             0               0          0              0  \n",
       "8             0               0          0              0  \n",
       "9             0               1          0              0  \n",
       "\n",
       "[10 rows x 21 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_windic[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "de0f9099-d890-4c1a-b2a9-1533b0db5063",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "movie_id                                       1\n",
       "title                           Toy Story (1995)\n",
       "genres               Animation|Children's|Comedy\n",
       "Genre_Action                                   0\n",
       "Genre_Adventure                                0\n",
       "Genre_Animation                                1\n",
       "Genre_Children's                               1\n",
       "Genre_Comedy                                   1\n",
       "Genre_Crime                                    0\n",
       "Genre_Documentary                              0\n",
       "Genre_Drama                                    0\n",
       "Genre_Fantasy                                  0\n",
       "Genre_Film-Noir                                0\n",
       "Genre_Horror                                   0\n",
       "Genre_Musical                                  0\n",
       "Genre_Mystery                                  0\n",
       "Genre_Romance                                  0\n",
       "Genre_Sci-Fi                                   0\n",
       "Genre_Thriller                                 0\n",
       "Genre_War                                      0\n",
       "Genre_Western                                  0\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_windic.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f56ea071-132c-4e78-9ac4-f4219127374f",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12345) # to make the example repeatable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "365d9dd6-47e5-4d9d-9584-37c8a739d584",
   "metadata": {},
   "outputs": [],
   "source": [
    "values = np.random.uniform(size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "d99e4703-31cf-41ec-99c3-63dc6660163c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9296, 0.3164, 0.1839, 0.2046, 0.5677, 0.5955, 0.9645, 0.6532,\n",
       "       0.7489, 0.6536])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values.round(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "f4a27b09-2468-456f-b97d-1ab5568238f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [0, 0.2, 0.4, 0.6, 0.8, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "52bb1f5f-1829-46a8-a6ad-09bce6a7e891",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>(0.0, 0.2]</th>\n",
       "      <th>(0.2, 0.4]</th>\n",
       "      <th>(0.4, 0.6]</th>\n",
       "      <th>(0.6, 0.8]</th>\n",
       "      <th>(0.8, 1.0]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   (0.0, 0.2]  (0.2, 0.4]  (0.4, 0.6]  (0.6, 0.8]  (0.8, 1.0]\n",
       "0       False       False       False       False        True\n",
       "1       False        True       False       False       False\n",
       "2        True       False       False       False       False\n",
       "3       False        True       False       False       False\n",
       "4       False       False        True       False       False\n",
       "5       False       False        True       False       False\n",
       "6       False       False       False       False        True\n",
       "7       False       False       False        True       False\n",
       "8       False       False       False        True       False\n",
       "9       False       False       False        True       False"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(pd.cut(values, bins))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "739d23a1-d80f-429b-9a03-b22671bf45c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series([1, 2, 3, None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fd08d4d4-6176-4f56-8487-b6668da874e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    2.0\n",
       "2    3.0\n",
       "3    NaN\n",
       "dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b0cb6e3f-032b-47b2-b49e-53985a6c8e27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "952702ef-1d59-401c-bfc9-f8580a145391",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series([1, 2, 3, None], dtype=pd.Int64Dtype())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "33e8bfaf-da91-4a8f-8b76-38a7ab642a7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1\n",
       "1       2\n",
       "2       3\n",
       "3    <NA>\n",
       "dtype: Int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "644fdec0-fb8c-451c-a21f-88020d1e1043",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1    False\n",
       "2    False\n",
       "3     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "773be87f-e7f7-4423-a265-a2df4ca16b00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Dtype()"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bd021b08-4daf-449b-9fe5-594064d1d8eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<NA>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "47749c46-5345-4aa5-8abc-18f8a3b6fee8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s[3] is pd.NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "13ca124f-4512-4283-8a38-e0244fad235f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The capitalization is necessary.\n",
    "s = pd.Series([1, 2, 3, None], dtype=\"Int64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "21adb0f0-a607-4ea2-b34d-736916774498",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       1\n",
       "1       2\n",
       "2       3\n",
       "3    <NA>\n",
       "dtype: Int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6bd1f863-c660-4af4-8091-6452af05160f",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(['one', 'two', None, 'three'], dtype=pd.StringDtype())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a04240e4-03a1-4811-a4b1-4e9335d99573",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      one\n",
       "1      two\n",
       "2     <NA>\n",
       "3    three\n",
       "dtype: string"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "224a0aeb-722f-4b10-8fb3-616b8353ead3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\"A\": [1, 2, None, 4],\n",
    "                   \"B\": [\"one\", \"two\", \"three\", None],\n",
    "                   \"C\": [False, None, False, True]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c50f5c84-b22f-4cec-b8de-e549df26cb56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>one</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>two</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>three</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     A      B      C\n",
       "0  1.0    one  False\n",
       "1  2.0    two   None\n",
       "2  NaN  three  False\n",
       "3  4.0   None   True"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "c244d1b2-b545-4915-aaf7-c4ca761e6206",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"A\"] = df[\"A\"].astype(\"Int64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "bbbf14d7-fc0b-4cf1-af41-8d123b61b761",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"B\"] = df[\"B\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "23b5b2a6-c79c-4f7f-9722-2e6cc4c585d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"C\"] = df[\"C\"].astype(\"boolean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "82dded00-bdc3-4ebb-a21f-4af7bdf0d787",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>one</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>two</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>three</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      A      B      C\n",
       "0     1    one  False\n",
       "1     2    two   <NA>\n",
       "2  <NA>  three  False\n",
       "3     4   <NA>   True"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "6e03b206-de0d-48c0-9240-d5dc6e5c5f14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Dtype()"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"A\"].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "b75d3a23-18be-46f9-8d4d-1c3435606757",
   "metadata": {},
   "outputs": [],
   "source": [
    "val = \"a,b,  guido\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "3b9460b7-2dd7-4d72-b5dd-d1704ef9e4ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'b', '  guido']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.split(\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "1f5a03b4-2dfc-49d9-8a4e-567b32df5b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pieces = [x.strip() for x in val.split(\",\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "227c0c38-2527-45fc-a102-dfd4f4177ef6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'b', 'guido']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pieces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "f2fa45a5-3532-4d19-bdbe-add06317c4a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "first, second, third = pieces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "3b0c032d-cf12-429a-b03a-17fb176143db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a::b::guido'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first + \"::\" + second + \"::\" + third"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "39a2774f-a263-4650-be30-955619963565",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a::b::guido'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"::\".join(pieces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "5f02f1e8-ba25-4aff-8985-7261090242a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"guido\" in val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "efc349e6-2811-4ff9-825c-e2ad638fc814",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.index(\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "e7c8cbb2-2ab9-4bc5-ab6f-3edee6eaa84e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.find(\":\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "108f2cd3-d16c-4757-8c70-74356acc3874",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The difference between find and index is that index raises an exception \n",
    "# if the string isn’t found (versus returning –1)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "b7ab7fc2-7f67-4dd5-89f8-6f226a157288",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a,b,  guido'"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "b9d74278-91b5-4982-bcea-d26b35136dc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.count(\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "cb347e17-c3e5-4b6a-b83c-0d8a1643f5a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a::b::  guido'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.replace(\",\", \"::\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "9532cde3-5215-4260-be03-317115e2652b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ab  guido'"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.replace(\",\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6a840756-99d2-440d-97c5-708530e8b292",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3f5800e2-9f7e-4f2e-8ea6-08a9a9d46d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"foo    bar\\t baz  \\tqux\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "33cf3ca1-0d2b-40c1-a9f6-a6c16041dfc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'qux']"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The regex describing one or more whitespace characters is \\s+.\n",
    "re.split(r\"\\s+\", text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f1f49bf6-c923-4331-a9a9-15d2c8aed86a",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex = re.compile(r\"\\s+\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c145d6d5-fc5f-4d0f-85e1-5fed8d07327d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'qux']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.split(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7fa32cc5-8b6b-4f55-8c15-7a93bd3055b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['    ', '\\t ', '  \\t']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b3a8dfe9-bb50-4650-bd2a-11bbeff32536",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\"\"Dave dave@google.com\n",
    "Steve steve@gmail.com\n",
    "Rob rob@gmail.com\n",
    "Ryan ryan@yahoo.com\"\"\"\n",
    "pattern = r\"[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\"\n",
    "\n",
    "# re.IGNORECASE makes the regex case insensitive\n",
    "regex = re.compile(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cddc9d41-67f5-432f-9ac9-ab1c22651e6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dave@google.com', 'steve@gmail.com', 'rob@gmail.com', 'ryan@yahoo.com']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a37137c5-9728-4c1d-aca1-760aad969e15",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = regex.search(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9c980d47-8e5c-4355-a91e-7180625a6f6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<re.Match object; span=(5, 20), match='dave@google.com'>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "056b7911-d13e-4c40-ad99-e3c32997813e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dave@google.com'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[m.start():m.end()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "558bf69f-3d1d-4034-94ab-1830034df949",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(regex.match(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9e209a7a-1971-4559-9bd9-f6c15bb8f8ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dave REDACTED\n",
      "Steve REDACTED\n",
      "Rob REDACTED\n",
      "Ryan REDACTED\n"
     ]
    }
   ],
   "source": [
    "print(regex.sub(\"REDACTED\", text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f26b2808-5715-4b36-89bb-b5138ac32d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7d698653-093c-434d-b66d-5cb026ff762e",
   "metadata": {},
   "outputs": [],
   "source": [
    "regex = re.compile(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "afb33b19-85fe-40f0-b14e-279652a24375",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('dave', 'google', 'com'),\n",
       " ('steve', 'gmail', 'com'),\n",
       " ('rob', 'gmail', 'com'),\n",
       " ('ryan', 'yahoo', 'com')]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4c80b708-7bc6-4783-af79-64a33cd2a5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = regex.match(\"wesm@bright.net\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "75cf6509-9d9c-4472-bb4d-02cba79654e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('wesm', 'bright', 'net')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.groups()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "05376186-06d9-493a-9606-09db32392507",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dave Username: dave, Domain: google, Suffix: com\n",
      "Steve Username: steve, Domain: gmail, Suffix: com\n",
      "Rob Username: rob, Domain: gmail, Suffix: com\n",
      "Ryan Username: ryan, Domain: yahoo, Suffix: com\n"
     ]
    }
   ],
   "source": [
    "print(regex.sub(r\"Username: \\1, Domain: \\2, Suffix: \\3\", text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "de449ed4-2ac8-483d-aede-966a46b31f54",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\"Dave\": \"dave@google.com\", \"Steve\": \"steve@gmail.com\",\n",
    "        \"Rob\": \"rob@gmail.com\", \"Wes\": np.nan}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "8fcedb5f-8f7b-4583-a900-542dc34bd821",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.Series(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "fff996fa-e40b-4426-92ed-eba7466dccf9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     dave@google.com\n",
       "Steve    steve@gmail.com\n",
       "Rob        rob@gmail.com\n",
       "Wes                  NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "62f140bd-b8f5-4ac5-82bf-67ee2af6b6a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     False\n",
       "Steve    False\n",
       "Rob      False\n",
       "Wes       True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "a5d0c923-9d9c-4ea5-8783-3120c56f07d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     False\n",
       "Steve     True\n",
       "Rob       True\n",
       "Wes        NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.contains(\"gmail\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2576932b-3cd3-4b05-bca6-07ab4ce9b6cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_as_string_ext = data.astype('string')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ee0267b5-6adc-491b-8243-19436d6c9eee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     dave@google.com\n",
       "Steve    steve@gmail.com\n",
       "Rob        rob@gmail.com\n",
       "Wes                 <NA>\n",
       "dtype: string"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_as_string_ext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "30bbbb83-d21f-4e1b-8150-d62e2dcf8978",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     False\n",
       "Steve     True\n",
       "Rob       True\n",
       "Wes       <NA>\n",
       "dtype: boolean"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_as_string_ext.str.contains(\"gmail\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "dd8c32f2-bbd0-4cf8-b3c1-cb2bcb7d2352",
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "12e656c1-7d3d-482c-9ece-e985a4cd5de5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     [(dave, google, com)]\n",
       "Steve    [(steve, gmail, com)]\n",
       "Rob        [(rob, gmail, com)]\n",
       "Wes                        NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.findall(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "5c43cdb3-4f69-4561-9766-4640c57b0c4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "matches = data.str.findall(pattern, flags=re.IGNORECASE).str[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "e80918ef-2275-4a73-9acf-102c3142b639",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     (dave, google, com)\n",
       "Steve    (steve, gmail, com)\n",
       "Rob        (rob, gmail, com)\n",
       "Wes                      NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "a005b8eb-1e28-4446-bb2f-6df9eb741ff3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     google\n",
       "Steve     gmail\n",
       "Rob       gmail\n",
       "Wes         NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches.str.get(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "22e3d389-3e35-4646-81bb-8f05d3458032",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dave     dave@\n",
       "Steve    steve\n",
       "Rob      rob@g\n",
       "Wes        NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "11b41fd5-ac45-4811-bd80-e3b01335ec95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Dave</th>\n",
       "      <td>dave</td>\n",
       "      <td>google</td>\n",
       "      <td>com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Steve</th>\n",
       "      <td>steve</td>\n",
       "      <td>gmail</td>\n",
       "      <td>com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rob</th>\n",
       "      <td>rob</td>\n",
       "      <td>gmail</td>\n",
       "      <td>com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wes</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0       1    2\n",
       "Dave    dave  google  com\n",
       "Steve  steve   gmail  com\n",
       "Rob      rob   gmail  com\n",
       "Wes      NaN     NaN  NaN"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.extract(pattern, flags=re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "925b625c-3bbe-47b2-9497-65d7014a4fda",
   "metadata": {},
   "outputs": [],
   "source": [
    "values = pd.Series(['apple', 'orange', 'apple',\n",
    "                    'apple'] * 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "d5a0c737-d28b-4e90-8da5-3d7cb76283e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "2     apple\n",
       "3     apple\n",
       "4     apple\n",
       "5    orange\n",
       "6     apple\n",
       "7     apple\n",
       "dtype: object"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "9cde9349-a100-49b6-b3e4-4922c6e316bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['apple', 'orange'], dtype=object)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "d59a939e-a0cd-44a9-b728-830bde8eb48c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Locke\\AppData\\Local\\Temp\\ipykernel_17356\\3297668723.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(values)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "apple     6\n",
       "orange    2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "6b7f50f3-2c33-429b-9cdb-4e74113b366b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "apple     6\n",
       "orange    2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(values).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "e1892d0f-56ca-401f-b4e5-7d6593ae926a",
   "metadata": {},
   "outputs": [],
   "source": [
    "values = pd.Series([0, 1, 0, 0] * 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "6dfc897a-5b2d-434d-8dcd-3847b577f67e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dim = pd.Series(['apple', 'orange'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "2cff2c75-deee-4316-ac17-b66a4cce7c46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    1\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "5    1\n",
       "6    0\n",
       "7    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "d665d0da-5d4e-42a4-8ea6-bd0a8e474593",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "dtype: object"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e3c081ca-5476-4737-95d5-5c75dcbb4596",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "0     apple\n",
       "0     apple\n",
       "0     apple\n",
       "1    orange\n",
       "0     apple\n",
       "0     apple\n",
       "dtype: object"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim.take(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "0d836080-cd6d-495d-a773-a064bbffaf6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "fruits = ['apple', 'orange', 'apple', 'apple'] * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "9e517651-3135-4ff7-9586-690fd0b0227f",
   "metadata": {},
   "outputs": [],
   "source": [
    "N = len(fruits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "61eea19e-f5c2-4561-a2fc-75c058a06e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng(seed=12345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "12a7c958-451c-4d86-a730-b703ffd9ccd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'fruit': fruits,\n",
    "                   'basket_id': np.arange(N),\n",
    "                   'count': rng.integers(3, 15, size=N),\n",
    "                   'weight': rng.uniform(0, 4, size=N)},\n",
    "                  columns=['basket_id', 'fruit', 'count', 'weight'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "c4461c72-e868-4883-bca4-e141031e10f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>basket_id</th>\n",
       "      <th>fruit</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>apple</td>\n",
       "      <td>11</td>\n",
       "      <td>1.564438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>orange</td>\n",
       "      <td>5</td>\n",
       "      <td>1.331256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>apple</td>\n",
       "      <td>12</td>\n",
       "      <td>2.393235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>apple</td>\n",
       "      <td>6</td>\n",
       "      <td>0.746937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>apple</td>\n",
       "      <td>5</td>\n",
       "      <td>2.691024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>orange</td>\n",
       "      <td>12</td>\n",
       "      <td>3.767211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>apple</td>\n",
       "      <td>10</td>\n",
       "      <td>0.992983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>apple</td>\n",
       "      <td>11</td>\n",
       "      <td>3.795525</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   basket_id   fruit  count    weight\n",
       "0          0   apple     11  1.564438\n",
       "1          1  orange      5  1.331256\n",
       "2          2   apple     12  2.393235\n",
       "3          3   apple      6  0.746937\n",
       "4          4   apple      5  2.691024\n",
       "5          5  orange     12  3.767211\n",
       "6          6   apple     10  0.992983\n",
       "7          7   apple     11  3.795525"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "54ff3fdc-bb20-4a69-8ba9-b459b8c98d20",
   "metadata": {},
   "outputs": [],
   "source": [
    "fruit_cat = df['fruit'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "fc37f278-a449-478b-867d-09d857b1c500",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "2     apple\n",
       "3     apple\n",
       "4     apple\n",
       "5    orange\n",
       "6     apple\n",
       "7     apple\n",
       "Name: fruit, dtype: category\n",
       "Categories (2, object): ['apple', 'orange']"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fruit_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "793cffa4-ee63-4702-8f6b-29ef9b9ff79b",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = fruit_cat.array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "077cef5d-536e-4469-9659-8348160268a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.arrays.categorical.Categorical"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "71825cd3-a7ea-496e-86a1-e0bf528e731f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['apple', 'orange'], dtype='object')"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "d9e3854f-94e2-4b5b-b53d-783589d6f9ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 0, 0, 1, 0, 0], dtype=int8)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "a403b7fc-6e99-4b0c-9994-cbd1bc44050b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'apple', 1: 'orange'}"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(enumerate(c.categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "72ff366d-8e86-4029-9391-b26ba712a011",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['fruit'] = df['fruit'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "944625d0-b041-41c0-8599-ad7591de284a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     apple\n",
       "1    orange\n",
       "2     apple\n",
       "3     apple\n",
       "4     apple\n",
       "5    orange\n",
       "6     apple\n",
       "7     apple\n",
       "Name: fruit, dtype: category\n",
       "Categories (2, object): ['apple', 'orange']"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['fruit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "0de3fe34-9c7a-4bf6-8dd6-c346140d5a19",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_categories = pd.Categorical(['foo', 'bar', 'baz', 'foo', 'bar'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "f98f0265-44ae-41be-9a10-99018d3683c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'bar']\n",
       "Categories (3, object): ['bar', 'baz', 'foo']"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "eef1d46a-cca0-4892-9e2f-0303bb585ef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['foo', 'bar', 'baz']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "b8de67a3-8987-45fa-9eab-a31334cb7668",
   "metadata": {},
   "outputs": [],
   "source": [
    "codes = [0, 1, 2, 0, 0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "73cdc52e-d362-4747-bedf-98d62b53c229",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_cats_2 = pd.Categorical.from_codes(codes, categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "3484b5fe-e1e4-417b-890e-5e2e7a77b97a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'foo', 'bar']\n",
       "Categories (3, object): ['foo', 'bar', 'baz']"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cats_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "4d7b5f88-8b37-4b2c-8636-d81418fbabbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "ordered_cat = pd.Categorical.from_codes(codes, categories,\n",
    "                                        ordered=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "2c3d5dd8-0b51-446c-b9e9-83b3fe53a0e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'foo', 'bar']\n",
       "Categories (3, object): ['foo' < 'bar' < 'baz']"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "12b10c04-f1e7-43f3-a424-f6bcf67c53ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', 'bar', 'baz', 'foo', 'foo', 'bar']\n",
       "Categories (3, object): ['foo' < 'bar' < 'baz']"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_cats_2.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "0dd8b10c-b037-40cb-a5ad-5581a0326601",
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng(seed=12345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "6f214a2b-1a86-4649-a87d-4f2397f99906",
   "metadata": {},
   "outputs": [],
   "source": [
    "draws = rng.standard_normal(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "34129524-e6f9-4049-b2a0-86a987f0735d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.42382504,  1.26372846, -0.87066174, -0.25917323, -0.07534331])"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draws[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "f5f565dc-4ebf-43f6-b880-896d981c1fad",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = pd.qcut(draws, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "6e65f4de-62c3-4447-8c1e-0020e07150a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(-3.121, -0.675], (0.687, 3.211], (-3.121, -0.675], (-0.675, 0.0134], (-0.675, 0.0134], ..., (0.0134, 0.687], (0.0134, 0.687], (-0.675, 0.0134], (0.0134, 0.687], (-0.675, 0.0134]]\n",
       "Length: 1000\n",
       "Categories (4, interval[float64, right]): [(-3.121, -0.675] < (-0.675, 0.0134] < (0.0134, 0.687] < (0.687, 3.211]]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "72cc0223-becb-4c4d-b0b2-79b9e70e5dae",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = pd.qcut(draws, 4, labels=['Q1', 'Q2', 'Q3', 'Q4'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "194bfd0e-1d3d-4a94-8ed4-ffdf279c65e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Q1', 'Q4', 'Q1', 'Q2', 'Q2', ..., 'Q3', 'Q3', 'Q2', 'Q3', 'Q2']\n",
       "Length: 1000\n",
       "Categories (4, object): ['Q1' < 'Q2' < 'Q3' < 'Q4']"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "57770c20-b522-4457-8e05-b5d2e7018404",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 3, 0, 1, 1, 0, 0, 2, 2, 0], dtype=int8)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins.codes[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "feb51d0e-71ac-45d3-9bf3-5f0f8d9e8edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = pd.Series(bins, name='quartile')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "56fd80db-d944-4d7d-a726-2ef4cf297141",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Locke\\AppData\\Local\\Temp\\ipykernel_17356\\1248366115.py:2: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  .groupby(bins)\n"
     ]
    }
   ],
   "source": [
    "results = (pd.Series(draws)\n",
    "           .groupby(bins)\n",
    "           .agg(['count', 'min', 'max'])\n",
    "           .reset_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "5480527a-aa6e-4117-91bb-49eef1f01075",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quartile</th>\n",
       "      <th>count</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>250</td>\n",
       "      <td>-3.119609</td>\n",
       "      <td>-0.678494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>250</td>\n",
       "      <td>-0.673305</td>\n",
       "      <td>0.008009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q3</td>\n",
       "      <td>250</td>\n",
       "      <td>0.018753</td>\n",
       "      <td>0.686183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q4</td>\n",
       "      <td>250</td>\n",
       "      <td>0.688282</td>\n",
       "      <td>3.211418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  quartile  count       min       max\n",
       "0       Q1    250 -3.119609 -0.678494\n",
       "1       Q2    250 -0.673305  0.008009\n",
       "2       Q3    250  0.018753  0.686183\n",
       "3       Q4    250  0.688282  3.211418"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "6359bc00-20bb-44b6-955e-73cf03482fca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Q1\n",
       "1    Q2\n",
       "2    Q3\n",
       "3    Q4\n",
       "Name: quartile, dtype: category\n",
       "Categories (4, object): ['Q1' < 'Q2' < 'Q3' < 'Q4']"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results['quartile']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "b8b45a92-ca28-4ee6-8e30-cb6fff179086",
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 10_000_000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "d0c218c0-53e4-43c0-a4b5-ffa6a780ea6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = pd.Series(['foo', 'bar', 'baz', 'qux'] * (N // 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "ab3ca449-b736-45f2-aed6-4d23404b53a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = labels.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "eb0fe624-d889-484d-aaa4-fc094791ca22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "520000132"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "7790fbdc-1a50-4257-b094-1920e2533c19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000512"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "32120c29-c4a7-485d-9fcc-fec036080354",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 234 ms\n",
      "Wall time: 565 ms\n"
     ]
    }
   ],
   "source": [
    "%time _ = labels.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "4b08bb2a-0cae-4742-ad07-daefdf6d3388",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "475 ms ± 56.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%timeit labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "3594ec75-15ac-4669-85c3-c664ee0639a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.6 ms ± 3.35 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit categories.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "5241639f-6a8b-4102-9d5f-4b94f73a77c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(['a', 'b', 'c', 'd'] * 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "1404021d-f2ff-48f1-80d4-0d2a8747c267",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "2    c\n",
       "3    d\n",
       "4    a\n",
       "5    b\n",
       "6    c\n",
       "7    d\n",
       "dtype: object"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "2996f61e-e9fc-44f4-8ad4-984286f1a72f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_s = s.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "28c567aa-8200-440c-8b29-a3814a4708eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "2    c\n",
       "3    d\n",
       "4    a\n",
       "5    b\n",
       "6    c\n",
       "7    d\n",
       "dtype: category\n",
       "Categories (4, object): ['a', 'b', 'c', 'd']"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "ee17f239-0b97-41a3-8753-0b67298ec55b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    1\n",
       "2    2\n",
       "3    3\n",
       "4    0\n",
       "5    1\n",
       "6    2\n",
       "7    3\n",
       "dtype: int8"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "216a4023-1284-4c44-9a08-4bd4ad1ce3de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['a', 'b', 'c', 'd'], dtype='object')"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.cat.categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "fc31e458-0f3f-446a-b604-9861e10357e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_categories = ['a', 'b', 'c', 'd', 'e']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "8127f1ae-3b23-4a9b-a8f1-ccd03bc25abd",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_s2 = cat_s.cat.set_categories(actual_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "33ae1e31-2843-4ed5-ae77-717362770644",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "2    c\n",
       "3    d\n",
       "4    a\n",
       "5    b\n",
       "6    c\n",
       "7    d\n",
       "dtype: category\n",
       "Categories (5, object): ['a', 'b', 'c', 'd', 'e']"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "da446f3f-aa21-4350-8822-5f800b7a5fcb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a    2\n",
       "b    2\n",
       "c    2\n",
       "d    2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "9d161ac7-6178-48f9-9193-f2fd7203f7ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a    2\n",
       "b    2\n",
       "c    2\n",
       "d    2\n",
       "e    0\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s2.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "e90a422d-4122-4795-ac5f-5b076a102a40",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_s3 = cat_s[cat_s.isin(['a', 'b'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "755dd7d5-1238-46b0-b486-ac738a60ba6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "4    a\n",
       "5    b\n",
       "dtype: category\n",
       "Categories (4, object): ['a', 'b', 'c', 'd']"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "699c7071-84d8-4baa-b97e-514d9736a886",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    a\n",
       "1    b\n",
       "4    a\n",
       "5    b\n",
       "dtype: category\n",
       "Categories (2, object): ['a', 'b']"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_s3.cat.remove_unused_categories()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "ea76893b-467a-4703-b9ff-62d3a44c7ae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_s = pd.Series(['a', 'b', 'c', 'd'] * 2, dtype='category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "9ab7f730-acf7-41ff-b586-11a947d96258",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     a    b    c    d\n",
       "0  1.0  0.0  0.0  0.0\n",
       "1  0.0  1.0  0.0  0.0\n",
       "2  0.0  0.0  1.0  0.0\n",
       "3  0.0  0.0  0.0  1.0\n",
       "4  1.0  0.0  0.0  0.0\n",
       "5  0.0  1.0  0.0  0.0\n",
       "6  0.0  0.0  1.0  0.0\n",
       "7  0.0  0.0  0.0  1.0"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(cat_s, dtype=float)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}