06-pandas-dataframes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cb3b5773",
   "metadata": {},
   "source": [
    "# Pandas DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "292515eb-3539-43e4-aa2e-0c4128a0de5d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sunpy in /Users/pgm5/anaconda3/lib/python3.10/site-packages (5.0.0)\n",
      "Requirement already satisfied: drms in /Users/pgm5/anaconda3/lib/python3.10/site-packages (0.6.4)\n",
      "Requirement already satisfied: cdflib in /Users/pgm5/anaconda3/lib/python3.10/site-packages (1.1.0)\n",
      "Requirement already satisfied: zeep in /Users/pgm5/anaconda3/lib/python3.10/site-packages (4.1.0)\n",
      "Requirement already satisfied: h5netcdf in /Users/pgm5/anaconda3/lib/python3.10/site-packages (1.2.0)\n",
      "Requirement already satisfied: matplotlib in /Users/pgm5/anaconda3/lib/python3.10/site-packages (3.7.1)\n",
      "Requirement already satisfied: astropy>=5.0.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from sunpy) (5.3.2)\n",
      "Requirement already satisfied: numpy>=1.21.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from sunpy) (1.25.2)\n",
      "Requirement already satisfied: packaging>=19.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from sunpy) (23.0)\n",
      "Requirement already satisfied: parfive[ftp]>=2.0.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from sunpy) (2.0.2)\n",
      "Requirement already satisfied: pandas in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from drms) (2.0.3)\n",
      "Requirement already satisfied: attrs>=17.2.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (22.1.0)\n",
      "Requirement already satisfied: cached-property>=1.3.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (1.5.2)\n",
      "Requirement already satisfied: isodate>=0.5.4 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (0.6.1)\n",
      "Requirement already satisfied: lxml>=4.6.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (4.9.2)\n",
      "Requirement already satisfied: platformdirs>=1.4.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (3.10.0)\n",
      "Requirement already satisfied: requests>=2.7.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (2.31.0)\n",
      "Requirement already satisfied: requests-toolbelt>=0.7.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (1.0.0)\n",
      "Requirement already satisfied: requests-file>=1.5.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (1.5.1)\n",
      "Requirement already satisfied: pytz in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from zeep) (2022.1)\n",
      "Requirement already satisfied: h5py in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from h5netcdf) (3.9.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from matplotlib) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from matplotlib) (4.25.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from matplotlib) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from matplotlib) (9.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from matplotlib) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: pyerfa>=2.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from astropy>=5.0.1->sunpy) (2.0.0.3)\n",
      "Requirement already satisfied: PyYAML>=3.13 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from astropy>=5.0.1->sunpy) (6.0)\n",
      "Requirement already satisfied: six in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from isodate>=0.5.4->zeep) (1.16.0)\n",
      "Requirement already satisfied: tqdm>=4.27.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from parfive[ftp]>=2.0.0->sunpy) (4.65.0)\n",
      "Requirement already satisfied: aiohttp in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from parfive[ftp]>=2.0.0->sunpy) (3.8.5)\n",
      "Requirement already satisfied: aioftp>=0.17.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from parfive[ftp]>=2.0.0->sunpy) (0.21.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from requests>=2.7.0->zeep) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from requests>=2.7.0->zeep) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from requests>=2.7.0->zeep) (1.26.16)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from requests>=2.7.0->zeep) (2023.7.22)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from pandas->drms) (2023.3)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from aiohttp->parfive[ftp]>=2.0.0->sunpy) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from aiohttp->parfive[ftp]>=2.0.0->sunpy) (4.0.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from aiohttp->parfive[ftp]>=2.0.0->sunpy) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from aiohttp->parfive[ftp]>=2.0.0->sunpy) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/pgm5/anaconda3/lib/python3.10/site-packages (from aiohttp->parfive[ftp]>=2.0.0->sunpy) (1.3.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/pgm5/anaconda3/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Files Downloaded: 100%|██████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00,  4.80file/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 2880 entries, 2021-07-01 00:00:29.999998 to 2021-07-02 23:59:30.000001\n",
      "Data columns (total 7 columns):\n",
      " #   Column                  Non-Null Count  Dtype  \n",
      "---  ------                  --------------  -----  \n",
      " 0   B_RTN_0                 2880 non-null   float32\n",
      " 1   B_RTN_1                 2880 non-null   float32\n",
      " 2   B_RTN_2                 2880 non-null   float32\n",
      " 3   QUALITY_BITMASK         2880 non-null   uint16 \n",
      " 4   QUALITY_FLAG            2880 non-null   uint8  \n",
      " 5   VECTOR_RANGE            2880 non-null   uint8  \n",
      " 6   VECTOR_TIME_RESOLUTION  2880 non-null   float32\n",
      "dtypes: float32(4), uint16(1), uint8(2)\n",
      "memory usage: 78.8 KB\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# First, we load the data we want to work with.\n",
    "# The code below pulls together the process we stepped through in the previous part.\n",
    "\n",
    "# Install a pip package in the current Jupyter kernel\n",
    "import sys\n",
    "!{sys.executable} -m pip install sunpy drms cdflib zeep h5netcdf matplotlib\n",
    "\n",
    "from sunpy.net import Fido\n",
    "from sunpy.net import attrs as a\n",
    "from sunpy.timeseries import TimeSeries\n",
    "\n",
    "date_range = a.Time('2021/07/01', '2021/07/08')\n",
    "dataset = a.cdaweb.Dataset('SOLO_L2_MAG-RTN-NORMAL-1-MINUTE')\n",
    "result = Fido.search(date_range, dataset)\n",
    "\n",
    "downloaded_files = Fido.fetch(result[0, 0:2])\n",
    "solo_mag = TimeSeries(downloaded_files, concatenate=True)\n",
    "solo_mag_data = solo_mag.to_dataframe()\n",
    "print(solo_mag_data.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bf7edaa-f49b-4e7f-bec0-5fb67224465e",
   "metadata": {},
   "source": [
    "## Note about Pandas DataFrames/Series\n",
    "\n",
    "A [DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html) is a collection of [Series](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.Series.html); The DataFrame is the way Pandas represents a table, and Series is the data-structure Pandas use to represent a column.\n",
    "\n",
    "Pandas is built on top of the [Numpy](https://www.numpy.org/) library, which in practice means that most of the methods definned for Numpy Arrays apply to Pandas Series/DataFrames.\n",
    "\n",
    "What makes Pandas so attractive is the powerful interface to access individual records of the table, proper handling of missing values, and relational-databases operations between DataFrames."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaed5980-a5c8-4ef1-b891-11502fee0301",
   "metadata": {},
   "source": [
    "## Selecting values\n",
    "\n",
    "To access a value at the position `[i,j]` of a DataFrame, we have two options, depending on what is the meaning of `i` in use. Remember that a DataFrame provides an _index_ as a way to identify the rows of the table; a row, then, has a _position_ inside the table as well as a _label_, which uniquely identifies its _entry_ in the DataFrame."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b66cb4d9-2a86-4a16-8388-70fd9fd66638",
   "metadata": {},
   "source": [
    "### Use `DataFrame.iloc[..., ...]` to select values by their (entry) position\n",
    "\n",
    "Can specify location by numerical index analogously to 2D version of character selection in strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5f2abbe8-94da-4d16-abb1-d2e1bfa8b6f1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2058102\n"
     ]
    }
   ],
   "source": [
    "print(solo_mag_data.iloc[0,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b105c77a-87ae-4eb6-a634-bb6c357cf556",
   "metadata": {},
   "source": [
    "### Use `DataFrame.loc[..., ...]` to select values by their (entry) label.\n",
    "\n",
    "Can specify location by row and/or column name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b3d558b3-bb8d-4876-a2c5-4c79920d2cb0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2058102\n"
     ]
    }
   ],
   "source": [
    "print(solo_mag_data.loc['2021-07-01 00:00:29.999998', 'B_RTN_0'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "262a91f3-3f50-4f17-af6d-0a3faa34bd0e",
   "metadata": {},
   "source": [
    "### Use `:` on its own to mean all columns or all rows.\n",
    "\n",
    "Just like Python’s usual slicing notation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62324599-91a7-45de-bb09-61f42946a937",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(solo_mag_data.loc['2021-07-01 00:00:29.999998', :])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3c92739-8faa-4b8b-bed2-457f42b3cd23",
   "metadata": {},
   "source": [
    "Would get the same result printing `data.loc['2021-07-01 00:00:29.999998']` (without a second index)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee5baf73-f608-4fef-8ffd-af1f393dc966",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(solo_mag_data.loc[:, 'B_RTN_0'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35bdf1ba-68e4-4f76-a530-20f78d3bf2d0",
   "metadata": {},
   "source": [
    "- Would get the same result printing `data['B_RTN_0']`\n",
    "- Also get the same result printing `data.B_RTN_0` (not recommended, because easily confused with . notation for methods)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83849d0a-8d5a-414d-928e-507bef181916",
   "metadata": {},
   "source": [
    "### Select multiple columns or rows using `DataFrame.loc` and a named slice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9861df05-bea4-407a-b6ee-64a712419914",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                             B_RTN_0   B_RTN_1   B_RTN_2\n",
      "EPOCH                                                   \n",
      "2021-07-01 00:01:30.000004  0.410739  9.685264 -0.023507\n",
      "2021-07-01 00:02:30.000001  0.758328  9.928966 -2.149603\n",
      "2021-07-01 00:03:29.999997 -0.541748  9.544032 -3.647228\n",
      "2021-07-01 00:04:30.000004  3.109208  8.734163  3.252406\n"
     ]
    }
   ],
   "source": [
    "print(solo_mag_data.loc['2021-07-01 00:01:30.000004':'2021-07-01 00:04:30.000004', 'B_RTN_0':'B_RTN_2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ad145ea-c1a5-405c-aa5f-98e16a6d492e",
   "metadata": {},
   "source": [
    "In the above code, we discover that **slicing using `loc` is inclusive at both ends**, which differs from **slicing using `iloc`**, where slicing indicates everything up to but not including the final index."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3f9c0e0-14eb-4dfc-adc9-f24089d8a69d",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Result of slicing can be used in further operations.\n",
    "\n",
    "- Usually don’t just print a slice.\n",
    "- All the statistical operators that work on entire dataframes work the same way on slices.\n",
    "- E.g., calculate max of a slice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f0dcf75-1a13-4924-86dc-fd2d8a898b9d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(solo_mag_data.loc['2021-07-01 00:01:30.000004':'2021-07-01 00:04:30.000004', 'B_RTN_0':'B_RTN_2'].max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d010e23-ccd7-43b7-b434-76ae5048f542",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(solo_mag_data.loc['2021-07-01 00:01:30.000004':'2021-07-01 00:04:30.000004', 'B_RTN_0':'B_RTN_2'].min())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a861963-167d-4ce6-9fa3-6cc291de9c2d",
   "metadata": {},
   "source": [
    "## Use comparisons to select data based on value.\n",
    "\n",
    "- Comparison is applied element by element.\n",
    "- Returns a similarly-shaped dataframe of `True` and `False`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16708c09-8a59-4c76-8d68-e9d3c5364772",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Use a subset of data to keep output readable.\n",
    "subset = solo_mag_data.loc['2021-07-01 00:01:30.000004':'2021-07-01 00:04:30.000004', 'B_RTN_0':'B_RTN_2']\n",
    "print('Subset of data:\\n', subset)\n",
    "\n",
    "# Which values were positive ?\n",
    "print('\\nWhere are values positive?\\n', subset > 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "818e573e-a8df-407b-98c1-ece7851e1fc5",
   "metadata": {},
   "source": [
    "### Select values or NaN using a Boolean mask.\n",
    "\n",
    "A frame full of Booleans is sometimes called a mask because of how it can be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a4b195b-19a7-44f4-94fa-66f6f317bfab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mask = subset > 0\n",
    "print(subset[mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e884f457-53b7-4229-b492-20924ec92f60",
   "metadata": {},
   "source": [
    "- Get the value where the mask is true, and NaN (Not a Number) where it is false.\n",
    "- Useful because NaNs are ignored by operations like max, min, average, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5d2cda3-0a65-4b4a-9445-efc7d21d3142",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(subset[subset > 0].describe())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cf862b8-fdec-472b-9bb5-132651c2c3eb",
   "metadata": {},
   "source": [
    "## Key Points\n",
    "\n",
    "- Use `DataFrame.iloc[..., ...]` to select values by integer location.\n",
    "- Use `:` on its own to mean all columns or all rows.\n",
    "- Select multiple columns or rows using `DataFrame.loc` and a named slice.\n",
    "- Result of slicing can be used in further operations.\n",
    "- Use comparisons to select data based on value.\n",
    "- Select values or NaN using a Boolean mask."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ad748c9-10b3-4438-83fc-7fccdf7cb506",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}