temp_rainfall.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import dependencies\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import os\n",
    "from os import listdir\n",
    "import zipfile\n",
    "import glob\n",
    "\n",
    "import pymongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create connection to MongoDB\n",
    "conn = 'mongodb://localhost:27017'\n",
    "client = pymongo.MongoClient(conn)\n",
    "\n",
    "db = client.australia_fire_db\n",
    "temp_rainfall = db.temp_rainfall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# unzip files in Resources folder.\n",
    "extension = \".zip\"\n",
    "extracted_dir_name = \".\"\n",
    "\n",
    "# Get the current working directory.\n",
    "# Need to be in root directory of this project for this to work.\n",
    "cwd_dir_name = os.getcwd()\n",
    "print(f\"The current working directory is {cwd_dir_name}.\")\n",
    "\n",
    "os.chdir(\"Resources\") # change directory from working dir to dir with zip file.\n",
    "# This should be the \"Resources\" folder.\n",
    "dir_name = os.getcwd()\n",
    "print(f\"You are now in the following directory: {dir_name}.\")\n",
    "\n",
    "for item in os.listdir(dir_name): # loop through the items in the directory.\n",
    "    if item.endswith(extension): # check for \".zip\" extension\"\n",
    "        try:\n",
    "            file_name = os.path.abspath(item) # get full path of files\n",
    "            zip_ref = zipfile.ZipFile(file_name) # create zipfile object\n",
    "            unzipped_directory = os.path.join(extracted_dir_name) # reference to the directory where the zip files will be extracted.\n",
    "            zip_ref.extractall(unzipped_directory) # extract file to dir\n",
    "            zip_ref.close() # close file\n",
    "            print(f\"Successfully unzipped {item} into the following folder:{dir_name}.\")\n",
    "        except Exception as error:\n",
    "            print(f\"Error trying to unzip data file(s).\")\n",
    "            print(error)\n",
    "            \n",
    "# Go up one directory into the project root directory.\n",
    "os.chdir(os.path.normpath(os.getcwd() + os.sep + os.pardir))\n",
    "print(os.path.normpath(os.getcwd() + os.sep + os.pardir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read in max temp csvs\n",
    "temp_path = r'Resources/australia_temp_data' \n",
    "all_temp_files = glob.glob(temp_path + \"/*.csv\")\n",
    "\n",
    "temp_li = []\n",
    "\n",
    "for filename in all_temp_files:\n",
    "    df = pd.read_csv(filename, index_col=None, header=0)\n",
    "    temp_li.append(df)\n",
    "\n",
    "temp_df = pd.concat(temp_li, axis=0, ignore_index=True)\n",
    "temp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read in rainfall csvs\n",
    "rainfall_path = r'Resources/australia_rainfall_data' \n",
    "all_rainfall_files = glob.glob(rainfall_path + \"/*.csv\")\n",
    "\n",
    "rainfall_li = []\n",
    "\n",
    "for filename in all_rainfall_files:\n",
    "    df = pd.read_csv(filename, index_col=None, header=0)\n",
    "    rainfall_li.append(df)\n",
    "\n",
    "rainfall_df = pd.concat(rainfall_li, axis=0, ignore_index=True)\n",
    "rainfall_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep neccessary columns from temp_df\n",
    "max_temp_df = temp_df[[\"Station Number\",\n",
    "                    \"Year\", \n",
    "                    \"Annual\"]]\n",
    "\n",
    "# Keep neccessary columns from rainfall_df\n",
    "new_rainfall_df = rainfall_df[[\"Station Number\",\n",
    "                    \"Year\", \n",
    "                    \"Annual\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Rename Annual column for anticipated join\n",
    "annual_temp_df = max_temp_df.rename(columns={\"Annual\": \"Annual Max Temp\"})\n",
    "annual_rainfall_df = new_rainfall_df.rename(columns={\"Annual\": \"Annual Rainfall\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop null values\n",
    "new_annual_temp_df = annual_temp_df.dropna(subset=['Annual Max Temp'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop null values\n",
    "new_annual_rainfall_df = annual_rainfall_df.dropna(subset=['Annual Rainfall'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter for data from year 1956-2019\n",
    "#most capitals have data from 1956-2019\n",
    "#2020 had an annual value of NaN since it isn't technically a complete year\n",
    "filtered_temp_df = new_annual_temp_df.loc[(new_annual_temp_df['Year'] >= 1956)]\n",
    "filtered_rainfall_df = new_annual_rainfall_df.loc[(new_annual_rainfall_df['Year'] >= 1956)]\n",
    "filtered_rainfall_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Overall mean temp and rainfall for future difference from mean calculation\n",
    "overall_mean_temp = filtered_temp_df[\"Annual Max Temp\"].mean()\n",
    "print(f'Overall Mean Temp from 1956 to 2019: {overall_mean_temp}')\n",
    "\n",
    "overall_mean_rainfall = filtered_rainfall_df[\"Annual Rainfall\"].mean()\n",
    "print(f'Overall Mean Rainfall from 1956 to 2019: {overall_mean_rainfall}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Find mean temp and rainfall for each year\n",
    "temps_by_year = filtered_temp_df.groupby(\"Year\")\n",
    "mean_temps = temps_by_year[\"Annual Max Temp\"].mean()\n",
    "\n",
    "rainfall_by_year = filtered_rainfall_df.groupby(\"Year\")\n",
    "mean_rainfall = rainfall_by_year[\"Annual Rainfall\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a new dataframe for just averages\n",
    "yearly_averages_df = pd.DataFrame({\"Avg Annual Temp\": mean_temps,\n",
    "                              \"Avg Annual Rainfall\": mean_rainfall})\n",
    "yearly_averages_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reset index so that the year will be included in a dictionary\n",
    "yearly_avg_df = yearly_averages_df.reset_index()\n",
    "yearly_avg_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yearly_avg_df['temp_difference'] = yearly_avg_df['Avg Annual Temp'] - overall_mean_temp\n",
    "yearly_avg_df['rainfall_difference'] = yearly_avg_df['Avg Annual Rainfall'] - overall_mean_rainfall\n",
    "yearly_avg_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform the dataframe into a dictionary\n",
    "yearly_avg_dict = yearly_avg_df.to_dict('records')\n",
    "yearly_avg_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# insert records into the MongoDB collection histroicalFires\n",
    "\n",
    "if (temp_rainfall.count() == 0):\n",
    "    temp_rainfall.insert(yearly_avg_dict)\n",
    "    \n",
    "else:\n",
    "    print(\"Data already exists\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}