TerraFusion_on_CC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5pyd\n",
    "import os\n",
    "import numpy as np\n",
    "import datetime\n",
    "import julian"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Credentials on Matias's VM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_user = 'terra'\n",
    "api_password = '4rr3T'\n",
    "api_endpoint = \"http://terra-condo01.ncsa.illinois.edu:8000\"\n",
    "api_file = \"terra.ncsa.illinois.edu\"\n",
    "api_path = f\"basicfusion.{api_file}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getProxy(bf_file):\n",
    "    year = bf_file.split('_')[4][:4]\n",
    "    month = bf_file.split('_')[4][4:6]\n",
    "    bf_path = f\"{bf_file}.{year}%2E{month}.{api_path}\"\n",
    "    print(bf_path)\n",
    "    h5f = h5pyd.File(bf_path, 'r', endpoint=api_endpoint, username=api_user, password=api_password)\n",
    "    return h5f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TERRA_BF_L1B_O66274_20120603081315_F000_V001.2012%2E06.basicfusion.terra.ncsa.illinois.edu\n"
     ]
    }
   ],
   "source": [
    "f = getProxy('TERRA_BF_L1B_O66274_20120603081315_F000_V001')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "SPATIAL_RESOLUTION = 0.5\n",
    "\n",
    "NUM_POINTS = 1 / SPATIAL_RESOLUTION\n",
    "NUM_LATS = int(180 / SPATIAL_RESOLUTION)\n",
    "NUM_LONS = int(360 / SPATIAL_RESOLUTION)\n",
    "\n",
    "# \n",
    "orbit_sw_sum  = np.zeros((NUM_LATS, NUM_LONS))\n",
    "orbit_sw_num  = np.zeros((NUM_LATS, NUM_LONS), dtype='int16')\n",
    "orbit_lw_sum  = np.zeros((NUM_LATS, NUM_LONS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_descending(h5f, instrument):\n",
    "    \"\"\"\n",
    "    Given a BF instance, return the starting and ending time of the descending node.\n",
    "    Note that both times are stick to the MODIS 5-min granules.\n",
    "\n",
    "    For CERES, return the julian time of the first/last descending MODIS granule in the bf file;\n",
    "    For MODIS, return the first/last descending MODIS granule in the bf file;\n",
    "    For MISR, return the first/last block of the descending MODIS granule in the bf file.\n",
    "    \n",
    "    Args:\n",
    "        h5f (hdf5 instance): instance of a basic fusion file\n",
    "        instrument (str)   : instrument from 'CERES', 'MISR', and 'MODIS'\n",
    "    \n",
    "    Returns:\n",
    "        out_array (array): return MODIS granules, CERES start/end julian times, \n",
    "                                     or MISR start/end blocks depending on the instrument\n",
    "    \"\"\"\n",
    "\n",
    "    MODIS_granules = [item[0] for item in h5f['MODIS'].items()]\n",
    "    if len(MODIS_granules) == 0:\n",
    "        print(\">> IOError( no available MODIS granule in orbit {} )\".format(bf_file))\n",
    "        out_array = np.array([0, 0])\n",
    "\n",
    "    else:\n",
    "        descending_granules = []\n",
    "        for igranule in MODIS_granules:     \n",
    "            try:\n",
    "                lats = h5f['MODIS/{}/_1KM/Geolocation/Latitude'.format(igranule)][:]\n",
    "                lons = h5f['MODIS/{}/_1KM/Geolocation/Longitude'.format(igranule)][:]\n",
    "            except KeyError:\n",
    "                print(\">> KeyError( cannot access lat/lon in {} )\".format(igranule))\n",
    "                continue\n",
    "\n",
    "            # Process descending granules only (neutral granules are omitted)\n",
    "            # May be improved by using a better criteria (?)\n",
    "            cnt = 0\n",
    "            for i in range(1, len(lats)):\n",
    "                if all(lats[i]<lats[i-1]) == False:\n",
    "                    cnt += 1\n",
    "            if cnt >= 1000:\n",
    "                # print(\">> CriteriaError( this is not a descending granule {} )\".format(igranule))\n",
    "                continue\n",
    "            else:\n",
    "                descending_granules.append(igranule)\n",
    "\n",
    "        descending_julian_bound = np.array([granuletime_to_jd(descending_granules[0]), granuletime_to_jd(descending_granules[-1], offset_mins=5)])\n",
    "\n",
    "        if instrument.startswith('MODIS'):\n",
    "            out_array = descending_granules\n",
    "        elif instrument.startswith('CERES'):\n",
    "            out_array = descending_julian_bound\n",
    "        elif instrument.startswith('MISR'):\n",
    "            # get MISR camera\n",
    "            icam = instrument.split('.')[1]\n",
    "            # get MISR BlockCenterTime\n",
    "            try:\n",
    "                bct = h5f['MISR/AN/BlockCenterTime'][:]\n",
    "            except:\n",
    "                print(\">> IOError( cannot access MISR data )\")\n",
    "                out_array = np.array([0, 0])\n",
    "            else:\n",
    "                misr_block_julian = []\n",
    "                for ibct in bct: \n",
    "                    yr, mon, day = str(ibct).split('-') # ibct: \"b'2012-06-03T07:34:23.000532Z\"\n",
    "                    yr = int(yr[2:])                    # 2012\n",
    "                    if yr == 0:\n",
    "                        misr_block_julian.append(0)\n",
    "                    else:\n",
    "                        hr, mn, sec_decimal = day[3:].split(':') # \"07:34:23.000532Z'\"\n",
    "                        sec = int(float(sec_decimal[:-2]))       # 23.000532\n",
    "                        millisec = int(1000*(float(sec_decimal[:-2]) - sec)) # .000532 * 1000\n",
    "\n",
    "                        dt = datetime.datetime(yr, int(mon), int(day[:2]), int(hr), int(mn), sec, millisec)\n",
    "                        misr_block_julian.append(julian.to_jd(dt, fmt='jd'))\n",
    "                \n",
    "                misr_block_julian = np.array(misr_block_julian)\n",
    "                misr_descending_blocks = np.where((misr_block_julian>=descending_julian_bound[0])&(misr_block_julian<=descending_julian_bound[1]))[0]\n",
    "\n",
    "                out_array = misr_descending_blocks\n",
    "    return out_array\n",
    "\n",
    "\n",
    "def granuletime_to_jd(mod_granule_string, offset_mins=0):\n",
    "    \"\"\"\n",
    "    Convert a MODIS granule string (e.g., 'granule_2012155_0700') to julian date (e.g., 2456081.7916666665)\n",
    "    \n",
    "    Args:\n",
    "        mod_granule_string (str): MODIS granule string\n",
    "        offset_mins (int, optional): offset minutes\n",
    "\n",
    "    Returns:\n",
    "        jd (int): julian date\n",
    "    \"\"\"\n",
    "    yr = int(mod_granule_string.split('_')[1][:4])\n",
    "    doy = int(mod_granule_string.split('_')[1][-3:])\n",
    "    hr = int(mod_granule_string.split('_')[2][:2])\n",
    "    mn = int(mod_granule_string.split('_')[2][-2:])\n",
    "\n",
    "    dt = datetime.datetime(yr, 1, 1, hr, mn) + datetime.timedelta(days=doy-1, minutes=offset_mins)\n",
    "    jd = julian.to_jd(dt, fmt='jd')    \n",
    "    return jd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,\n",
       "        40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,  52,\n",
       "        53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,  65,\n",
       "        66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,  78,\n",
       "        79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,  91,\n",
       "        92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103, 104,\n",
       "       105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,\n",
       "       118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130,\n",
       "       131, 132, 133, 134, 135, 136, 137, 138])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_descending(f, 'MISR.AN')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}