run.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Harvest and transform\n",
    "\n",
    "There is a considerable amount of data transform and processing, before we obtain a per-EZG, per-variable timeseries CSV.\n",
    "Most of the heavy lifiting is outsources to thrid party packages wherever possible, the remainder is implemented into a `EZG` class, which represents the EZG and makes processing easy. \n",
    "\n",
    "The first part of this example shows how a single EZG can be accessed, the second part, how the EZG can access data. The processed data is then reduced and saved to disk."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "from wetterdienst.provider.dwd import radar, observation\n",
    "from datetime import datetime as dt\n",
    "from datetime import timedelta as td\n",
    "\n",
    "from harvest.ezg import EZG\n",
    "from harvest.radolan import RadolanUtility\n",
    "from harvest.reducers.station import transpose_station_data\n",
    "from harvest.reducers.radolan import spatial_reduce"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All EZGs can easily be read from a geo-spatial file. Read a ESRI Shapefile, GeoJSON, whatever is convenient for you.\n",
    "It has to be a vector file and the coordinate reference system has to be defined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load all EZGs from the file\n",
    "all_ezgs = EZG.from_file('./EZG/Basiseinzugsgebiet (AWGN)_polygon.shp')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a single one from the list, print attribute table and show the shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('OBJECTID', 206), ('OBJECT_ID', 1464.0), ('FG_ID', 1464), ('FGKZ_NR', 2472600000000.0), ('LANGNAME', 'Ohrenbach'), ('FLAECHE', 51.011), ('EINZUGSGEB', 4), ('EINZUGSG00', 'Einzugsgebiet'), ('EINZUGSG01', 'B'), ('EINZUGSG02', 'Basis-Einzugsgebiet'), ('VOR_FGKZ_N', 2472000000000.0), ('VOR_FG_ID', 4217.0), ('VOR_FG_LAN', 'Mud'), ('WASSERKOER', None), ('WASSERKO00', None), ('AENDERUNGS', '20160818'), ('AENDERUN00', '20160818'), ('LENGTH', 0.0), ('MOD_BY', 'GEO'), ('LAST_MOD', '20160818'), ('SE_ANNO_CA', None), ('WASSERKO01', None), ('FLAECHE_GE', 378.145), ('STAND_FACH', 'Mai 2021')])\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"300\" height=\"300\" viewBox=\"505046.98200800014 5499707.774308001 10409.02098400006 10852.944083999842\" preserveAspectRatio=\"xMinYMin meet\"><g transform=\"matrix(1,0,0,-1,0,11010268.492700001)\"><path fill-rule=\"evenodd\" fill=\"#66cc99\" stroke=\"#555555\" stroke-width=\"72.35296055999895\" opacity=\"0.6\" d=\"M 513262.83999999985,5504514.114600001 L 513374.46949999966,5504522.1866999995 L 513399.80429999996,5504488.9364 L 513449.29949999973,5504450.999600001 L 513629.4768000003,5504364.533600001 L 513716.4419999998,5504315.053099999 L 513797.9090999998,5504257.066 L 513866.8810999999,5504185.104499999 L 513897.6185999997,5504145.620100001 L 513920.8591,5504101.637599999 L 513938.3618000001,5504055.156099999 L 513972.3373999996,5503917.221100001 L 513991.33949999977,5503863.2325 L 514014.8196999999,5503818.250399999 L 514043.80819999985,5503777.7667 L 514102.2843000004,5503708.294 L 514285.70990000013,5503525.8662 L 514325.9434000002,5503496.877699999 L 514369.4260999998,5503471.887499999 L 514505.37179999985,5503408.4125 L 514579.09190000035,5503361.431 L 514666.05680000037,5503290.459000001 L 514733.0296,5503229.493100001 L 514816.73589999974,5503159.510600001 L 514849.9824000001,5503128.5229 L 514884.2185000004,5503086.039899999 L 514909.20830000006,5503043.057 L 514926.70100000035,5502996.0757 L 514941.9444000004,5502911.609099999 L 514942.4433000004,5502797.6548999995 L 514991.92260000017,5502604.7316 L 515011.6644000001,5502559.2497000005 L 515024.90859999973,5502466.786599999 L 515004.4149000002,5502211.8884 L 514986.9216,5502125.912699999 L 514978.42459999956,5502064.4472 L 514979.9232999999,5501970.9845 L 514975.68489999976,5501915.006899999 L 514976.4243000001,5501865.026900001 L 514982.4215000002,5501819.0451 L 514994.1666000001,5501785.058599999 L 515021.65550000034,5501754.071 L 515054.0421000002,5501701.8814 L 515031.15139999986,5501696.0940000005 L 514965.1774000004,5501644.115 L 514880.9605999999,5501599.133199999 L 514837.97759999987,5501566.146500001 L 514742.0151000004,5501460.188999999 L 514679.78949999996,5501373.223999999 L 514645.8026999999,5501336.738600001 L 514605.31879999954,5501308.2501 L 514567.08389999997,5501286.259 L 514512.85539999977,5501271.764900001 L 514413.39499999955,5501267.766899999 L 514314.4444000004,5501256.2717 L 514269.20239999983,5501254.272600001 L 514114.51400000043,5501256.272399999 L 514065.5334999999,5501246.7762 L 513934.33550000004,5501213.289999999 L 513869.8613,5501213.290200001 L 513820.6309000002,5501206.293099999 L 513673.68929999974,5501177.3050999995 L 513538.49289999995,5501138.3211 L 513484.50430000015,5501114.3311 L 513445.02979999967,5501083.8431 L 513319.57909999974,5501025.366900001 L 513266.8501000004,5500994.3794 L 513030.6930999998,5500818.460200001 L 512888.99899999984,5500755.9756000005 L 512844.76659999974,5500732.485099999 L 512804.78249999974,5500709.4944 L 512720.81549999956,5500648.518999999 L 512687.3185999999,5500612.533399999 L 512659.33949999977,5500571.050100001 L 512276.48900000006,5500109.735200001 L 512234.0060999999,5500129.727399999 L 512061.0756000001,5500230.187899999 L 511986.1058,5500277.6691 L 511934.1266000001,5500304.668500001 L 511885.3958999999,5500324.650699999 L 511836.91500000004,5500336.146500001 L 511787.6843999997,5500343.1439 L 511738.2039000001,5500343.643999999 L 511638.2429999998,5500334.638 L 511538.53199999966,5500337.647399999 L 511440.82079999987,5500357.649900001 L 511199.4160000002,5500422.6151 L 511103.4539000001,5500434.6109 L 511000.74430000037,5500441.1087 L 510653.1310999999,5500480.594699999 L 510606.64940000046,5500488.591700001 L 510555.66949999984,5500502.5864 L 510413.72559999954,5500550.568 L 510314.7644999996,5500561.064200001 L 510065.8622000003,5500540.073899999 L 510018.3808000004,5500526.579500001 L 509890.9304999998,5500472.6017 L 509681.0120000001,5500316.664899999 L 509613.0374999996,5500148.732000001 L 509482.0898000002,5500253.691 L 509323.1523000002,5500273.6839000005 L 509079.74789999984,5500277.6833999995 L 508879.07650000043,5500238.200099999 L 508732.6341000004,5500242.199200001 L 508609.9327999996,5500317.6697 L 508519.9687999999,5500404.1357 L 508498.2181000002,5500530.5855 L 508498.21910000034,5500681.5252 L 508490.9829000002,5500822.4691 L 508472.9907999998,5500944.920399999 L 508383.0268000001,5501041.882099999 L 508249.57990000024,5501150.339500001 L 508170.3618999999,5501294.282500001 L 508051.40940000024,5501413.2357 L 507943.45210000034,5501445.723200001 L 507788.51329999976,5501492.7051 L 507676.8075000001,5501539.1874 L 507497.3788999999,5501690.1281 L 506952.84520000033,5502043.989499999 L 506910.36199999973,5502069.979499999 L 506864.8799999999,5502090.9714 L 506779.1640999997,5502146.9495 L 506741.9288999997,5502175.938100001 L 506670.20749999955,5502245.4109000005 L 506260.6211000001,5502683.7381 L 506196.89690000005,5502760.208000001 L 506058.20239999983,5502904.1514 L 505969.23759999964,5502949.6236000005 L 505922.75600000005,5502968.126499999 L 505827.29360000044,5502997.1154 L 505778.56290000025,5503007.611500001 L 505524.66280000005,5503025.615599999 L 505479.93039999995,5503033.1029 L 505448.94290000014,5503063.091 L 505471.4341000002,5503106.5736 L 505481.1808000002,5503155.054099999 L 505504.92349999957,5503454.434900001 L 505511.4210999999,5503503.4155 L 505532.91309999954,5503600.8763999995 L 505549.6579,5503754.3149999995 L 505566.6518000001,5503855.274700001 L 505615.1331000002,5503934.743000001 L 505649.61980000045,5503970.728399999 L 505701.3498999998,5504056.194 L 505760.32749999966,5504136.661499999 L 505831.0499,5504207.6329 L 505915.5175999999,5504278.104499999 L 505993.48709999956,5504322.5864 L 506077.45440000016,5504380.062999999 L 506112.69049999956,5504413.0495 L 506180.1649000002,5504486.5199 L 506206.15479999967,5504528.5031 L 506222.1487999996,5504575.4846 L 506238.6429000003,5504673.945 L 506242.3920999998,5504731.9221 L 506234.3959999997,5504836.880000001 L 506218.6520999996,5504869.866900001 L 506193.17250000034,5504912.8499 L 506131.18740000017,5504991.3191 L 506113.44460000005,5505036.801100001 L 506011.2355000004,5505145.758400001 L 505871.04140000045,5505287.7027 L 505839.80399999954,5505327.187000001 L 505812.06510000024,5505368.670399999 L 505765.0845999997,5505456.1362 L 505738.84520000033,5505552.5978999995 L 505719.8535000002,5505649.5594 L 505743.3443,5505693.552100001 L 505774.8322999999,5505731.0272 L 505857.05040000007,5505788.0141 L 505956.51180000044,5505840.4826 L 506028.2340000002,5505888.9629 L 506059.2224000003,5505928.4471 L 506126.4463999998,5506001.9176 L 506182.9247000003,5506084.3847 L 506201.6676000003,5506130.8661 L 506245.9008999998,5506220.3302 L 506330.11950000003,5506455.2366 L 506362.35780000035,5506601.677999999 L 506368.1058,5506651.6581999995 L 506373.6041000001,5506751.118899999 L 506368.3574999999,5506901.0591 L 506364.85950000025,5506950.539899999 L 506353.3644000003,5507012.015699999 L 506342.8685999997,5507048.0012 L 506324.8761,5507094.482899999 L 506300.3855999997,5507139.965 L 506281.6434000004,5507184.447000001 L 506257.15359999985,5507227.430500001 L 506096.71800000034,5507418.8552 L 505983.26350000035,5507583.2904 L 505958.02359999996,5507626.273399999 L 505940.03110000025,5507672.755000001 L 505916.5404000003,5507716.7377 L 505898.5480000004,5507763.2192 L 505884.8036000002,5507811.2006 L 505864.56350000016,5508063.600199999 L 505863.31439999957,5508159.562000001 L 505874.06060000043,5508208.5427 L 505905.5493000001,5508303.004899999 L 506080.98539999966,5509031.7147 L 506112.72360000014,5509178.656199999 L 506144.4621000001,5509273.1186999995 L 506152.45990000013,5509385.073899999 L 506146.46279999986,5509480.046599999 L 506132.46949999966,5509621.98 L 506131.2204,5509721.441 L 506152.4633999998,5509869.881899999 L 506244.6771,5509908.366 L 506339.1403999999,5509939.3532 L 506384.6224999996,5509958.3456 L 506426.10649999976,5509985.8344 L 506515.8214999996,5510028.8169 L 506611.5340999998,5510056.805400001 L 506704.74789999984,5510092.7907 L 506790.2143000001,5510118.779999999 L 506885.1774000004,5510138.771500001 L 506948.40249999985,5510144.768999999 L 506997.8828999996,5510142.2697 L 507096.33440000005,5510124.7762 L 507192.5462999996,5510099.2859000005 L 507286.7690000003,5510066.2984 L 507434.45100000035,5510041.807600001 L 507533.92179999966,5510033.3104 L 507681.8635,5510013.8177000005 L 507781.0744000003,5510004.321 L 507830.8048999999,5510009.318700001 L 507918.7706000004,5510034.8083999995 L 508254.88939999975,5510158.7575 L 508314.6158999996,5510155.2586 L 508354.1003999999,5510148.2612 L 508393.8345999997,5510135.765900001 L 508441.31549999956,5510101.779200001 L 508473.05260000005,5510007.316500001 L 508525.5306000002,5509814.392999999 L 508551.0195000004,5509666.951400001 L 508581.7570000002,5509571.988700001 L 508601.2489,5509526.006899999 L 508633.73560000025,5509431.544199999 L 508676.21829999983,5509341.0802 L 508748.68840000033,5509155.1634 L 508792.6705,5509065.1888999995 L 509077.05510000046,5508537.3969 L 509092.04839999974,5508440.945 L 509069.8065999998,5508343.9736 L 509011.5784,5508206.0285 L 508985.5878999997,5508109.567 L 508976.341,5508015.6041 L 508994.3331000004,5507912.6449 L 509007.3278999999,5507864.664000001 L 509093.29289999977,5507698.7294 L 509315.2026000004,5507292.8892 L 509360.1843999997,5507203.9244 L 509384.1743000001,5507117.9582 L 509433.65440000035,5507033.9914 L 509484.2093000002,5506916.003900001 L 509511.51889999956,5506846.1522 L 509540.6974999998,5506784.203600001 L 509573.9484000001,5506741.8379999995 L 509623.0991000002,5506726.8101 L 509686.1133000003,5506729.3289 L 509814.46889999975,5506736.8869 L 509875.4473000001,5506729.6142 L 509917.7145999996,5506704.6011 L 510005.8362999996,5506655.9322 L 510087.07359999977,5506566.9344999995 L 510168.1171000004,5506482.7839 L 510276.1113999998,5506392.4277 L 510382.2631000001,5506287.433 L 510441.98029999994,5506250.979699999 L 510484.2466000002,5506195.6227 L 510481.8189000003,5506074.150699999 L 510473.4784000004,5505979.1446 L 510473.0865000002,5505867.4639 L 510485.88040000014,5505790.2951 L 510498.09339999966,5505727.668099999 L 510555.96690000035,5505646.2322 L 510611.41779999994,5505595.043099999 L 510692.65479999967,5505506.044500001 L 510789.30539999995,5505395.8145 L 510880.0434999997,5505312.0502 L 510987.94049999956,5505224.116900001 L 511129.5752999997,5505142.386499999 L 511190.06709999964,5505086.5439 L 511235.9204000002,5505032.5436 L 511255.1135,5504977.478 L 511254.14130000025,5504910.682600001 L 511340.32490000036,5504880.1417 L 511471.3934000004,5504819.8365 L 511627.1827999996,5504748.3815 L 511835.1277999999,5504647.4529 L 512093.0976,5504570.370999999 L 512375.40029999986,5504491.833900001 L 512480.1014999999,5504483.8802000005 L 512581.79789999966,5504490.3716 L 512671.2795000002,5504498.802200001 L 512765.31790000014,5504514.697899999 L 512832.88960000034,5504524.680600001 L 512928.49459999986,5504525.170700001 L 512985.98699999973,5504510.8662 L 513069.24230000004,5504504.706499999 L 513220.5050999997,5504517.866 L 513262.83999999985,5504514.114600001 z\" /></g></svg>",
      "text/plain": [
       "<shapely.geometry.polygon.Polygon at 0x7f3c039a6410>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use just one and download radolan data\n",
    "ezg = all_ezgs[13]\n",
    "\n",
    "print(ezg.properties)\n",
    "\n",
    "ezg.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameter\n",
    "\n",
    "The next cell is using the Radolan utility to specify the request and hot-load the data. This takes a while, but it is cached for this session and the EZGs use the cache instead of re-downloading radolan data all over the place.\n",
    "\n",
    "This is a possbile TODO, as it would be even more convenient to save the cache to disk, if the user wishes so. One has to trade-off disk-space for memory here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hot-load RADOLAN, to make stuff faster\n",
    "radolan = RadolanUtility(\n",
    "    parameter=radar.DwdRadarParameter.RADOLAN_CDC,\n",
    "    period=radar.DwdRadarPeriod.RECENT,\n",
    "    resolution=radar.DwdRadarResolution.DAILY,\n",
    "    start_date=dt.now() - td(days=365),\n",
    "    end_date=dt.now()\n",
    ")\n",
    "\n",
    "# hot load\n",
    "radolan._load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/240 [00:00<?, ?it/s]/home/mirko/anaconda3/envs/gdal/lib/python3.10/site-packages/dateparser/utils/__init__.py:130: PytzUsageWarning: The localize method is no longer necessary, as this time zone supports the fold attribute (PEP 495). For more details on migrating to a PEP 495-compliant implementation, see https://pytz-deprecation-shim.readthedocs.io/en/latest/migration.html\n",
      "  date_obj = tz.localize(date_obj)\n",
      "  0%|          | 1/240 [00:21<1:27:18, 21.92s/it]\n"
     ]
    }
   ],
   "source": [
    "historical = RadolanUtility(\n",
    "    parameter=radar.DwdRadarParameter.RADOLAN_CDC,\n",
    "    period=radar.DwdRadarPeriod.HISTORICAL,\n",
    "    resolution=radar.DwdRadarResolution.DAILY\n",
    ")\n",
    "historical._load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, the main loop for all requested ezgs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "No weather stations were found for coordinate 49.67015244651128°N and 8.548990660513383°E and distance 15.0km\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "No weather stations were found for coordinate 49.026501650686654°N and 10.389943924247676°E and distance 15.0km\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "No weather stations were found for coordinate 49.026501650686654°N and 10.389943924247676°E and distance 15.0km\n",
      "parameters {'end_of_interval', 'true_local_time'} are skipped in tidy format as the date parameters are currently not converted to floats\n",
      "/home/mirko/Dropbox/hydrocode/camels/harvest/harvest/reducers/radolan.py:28: UserWarning: Warning: converting a masked element to nan.\n",
      "  data['mean'] = np.fromiter((chunk.mean() for chunk in radolan_chunks), dtype=float)\n",
      "/home/mirko/Dropbox/hydrocode/camels/harvest/harvest/reducers/radolan.py:40: UserWarning: Warning: converting a masked element to nan.\n",
      "  data['sum'] = np.fromiter((chunk.sum() for chunk in radolan_chunks), dtype=float)\n"
     ]
    }
   ],
   "source": [
    "# Some settings\n",
    "OUT_DIR = './temp/'                         # output directory\n",
    "name_properties = ['FG_ID', 'LANGNAME']     # properites, that should make up the folder names\n",
    "if_exists = 'overwirte'                     # if folder exists, either 'skip' downlaod or 'overwrite' old folder\n",
    "STATION_DISTANCE = 15                       # distance in km - if there are no stations inside the EZG, search within this distance\n",
    "STATION_CLOSEST_N = 1                       # if there are no stations inside and not within DISTANCE, take the N closest stations\n",
    "OMIT_QUALITY_FLAG = True                    # omit the quality flag in the station data? - set to False if not in development\n",
    "\n",
    "# we will iterate ezgs, set it to all_ezgs if you want to iterate all\n",
    "ezgs = all_ezgs[:5]\n",
    "\n",
    "# dirty fix, cause I am too stupid\n",
    "#EZG._dwd_request_params['period'] = observation.DwdObservationPeriod.RECENT\n",
    "#PAR = [observation.DwdObservationDataset.TEMPERATURE_AIR, observation.DwdObservationDataset.SOLAR, observation.DwdObservationDataset.WIND]\n",
    "#EZG._dwd_request_params['period'] = observation.DwdObservationPeriod.HISTORICAL\n",
    "EZG._dwd_request_params['resolution'] = observation.DwdObservationResolution.DAILY\n",
    "PARAMETER = [observation.DwdObservationDataset.CLIMATE_SUMMARY]\n",
    "PERIODS = [observation.DwdObservationPeriod.HISTORICAL, observation.DwdObservationPeriod.RECENT]\n",
    "\n",
    "# yippie\n",
    "for ezg in ezgs:\n",
    "    # build the name\n",
    "    name = '_'.join([str(ezg.properties[prop]) for prop in name_properties])\n",
    "\n",
    "    # check for this folder\n",
    "    if os.path.exists(os.path.join(OUT_DIR, name)):\n",
    "        if if_exists == 'skip':\n",
    "            continue\n",
    "    else:\n",
    "        os.mkdir(os.path.join(OUT_DIR, name))\n",
    "    \n",
    "    # --------------\n",
    "    # DWD STATIONS\n",
    "    for P in PARAMETER:\n",
    "        EZG._dwd_request_params['parameter'] = P\n",
    "        data_cache = dict()\n",
    "        for period in PERIODS:\n",
    "            EZG._dwd_request_params['period'] = period\n",
    "            # load DWD station data\n",
    "            if not ezg.get_dwd_within_ezg().df.empty:\n",
    "                stations = ezg.get_dwd_within_ezg()\n",
    "            elif not ezg.get_dwd_around_centroid(STATION_DISTANCE, 'km').df.empty:\n",
    "                stations = ezg.get_dwd_around_centroid(STATION_DISTANCE, 'km')\n",
    "            else:\n",
    "                stations = ezg.get_dwd_by_rank(STATION_CLOSEST_N)\n",
    "                    \n",
    "            # reduce the data\n",
    "            station_data = transpose_station_data(stations, variables='all', omit_quality_flag=OMIT_QUALITY_FLAG,)\n",
    "\n",
    "            # cache the data\n",
    "            for param_name, df in station_data.items():\n",
    "                data_cache[param_name] = df if param_name not in data_cache else pd.concat([data_cache[param_name], df])\n",
    "        \n",
    "        # save the data\n",
    "        for param_name, df in data_cache.items():\n",
    "            df.to_csv(os.path.join(OUT_DIR, name, param_name + '.csv'), index=True)\n",
    "\n",
    "    # --------------\n",
    "    # DWD RADOLAN\n",
    "    # load the chunks masked by this ezg\n",
    "    radolan_chunks = ezg.dwd_radolan_load(util=radolan)\n",
    "    \n",
    "    # reduce\n",
    "    df = spatial_reduce(radolan_chunks, targets=['sum', 'mean'], utility=radolan)\n",
    "    df.to_csv(os.path.join(OUT_DIR, name, 'radolan.csv'), index=True)\n",
    "\n",
    "    # finally add the geojson\n",
    "    with open(os.path.join(OUT_DIR, name, 'ezg.geojson'), 'w') as fp:\n",
    "        json.dump(ezg._geojson, fp)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "bb08d72d89bcc8e10696a4082fac27d5286fc9c79985d06a05b3b03835e47841"
  },
  "kernelspec": {
   "display_name": "Python 3.10.4 ('gdal')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}