Finish ecmwf rework

ecohealthalliance · Sep 20, 2024 · f9e342b · f9e342b
1 parent 4119d81
commit f9e342b
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 37 deletions.
diff --git a/R/get_grib_metadata.R b/R/get_grib_metadata.R
@@ -12,11 +12,12 @@ get_grib_metadata <- function(raw_file) {
 
   # options = "json" works in targets but not during live testing
   # I have no idea why I can't get it to work in the console.
+  # It's a path problem linking terra to an old version of gdal.
+  # Since options="json" only works for newer gdal go with the 
+  # conservative option. Even though it would be super nice!
   # gdalinfo_text <- terra::describe(raw_file, options = "json") 
 
   gdalinfo_text <- terra::describe(raw_file) 
-  return(list(file = raw_file, 
-              text = gdalinfo_text))
 
   # Remove all text up to first BAND ^GEOGCRS
   metadata_start_index <- grep("^Band|^BAND", gdalinfo_text)[1]

diff --git a/R/transform_ecmwf_forecasts.R b/R/transform_ecmwf_forecasts.R
@@ -18,9 +18,7 @@
 #' @export
 transform_ecmwf_forecasts <- function(ecmwf_forecasts_api_parameters,
                                       local_folder = ecmwf_forecasts_transformed_directory,
-                                      continent_raster_template,
-                                      n_workers = 2,
-                                      time_out = 3600) {
+                                      continent_raster_template) {
 
   # Check that ecmwf_forecasts_api_parameters is only one row
   stopifnot(nrow(ecmwf_forecasts_api_parameters) == 1)
@@ -47,13 +45,11 @@ transform_ecmwf_forecasts <- function(ecmwf_forecasts_api_parameters,
                            variable = unlist(ecmwf_forecasts_api_parameters$variables)) |>
     rowwise() |>
     mutate(raw_file = file.path(local_folder, glue::glue("ecmwf_seasonal_forecast_sys{system}_{year}_{product_type}_{variable}.grib")))
-  # return(terra::describe(raw_files$raw_file[1], options = "json"))
-  return(get_grib_metadata(raw_files$raw_file[1]))
 
   # Check if raw files are already present and can be opened
   # If not re-download them all.
   error_safe_read_rast <- possibly(terra::rast, NULL)
-  raw_gribs = map(raw_files$raw_file, ~error_safe_read_rast(.x))
+  raw_gribs = map(raw_files$raw_file, ~error_safe_read_rast(.x)) |> suppressWarnings()
 
   if(any(map_vec(raw_gribs, is.null))) {
 
@@ -78,12 +74,12 @@ transform_ecmwf_forecasts <- function(ecmwf_forecasts_api_parameters,
     ecmwfr::wf_set_key(user = Sys.getenv("ECMWF_USERID"), key = Sys.getenv("ECMWF_TOKEN"))
 
     # https://cds-beta.climate.copernicus.eu/datasets/seasonal-postprocessed-single-levels?tab=overview
-    ecmwfr::wf_request_batch(request = request_list,
-                             user = Sys.getenv("ECMWF_USERID"), 
-                             workers = n_workers,
-                             path = local_folder,
-                             time_out = time_out,
-                             total_timeout = length(request_list) * time_out/n_workers)
+    purrr::walk(request_list, 
+                .progress = TRUE,
+                ~ecmwfr::wf_request(request = .x,
+                                 user = Sys.getenv("ECMWF_USERID"), 
+                                 path = local_folder,
+                                 verbose = F))
 
     # Verify that terra can open all the saved grib files. If not return NULL to try again next time
     raw_gribs = map(raw_files$raw_file, ~error_safe_read_rast(.x))
@@ -97,8 +93,6 @@ transform_ecmwf_forecasts <- function(ecmwf_forecasts_api_parameters,
     message(glue::glue("ecmwf_seasonal_forecast_sys{system}_{year} raw files successfully downloaded."))
   }
 
-  return(get_grib_metadata(raw_files$raw_file[1]))
-
   meta <- map_dfr(1:length(raw_files), function(i) {
     get_grib_metadata(raw_files$raw_file[i]) |>
     mutate(step_range = as.numeric(GRIB_FORECAST_SECONDS) / 3600, # forecast step in hours from seconds
@@ -160,7 +154,9 @@ transform_ecmwf_forecasts <- function(ecmwf_forecasts_api_parameters,
     rh <- 100 * exp((17.625 * dp)/(243.04 + dp))/exp((17.625 * t)/(243.04 + t))
     assertthat::assert_that(all(rh <= 100 & rh >=0))
     return(rh)
-  }) |> set_names(str_replace(dp_cols, "_2d", "_rh"))
+  }) |> 
+    suppressMessages() |> 
+    set_names(str_replace(dp_cols, "_2d", "_rh"))
 
   # Remove dewpoint temperature and add relative humidity
   grib_means <- grib_means |> 

diff --git a/_targets.R b/_targets.R
@@ -352,9 +352,8 @@ dynamic_targets <- tar_plan(
   tar_target(ecmwf_forecasts_transformed, 
              transform_ecmwf_forecasts(ecmwf_forecasts_api_parameters,
                                        local_folder = ecmwf_forecasts_transformed_directory,
-                                       continent_raster_template,
-                                       n_workers = 2),
-             pattern = tail(ecmwf_forecasts_api_parameters, 1),
+                                       continent_raster_template),
+             pattern = map(ecmwf_forecasts_api_parameters),
              error = "null",
              # format = "file",
              repository = "local",