Fix super annoying inclusive date range bug with sentinel ndvi data

ecohealthalliance · Dec 8, 2024 · e6d3141 · e6d3141
1 parent b2449f6
commit e6d3141
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 8 deletions.
diff --git a/R/get_sentinel_ndvi_api_parameters.R b/R/get_sentinel_ndvi_api_parameters.R
@@ -25,7 +25,14 @@ get_sentinel_ndvi_api_parameters <- function() {
   # 229 results as of 2023-03-20, so max records of 500 is safe
   url <- "https://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel3/search.json?maxRecords=500&productType=SY_2_V10___&platform=S3A&box=13.4,7.46,24.0,23.4&timeliness=NT" 
   resp <- httr::GET(url)
-  out <- jsonlite::fromJSON(rawToChar(resp$content))
-  return(out$features)
+  out <- jsonlite::fromJSON(rawToChar(resp$content)) |> 
+    pluck("features") |>
+    arrange(properties$startDate) |>
+    mutate(start_date = lubridate::floor_date(lubridate::as_date(properties$startDate), unit = "day"), 
+           end_date = lead(start_date - 1))
+
+  out$end_date[nrow(out)] <- lubridate::floor_date(lubridate::as_date(out$properties$completionDate[nrow(out)]), unit = "day")
+
+  return(out)
 
 }
diff --git a/R/transform_sentinel_ndvi.R b/R/transform_sentinel_ndvi.R
@@ -45,8 +45,11 @@ transform_sentinel_ndvi <- function(sentinel_ndvi_api_parameters,
   # Extract start and end dates from the raw downloaded file name
   # naming conventions
   # https://sentinels.copernicus.eu/web/sentinel/user-guides/sentinel-3-synergy/naming-conventions
-  start_date <- sentinel_ndvi_api_parameters$properties$startDate |> as.Date()
-  end_date <-  sentinel_ndvi_api_parameters$properties$completionDate |> as.Date()
+  # Darned sentinel data has INCLUSIVE startDate and completionDate. Collection finishes at ~noon UTC. 
+  # Solve by shifting end date back one day. This slightly changes range
+  # Do this in sentinel_ndvi_api_parameters step
+  start_date <- sentinel_ndvi_api_parameters$start_date
+  end_date <-  sentinel_ndvi_api_parameters$end_date
 
   sentinel_ndvi_filename <- file.path(sentinel_ndvi_transformed_directory, glue::glue("transformed_sentinel_NDVI_{start_date}_to_{end_date}.gz.parquet"))