From bfbabec6502888baa7d112a85a7f7e96a5328946 Mon Sep 17 00:00:00 2001 From: afwillia Date: Mon, 26 Feb 2024 15:04:00 -0800 Subject: [PATCH] add endpoint for downloading an asset view --- R/synapse_rest_api.R | 46 ++++++++++++++++++++++++++++++++---- functions/synapse_rest_api.R | 46 ++++++++++++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/R/synapse_rest_api.R b/R/synapse_rest_api.R index e5956e62..21435858 100644 --- a/R/synapse_rest_api.R +++ b/R/synapse_rest_api.R @@ -62,8 +62,9 @@ synapse_get <- function(url = "https://repo-prod.prod.sagebase.org/repo/v1/entit resp <- req |> httr2::req_retry( max_tries = 5, - is_transient = \(resp) httr2::resp_status(resp) %in% c(429, 500, 503) + is_transient = \(resp) httr2::resp_status(resp) %in% c(429, 500, 503, 403) ) |> + httr2::req_throttle(1/2) |> httr2::req_headers(Authorization = sprintf("Bearer %s", auth)) |> httr2::req_perform() resp |> httr2::resp_body_json() @@ -214,9 +215,17 @@ synapse_table_query <- function(id, auth, query, partMask=0x7F) { #' @param auth Synapse token synapse_table_get <- function(id, async_token, auth) { url <- file.path("https://repo-prod.prod.sagebase.org/repo/v1/entity", id,"table/query/async/get", async_token) - req <- httr::GET(url = url, - httr::add_headers(Authorization=paste0("Bearer ", auth))) - httr::content(req) + request <- httr2::request(url) + response <- request |> + httr2::req_retry( + max_tries = 5, + is_transient = \(r) httr2::resp_status(r) %in% c(429, 500, 503, 202, 403) + ) |> + httr2::req_throttle(1/2) |> + httr2::req_headers(Authorization = sprintf("Bearer %s", auth)) |> + httr2::req_perform() + httr2::resp_body_json(response) + } #' @title Get column names from a Synapse table @@ -243,7 +252,6 @@ synapse_storage_projects <- function(id, auth, select_cols = c("id", "name", "pa select_cols_format <- paste(select_cols, collapse = ", ") query <- sprintf("select distinct %s from %s", select_cols_format, id) request <- synapse_table_query(id, auth, query, partMask = 0x1) - Sys.sleep(1) response <- synapse_table_get(id, request$token, auth) setNames( @@ -279,3 +287,31 @@ synapse_download_file_handle <- function(dataFileHandleId, id, auth, filepath=NU if (is.null(filepath)) readr::read_csv(destfile, show_col_types = FALSE) } + +#' @title Download the storage manifest records from an asset view table +synapse_get_manifests_in_asset_view <- function(id, auth) { + request <- synapse_table_query( + id = id, + auth = auth, + query = paste("select * from", + id, + "where name like 'synapse|_storage|_manifest|_%' escape '|'"), + partMask = 0x11) + response <- synapse_table_get( + id = id, + async_token = request$token, + auth = auth) + # Format the query results by reshaping the results list and getting column + # names. partMask 0x11 gets queryResults and column names + setNames( + tibble::as_tibble( + t( + vapply( + response$queryResult$queryResults$rows, function(x) { + null_ind <- which(sapply(x$values, is.null)) + x$values[null_ind] <- NA + unlist(x$values) + }, + character(length(response$columnModels))))), + vapply(response$columnModels, function(x) x$name,character(1L))) +} diff --git a/functions/synapse_rest_api.R b/functions/synapse_rest_api.R index e5956e62..21435858 100644 --- a/functions/synapse_rest_api.R +++ b/functions/synapse_rest_api.R @@ -62,8 +62,9 @@ synapse_get <- function(url = "https://repo-prod.prod.sagebase.org/repo/v1/entit resp <- req |> httr2::req_retry( max_tries = 5, - is_transient = \(resp) httr2::resp_status(resp) %in% c(429, 500, 503) + is_transient = \(resp) httr2::resp_status(resp) %in% c(429, 500, 503, 403) ) |> + httr2::req_throttle(1/2) |> httr2::req_headers(Authorization = sprintf("Bearer %s", auth)) |> httr2::req_perform() resp |> httr2::resp_body_json() @@ -214,9 +215,17 @@ synapse_table_query <- function(id, auth, query, partMask=0x7F) { #' @param auth Synapse token synapse_table_get <- function(id, async_token, auth) { url <- file.path("https://repo-prod.prod.sagebase.org/repo/v1/entity", id,"table/query/async/get", async_token) - req <- httr::GET(url = url, - httr::add_headers(Authorization=paste0("Bearer ", auth))) - httr::content(req) + request <- httr2::request(url) + response <- request |> + httr2::req_retry( + max_tries = 5, + is_transient = \(r) httr2::resp_status(r) %in% c(429, 500, 503, 202, 403) + ) |> + httr2::req_throttle(1/2) |> + httr2::req_headers(Authorization = sprintf("Bearer %s", auth)) |> + httr2::req_perform() + httr2::resp_body_json(response) + } #' @title Get column names from a Synapse table @@ -243,7 +252,6 @@ synapse_storage_projects <- function(id, auth, select_cols = c("id", "name", "pa select_cols_format <- paste(select_cols, collapse = ", ") query <- sprintf("select distinct %s from %s", select_cols_format, id) request <- synapse_table_query(id, auth, query, partMask = 0x1) - Sys.sleep(1) response <- synapse_table_get(id, request$token, auth) setNames( @@ -279,3 +287,31 @@ synapse_download_file_handle <- function(dataFileHandleId, id, auth, filepath=NU if (is.null(filepath)) readr::read_csv(destfile, show_col_types = FALSE) } + +#' @title Download the storage manifest records from an asset view table +synapse_get_manifests_in_asset_view <- function(id, auth) { + request <- synapse_table_query( + id = id, + auth = auth, + query = paste("select * from", + id, + "where name like 'synapse|_storage|_manifest|_%' escape '|'"), + partMask = 0x11) + response <- synapse_table_get( + id = id, + async_token = request$token, + auth = auth) + # Format the query results by reshaping the results list and getting column + # names. partMask 0x11 gets queryResults and column names + setNames( + tibble::as_tibble( + t( + vapply( + response$queryResult$queryResults$rows, function(x) { + null_ind <- which(sapply(x$values, is.null)) + x$values[null_ind] <- NA + unlist(x$values) + }, + character(length(response$columnModels))))), + vapply(response$columnModels, function(x) x$name,character(1L))) +}