From a2cde40b4af9ddbe7745d7bc028cd922f5d43078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Roycourt?= Date: Fri, 20 Oct 2023 14:22:03 +0200 Subject: [PATCH] fix(ark-metadata): save raw_metadata --- crates/ark-metadata/src/metadata_manager.rs | 16 ++-- crates/ark-metadata/src/types.rs | 12 ++- crates/ark-metadata/src/utils.rs | 97 +++++++++++++++------ 3 files changed, 91 insertions(+), 34 deletions(-) diff --git a/crates/ark-metadata/src/metadata_manager.rs b/crates/ark-metadata/src/metadata_manager.rs index bd7b1495f..a0ccc26da 100644 --- a/crates/ark-metadata/src/metadata_manager.rs +++ b/crates/ark-metadata/src/metadata_manager.rs @@ -80,13 +80,19 @@ impl<'a, T: Storage, C: StarknetClient, F: FileManager> MetadataManager<'a, T, C .await .map_err(|_| MetadataError::ParsingError)?; - let token_metadata = get_token_metadata(&self.request_client, token_uri.as_str()) - .await - .map_err(|_| MetadataError::RequestTokenUriError)?; - - if token_metadata.image.is_some() { + let token_metadata = get_token_metadata( + &self.request_client, + token_uri.as_str(), + ipfs_gateway_uri, + image_timeout, + ) + .await + .map_err(|_| MetadataError::RequestTokenUriError)?; + + if token_metadata.metadata.image.is_some() { let ipfs_url = ipfs_gateway_uri.to_string(); let url = token_metadata + .metadata .image .as_ref() .map(|s| s.replace("ipfs://", &ipfs_url)) diff --git a/crates/ark-metadata/src/types.rs b/crates/ark-metadata/src/types.rs index d30d5171f..2c172c5e4 100644 --- a/crates/ark-metadata/src/types.rs +++ b/crates/ark-metadata/src/types.rs @@ -19,9 +19,13 @@ pub enum StorageError { #[derive(Debug, Deserialize, Serialize)] pub enum DisplayType { + #[serde(rename = "number")] Number, + #[serde(rename = "boost_percentage")] BoostPercentage, + #[serde(rename = "boost_number")] BoostNumber, + #[serde(rename = "date")] Date, } @@ -56,8 +60,14 @@ pub struct MetadataAttribute { #[derive(Debug, Default, Deserialize, Serialize)] pub struct TokenMetadata { + pub raw_metadata: String, + pub metadata: NormalizedMetadata, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct NormalizedMetadata { pub image: Option, - pub image_data: Option, + pub image_data: Option, // Raw SVG image data, if you want to generate images on the fly (not recommended). Only use this if you're not including the image parameter. pub external_url: Option, pub description: Option, pub name: Option, diff --git a/crates/ark-metadata/src/utils.rs b/crates/ark-metadata/src/utils.rs index 6977b23a4..abd1b3bca 100644 --- a/crates/ark-metadata/src/utils.rs +++ b/crates/ark-metadata/src/utils.rs @@ -1,17 +1,33 @@ -use crate::types::{MetadataType, TokenMetadata}; +use crate::types::{MetadataType, NormalizedMetadata, TokenMetadata}; use anyhow::{anyhow, Result}; use base64::{engine::general_purpose, Engine as _}; use reqwest::header::{HeaderMap, CONTENT_LENGTH, CONTENT_TYPE}; use reqwest::Client; -use std::{env, time::Duration}; -use tracing::debug; - -pub async fn get_token_metadata(client: &Client, uri: &str) -> Result { +use std::time::Duration; +use tracing::{debug, error, info}; + +pub async fn get_token_metadata( + client: &Client, + uri: &str, + ipfs_gateway_uri: &str, + request_timeout_duration: Duration, +) -> Result { let metadata_type = get_metadata_type(uri); let metadata = match metadata_type { - MetadataType::Ipfs(uri) => get_ipfs_metadata(&uri, client).await?, - MetadataType::Http(uri) => get_http_metadata(&uri, client).await?, - MetadataType::OnChain(uri) => get_onchain_metadata(&uri)?, + MetadataType::Ipfs(uri) => { + let ipfs_hash = uri.trim_start_matches("ipfs://"); + let complete_uri = format!("{}{}", ipfs_gateway_uri, ipfs_hash); + info!("Fetching metadata from IPFS: {}", complete_uri.as_str()); + fetch_metadata(complete_uri.as_str(), client, request_timeout_duration).await? + } + MetadataType::Http(uri) => { + info!("Fetching metadata from HTTPS: {}", uri.as_str()); + fetch_metadata(&uri, client, request_timeout_duration).await? + } + MetadataType::OnChain(uri) => { + info!("Fetching on-chain metadata: {}", uri); + get_onchain_metadata(&uri)? + } }; Ok(metadata) } @@ -26,20 +42,33 @@ pub fn get_metadata_type(uri: &str) -> MetadataType { } } -async fn get_ipfs_metadata(uri: &str, client: &Client) -> Result { - let mut ipfs_url = env::var("IPFS_GATEWAY_URI").expect("IPFS_GATEWAY_URI must be set"); - let ipfs_hash = uri.trim_start_matches("ipfs://"); - ipfs_url.push_str(ipfs_hash); - let request = client.get(ipfs_url).timeout(Duration::from_secs(3)); - let response = request.send().await?; - let metadata = response.json::().await?; - Ok(metadata) -} - -async fn get_http_metadata(uri: &str, client: &Client) -> Result { - let resp = client.get(uri).send().await?; - let metadata: TokenMetadata = resp.json().await?; - Ok(metadata) +async fn fetch_metadata( + uri: &str, + client: &Client, + request_timeout_duration: Duration, +) -> Result { + let request = client.get(uri).timeout(request_timeout_duration); + let response = request.send().await; + + match response { + Ok(response) => { + if response.status().is_success() { + let raw_metadata = response.text().await?; + let metadata = serde_json::from_str::(raw_metadata.as_str())?; + Ok(TokenMetadata { + raw_metadata, + metadata, + }) + } else { + error!("Failed to get ipfs metadata. URI: {}", uri); + Err(anyhow!("Failed to get ipfs metadata")) + } + } + Err(e) => { + error!("Failed to get ipfs metadata: {:?}", e); + Err(anyhow!("Failed to get ipfs metadata")) + } + } } pub fn file_extension_from_mime_type(mime_type: &str) -> &str { @@ -73,14 +102,19 @@ fn get_onchain_metadata(uri: &str) -> Result { // If it is base64 encoded, decode it, parse and return let decoded = general_purpose::STANDARD.decode(uri)?; let decoded = std::str::from_utf8(&decoded)?; - let metadata: TokenMetadata = serde_json::from_str(decoded)?; - Ok(metadata) + + let metadata: NormalizedMetadata = serde_json::from_str::(decoded)?; + Ok(TokenMetadata { + raw_metadata: decoded.to_string(), + metadata, + }) } Some(("data:application/json", uri)) => { - // If it is plain json, parse it and return - //println!("Handling {:?}", uri); - let metadata: TokenMetadata = serde_json::from_str(uri)?; - Ok(metadata) + let metadata = serde_json::from_str::(uri)?; + Ok(TokenMetadata { + raw_metadata: uri.to_string(), + metadata, + }) } _ => match serde_json::from_str(uri) { // If it is only the URI without the data format information, try to format it @@ -123,6 +157,13 @@ mod tests { use super::*; use reqwest::header::{HeaderMap, HeaderValue, CONTENT_LENGTH, CONTENT_TYPE}; + #[test] + fn test_file_extension_from_mime_type() { + assert_eq!(file_extension_from_mime_type("image/png"), "png"); + assert_eq!(file_extension_from_mime_type("image/jpeg"), "jpg"); + assert_eq!(file_extension_from_mime_type("video/mp4"), "mp4"); + } + #[tokio::test] async fn test_determining_metadata_type() { let metadata_type =