From 4a34c828cc2bc91ce8d11faae475df8bb8ec35d9 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Fri, 22 Nov 2024 21:35:34 +0545 Subject: [PATCH] feat(download): add gitlab support --- src/misc/download.rs | 399 ------------------------------------ src/misc/download/github.rs | 229 +++++++++++++++++++++ src/misc/download/gitlab.rs | 231 +++++++++++++++++++++ src/misc/download/mod.rs | 222 ++++++++++++++++++++ 4 files changed, 682 insertions(+), 399 deletions(-) delete mode 100644 src/misc/download.rs create mode 100644 src/misc/download/github.rs create mode 100644 src/misc/download/gitlab.rs create mode 100644 src/misc/download/mod.rs diff --git a/src/misc/download.rs b/src/misc/download.rs deleted file mode 100644 index 1070a41..0000000 --- a/src/misc/download.rs +++ /dev/null @@ -1,399 +0,0 @@ -use std::{env, fs::Permissions, os::unix::fs::PermissionsExt, path::Path}; - -use anyhow::{Context, Result}; -use chrono::Utc; -use futures::StreamExt; -use indicatif::ProgressBar; -use regex::Regex; -use reqwest::{ - header::{HeaderMap, AUTHORIZATION, USER_AGENT}, - Response, StatusCode, Url, -}; -use serde::{Deserialize, Serialize}; -use tokio::{ - fs::{self, File}, - io::{AsyncReadExt, AsyncWriteExt, BufReader}, -}; -use tracing::{debug, error, info, trace}; - -use crate::{ - core::{ - color::{Color, ColorExt}, - constant::ELF_MAGIC_BYTES, - util::{download_progress_style, format_bytes, interactive_ask, AskType}, - }, - package::parse_package_query, - registry::{select_single_package, PackageRegistry}, -}; - -#[derive(Clone, Debug, Deserialize, Serialize)] -struct GithubAsset { - name: String, - size: u64, - browser_download_url: String, -} - -#[derive(Debug, Deserialize, Serialize)] -struct GithubRelease { - tag_name: String, - draft: bool, - prerelease: bool, - published_at: String, - assets: Vec, -} - -static GITHUB_URL_REGEX: &str = - r"^(?i)(?:https?://)?(?:github(?:\.com)?[:/])([^/@]+/[^/@]+)(?:@([^/\s]*)?)?$"; - -fn extract_filename(url: &str) -> String { - Path::new(url) - .file_name() - .map(|name| name.to_string_lossy().to_string()) - .unwrap_or_else(|| { - let dt = Utc::now().timestamp(); - dt.to_string() - }) -} - -async fn is_elf(file_path: &Path) -> bool { - let Ok(file) = File::open(file_path).await else { - return false; - }; - let mut file = BufReader::new(file); - - let mut magic_bytes = [0_u8; 4]; - if file.read_exact(&mut magic_bytes).await.is_ok() { - return magic_bytes == ELF_MAGIC_BYTES; - } - false -} - -pub async fn download(url: &str, output: Option) -> Result<()> { - let client = reqwest::Client::new(); - let response = client - .get(url) - .header(USER_AGENT, "pkgforge/soar") - .send() - .await?; - - if !response.status().is_success() { - return Err(anyhow::anyhow!( - "Error fetching {} [{}]", - url.color(Color::Blue), - response.status().color(Color::Red) - )); - } - - let filename = output.unwrap_or(extract_filename(url)); - let filename = if filename.ends_with("/") { - format!( - "{}/{}", - filename.trim_end_matches("/"), - extract_filename(url) - ) - } else { - filename - }; - let output_path = Path::new(&filename); - - if let Some(output_dir) = output_path.parent() { - if !output_dir.exists() { - fs::create_dir_all(&output_dir).await.context(format!( - "Failed to create directory: {}", - output_dir.display() - ))?; - } - } - - let temp_path = format!("{}.tmp", output_path.display()); - - info!( - "Downloading file from {} [{}]", - url.color(Color::Blue), - format_bytes(response.content_length().unwrap_or_default()).color(Color::Yellow) - ); - - let content_length = response.content_length().unwrap_or(0); - let progress_bar = ProgressBar::new(content_length); - progress_bar.set_style(download_progress_style(false)); - - let mut stream = response.bytes_stream(); - let mut file = fs::OpenOptions::new() - .write(true) - .create(true) - .append(true) - .open(&temp_path) - .await - .context("Failed to open temp file for writing")?; - - let mut downloaded_bytes = 0u64; - while let Some(chunk) = stream.next().await { - let chunk = chunk.context("Failed to read chunk")?; - file.write_all(&chunk).await?; - downloaded_bytes = downloaded_bytes.saturating_add(chunk.len() as u64); - progress_bar.set_position(downloaded_bytes); - if content_length == 0 { - progress_bar.set_length(downloaded_bytes); - } - } - progress_bar.finish(); - - fs::rename(&temp_path, &output_path).await?; - - if is_elf(output_path).await { - fs::set_permissions(&output_path, Permissions::from_mode(0o755)).await?; - } - - info!("Downloaded {}", output_path.display().color(Color::Blue)); - - Ok(()) -} - -#[derive(Debug)] -enum GithubApi { - PkgForge, - Github, -} - -async fn call_github_api(gh_api: &GithubApi, user_repo: &str) -> Result { - let client = reqwest::Client::new(); - let url = format!( - "{}/repos/{}/releases", - match gh_api { - GithubApi::PkgForge => "https://api.gh.pkgforge.dev", - GithubApi::Github => "https://api.github.com", - }, - user_repo - ); - let mut headers = HeaderMap::new(); - headers.insert(USER_AGENT, "pkgforge/soar".parse()?); - if matches!(gh_api, GithubApi::Github) { - if let Ok(token) = env::var("GITHUB_TOKEN") { - trace!("Using Github token: {}", token); - headers.insert(AUTHORIZATION, format!("Bearer {}", token).parse()?); - } - } - client - .get(&url) - .headers(headers) - .send() - .await - .context("Failed to fetch GitHub releases") -} - -fn should_fallback(status: StatusCode) -> bool { - status == StatusCode::TOO_MANY_REQUESTS - || status == StatusCode::UNAUTHORIZED - || status == StatusCode::FORBIDDEN - || status.is_server_error() -} - -async fn fetch_github_releases(gh_api: &GithubApi, user_repo: &str) -> Result> { - let response = match call_github_api(gh_api, user_repo).await { - Ok(resp) => { - let status = resp.status(); - if should_fallback(status) && matches!(gh_api, GithubApi::PkgForge) { - debug!("Failed to fetch Github asset using pkgforge API. Retrying request using Github API."); - call_github_api(&GithubApi::Github, user_repo).await? - } else { - resp - } - } - Err(e) => return Err(e), - }; - - if !response.status().is_success() { - anyhow::bail!( - "Error fetching releases for {}: {}", - user_repo, - response.status() - ); - } - - let releases: Vec = response - .json() - .await - .context("Failed to parse GitHub response")?; - - Ok(releases) -} - -fn select_asset_idx(assets: &[&GithubAsset], max: usize) -> Result { - for (i, asset) in assets.iter().enumerate() { - info!( - " [{}] {:#?} ({})", - i + 1, - asset.name, - format_bytes(asset.size), - ); - } - let selection = loop { - let response = interactive_ask( - &format!("Select an asset (1-{}): ", assets.len()), - AskType::Normal, - )?; - - match response.parse::() { - Ok(n) if n > 0 && n <= max => break n - 1, - _ => error!("Invalid selection, please try again."), - } - }; - Ok(selection) -} - -pub async fn download_and_save( - registry: PackageRegistry, - links: &[String], - yes: bool, - output: Option, - regex_patterns: Option<&[String]>, - match_keywords: Option<&[String]>, - exclude_keywords: Option<&[String]>, -) -> Result<()> { - let re = Regex::new(GITHUB_URL_REGEX).unwrap(); - let asset_regexes = regex_patterns - .map(|patterns| { - patterns - .iter() - .map(|pattern| Regex::new(pattern)) - .collect::, regex::Error>>() - }) - .transpose()? - .unwrap_or_default(); - - for link in links { - let link = link.trim(); - if re.is_match(link) { - info!( - "GitHub repository URL detected: {}", - link.color(Color::Blue) - ); - if let Some(caps) = re.captures(link) { - let user_repo = caps.get(1).unwrap().as_str(); - let tag = caps - .get(2) - .map(|tag| tag.as_str().trim()) - .filter(|&tag| !tag.is_empty()); - info!("Fetching releases for {}...", user_repo); - - let releases = fetch_github_releases(&GithubApi::PkgForge, user_repo).await?; - - let release = if let Some(tag_name) = tag { - releases - .iter() - .find(|release| release.tag_name.starts_with(tag_name)) - } else { - releases - .iter() - .find(|release| !release.prerelease && !release.draft) - }; - - let Some(release) = release else { - error!( - "No {} found for repository {}", - tag.map(|t| format!("tag {}", t)) - .unwrap_or("stable release".to_owned()), - user_repo - ); - continue; - }; - - let assets = &release.assets; - - if assets.is_empty() { - error!("No assets found for the release."); - continue; - } - - let selected_asset = { - let assets: Vec<&GithubAsset> = assets - .iter() - .filter(|asset| { - asset_regexes - .iter() - .all(|regex| regex.is_match(&asset.name)) - && match_keywords.map_or(true, |keywords| { - keywords.iter().all(|keyword| { - keyword - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .all(|part| { - asset - .name - .to_lowercase() - .contains(&part.to_lowercase()) - }) - }) - }) - && exclude_keywords.map_or(true, |keywords| { - keywords.iter().all(|keyword| { - keyword - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .all(|part| { - !asset - .name - .to_lowercase() - .contains(&part.to_lowercase()) - }) - }) - }) - }) - .collect(); - - match assets.len() { - 0 => { - error!("No assets matched the provided criteria."); - continue; - } - 1 => assets[0], - _ => { - if yes { - assets[0] - } else { - info!( - "Multiple matching assets found for {}{}", - release.tag_name, - if release.prerelease { - " [prerelease]".color(Color::BrightRed) - } else { - " [stable]".color(Color::BrightCyan) - } - ); - - let asset_idx = select_asset_idx(&assets, assets.len())?; - assets[asset_idx] - } - } - } - }; - - let download_url = &selected_asset.browser_download_url; - download(download_url, output.clone()).await?; - } - } else if let Ok(url) = Url::parse(link) { - download(url.as_str(), output.clone()).await?; - } else { - error!("{} is not a valid URL", link.color(Color::Blue)); - info!("Searching for package instead.."); - - let query = parse_package_query(link); - let packages = registry.storage.get_packages(&query); - - if let Some(packages) = packages { - let resolved_pkg = if yes || packages.len() == 1 { - &packages[0] - } else { - select_single_package(&packages)? - }; - download(&resolved_pkg.package.download_url, output.clone()).await?; - } else { - error!("No packages found."); - } - }; - } - - Ok(()) -} diff --git a/src/misc/download/github.rs b/src/misc/download/github.rs new file mode 100644 index 0000000..299e8b2 --- /dev/null +++ b/src/misc/download/github.rs @@ -0,0 +1,229 @@ +use std::env; + +use anyhow::{Context, Result}; +use regex::Regex; +use reqwest::{ + header::{HeaderMap, AUTHORIZATION, USER_AGENT}, + Response, +}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, error, info, trace}; + +use crate::{ + core::{ + color::{Color, ColorExt}, + util::{format_bytes, interactive_ask, AskType}, + }, + misc::download::download, +}; + +use super::{should_fallback, ApiType}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct GithubAsset { + name: String, + size: u64, + browser_download_url: String, +} + +#[derive(Debug, Deserialize, Serialize)] +struct GithubRelease { + tag_name: String, + draft: bool, + prerelease: bool, + published_at: String, + assets: Vec, +} + +pub static GITHUB_URL_REGEX: &str = + r"^(?i)(?:https?://)?(?:github(?:\.com)?[:/])([^/@]+/[^/@]+)(?:@([^/\s]*)?)?$"; + +async fn call_github_api(gh_api: &ApiType, user_repo: &str) -> Result { + let client = reqwest::Client::new(); + let url = format!( + "{}/repos/{}/releases?per_page=100", + match gh_api { + ApiType::PkgForge => "https://api.gh.pkgforge.dev", + ApiType::Primary => "https://api.github.com", + }, + user_repo + ); + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, "pkgforge/soar".parse()?); + if matches!(gh_api, ApiType::Primary) { + if let Ok(token) = env::var("GITHUB_TOKEN") { + trace!("Using Github token: {}", token); + headers.insert(AUTHORIZATION, format!("Bearer {}", token).parse()?); + } + } + client + .get(&url) + .headers(headers) + .send() + .await + .context("Failed to fetch GitHub releases") +} + +async fn fetch_github_releases(gh_api: &ApiType, user_repo: &str) -> Result> { + let response = match call_github_api(gh_api, user_repo).await { + Ok(resp) => { + let status = resp.status(); + if should_fallback(status) && matches!(gh_api, ApiType::PkgForge) { + debug!("Failed to fetch Github asset using pkgforge API. Retrying request using Github API."); + call_github_api(&ApiType::Primary, user_repo).await? + } else { + resp + } + } + Err(e) => return Err(e), + }; + + if !response.status().is_success() { + anyhow::bail!( + "Error fetching releases for {}: {}", + user_repo, + response.status() + ); + } + + let releases: Vec = response + .json() + .await + .context("Failed to parse GitHub response")?; + + Ok(releases) +} + +fn select_asset_idx(assets: &[&GithubAsset], max: usize) -> Result { + for (i, asset) in assets.iter().enumerate() { + info!( + " [{}] {:#?} ({})", + i + 1, + asset.name, + format_bytes(asset.size), + ); + } + let selection = loop { + let response = interactive_ask( + &format!("Select an asset (1-{}): ", assets.len()), + AskType::Normal, + )?; + + match response.parse::() { + Ok(n) if n > 0 && n <= max => break n - 1, + _ => error!("Invalid selection, please try again."), + } + }; + Ok(selection) +} + +pub async fn handle_github_download( + re: &Regex, + link: &str, + output: Option, + match_keywords: Option<&[String]>, + exclude_keywords: Option<&[String]>, + asset_regexes: &Vec, + yes: bool, +) -> Result<()> { + if let Some(caps) = re.captures(link) { + let user_repo = caps.get(1).unwrap().as_str(); + let tag = caps + .get(2) + .map(|tag| tag.as_str().trim()) + .filter(|&tag| !tag.is_empty()); + info!("Fetching releases for {}...", user_repo); + + let releases = fetch_github_releases(&ApiType::PkgForge, user_repo).await?; + + let release = if let Some(tag_name) = tag { + releases + .iter() + .find(|release| release.tag_name.starts_with(tag_name)) + } else { + releases + .iter() + .find(|release| !release.prerelease && !release.draft) + }; + + let Some(release) = release else { + error!( + "No {} found for repository {}", + tag.map(|t| format!("tag {}", t)) + .unwrap_or("stable release".to_owned()), + user_repo + ); + return Ok(()); + }; + + let assets = &release.assets; + + if assets.is_empty() { + error!("No assets found for the release."); + return Ok(()); + } + + let selected_asset = { + let assets: Vec<&GithubAsset> = assets + .iter() + .filter(|asset| { + asset_regexes + .iter() + .all(|regex| regex.is_match(&asset.name)) + && match_keywords.map_or(true, |keywords| { + keywords.iter().all(|keyword| { + keyword + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .all(|part| { + asset.name.to_lowercase().contains(&part.to_lowercase()) + }) + }) + }) + && exclude_keywords.map_or(true, |keywords| { + keywords.iter().all(|keyword| { + keyword + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .all(|part| { + !asset.name.to_lowercase().contains(&part.to_lowercase()) + }) + }) + }) + }) + .collect(); + + match assets.len() { + 0 => { + error!("No assets matched the provided criteria."); + return Ok(()); + } + 1 => assets[0], + _ => { + if yes { + assets[0] + } else { + info!( + "Multiple matching assets found for {}{}", + release.tag_name, + if release.prerelease { + " [prerelease]".color(Color::BrightRed) + } else { + " [stable]".color(Color::BrightCyan) + } + ); + + let asset_idx = select_asset_idx(&assets, assets.len())?; + assets[asset_idx] + } + } + } + }; + + let download_url = &selected_asset.browser_download_url; + download(download_url, output.clone()).await?; + } + Ok(()) +} diff --git a/src/misc/download/gitlab.rs b/src/misc/download/gitlab.rs new file mode 100644 index 0000000..fbb4608 --- /dev/null +++ b/src/misc/download/gitlab.rs @@ -0,0 +1,231 @@ +use std::env; + +use anyhow::{Context, Result}; +use regex::Regex; +use reqwest::{ + header::{HeaderMap, AUTHORIZATION, USER_AGENT}, + Response, +}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, error, info, trace}; + +use crate::{ + core::{ + color::{Color, ColorExt}, + util::{interactive_ask, AskType}, + }, + misc::download::download, +}; + +use super::should_fallback; + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct GitlabAsset { + name: String, + direct_asset_url: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct GitlabAssets { + links: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct GitlabRelease { + tag_name: String, + upcoming_release: bool, + released_at: String, + assets: GitlabAssets, +} + +pub static GITLAB_URL_REGEX: &str = + r"^(?i)(?:https?://)?(?:gitlab(?:\.com)?[:/])([^/@]+/[^/@]+)(?:@([^/\s]*)?)?$"; + +#[derive(Debug)] +enum GitlabApi { + PkgForge, + Gitlab, +} + +async fn call_gitlab_api(gh_api: &GitlabApi, user_repo: &str) -> Result { + let client = reqwest::Client::new(); + let url = format!( + "{}/api/v4/projects/{}/releases", + match gh_api { + GitlabApi::PkgForge => "https://api.gl.pkgforge.dev", + GitlabApi::Gitlab => "https://gitlab.com", + }, + user_repo.replace("/", "%2F") + ); + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, "pkgforge/soar".parse()?); + if matches!(gh_api, GitlabApi::Gitlab) { + if let Ok(token) = env::var("GITLAB_TOKEN") { + trace!("Using Gitlab token: {}", token); + headers.insert(AUTHORIZATION, format!("Bearer {}", token).parse()?); + } + } + client + .get(&url) + .headers(headers) + .send() + .await + .context("Failed to fetch Gitlab releases") +} + +async fn fetch_gitlab_releases(gh_api: &GitlabApi, user_repo: &str) -> Result> { + let response = match call_gitlab_api(gh_api, user_repo).await { + Ok(resp) => { + let status = resp.status(); + if should_fallback(status) && matches!(gh_api, GitlabApi::PkgForge) { + debug!("Failed to fetch Gitlab asset using pkgforge API. Retrying request using Gitlab API."); + call_gitlab_api(&GitlabApi::Gitlab, user_repo).await? + } else { + resp + } + } + Err(e) => return Err(e), + }; + + if !response.status().is_success() { + anyhow::bail!( + "Error fetching releases for {}: {}", + user_repo, + response.status() + ); + } + + let releases: Vec = response + .json() + .await + .context("Failed to parse Gitlab response")?; + + Ok(releases) +} + +fn select_asset_idx(assets: &[&GitlabAsset], max: usize) -> Result { + for (i, asset) in assets.iter().enumerate() { + info!(" [{}] {}", i + 1, asset.name); + } + let selection = loop { + let response = interactive_ask( + &format!("Select an asset (1-{}): ", assets.len()), + AskType::Normal, + )?; + + match response.parse::() { + Ok(n) if n > 0 && n <= max => break n - 1, + _ => error!("Invalid selection, please try again."), + } + }; + Ok(selection) +} + +pub async fn handle_gitlab_download( + re: &Regex, + link: &str, + output: Option, + match_keywords: Option<&[String]>, + exclude_keywords: Option<&[String]>, + asset_regexes: &Vec, + yes: bool, +) -> Result<()> { + if let Some(caps) = re.captures(link) { + let user_repo = caps.get(1).unwrap().as_str(); + let tag = caps + .get(2) + .map(|tag| tag.as_str().trim()) + .filter(|&tag| !tag.is_empty()); + info!("Fetching releases for {}...", user_repo); + + let releases = fetch_gitlab_releases(&GitlabApi::PkgForge, user_repo).await?; + + let release = if let Some(tag_name) = tag { + releases + .iter() + .find(|release| release.tag_name.starts_with(tag_name)) + } else { + releases.iter().find(|release| !release.upcoming_release) + }; + + let Some(release) = release else { + error!( + "No {} found for repository {}", + tag.map(|t| format!("tag {}", t)) + .unwrap_or("stable release".to_owned()), + user_repo + ); + return Ok(()); + }; + + let assets = &release.assets.links; + + if assets.is_empty() { + error!("No assets found for the release."); + return Ok(()); + } + + let selected_asset = { + let assets: Vec<&GitlabAsset> = assets + .iter() + .filter(|asset| { + asset_regexes + .iter() + .all(|regex| regex.is_match(&asset.name)) + && match_keywords.map_or(true, |keywords| { + keywords.iter().all(|keyword| { + keyword + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .all(|part| { + asset.name.to_lowercase().contains(&part.to_lowercase()) + }) + }) + }) + && exclude_keywords.map_or(true, |keywords| { + keywords.iter().all(|keyword| { + keyword + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .all(|part| { + !asset.name.to_lowercase().contains(&part.to_lowercase()) + }) + }) + }) + }) + .collect(); + + match assets.len() { + 0 => { + error!("No assets matched the provided criteria."); + return Ok(()); + } + 1 => assets[0], + _ => { + if yes { + assets[0] + } else { + info!( + "Multiple matching assets found for {}{}", + release.tag_name, + if release.upcoming_release { + " [prerelease]".color(Color::BrightRed) + } else { + " [stable]".color(Color::BrightCyan) + } + ); + + let asset_idx = select_asset_idx(&assets, assets.len())?; + assets[asset_idx] + } + } + } + }; + + let download_url = &selected_asset.direct_asset_url; + download(download_url, output.clone()).await?; + } + Ok(()) +} diff --git a/src/misc/download/mod.rs b/src/misc/download/mod.rs new file mode 100644 index 0000000..2d1adae --- /dev/null +++ b/src/misc/download/mod.rs @@ -0,0 +1,222 @@ +use std::{fs::Permissions, os::unix::fs::PermissionsExt, path::Path}; + +use anyhow::{Context, Result}; +use chrono::Utc; +use futures::StreamExt; +use github::{handle_github_download, GITHUB_URL_REGEX}; +use gitlab::{handle_gitlab_download, GITLAB_URL_REGEX}; +use indicatif::ProgressBar; +use regex::Regex; +use reqwest::{header::USER_AGENT, StatusCode, Url}; +use tokio::{ + fs::{self, File}, + io::{AsyncReadExt, AsyncWriteExt, BufReader}, +}; +use tracing::{error, info}; + +mod github; +mod gitlab; + +use crate::{ + core::{ + color::{Color, ColorExt}, + constant::ELF_MAGIC_BYTES, + util::{download_progress_style, format_bytes}, + }, + package::parse_package_query, + registry::{select_single_package, PackageRegistry}, +}; + +pub enum ApiType { + PkgForge, + Primary, +} + +fn extract_filename(url: &str) -> String { + Path::new(url) + .file_name() + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_else(|| { + let dt = Utc::now().timestamp(); + dt.to_string() + }) +} + +async fn is_elf(file_path: &Path) -> bool { + let Ok(file) = File::open(file_path).await else { + return false; + }; + let mut file = BufReader::new(file); + + let mut magic_bytes = [0_u8; 4]; + if file.read_exact(&mut magic_bytes).await.is_ok() { + return magic_bytes == ELF_MAGIC_BYTES; + } + false +} + +pub async fn download(url: &str, output: Option) -> Result<()> { + let client = reqwest::Client::new(); + let response = client + .get(url) + .header(USER_AGENT, "pkgforge/soar") + .send() + .await?; + + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Error fetching {} [{}]", + url.color(Color::Blue), + response.status().color(Color::Red) + )); + } + + let filename = output.unwrap_or(extract_filename(url)); + let filename = if filename.ends_with("/") { + format!( + "{}/{}", + filename.trim_end_matches("/"), + extract_filename(url) + ) + } else { + filename + }; + let output_path = Path::new(&filename); + + if let Some(output_dir) = output_path.parent() { + if !output_dir.exists() { + fs::create_dir_all(&output_dir).await.context(format!( + "Failed to create directory: {}", + output_dir.display() + ))?; + } + } + + let temp_path = format!("{}.tmp", output_path.display()); + + info!( + "Downloading file from {} [{}]", + url.color(Color::Blue), + format_bytes(response.content_length().unwrap_or_default()).color(Color::Yellow) + ); + + let content_length = response.content_length().unwrap_or(0); + let progress_bar = ProgressBar::new(content_length); + progress_bar.set_style(download_progress_style(false)); + + let mut stream = response.bytes_stream(); + let mut file = fs::OpenOptions::new() + .write(true) + .create(true) + .append(true) + .open(&temp_path) + .await + .context("Failed to open temp file for writing")?; + + let mut downloaded_bytes = 0u64; + while let Some(chunk) = stream.next().await { + let chunk = chunk.context("Failed to read chunk")?; + file.write_all(&chunk).await?; + downloaded_bytes = downloaded_bytes.saturating_add(chunk.len() as u64); + progress_bar.set_position(downloaded_bytes); + if content_length == 0 { + progress_bar.set_length(downloaded_bytes); + } + } + progress_bar.finish(); + + fs::rename(&temp_path, &output_path).await?; + + if is_elf(output_path).await { + fs::set_permissions(&output_path, Permissions::from_mode(0o755)).await?; + } + + info!("Downloaded {}", output_path.display().color(Color::Blue)); + + Ok(()) +} + +fn should_fallback(status: StatusCode) -> bool { + status == StatusCode::TOO_MANY_REQUESTS + || status == StatusCode::UNAUTHORIZED + || status == StatusCode::FORBIDDEN + || status.is_server_error() +} + +pub async fn download_and_save( + registry: PackageRegistry, + links: &[String], + yes: bool, + output: Option, + regex_patterns: Option<&[String]>, + match_keywords: Option<&[String]>, + exclude_keywords: Option<&[String]>, +) -> Result<()> { + let github_re = Regex::new(GITHUB_URL_REGEX).unwrap(); + let gitlab_re = Regex::new(GITLAB_URL_REGEX).unwrap(); + let asset_regexes = regex_patterns + .map(|patterns| { + patterns + .iter() + .map(|pattern| Regex::new(pattern)) + .collect::, regex::Error>>() + }) + .transpose()? + .unwrap_or_default(); + + for link in links { + let link = link.trim(); + if github_re.is_match(link) { + info!( + "GitHub repository URL detected: {}", + link.color(Color::Blue) + ); + handle_github_download( + &github_re, + link, + output.clone(), + match_keywords, + exclude_keywords, + &asset_regexes, + yes, + ) + .await?; + } else if gitlab_re.is_match(link) { + info!( + "Gitlab repository URL detected: {}", + link.color(Color::Blue) + ); + handle_gitlab_download( + &gitlab_re, + link, + output.clone(), + match_keywords, + exclude_keywords, + &asset_regexes, + yes, + ) + .await?; + } else if let Ok(url) = Url::parse(link) { + download(url.as_str(), output.clone()).await?; + } else { + error!("{} is not a valid URL", link.color(Color::Blue)); + info!("Searching for package instead.."); + + let query = parse_package_query(link); + let packages = registry.storage.get_packages(&query); + + if let Some(packages) = packages { + let resolved_pkg = if yes || packages.len() == 1 { + &packages[0] + } else { + select_single_package(&packages)? + }; + download(&resolved_pkg.package.download_url, output.clone()).await?; + } else { + error!("No packages found."); + } + }; + } + + Ok(()) +}