diff --git a/distrod/distrod/Cargo.toml b/distrod/distrod/Cargo.toml index 12a3e3e..9f8c78e 100644 --- a/distrod/distrod/Cargo.toml +++ b/distrod/distrod/Cargo.toml @@ -13,10 +13,12 @@ log = "0.4" env_logger = "0.8" anyhow = "1.0" nix = "0.20.0" +indicatif = "0.16" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" scraper = "0.12" -reqwest = { version = "0.11", features = ["blocking", "json"] } +reqwest = { version = "0.11" } +tokio = { version = "1.10", features = ["rt", "rt-multi-thread", "macros"] } chrono = "0.4" xz2 = "0.1" tar = "0.4" diff --git a/distrod/distrod/src/main.rs b/distrod/distrod/src/main.rs index 17bf810..cd3cb66 100644 --- a/distrod/distrod/src/main.rs +++ b/distrod/distrod/src/main.rs @@ -1,12 +1,12 @@ use anyhow::{anyhow, bail, Context, Result}; use distro::Distro; -use libs::cli_ui::{choose_from_list, init_logger, prompt_path, LogLevel}; +use libs::cli_ui::{build_progress_bar, choose_from_list, init_logger, prompt_path, LogLevel}; use libs::distrod_config::{self, DistrodConfig}; use libs::local_image::LocalDistroImage; use libs::multifork::set_noninheritable_sig_ign; use std::ffi::{CString, OsString}; use std::fs::File; -use std::io::{stdin, Read}; +use std::io::{stdin, Cursor, Read}; use std::os::unix::prelude::OsStrExt; use std::path::Path; use structopt::StructOpt; @@ -15,7 +15,8 @@ use xz2::read::XzDecoder; use libs::command_alias::CommandAlias; use libs::distro; use libs::distro_image::{ - self, DistroImage, DistroImageFetcher, DistroImageFetcherGen, DistroImageFile, + self, download_file_with_progress, DistroImage, DistroImageFetcher, DistroImageFetcherGen, + DistroImageFile, }; use libs::lxd_image::LxdDistroImageList; use libs::passwd::IdCredential; @@ -64,7 +65,7 @@ pub struct ExecOpts { #[structopt(short, long)] user: Option, - #[structopt(short, long)] + #[structopt(short = "i", long)] uid: Option, #[structopt(short, long)] @@ -227,7 +228,8 @@ fn disable_wsl_exec_hook(_opts: DisableOpts) -> Result<()> { Ok(()) } -fn create_distro(opts: CreateOpts) -> Result<()> { +#[tokio::main] +async fn create_distro(opts: CreateOpts) -> Result<()> { let image = match opts.image_path { None => { let local_image_fetcher = @@ -235,10 +237,11 @@ fn create_distro(opts: CreateOpts) -> Result<()> { let lxd_image_fetcher = || Ok(Box::new(LxdDistroImageList::default()) as Box); let fetchers = vec![ - Box::new(local_image_fetcher) as Box, - Box::new(lxd_image_fetcher) as Box, + Box::new(local_image_fetcher) as DistroImageFetcherGen, + Box::new(lxd_image_fetcher) as DistroImageFetcherGen, ]; distro_image::fetch_image(fetchers, choose_from_list, 1) + .await .with_context(|| "Failed to fetch the image list.")? } Some(path) => DistroImage { @@ -255,13 +258,10 @@ fn create_distro(opts: CreateOpts) -> Result<()> { ) as Box, DistroImageFile::Url(url) => { log::info!("Downloading '{}'...", url); - let client = reqwest::blocking::Client::builder().timeout(None).build()?; - let response = client - .get(&url) - .send() - .with_context(|| format!("Failed to download {}.", &url))?; + let mut bytes = vec![]; + download_file_with_progress(&url, build_progress_bar, &mut bytes).await?; log::info!("Download done."); - Box::new(std::io::Cursor::new(response.bytes()?)) as Box + Box::new(Cursor::new(bytes)) as Box } }; diff --git a/distrod/distrod_wsl_launcher/Cargo.toml b/distrod/distrod_wsl_launcher/Cargo.toml index ceb72cf..239352e 100644 --- a/distrod/distrod_wsl_launcher/Cargo.toml +++ b/distrod/distrod_wsl_launcher/Cargo.toml @@ -17,9 +17,11 @@ anyhow = "1.0" xz2 = "0.1" tar = "0.4" flate2 = "1.0" +indicatif = "0.16" chrono = "0.4" scraper = "0.12" -reqwest = { version = "0.11", features = ["blocking", "json"] } +reqwest = { version = "0.11" } +tokio = { version = "1.10", features = ["rt", "rt-multi-thread", "macros"] } colored = "2" tempfile = "3" bytes = "1.0" diff --git a/distrod/distrod_wsl_launcher/src/main.rs b/distrod/distrod_wsl_launcher/src/main.rs index 6352795..e3f2611 100644 --- a/distrod/distrod_wsl_launcher/src/main.rs +++ b/distrod/distrod_wsl_launcher/src/main.rs @@ -1,9 +1,12 @@ use anyhow::{bail, Context, Result}; use flate2::read::GzDecoder; use flate2::write::GzEncoder; -use libs::cli_ui::{self, LogLevel}; +use indicatif::ProgressBar; +use libs::cli_ui::{self, build_progress_bar, LogLevel}; use libs::cli_ui::{init_logger, prompt_string}; -use libs::distro_image::{self, DistroImageFetcher, DistroImageFetcherGen, DistroImageFile}; +use libs::distro_image::{ + self, download_file_with_progress, DistroImageFetcher, DistroImageFetcherGen, DistroImageFile, +}; use libs::distrod_config; use libs::local_image::LocalDistroImage; use libs::lxd_image::LxdDistroImageList; @@ -137,7 +140,8 @@ fn config_distro(distro_name: &str, opts: ConfigOpts) -> Result<()> { Ok(()) } -fn install_distro(distro_name: &str, opts: InstallOpts) -> Result<()> { +#[tokio::main] +async fn install_distro(distro_name: &str, opts: InstallOpts) -> Result<()> { let wsl = WslApi::new() .with_context(|| "Failed to retrieve WSL API. Have you enabled the WSL2 feature?")?; @@ -158,7 +162,9 @@ You can install a local .tar.xz, or download an image from linuxcontainers.org. BTW, you can run Systemd with distrod, so you can try LXC/LXD with distrod! =================================================================================" ); - let lxd_root_tarxz = fetch_distro_image().with_context(|| "Failed to fetch a distro image.")?; + let lxd_root_tarxz = fetch_distro_image() + .await + .with_context(|| "Failed to fetch a distro image.")?; let lxd_tar = tar::Archive::new(XzDecoder::new(lxd_root_tarxz)); log::info!( @@ -203,16 +209,17 @@ You can install a local .tar.xz, or download an image from linuxcontainers.org. Ok(()) } -fn fetch_distro_image() -> Result> { +async fn fetch_distro_image() -> Result> { let local_image_fetcher = || Ok(Box::new(LocalDistroImage::new(cli_ui::prompt_path)) as Box); let lxd_image_fetcher = || Ok(Box::new(LxdDistroImageList::default()) as Box); let fetchers = vec![ - Box::new(local_image_fetcher) as Box, - Box::new(lxd_image_fetcher) as Box, + Box::new(local_image_fetcher) as DistroImageFetcherGen, + Box::new(lxd_image_fetcher) as DistroImageFetcherGen, ]; let image = distro_image::fetch_image(fetchers, cli_ui::choose_from_list, 1) + .await .with_context(|| "Failed to fetch the image list.")?; match image.image { DistroImageFile::Local(path) => { @@ -222,14 +229,10 @@ fn fetch_distro_image() -> Result> { } DistroImageFile::Url(url) => { log::info!("Downloading '{}'...", url); - let client = reqwest::blocking::Client::builder().timeout(None).build()?; - let response = client - .get(&url) - .send() - .with_context(|| format!("Failed to download {}.", &url))?; - let bytes = response.bytes().with_context(|| "Download failed.")?; + let mut bytes = vec![]; + download_file_with_progress(&url, build_progress_bar, &mut bytes).await?; log::info!("Download done."); - Ok(Box::new(Cursor::new(bytes))) + Ok(Box::new(Cursor::new(bytes)) as Box) } } } @@ -349,6 +352,7 @@ fn add_user( ), true, )?; + log::info!("Querying the generated uid. This may take some time depending on your machine."); query_uid(wsl, distro_name, user_name, tmp_dir) } diff --git a/distrod/libs/Cargo.toml b/distrod/libs/Cargo.toml index 3b447de..7f994ec 100644 --- a/distrod/libs/Cargo.toml +++ b/distrod/libs/Cargo.toml @@ -7,13 +7,15 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-trait = "0.1.51" anyhow = "1.0" chrono = "0.4" colored = "2" log = "0.4" env_logger = "0.8" scraper = "0.12" -reqwest = { version = "0.11", features = ["blocking", "json"] } +indicatif = "0.16" +reqwest = { version = "0.11" } glob = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/distrod/libs/src/cli_ui.rs b/distrod/libs/src/cli_ui.rs index f5c1efc..8950cf0 100644 --- a/distrod/libs/src/cli_ui.rs +++ b/distrod/libs/src/cli_ui.rs @@ -118,3 +118,11 @@ pub fn prompt_string(message: &str, target_name: &str, default: Option<&str>) -> choice = choice.trim_end().to_owned(); Ok(choice) } + +pub fn build_progress_bar(total_size: u64) -> indicatif::ProgressBar { + let bar = indicatif::ProgressBar::new(total_size); + bar.set_style(indicatif::ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})") + .progress_chars("#>-")); + bar +} diff --git a/distrod/libs/src/distro_image.rs b/distrod/libs/src/distro_image.rs index 05453e4..307033a 100644 --- a/distrod/libs/src/distro_image.rs +++ b/distrod/libs/src/distro_image.rs @@ -1,13 +1,15 @@ use std::ffi::OsString; -use anyhow::Result; +use anyhow::{Context, Result}; +use async_trait::async_trait; pub type ListChooseFn = fn(list: DistroImageList) -> Result>; pub type PromptPath = fn(message: &str, default: Option<&str>) -> Result; +#[async_trait] pub trait DistroImageFetcher { fn get_name(&self) -> &str; - fn fetch(&self) -> Result; + async fn fetch(&self) -> Result; } pub enum DistroImageList { @@ -37,10 +39,10 @@ pub enum DistroImageFile { Url(String), } -pub type DistroImageFetcherGen = dyn Fn() -> Result>; +pub type DistroImageFetcherGen = Box Result> + Sync>; -pub fn fetch_image( - fetchers: Vec>, +pub async fn fetch_image( + fetchers: Vec, choose_from_list: ListChooseFn, default_index: usize, ) -> Result { @@ -49,7 +51,7 @@ pub fn fetch_image( default_index, }) as Box; loop { - let fetched_image_list = distro_image_list.fetch()?; + let fetched_image_list = distro_image_list.fetch().await?; match fetched_image_list { DistroImageList::Fetcher(_, _, _) => { distro_image_list = choose_from_list(fetched_image_list)?; @@ -62,16 +64,17 @@ pub fn fetch_image( } struct DistroImageFetchersList { - fetchers: Vec>, + fetchers: Vec, default_index: usize, } +#[async_trait] impl DistroImageFetcher for DistroImageFetchersList { fn get_name(&self) -> &str { "Image candidates" } - fn fetch(&self) -> Result { + async fn fetch(&self) -> Result { let fetchers: Result>> = self.fetchers.iter().map(|f| f()).collect(); Ok(DistroImageList::Fetcher( "the way to get a distro image".to_owned(), @@ -80,3 +83,35 @@ impl DistroImageFetcher for DistroImageFetchersList { )) } } + +pub async fn download_file_with_progress( + url: &str, + progress_bar_builder: F, + out: &mut W, +) -> Result<()> +where + F: FnOnce(u64) -> indicatif::ProgressBar, + W: std::io::Write, +{ + let client = reqwest::Client::builder().build()?; + let mut response = client + .get(url) + .send() + .await + .with_context(|| format!("Failed to download {}.", &url))?; + let total_size = response + .content_length() + .with_context(|| format!("Failed to get the content length of {}.", &url))?; + + let progress_bar = progress_bar_builder(total_size); + let mut downloaded_size = 0; + + while let Some(bytes) = response.chunk().await? { + out.write_all(&bytes)?; + downloaded_size = std::cmp::min(downloaded_size + bytes.len(), total_size as usize); + progress_bar.set_position(downloaded_size as u64); + } + + progress_bar.finish(); + Ok(()) +} diff --git a/distrod/libs/src/local_image.rs b/distrod/libs/src/local_image.rs index c5c772e..7adbd0e 100644 --- a/distrod/libs/src/local_image.rs +++ b/distrod/libs/src/local_image.rs @@ -1,3 +1,4 @@ +use async_trait::async_trait; use std::path::{Path, PathBuf}; use crate::distro_image::{ @@ -15,12 +16,13 @@ impl LocalDistroImage { } } +#[async_trait] impl DistroImageFetcher for LocalDistroImage { fn get_name(&self) -> &str { "Use a local tar.xz file" } - fn fetch(&self) -> Result { + async fn fetch(&self) -> Result { let mut path; loop { path = (self.prompt_path)("Please input the path to your .tar.xz image file.", None)?; diff --git a/distrod/libs/src/lxd_image.rs b/distrod/libs/src/lxd_image.rs index 9e9d01e..7b715d9 100644 --- a/distrod/libs/src/lxd_image.rs +++ b/distrod/libs/src/lxd_image.rs @@ -3,14 +3,15 @@ use crate::distro_image::{ ListChooseFn, }; use anyhow::{anyhow, Context, Result}; +use async_trait::async_trait; use chrono::NaiveDateTime; static LINUX_CONTAINERS_ORG_BASE: &str = "https://uk.images.linuxcontainers.org/"; -pub fn fetch_lxd_image(choose_from_list: ListChooseFn) -> Result { +pub async fn fetch_lxd_image(choose_from_list: ListChooseFn) -> Result { let mut distro_image_list = Box::new(LxdDistroImageList {}) as Box; loop { - let fetched_image_list = distro_image_list.fetch()?; + let fetched_image_list = distro_image_list.fetch().await?; match fetched_image_list { DistroImageList::Fetcher(_, _, _) => { distro_image_list = choose_from_list(fetched_image_list)?; @@ -25,13 +26,15 @@ pub fn fetch_lxd_image(choose_from_list: ListChooseFn) -> Result { #[derive(Default)] pub struct LxdDistroImageList; +#[async_trait] impl DistroImageFetcher for LxdDistroImageList { fn get_name(&self) -> &str { "Download a LXD image" } - fn fetch(&self) -> Result { + async fn fetch(&self) -> Result { let distros: Vec<_> = fetch_apache_file_list("images/") + .await .map(|links| { links .into_iter() @@ -44,6 +47,7 @@ impl DistroImageFetcher for LxdDistroImageList { .collect() }) .with_context(|| "Failed to parse the distro image list of the LXD image server.")?; + Ok(DistroImageList::Fetcher( "a LXD image".to_owned(), distros, @@ -58,13 +62,15 @@ pub struct LxdDistroVersionList { version_list_url: String, } +#[async_trait] impl DistroImageFetcher for LxdDistroVersionList { fn get_name(&self) -> &str { self.name.as_str() } - fn fetch(&self) -> Result { + async fn fetch(&self) -> Result { let mut links = fetch_apache_file_list(&self.version_list_url) + .await .with_context(|| "Failed to parse the version list.")?; links.sort_by(|a, b| a.last_modified.cmp(&b.last_modified)); let versions: Vec<_> = links @@ -96,13 +102,15 @@ pub struct LxdDistroVersion { platform_list_url: String, } +#[async_trait] impl DistroImageFetcher for LxdDistroVersion { fn get_name(&self) -> &str { self.version_name.as_str() } - fn fetch(&self) -> Result { + async fn fetch(&self) -> Result { let mut dates = fetch_apache_file_list(&format!("{}amd64/cloud", &self.platform_list_url)) + .await .with_context(|| format!("Failed to get the image for amd64/cloud. Perhaps '{}amd64/cloud' is not found?", &self.platform_list_url))?; dates.sort_by(|a, b| b.last_modified.cmp(&a.last_modified)); let latest = &dates[0]; @@ -117,16 +125,19 @@ impl DistroImageFetcher for LxdDistroVersion { } } -fn fetch_apache_file_list(relative_url: &str) -> Result> { +async fn fetch_apache_file_list(relative_url: &str) -> Result> { let url = LINUX_CONTAINERS_ORG_BASE.to_owned() + relative_url; let date_selector = scraper::Selector::parse("body > table > tbody > tr > td:nth-child(3)").unwrap(); let a_link_selector = scraper::Selector::parse("body > table > tbody > tr > td:nth-child(2) > a").unwrap(); log::info!("Fetching from linuxcontainers.org..."); - let apache_file_list_body = reqwest::blocking::get(&url) + let apache_file_list_body = reqwest::get(&url) + .await .with_context(|| format!("Failed to fetch {}", &url))? - .text()?; + .text() + .await + .with_context(|| format!("Failed to get the text of {}", &url))?; let doc = scraper::Html::parse_document(&apache_file_list_body); let dates: Vec<_> = doc.select(&date_selector).collect(); let a_links: Vec<_> = doc.select(&a_link_selector).collect(); diff --git a/distrod/portproxy/Cargo.toml b/distrod/portproxy/Cargo.toml index 6303a71..064f42a 100644 --- a/distrod/portproxy/Cargo.toml +++ b/distrod/portproxy/Cargo.toml @@ -7,9 +7,9 @@ edition = "2018" [dependencies] libs = { path = "../libs" } anyhow = "1" -#tokio = { version = "1", features = ["io-util", "net", "rt", "rt-multi-thread"] } tokio = { version = "1", features = ["full"] } -structopt = { version = "0.3" } +# Use my fork until https://github.com/TeXitoi/structopt/issues/490 is resolved +structopt = { git = "https://github.com/nullpo-head/structopt.git" } log = "0.4" env_logger = "0.8" strum = { version = "0.20", features = ["derive"] }