diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fef45d0c7..ddc0154e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,6 +82,12 @@ test instance::wasitest::test_delete_after_create ... ok test instance::wasitest::test_wasi ... ok ``` +Run individual test via cargo adding `RUST_LOG=trace` (adjust the level of logging as needed) to see shim output. Also adjust the test name as needed. + +``` +RUST_LOG=DEBUG cargo test --package containerd-shim-wasmtime --lib -- wasmtime_tests::test_hello_world --exact --nocapture +``` + ### End to End tests The e2e test run on [k3s](https://k3s.io/) and [kind](https://kind.sigs.k8s.io/). A test image is built using [oci-tar-builder](./crates/oci-tar-builder/) and is loaded onto the clusters. This test image is not pushed to an external registry so be sure to use the Makefile targets to build the image and load it on the cluster. diff --git a/Cargo.lock b/Cargo.lock index b6679c1b6..613cf64b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.7" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -628,12 +628,16 @@ dependencies = [ "log", "nix 0.27.1", "oci-spec", + "oci-tar-builder", + "prost-types 0.11.9", "protobuf 3.2.0", "serde", "serde_json", + "sha256", "tempfile", "thiserror", "tokio", + "tokio-stream", "ttrpc", "ttrpc-codegen", "wasmparser 0.121.0", diff --git a/Cargo.toml b/Cargo.toml index 0180a5f95..d750f4eba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } containerd-shim = "0.6.0" containerd-shim-wasm = { path = "crates/containerd-shim-wasm", version = "0.4.0" } containerd-shim-wasm-test-modules = { path = "crates/containerd-shim-wasm-test-modules", version = "0.3.1"} +oci-tar-builder = { path = "crates/oci-tar-builder", version = "0.3.1" } crossbeam = { version = "0.8.4", default-features = false } env_logger = "0.10" libc = "0.2.153" diff --git a/Makefile b/Makefile index cec2f64b3..5158f8ea6 100644 --- a/Makefile +++ b/Makefile @@ -238,6 +238,13 @@ test/k8s/deploy-workload-oci-%: test/k8s/clean test/k8s/cluster-% dist/img-oci.t # verify that we are still running after some time sleep 5s kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=5s + @if [ "$*" = "wasmtime" ]; then \ + set -e; \ + echo "checking for pre-compiled label and ensuring can scale"; \ + docker exec $(KIND_CLUSTER_NAME)-control-plane ctr -n k8s.io i ls | grep "runwasi.io/precompiled"; \ + kubectl --context=kind-$(KIND_CLUSTER_NAME) scale deployment wasi-demo --replicas=4; \ + kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=5s; \ + fi .PHONY: test/k8s-% test/k8s-%: test/k8s/deploy-workload-% @@ -289,6 +296,13 @@ test/k3s-oci-%: dist/img-oci.tar bin/k3s dist-% sleep 5s sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=5s sudo bin/k3s kubectl get pods -o wide + @if [ "$*" = "wasmtime" ]; then \ + set -e; \ + echo "checking for pre-compiled label and ensuring can scale"; \ + sudo bin/k3s ctr -n k8s.io i ls | grep "runwasi.io/precompiled"; \ + sudo bin/k3s kubectl scale deployment wasi-demo --replicas=4; \ + sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=5s; \ + fi sudo bin/k3s kubectl delete -f test/k8s/deploy.oci.yaml sudo bin/k3s kubectl wait deployment wasi-demo --for delete --timeout=60s @@ -299,6 +313,7 @@ test/k3s/clean: bin/k3s/clean; clean: -rm -rf dist -rm -rf bin + -rm -rf test/k8s/_out -$(MAKE) test-image/clean -$(MAKE) test/k8s/clean -$(MAKE) test/k3s/clean diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index cc891a94e..ba29873eb 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -16,6 +16,7 @@ anyhow = { workspace = true } chrono = { workspace = true } containerd-shim = { workspace = true } containerd-shim-wasm-test-modules = { workspace = true, optional = true } +oci-tar-builder = { workspace = true, optional = true } crossbeam = { workspace = true } env_logger = { workspace = true, optional = true } git-version = "0.3.9" @@ -32,6 +33,9 @@ wat = { workspace = true } tokio = { version = "1.36.0", features = [ "full" ] } futures = { version = "0.3.30" } wasmparser = "0.121.0" +tokio-stream = { version = "0.1" } +prost-types = "0.11" # should match version in containerd-shim +sha256 = "1.4.0" [target.'cfg(unix)'.dependencies] caps = "0.5" @@ -51,8 +55,9 @@ ttrpc-codegen = { version = "0.4.2", optional = true } containerd-shim-wasm-test-modules = { workspace = true } env_logger = { workspace = true } tempfile = { workspace = true } +oci-tar-builder = { workspace = true} [features] -testing = ["dep:containerd-shim-wasm-test-modules", "dep:env_logger", "dep:tempfile"] +testing = ["dep:containerd-shim-wasm-test-modules", "dep:env_logger", "dep:tempfile", "dep:oci-tar-builder"] generate_bindings = ["ttrpc-codegen"] generate_doc = [] diff --git a/crates/containerd-shim-wasm/src/container/context.rs b/crates/containerd-shim-wasm/src/container/context.rs index 5f4475a5b..e5b6881c8 100644 --- a/crates/containerd-shim-wasm/src/container/context.rs +++ b/crates/containerd-shim-wasm/src/container/context.rs @@ -34,6 +34,7 @@ pub trait RuntimeContext { } /// The source for a WASI module / components. +#[derive(Debug)] pub enum Source<'a> { // The WASI module is a file in the file system. File(PathBuf), diff --git a/crates/containerd-shim-wasm/src/container/engine.rs b/crates/containerd-shim-wasm/src/container/engine.rs index 77f10cc3f..f3461f6c8 100644 --- a/crates/containerd-shim-wasm/src/container/engine.rs +++ b/crates/containerd-shim-wasm/src/container/engine.rs @@ -1,7 +1,7 @@ use std::fs::File; use std::io::Read; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use super::Source; use crate::container::{PathResolve, RuntimeContext}; @@ -52,4 +52,27 @@ pub trait Engine: Clone + Send + Sync + 'static { fn supported_layers_types() -> &'static [&'static str] { &["application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"] } + + /// Precompiles a module that is in the WASM OCI layer format + /// This is used to precompile a module before it is run and will be called if can_precompile returns true. + /// It is called only the first time a module is run and the resulting bytes will be cached in the containerd content store. + /// The cached, precompiled module will be reloaded on subsequent runs. + fn precompile(&self, _layers: &[Vec]) -> Result> { + bail!("precompilation not supported for this runtime") + } + + /// Can_precompile lets the shim know if the runtime supports precompilation. + /// When it returns Some(unique_string) the `unique_string` will be used as a cache key for the precompiled module. + /// + /// `unique_string` should at least include the version of the shim running but could include other information such as a hash + /// of the version and cpu type and other important information in the validation of being able to use precompiled module. + /// If the string doesn't match then the module will be recompiled and cached with the new `unique_string`. + /// + /// This string will be used in the following way: + /// "runwasi.io/precompiled//" + /// + /// When it returns None the runtime will not be asked to precompile the module. This is the default value. + fn can_precompile(&self) -> Option { + None + } } diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs deleted file mode 100644 index 8a5d2782b..000000000 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ /dev/null @@ -1,159 +0,0 @@ -#![cfg(unix)] - -use std::path::Path; - -use containerd_client; -use containerd_client::services::v1::containers_client::ContainersClient; -use containerd_client::services::v1::content_client::ContentClient; -use containerd_client::services::v1::images_client::ImagesClient; -use containerd_client::services::v1::{GetContainerRequest, GetImageRequest, ReadContentRequest}; -use containerd_client::tonic::transport::Channel; -use containerd_client::{tonic, with_namespace}; -use futures::TryStreamExt; -use oci_spec::image::{Arch, ImageManifest, MediaType, Platform}; -use tokio::runtime::Runtime; -use tonic::Request; - -use crate::sandbox::error::{Error as ShimError, Result}; -use crate::sandbox::oci::{self, WasmLayer}; - -pub(crate) struct Client { - inner: Channel, - rt: Runtime, - namespace: String, -} - -// sync wrapper implementation from https://tokio.rs/tokio/topics/bridging -impl Client { - // wrapper around connection that will establish a connection and create a client - pub fn connect(address: impl AsRef, namespace: impl ToString) -> Result { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - let inner = rt - .block_on(containerd_client::connect(address)) - .map_err(|err| ShimError::Containerd(err.to_string()))?; - - Ok(Client { - inner, - rt, - namespace: namespace.to_string(), - }) - } - - // wrapper around read that will read the entire content file - pub fn read_content(&self, digest: impl ToString) -> Result> { - self.rt.block_on(async { - let req = ReadContentRequest { - digest: digest.to_string(), - ..Default::default() - }; - let req = with_namespace!(req, self.namespace); - ContentClient::new(self.inner.clone()) - .read(req) - .await - .map_err(|err| ShimError::Containerd(err.to_string()))? - .into_inner() - .map_ok(|msg| msg.data) - .try_concat() - .await - .map_err(|err| ShimError::Containerd(err.to_string())) - }) - } - - pub fn get_image_content_sha(&self, image_name: impl ToString) -> Result { - self.rt.block_on(async { - let name = image_name.to_string(); - let req = GetImageRequest { name }; - let req = with_namespace!(req, self.namespace); - let digest = ImagesClient::new(self.inner.clone()) - .get(req) - .await - .map_err(|err| ShimError::Containerd(err.to_string()))? - .into_inner() - .image - .ok_or_else(|| { - ShimError::Containerd(format!( - "failed to get image content sha for image {}", - image_name.to_string() - )) - })? - .target - .ok_or_else(|| { - ShimError::Containerd(format!( - "failed to get image content sha for image {}", - image_name.to_string() - )) - })? - .digest; - Ok(digest) - }) - } - - pub fn get_image(&self, container_name: impl ToString) -> Result { - self.rt.block_on(async { - let id = container_name.to_string(); - let req = GetContainerRequest { id }; - let req = with_namespace!(req, self.namespace); - let image = ContainersClient::new(self.inner.clone()) - .get(req) - .await - .map_err(|err| ShimError::Containerd(err.to_string()))? - .into_inner() - .container - .ok_or_else(|| { - ShimError::Containerd(format!( - "failed to get image for container {}", - container_name.to_string() - )) - })? - .image; - Ok(image) - }) - } - - // load module will query the containerd store to find an image that has an OS of type 'wasm' - // If found it continues to parse the manifest and return the layers that contains the WASM modules - // and possibly other configuration layers. - pub fn load_modules( - &self, - containerd_id: impl ToString, - supported_layer_types: &[&str], - ) -> Result<(Vec, Platform)> { - let image_name = self.get_image(containerd_id.to_string())?; - let digest = self.get_image_content_sha(image_name)?; - let manifest = self.read_content(digest)?; - let manifest = manifest.as_slice(); - let manifest = ImageManifest::from_reader(manifest)?; - - let image_config_descriptor = manifest.config(); - let image_config = self.read_content(image_config_descriptor.digest())?; - let image_config = image_config.as_slice(); - - // the only part we care about here is the platform values - let platform: Platform = serde_json::from_slice(image_config)?; - let Arch::Wasm = platform.architecture() else { - log::info!("manifest is not in WASM OCI image format"); - return Ok((vec![], platform)); - }; - log::info!("found manifest with WASM OCI image format."); - - let layers = manifest - .layers() - .iter() - .filter(|x| is_wasm_layer(x.media_type(), supported_layer_types)) - .map(|config| { - self.read_content(config.digest()).map(|module| WasmLayer { - config: config.clone(), - layer: module, - }) - }) - .collect::>>()?; - Ok((layers, platform)) - } -} - -fn is_wasm_layer(media_type: &MediaType, supported_layer_types: &[&str]) -> bool { - supported_layer_types.contains(&media_type.to_string().as_str()) -} diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs new file mode 100644 index 000000000..37e83f28e --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/client.rs @@ -0,0 +1,545 @@ +#![cfg(unix)] + +use std::collections::HashMap; +use std::path::Path; + +use containerd_client; +use containerd_client::services::v1::containers_client::ContainersClient; +use containerd_client::services::v1::content_client::ContentClient; +use containerd_client::services::v1::images_client::ImagesClient; +use containerd_client::services::v1::leases_client::LeasesClient; +use containerd_client::services::v1::{ + Container, DeleteContentRequest, GetContainerRequest, GetImageRequest, Image, Info, + InfoRequest, ReadContentRequest, UpdateImageRequest, UpdateRequest, WriteAction, + WriteContentRequest, +}; +use containerd_client::tonic::transport::Channel; +use containerd_client::{tonic, with_namespace}; +use futures::TryStreamExt; +use oci_spec::image::{Arch, ImageManifest, MediaType, Platform}; +use prost_types::FieldMask; +use sha256::digest; +use tokio::runtime::Runtime; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Code, Request}; + +use super::lease::LeaseGuard; +use crate::container::Engine; +use crate::sandbox::error::{Error as ShimError, Result}; +use crate::sandbox::oci::{self, WasmLayer}; +use crate::with_lease; + +static PRECOMPILE_PREFIX: &str = "runwasi.io/precompiled"; + +pub struct Client { + inner: Channel, + rt: Runtime, + namespace: String, + address: String, +} + +#[derive(Debug)] +pub(crate) struct WriteContent { + _lease: LeaseGuard, + pub digest: String, +} + +// sync wrapper implementation from https://tokio.rs/tokio/topics/bridging +impl Client { + // wrapper around connection that will establish a connection and create a client + pub fn connect( + address: impl AsRef + ToString, + namespace: impl ToString, + ) -> Result { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let inner = rt + .block_on(containerd_client::connect(address.as_ref())) + .map_err(|err| ShimError::Containerd(err.to_string()))?; + + Ok(Client { + inner, + rt, + namespace: namespace.to_string(), + address: address.to_string(), + }) + } + + // wrapper around read that will read the entire content file + fn read_content(&self, digest: impl ToString) -> Result> { + self.rt.block_on(async { + let req = ReadContentRequest { + digest: digest.to_string(), + ..Default::default() + }; + let req = with_namespace!(req, self.namespace); + ContentClient::new(self.inner.clone()) + .read(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .map_ok(|msg| msg.data) + .try_concat() + .await + .map_err(|err| ShimError::Containerd(err.to_string())) + }) + } + + // used in tests to clean up content + #[allow(dead_code)] + fn delete_content(&self, digest: impl ToString) -> Result<()> { + self.rt.block_on(async { + let req = DeleteContentRequest { + digest: digest.to_string(), + }; + let req = with_namespace!(req, self.namespace); + ContentClient::new(self.inner.clone()) + .delete(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))?; + Ok(()) + }) + } + + // wrapper around lease that will create a lease and return a guard that will delete the lease when dropped + fn lease(&self, reference: String) -> Result { + self.rt.block_on(async { + let mut lease_labels = HashMap::new(); + let expire = chrono::Utc::now() + chrono::Duration::hours(24); + lease_labels.insert("containerd.io/gc.expire".to_string(), expire.to_rfc3339()); + let lease_request = containerd_client::services::v1::CreateRequest { + id: reference.clone(), + labels: lease_labels, + }; + + let mut leases_client = LeasesClient::new(self.inner.clone()); + + let lease = leases_client + .create(with_namespace!(lease_request, self.namespace)) + .await + .map_err(|e| ShimError::Containerd(e.to_string()))? + .into_inner() + .lease + .ok_or_else(|| { + ShimError::Containerd(format!("unable to create lease for {}", reference)) + })?; + + Ok(LeaseGuard { + lease_id: lease.id, + address: self.address.clone(), + namespace: self.namespace.clone(), + }) + }) + } + + fn save_content( + &self, + data: Vec, + original_digest: String, + label: &str, + ) -> Result { + let expected = format!("sha256:{}", digest(data.clone())); + let reference = format!("precompile-{}", label); + let lease = self.lease(reference.clone())?; + + let digest = self.rt.block_on(async { + // create a channel to feed the stream; only sending one message at a time so we can set this to one + let (tx, rx) = mpsc::channel(1); + + let len = data.len() as i64; + log::debug!("Writing {} bytes to content store", len); + let mut client = ContentClient::new(self.inner.clone()); + + // Send write request with Stat action to containerd to let it know that we are going to write content + // if the content is already there, it will return early with AlreadyExists + log::debug!("Sending stat request to containerd"); + let req = WriteContentRequest { + r#ref: reference.clone(), + action: WriteAction::Stat.into(), + total: len, + expected: expected.clone(), + ..Default::default() + }; + tx.send(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))?; + let request_stream = ReceiverStream::new(rx); + let request_stream = + with_lease!(request_stream, self.namespace, lease.lease_id.clone()); + let mut response_stream = match client.write(request_stream).await { + Ok(response_stream) => response_stream.into_inner(), + Err(e) if e.code() == Code::AlreadyExists => { + log::info!("content already exists {}", expected.clone().to_string()); + return Ok(expected); + } + Err(e) => return Err(ShimError::Containerd(e.to_string())), + }; + let response = response_stream + .message() + .await + .map_err(|e| ShimError::Containerd(e.to_string()))? + .ok_or_else(|| { + ShimError::Containerd(format!( + "no response received after write request for {}", + expected + )) + })?; + + // There is a scenario where the content might have been removed manually + // but the content isn't removed from the containerd file system yet. + // In this case if we re-add it at before its removed from file system + // we don't need to copy the content again. Container tells us it found the blob + // by returning the offset of the content that was found. + let data_to_write = data[response.offset as usize..].to_vec(); + + // Write and commit at same time + let mut labels = HashMap::new(); + labels.insert(label.to_string(), original_digest.clone()); + let commit_request = WriteContentRequest { + action: WriteAction::Commit.into(), + total: len, + offset: response.offset, + expected: expected.clone(), + labels, + data: data_to_write, + ..Default::default() + }; + log::debug!( + "Sending commit request to containerd with response: {:?}", + response + ); + tx.send(commit_request) + .await + .map_err(|err| ShimError::Containerd(format!("commit request error: {}", err)))?; + let response = response_stream + .message() + .await + .map_err(|err| ShimError::Containerd(format!("response stream error: {}", err)))? + .ok_or_else(|| { + ShimError::Containerd(format!( + "no response received after write request for {}", + expected.clone() + )) + })?; + + log::debug!("Validating response"); + // client should validate that all bytes were written and that the digest matches + if response.offset != len { + return Err(ShimError::Containerd(format!( + "failed to write all bytes, expected {} got {}", + len, response.offset + ))); + } + if response.digest != expected { + return Err(ShimError::Containerd(format!( + "unexpected digest, expected {} got {}", + expected, response.digest + ))); + } + Ok(response.digest) + })?; + + Ok(WriteContent { + _lease: lease, + digest: digest.clone(), + }) + } + + fn get_info(&self, content_digest: String) -> Result { + self.rt.block_on(async { + let req = InfoRequest { + digest: content_digest.clone(), + }; + let req = with_namespace!(req, self.namespace); + let info = ContentClient::new(self.inner.clone()) + .info(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .info + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get info for content {}", + content_digest + )) + })?; + Ok(info) + }) + } + + fn update_info(&self, info: Info) -> Result { + self.rt.block_on(async { + let req = UpdateRequest { + info: Some(info.clone()), + update_mask: Some(FieldMask { + paths: vec!["labels".to_string()], + }), + }; + let req = with_namespace!(req, self.namespace); + let info = ContentClient::new(self.inner.clone()) + .update(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .info + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to update info for content {}", + info.digest + )) + })?; + Ok(info) + }) + } + + fn get_image(&self, image_name: impl ToString) -> Result { + self.rt.block_on(async { + let name = image_name.to_string(); + let req = GetImageRequest { name }; + let req = with_namespace!(req, self.namespace); + let image = ImagesClient::new(self.inner.clone()) + .get(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .image + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get image for image {}", + image_name.to_string() + )) + })?; + Ok(image) + }) + } + + fn update_image(&self, image: Image) -> Result { + self.rt.block_on(async { + let req = UpdateImageRequest { + image: Some(image.clone()), + update_mask: Some(FieldMask { + paths: vec!["labels".to_string()], + }), + }; + + let req = with_namespace!(req, self.namespace); + let image = ImagesClient::new(self.inner.clone()) + .update(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .image + .ok_or_else(|| { + ShimError::Containerd(format!("failed to update image {}", image.name)) + })?; + Ok(image) + }) + } + + fn extract_image_content_sha(&self, image: &Image) -> Result { + let digest = image + .target + .as_ref() + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get image content sha for image {}", + image.name + )) + })? + .digest + .clone(); + Ok(digest) + } + + fn get_container(&self, container_name: impl ToString) -> Result { + self.rt.block_on(async { + let id = container_name.to_string(); + let req = GetContainerRequest { id }; + let req = with_namespace!(req, self.namespace); + let container = ContainersClient::new(self.inner.clone()) + .get(req) + .await + .map_err(|err| ShimError::Containerd(err.to_string()))? + .into_inner() + .container + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get image for container {}", + container_name.to_string() + )) + })?; + Ok(container) + }) + } + + // load module will query the containerd store to find an image that has an OS of type 'wasm' + // If found it continues to parse the manifest and return the layers that contains the WASM modules + // and possibly other configuration layers. + pub fn load_modules( + &self, + containerd_id: impl ToString, + engine: &T, + ) -> Result<(Vec, Platform)> { + let container = self.get_container(containerd_id.to_string())?; + let mut image = self.get_image(container.image)?; + let image_digest = self.extract_image_content_sha(&image)?; + let manifest = self.read_content(image_digest.clone())?; + let manifest = manifest.as_slice(); + let manifest = ImageManifest::from_reader(manifest)?; + + let image_config_descriptor = manifest.config(); + let image_config = self.read_content(image_config_descriptor.digest())?; + let image_config = image_config.as_slice(); + + // the only part we care about here is the platform values + let platform: Platform = serde_json::from_slice(image_config)?; + let Arch::Wasm = platform.architecture() else { + log::info!("manifest is not in WASM OCI image format"); + return Ok((vec![], platform)); + }; + + log::info!("found manifest with WASM OCI image format."); + // This label is unique across runtimes and version of the shim running + // a precompiled component/module will not work across different runtimes or versions + let (can_precompile, precompile_id) = match engine.can_precompile() { + Some(precompile_id) => (true, precompile_label(T::name(), &precompile_id)), + None => (false, "".to_string()), + }; + + match image.labels.get(&precompile_id) { + Some(precompile_digest) if can_precompile => { + log::info!("found precompiled label: {} ", &precompile_id); + match self.read_content(precompile_digest) { + Ok(precompiled) => { + log::info!("found precompiled module in cache: {} ", &precompile_digest); + return Ok(( + vec![WasmLayer { + config: image_config_descriptor.clone(), + layer: precompiled, + }], + platform, + )); + } + Err(e) => { + // log and continue + log::warn!("failed to read precompiled module from cache: {}. Content may have been removed manually, will attempt to recompile", e); + } + } + } + _ => {} + } + + let layers = manifest + .layers() + .iter() + .filter(|x| is_wasm_layer(x.media_type(), T::supported_layers_types())) + .map(|config| self.read_content(config.digest())) + .collect::>>()?; + + if layers.is_empty() { + log::info!("no WASM modules found in OCI layers"); + return Ok((vec![], platform)); + } + + if can_precompile { + log::info!("precompiling module"); + let precompiled = engine.precompile(layers.as_slice())?; + log::info!("precompiling module: {}", image_digest.clone()); + let precompiled_content = + self.save_content(precompiled.clone(), image_digest.clone(), &precompile_id)?; + + log::debug!("updating image with compiled content digest"); + image + .labels + .insert(precompile_id, precompiled_content.digest.clone()); + self.update_image(image)?; + + // The original image is considered a root object, by adding a ref to the new compiled content + // We tell containerd to not garbage collect the new content until this image is removed from the system + // this ensures that we keep the content around after the lease is dropped + log::debug!("updating content with precompile digest to avoid garbage collection"); + let mut image_content = self.get_info(image_digest.clone())?; + image_content.labels.insert( + "containerd.io/gc.ref.content.precompile".to_string(), + precompiled_content.digest.clone(), + ); + self.update_info(image_content)?; + + return Ok(( + vec![WasmLayer { + config: image_config_descriptor.clone(), + layer: precompiled, + }], + platform, + )); + } + + log::info!("using module from OCI layers"); + let layers = layers + .into_iter() + .map(|module| WasmLayer { + config: image_config_descriptor.clone(), + layer: module, + }) + .collect::>(); + Ok((layers, platform)) + } +} + +fn precompile_label(name: &str, version: &str) -> String { + format!("{}/{}/{}", PRECOMPILE_PREFIX, name, version) +} + +fn is_wasm_layer(media_type: &MediaType, supported_layer_types: &[&str]) -> bool { + supported_layer_types.contains(&media_type.to_string().as_str()) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + + #[test] + fn test_save_content() { + let path = PathBuf::from("/run/containerd/containerd.sock"); + let path = path.to_str().unwrap(); + let client = Client::connect(path, "test-ns").unwrap(); + let data = b"hello world".to_vec(); + + let expected = digest(data.clone()); + let expected = format!("sha256:{}", expected); + + let label = precompile_label("test", "hasdfh"); + let returned = client + .save_content(data, "original".to_string(), &label) + .unwrap(); + assert_eq!(expected, returned.digest.clone()); + + let data = client.read_content(returned.digest.clone()).unwrap(); + assert_eq!(data, b"hello world"); + + client + .save_content(data.clone(), "original".to_string(), &label) + .expect_err("Should not be able to save when lease is open"); + + // need to drop the lease to be able to create a second one + // a second call should be successful since it already exists + drop(returned); + + // a second call should be successful since it already exists + let returned = client + .save_content(data, "original".to_string(), &label) + .unwrap(); + assert_eq!(expected, returned.digest); + + client.delete_content(expected.clone()).unwrap(); + + client + .read_content(expected) + .expect_err("content should not exist"); + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs new file mode 100644 index 000000000..de1414315 --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/lease.rs @@ -0,0 +1,63 @@ +#![cfg(unix)] + +use containerd_client::services::v1::leases_client::LeasesClient; +use containerd_client::{tonic, with_namespace}; +use tonic::Request; + +// Adds lease info to grpc header +// https://github.com/containerd/containerd/blob/8459273f806e068e1a6bacfaf1355bbbad738d5e/docs/garbage-collection.md#using-grpc +#[macro_export] +macro_rules! with_lease { + ($req : ident, $ns: expr, $lease_id: expr) => {{ + let mut req = Request::new($req); + let md = req.metadata_mut(); + // https://github.com/containerd/containerd/blob/main/namespaces/grpc.go#L27 + md.insert("containerd-namespace", $ns.parse().unwrap()); + md.insert("containerd-lease", $lease_id.parse().unwrap()); + req + }}; +} + +#[derive(Debug)] +pub(crate) struct LeaseGuard { + pub(crate) lease_id: String, + pub(crate) namespace: String, + pub(crate) address: String, +} + +// Provides a best effort for dropping a lease of the content. If the lease cannot be dropped, it will log a warning +impl Drop for LeaseGuard { + fn drop(&mut self) { + let id = self.lease_id.clone(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let client = rt.block_on(containerd_client::connect(self.address.clone())); + + let channel = match client { + Ok(channel) => channel, + Err(e) => { + log::error!( + "failed to connect to containerd: {}. lease may not be deleted", + e + ); + return; + } + }; + + let mut client = LeasesClient::new(channel); + + rt.block_on(async { + let req = containerd_client::services::v1::DeleteRequest { id, sync: false }; + let req = with_namespace!(req, self.namespace); + let result = client.delete(req).await; + + match result { + Ok(_) => log::debug!("removed lease"), + Err(e) => log::error!("failed to remove lease: {}", e), + } + }); + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs b/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs new file mode 100644 index 000000000..97c4d7083 --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/containerd/mod.rs @@ -0,0 +1,6 @@ +#![cfg(unix)] + +mod client; +mod lease; + +pub(crate) use client::Client; diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index c7edfcf55..0d3478007 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -11,7 +11,7 @@ use oci_spec::image::Descriptor; use super::error::Result; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct WasmLayer { pub config: Descriptor, pub layer: Vec, diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs index 8b228066f..628be4ef8 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs @@ -44,8 +44,8 @@ impl SandboxInstance for Instance { let stdio = Stdio::init_from_cfg(cfg)?; // check if container is OCI image with wasm layers and attempt to read the module - let (modules, platform) = containerd::Client::connect(cfg.get_containerd_address(), &namespace)? - .load_modules(&id, E::supported_layers_types()) + let (modules, platform) = containerd::Client::connect(cfg.get_containerd_address().as_str(), &namespace)? + .load_modules(&id, &engine) .unwrap_or_else(|e| { log::warn!("Error obtaining wasm layers for container {id}. Will attempt to use files inside container image. Error: {e}"); (vec![], Platform::default()) diff --git a/crates/containerd-shim-wasm/src/testing.rs b/crates/containerd-shim-wasm/src/testing.rs index 348668b26..1c1131386 100644 --- a/crates/containerd-shim-wasm/src/testing.rs +++ b/crates/containerd-shim-wasm/src/testing.rs @@ -1,22 +1,28 @@ //! Testing utilities used across different modules use std::collections::HashMap; -use std::fs::{create_dir, read_to_string, write, File}; +use std::fs::{self, create_dir, read_to_string, write, File}; use std::marker::PhantomData; use std::ops::Add; +use std::process::Command; use std::time::Duration; use anyhow::{bail, Result}; pub use containerd_shim_wasm_test_modules as modules; +use oci_spec::image::{self as spec, Arch}; use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; +use oci_tar_builder::{Builder, WASM_LAYER_MEDIA_TYPE}; use crate::sandbox::{Instance, InstanceConfig}; use crate::sys::signals::SIGKILL; +const TEST_NAMESPACE: &str = "runwasi-test"; + pub struct WasiTestBuilder where WasiInstance::Engine: Default + Send + Sync + Clone, { + container_name: String, tempdir: tempfile::TempDir, _phantom: PhantomData, } @@ -55,6 +61,7 @@ where write(dir.join("stderr"), "")?; let builder = Self { + container_name: "test".to_string(), tempdir, _phantom: Default::default(), } @@ -114,6 +121,85 @@ where Ok(self) } + pub fn as_oci_image( + mut self, + image_name: Option, + container_name: Option, + ) -> Result<(Self, oci_helpers::OCICleanup)> { + let mut builder = Builder::default(); + + let dir = self.tempdir.path(); + let wasm_path = dir.join("rootfs").join("hello.wasm"); + builder.add_layer_with_media_type(&wasm_path, WASM_LAYER_MEDIA_TYPE.to_string()); + + let config = spec::ConfigBuilder::default() + .entrypoint(vec!["_start".to_string()]) + .build() + .unwrap(); + + let img = spec::ImageConfigurationBuilder::default() + .config(config) + .os("wasip1") + .architecture(Arch::Wasm) + .rootfs( + spec::RootFsBuilder::default() + .diff_ids(vec![]) + .build() + .unwrap(), + ) + .build()?; + + let image_name = image_name.unwrap_or("localhost/hello:latest".to_string()); + builder.add_config(img, image_name.clone()); + + let img = dir.join("img.tar"); + let f = File::create(img.clone())?; + builder.build(f)?; + + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("image") + .arg("import") + .arg("--all-platforms") + .arg(img) + .spawn()? + .wait()? + .success(); + + if !success { + // if the container still exists try cleaning it up + bail!(" failed to import image"); + } + + fs::remove_file(&wasm_path)?; + + let container_name = container_name.unwrap_or("test".to_string()); + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("c") + .arg("create") + .arg(&image_name) + .arg(&container_name) + .spawn()? + .wait()? + .success(); + + if !success { + bail!(" failed to create container for image"); + } + + self.container_name = container_name.clone(); + Ok(( + self, + oci_helpers::OCICleanup { + image_name, + container_name, + }, + )) + } + pub fn build(self) -> Result> { let tempdir = self.tempdir; let dir = tempdir.path(); @@ -122,7 +208,7 @@ where let mut cfg = InstanceConfig::new( WasiInstance::Engine::default(), - "test_namespace", + TEST_NAMESPACE, "/run/containerd/containerd.sock", ); cfg.set_bundle(dir) @@ -130,7 +216,7 @@ where .set_stderr(dir.join("stderr")) .set_stdin(dir.join("stdin")); - let instance = WasiInstance::new("test".to_string(), Some(&cfg))?; + let instance = WasiInstance::new(self.container_name, Some(&cfg))?; Ok(WasiTest { instance, tempdir }) } } @@ -181,3 +267,136 @@ where Ok((status, stdout, stderr)) } } + +pub mod oci_helpers { + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + use anyhow::{bail, Result}; + + use super::TEST_NAMESPACE; + + pub struct OCICleanup { + pub image_name: String, + pub container_name: String, + } + + impl Drop for OCICleanup { + fn drop(&mut self) { + log::debug!("dropping OCIGuard"); + clean_container(self.container_name.clone()).unwrap(); + clean_image(self.image_name.clone()).unwrap(); + } + } + + pub fn clean_container(container_name: String) -> Result<()> { + log::debug!("deleting container '{}'", container_name); + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("c") + .arg("rm") + .arg(container_name) + .spawn()? + .wait()? + .success(); + + if !success { + bail!("failed to clean container") + } + + Ok(()) + } + + pub fn clean_image(image_name: String) -> Result<()> { + log::debug!("deleting image '{}'", image_name); + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("i") + .arg("rm") + .arg(image_name) + .spawn()? + .wait()? + .success(); + + if !success { + bail!("failed to clean image"); + } + + // the content isn't removed immediately, so we need to wait for it to be removed + // otherwise the next test will not behave as expected + let start = Instant::now(); + let timeout = Duration::from_secs(300); + loop { + let output = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("content") + .arg("ls") + .arg("-q") + .output()?; + + if output.stdout.is_empty() { + break; + } + + if start.elapsed() > timeout { + bail!("timed out waiting for content to be removed"); + } + + log::trace!("waiting for content to be removed"); + } + + Ok(()) + } + + pub fn get_image_label() -> Result<(String, String)> { + let mut grep = Command::new("grep") + .arg("-ohE") + .arg("runwasi.io/precompiled/.*") + .stdout(Stdio::piped()) + .stdin(Stdio::piped()) + .spawn()?; + + Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("i") + .arg("ls") + .stdout(grep.stdin.take().unwrap()) + .spawn()?; + + let output = grep.wait_with_output()?; + + let stdout = String::from_utf8(output.stdout)?; + + log::info!("stdout: {}", stdout); + + let label: Vec<&str> = stdout.split('=').collect(); + + Ok(( + label.first().unwrap().trim().to_string(), + label.last().unwrap().trim().to_string(), + )) + } + + pub fn remove_content(digest: String) -> Result<()> { + log::debug!("cleaning content '{}'", digest); + let success = Command::new("ctr") + .arg("-n") + .arg(TEST_NAMESPACE) + .arg("content") + .arg("rm") + .arg(digest) + .spawn()? + .wait()? + .success(); + + if !success { + bail!("failed to remove content"); + } + + Ok(()) + } +} diff --git a/crates/containerd-shim-wasmedge/src/tests.rs b/crates/containerd-shim-wasmedge/src/tests.rs index e292f4bd5..050ec3267 100644 --- a/crates/containerd-shim-wasmedge/src/tests.rs +++ b/crates/containerd-shim-wasmedge/src/tests.rs @@ -29,6 +29,21 @@ fn test_hello_world() -> anyhow::Result<()> { Ok(()) } +#[test] +#[serial] +fn test_hello_world_oci() -> anyhow::Result<()> { + let (builder, _oci_cleanup) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, None)?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + #[test] #[serial] fn test_custom_entrypoint() -> anyhow::Result<()> { diff --git a/crates/containerd-shim-wasmer/src/tests.rs b/crates/containerd-shim-wasmer/src/tests.rs index 15ee5b478..591a4c5c0 100644 --- a/crates/containerd-shim-wasmer/src/tests.rs +++ b/crates/containerd-shim-wasmer/src/tests.rs @@ -29,6 +29,21 @@ fn test_hello_world() -> anyhow::Result<()> { Ok(()) } +#[test] +#[serial] +fn test_hello_world_oci() -> anyhow::Result<()> { + let (builder, _oci_cleanup) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, None)?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + #[test] #[serial] fn test_custom_entrypoint() -> anyhow::Result<()> { diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index b1b10dda8..60585c818 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -1,4 +1,7 @@ +use std::collections::hash_map::DefaultHasher; use std::fs::File; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ @@ -6,25 +9,41 @@ use containerd_shim_wasm::container::{ }; use wasi_common::I32Exit; use wasmtime::component::{self as wasmtime_component, Component, ResourceTable}; -use wasmtime::{Module, Store}; +use wasmtime::{Config, Module, Precompiled, Store}; use wasmtime_wasi::preview2::{self as wasi_preview2}; use wasmtime_wasi::{self as wasi_preview1, Dir}; -pub type WasmtimeInstance = Instance; +pub type WasmtimeInstance = Instance>; #[derive(Clone)] -pub struct WasmtimeEngine { +pub struct WasmtimeEngine { engine: wasmtime::Engine, + config_type: PhantomData, } -impl Default for WasmtimeEngine { - fn default() -> Self { +#[derive(Clone)] +pub struct DefaultConfig {} + +impl WasiConfig for DefaultConfig { + fn new_config() -> Config { let mut config = wasmtime::Config::new(); config.wasm_component_model(true); // enable component linking + config + } +} + +pub trait WasiConfig: Clone + Sync + Send + 'static { + fn new_config() -> Config; +} + +impl Default for WasmtimeEngine { + fn default() -> Self { + let config = T::new_config(); Self { engine: wasmtime::Engine::new(&config) .context("failed to create wasmtime engine") .unwrap(), + config_type: PhantomData, } } } @@ -55,7 +74,7 @@ impl wasmtime_wasi::preview2::WasiView for WasiCtx { } } -impl Engine for WasmtimeEngine { +impl Engine for WasmtimeEngine { fn name() -> &'static str { "wasmtime" } @@ -77,11 +96,7 @@ impl Engine for WasmtimeEngine { let store = Store::new(&self.engine, wasi_ctx); let wasm_bytes = &source.as_bytes()?; - let status = match WasmBinaryType::from_bytes(wasm_bytes) { - Some(WasmBinaryType::Module) => self.execute_module(wasm_bytes, store, &func)?, - Some(WasmBinaryType::Component) => self.execute_component(wasm_bytes, store, func)?, - None => bail!("not a valid wasm binary format"), - }; + let status = self.execute(wasm_bytes, store, func)?; let status = status.map(|_| 0).or_else(|err| { match err.downcast_ref::() { @@ -96,21 +111,34 @@ impl Engine for WasmtimeEngine { Ok(status) } + + fn precompile(&self, layers: &[Vec]) -> Result> { + match layers { + [layer] => self.engine.precompile_module(layer), + _ => bail!("only a single module is supported when precompiling"), + } + } + + fn can_precompile(&self) -> Option { + let mut hasher = DefaultHasher::new(); + self.engine + .precompile_compatibility_hash() + .hash(&mut hasher); + Some(hasher.finish().to_string()) + } } -impl WasmtimeEngine { +impl WasmtimeEngine { /// Execute a wasm module. /// /// This function adds wasi_preview1 to the linker and can be utilized /// to execute a wasm module that uses wasi_preview1. fn execute_module( &self, - wasm_binary: &[u8], + module: Module, mut store: Store, func: &String, ) -> Result, anyhow::Error> { - log::debug!("loading wasm module"); - let module = Module::from_binary(&self.engine, wasm_binary)?; let mut module_linker = wasmtime::Linker::new(&self.engine); wasi_preview1::add_to_linker(&mut module_linker, |s: &mut WasiCtx| &mut s.wasi_preview1)?; @@ -134,12 +162,12 @@ impl WasmtimeEngine { /// to execute a wasm component that uses wasi_preview2. fn execute_component( &self, - wasm_binary: &[u8], + component: Component, mut store: Store, func: String, ) -> Result, anyhow::Error> { log::debug!("loading wasm component"); - let component = Component::from_binary(&self.engine, wasm_binary)?; + let mut linker = wasmtime_component::Linker::new(&self.engine); wasi_preview2::command::sync::add_to_linker(&mut linker)?; @@ -171,6 +199,40 @@ impl WasmtimeEngine { Ok(status) } } + + fn execute( + &self, + wasm_binary: &[u8], + store: Store, + func: String, + ) -> Result, anyhow::Error> { + match WasmBinaryType::from_bytes(wasm_binary) { + Some(WasmBinaryType::Module) => { + log::debug!("loading wasm module"); + let module = Module::from_binary(&self.engine, wasm_binary)?; + self.execute_module(module, store, &func) + } + Some(WasmBinaryType::Component) => { + let component = Component::from_binary(&self.engine, wasm_binary)?; + self.execute_component(component, store, func) + } + None => match &self.engine.detect_precompiled(wasm_binary) { + Some(Precompiled::Module) => { + log::info!("using precompiled module"); + let module = unsafe { Module::deserialize(&self.engine, wasm_binary) }?; + self.execute_module(module, store, &func) + } + Some(Precompiled::Component) => { + log::info!("using precompiled component"); + let component = unsafe { Component::deserialize(&self.engine, wasm_binary) }?; + self.execute_component(component, store, func) + } + None => { + bail!("invalid precompiled module") + } + }, + } + } } /// Prepare both wasi_preview1 and wasi_preview2 contexts. diff --git a/crates/containerd-shim-wasmtime/src/tests.rs b/crates/containerd-shim-wasmtime/src/tests.rs index 1073880ad..fcab3a50a 100644 --- a/crates/containerd-shim-wasmtime/src/tests.rs +++ b/crates/containerd-shim-wasmtime/src/tests.rs @@ -1,11 +1,31 @@ use std::time::Duration; -//use containerd_shim_wasm::sandbox::Instance; +use containerd_shim_wasm::container::Instance; use containerd_shim_wasm::testing::modules::*; -use containerd_shim_wasm::testing::WasiTest; +use containerd_shim_wasm::testing::{oci_helpers, WasiTest}; use serial_test::serial; - -use crate::instance::WasmtimeInstance as WasiInstance; +use wasmtime::Config; +use WasmtimeTestInstance as WasiInstance; + +use crate::instance::{WasiConfig, WasmtimeEngine}; + +// use test configuration to avoid dead locks when running tests +// https://github.com/containerd/runwasi/issues/357 +type WasmtimeTestInstance = Instance>; + +#[derive(Clone)] +struct WasiTestConfig {} + +impl WasiConfig for WasiTestConfig { + fn new_config() -> Config { + let mut config = wasmtime::Config::new(); + // Disable Wasmtime parallel compilation for the tests + // see https://github.com/containerd/runwasi/pull/405#issuecomment-1928468714 for details + config.parallel_compilation(false); + config.wasm_component_model(true); // enable component linking + config + } +} #[test] #[serial] @@ -29,6 +49,85 @@ fn test_hello_world() -> anyhow::Result<()> { Ok(()) } +#[test] +#[serial] +fn test_hello_world_oci() -> anyhow::Result<()> { + let (builder, _oci_cleanup) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, None)?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + +#[test] +#[serial] +fn test_hello_world_oci_uses_precompiled() -> anyhow::Result<()> { + let (builder, _oci_cleanup1) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c1".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + let (label, id) = oci_helpers::get_image_label()?; + assert!( + label.starts_with("runwasi.io/precompiled/wasmtime/"), + "was {}={}", + label, + id + ); + + // run second time, it should succeed without recompiling + let (builder, _oci_cleanup2) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c2".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + +#[test] +#[serial] +fn test_hello_world_oci_uses_precompiled_when_content_removed() -> anyhow::Result<()> { + let (builder, _oci_cleanup1) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c1".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + let (label, id) = oci_helpers::get_image_label()?; + + // remove the compiled content from the cache + assert!(label.starts_with("runwasi.io/precompiled/wasmtime/")); + oci_helpers::remove_content(id)?; + + // run second time, it should succeed + let (builder, _oci_cleanup2) = WasiTest::::builder()? + .with_wasm(HELLO_WORLD)? + .as_oci_image(None, Some("c2".to_string()))?; + + let (exit_code, stdout, _) = builder.build()?.start()?.wait(Duration::from_secs(10))?; + + assert_eq!(exit_code, 0); + assert_eq!(stdout, "hello world\n"); + + Ok(()) +} + #[test] #[serial] fn test_custom_entrypoint() -> anyhow::Result<()> { diff --git a/crates/oci-tar-builder/src/bin.rs b/crates/oci-tar-builder/src/bin.rs index 0d1ba80b8..d21e88aa1 100644 --- a/crates/oci-tar-builder/src/bin.rs +++ b/crates/oci-tar-builder/src/bin.rs @@ -5,10 +5,7 @@ use std::{env, fs}; use anyhow::Context; use clap::Parser; use oci_spec::image::{self as spec, Arch}; -use oci_tar_builder::Builder; - -pub const WASM_LAYER_MEDIA_TYPE: &str = - "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; +use oci_tar_builder::{Builder, WASM_LAYER_MEDIA_TYPE}; pub fn main() { let args = Args::parse(); diff --git a/crates/oci-tar-builder/src/lib.rs b/crates/oci-tar-builder/src/lib.rs index cabf785a1..b4f50e36e 100644 --- a/crates/oci-tar-builder/src/lib.rs +++ b/crates/oci-tar-builder/src/lib.rs @@ -41,6 +41,9 @@ struct DockerManifest { layers: Vec, } +pub const WASM_LAYER_MEDIA_TYPE: &str = + "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; + impl Builder { pub fn add_config(&mut self, config: ImageConfiguration, name: String) -> &mut Self { self.configs.push((config, name)); diff --git a/docs/oci-descision-flow.md b/docs/oci-descision-flow.md new file mode 100644 index 000000000..ac4525921 --- /dev/null +++ b/docs/oci-descision-flow.md @@ -0,0 +1,47 @@ +# OCI pre-compilation + +The OCI images layers are loaded from containerd. If the runtime supports pre-compilation the images will be precompiled and cached using the containerd content store. + +```mermaid +graph TD + start[Task new] + imgconfig[Load image config from containerd] + iswasm{Arch==wasm?} + alreadycompiled{Does image label for shim runtime version exist? runwasi.io/precompiled/runtime/version} + startcontainer[Create Container] + precompiledenabled{Is precompiling enabled in shim?} + precompiledenabled2{Is precompiling enabled in shim?} + fetchcache[Fetch cached precompiled layer from containerd content store] + precompile[Precompile using wasm runtime] + loadoci[Load OCI layers from containerd] + storecache[Store precompiled layer in containerd content store] + + start --> imgconfig --> iswasm + iswasm -- yes --> precompiledenabled + iswasm -- no. wasm will be loaded from file inside image --> startcontainer + + precompiledenabled -- yes --> alreadycompiled + precompiledenabled -- no --> loadoci --> precompiledenabled2 + + alreadycompiled -- yes --> fetchcache --> startcontainer + alreadycompiled -- no --> loadoci + + precompiledenabled2 -- yes --> precompile --> storecache --> startcontainer + precompiledenabled2 -- no --> startcontainer +``` + +Once a wasm module or component is pre-compiled it will remain in the containerd content store until the original image is removed from containerd. There is a small disk overhead associated with this but it reduces the complexity of managing stored versions during upgrades. + +To view the images in containerd that have associated pre-compilations: + +```bash +sudo ctr i ls | grep "runwasi.io" +ghcr.io/containerd/runwasi/wasi-demo-oci:latest application/vnd.oci.image.manifest.v1+json + sha256:60fccd77070dfeb682a1ebc742e9d677fc452b30a6b99188b081c968992394ce 2.4 MiB wasi/wasm +runwasi.io/precompiled/wasmtime/0.3.1=sha256:b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f1398706782e225fd0a98e + +# query for the sha in the label +sudo ctr content ls | grep "b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f139870" +sha256:60fccd77070dfeb682a1ebc742e9d677fc452b30a6b99188b081c968992394ce 561B 2 months containerd.io/gc.ref.content.0=sha256:a3c18cd551d54d3cfbf67acc9e8f7ef5761e76827fe7c1ae163fca0193be88b3,containerd.io/gc.ref.content.config=sha256:85b7f2b562fe8665ec9d9e6d47ab0b24e2315627f5f558d298475c4038d71e8b,containerd.io/gc.ref.content.precompile=sha256:b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f1398706782e225fd0a98e +sha256:b36753ab5a46f26f6bedb81b8a7b489cede8fc7386f1398706782e225fd0a98e 626.4kB 3 days runwasi.io/precompiled=sha256:60fccd77070dfeb682a1ebc742e9d677fc452b30a6b99188b081c968992394ce +``` \ No newline at end of file