Merge pull request #1160 from spkenv/workspaces

Workspaces Part I

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "crates/spk-solve",
     "crates/spk-solve/crates/*",
     "crates/spk-storage",
+    "crates/spk-workspace",
     "crates/spk",
 ]
 resolver = "2"
@@ -43,6 +44,7 @@ description = "SPK is a Package Manager for high-velocity software environments,
 [workspace.dependencies]
 arc-swap = "1.6.0"
 async-trait = "0.1"
+bracoxide = "0.1.4"
 bytes = "1.5"
 cached = "0.48.1"
 chrono = { version = "0.4.34", features = ["serde"] }
@@ -59,6 +61,7 @@ dunce = "1.0.4"
 dyn-clone = "1.0"
 enum_dispatch = "0.3.13"
 flatbuffers = "23.5.26"
+format_serde_error = { version = "0.3", default-features = false }
 fuser = "0.14.0"
 futures = "0.3.28"
 futures-core = "0.3.28"
@@ -135,6 +138,7 @@ spk-solve-package-iterator = { path = "crates/spk-solve/crates/package-iterator"
 spk-solve-solution = { path = "crates/spk-solve/crates/solution" }
 spk-solve-validation = { path = "crates/spk-solve/crates/validation" }
 spk-storage = { path = "crates/spk-storage" }
+spk-workspace = { path = "crates/spk-workspace" }
 static_assertions = "1.1"
 strip-ansi-escapes = "0.2.0"
 strum = { version = "0.26.3", features = ["derive"] }

Makefile

@@ -86,7 +86,7 @@ test:
 
 .PHONY: converters
 converters:
-	$(MAKE) -C packages spk-convert-pip/spk-convert-pip.spk
+	spk build spk-convert-pip
 
 .PHONY: rpms
 rpms: spk-rpm spfs-rpm

README.md

@@ -38,6 +38,8 @@ SPK/SPFS are distributed using the [Apache-2.0 license](LICENSE).
 
 `spk` is the software packaging system built on top of SPFS.
 
+The `packages` directory contains a collection of recipes that can be used as examples or to build out common open source software packages.
+
 ## Contributing
 
 Please read [Contributing to SPK](CONTRIBUTING.md).

crates/spk-cli/cmd-build/src/cmd_build.rs

@@ -4,8 +4,8 @@
 
 use clap::Args;
 use miette::Result;
-use spk_cli_common::{flags, CommandArgs, Run};
-use spk_cmd_make_binary::cmd_make_binary::PackageSpecifier;
+use spk_cli_common::flags::{self, PackageSpecifier};
+use spk_cli_common::{CommandArgs, Run};
 
 #[cfg(test)]
 #[path = "./cmd_build_test/mod.rs"]
@@ -37,9 +37,8 @@ pub struct Build {
     #[clap(long, short)]
     env: bool,
 
-    /// The package names or yaml spec files to build
-    #[clap(name = "NAME|SPEC_FILE")]
-    packages: Vec<String>,
+    #[clap(flatten)]
+    packages: flags::Packages,
 
     /// Build only the specified variants
     #[clap(flatten)]
@@ -77,13 +76,13 @@ impl Run for Build {
             .await?;
 
         // divide our packages into one for each iteration of mks/mkb
-        let mut runs: Vec<_> = self.packages.iter().map(|f| vec![f.to_owned()]).collect();
+        let mut runs: Vec<_> = self.packages.split();
         if runs.is_empty() {
-            runs.push(Vec::new());
+            runs.push(Default::default());
         }
 
         let mut builds_for_summary = spk_cli_common::BuildResult::default();
-        for packages in runs {
+        for mut packages in runs {
             let mut make_source = spk_cmd_make_source::cmd_make_source::MakeSource {
                 options: self.options.clone(),
                 verbose: self.verbose,
@@ -94,6 +93,17 @@ impl Run for Build {
             let idents = make_source.make_source().await?;
             builds_for_summary.extend(make_source.created_src);
 
+            // add the source ident specifier from the source build to ensure that
+            // the binary build operates over this exact source package
+            packages.packages = packages
+                .packages
+                .into_iter()
+                .zip(idents.into_iter())
+                .map(|(package, ident)| {
+                    PackageSpecifier::WithSourceIdent((package.into_specifier(), ident.into()))
+                })
+                .collect();
+
             let mut make_binary = spk_cmd_make_binary::cmd_make_binary::MakeBinary {
                 verbose: self.verbose,
                 runtime: self.runtime.clone(),
@@ -102,13 +112,7 @@ impl Run for Build {
                 here: self.here,
                 interactive: self.interactive,
                 env: self.env,
-                packages: packages
-                    .into_iter()
-                    .zip(idents.into_iter())
-                    .map(|(package, ident)| {
-                        PackageSpecifier::WithSourceIdent((package, ident.into()))
-                    })
-                    .collect(),
+                packages,
                 variant: self.variant.clone(),
                 formatter_settings: self.formatter_settings.clone(),
                 allow_circular_dependencies: self.allow_circular_dependencies,
@@ -139,6 +143,6 @@ impl Run for Build {
 impl CommandArgs for Build {
     // The important positional args for a build are the packages
     fn get_positional_args(&self) -> Vec<String> {
-        self.packages.clone()
+        self.packages.get_positional_args()
     }
 }

crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs

@@ -11,7 +11,7 @@ use miette::{bail, miette, Context, IntoDiagnostic, Report, Result};
 use spk_build::{BinaryPackageBuilder, BuildSource};
 use spk_cli_common::{flags, spk_exe, BuildArtifact, BuildResult, CommandArgs, Run};
 use spk_schema::foundation::format::FormatIdent;
-use spk_schema::ident::{PkgRequest, RangeIdent, RequestedBy};
+use spk_schema::ident::{PkgRequest, RequestedBy};
 use spk_schema::option_map::HOST_OPTIONS;
 use spk_schema::prelude::*;
 use spk_schema::OptionMap;
@@ -21,31 +21,6 @@ use spk_storage as storage;
 #[path = "./cmd_make_binary_test.rs"]
 mod cmd_make_binary_test;
 
-#[derive(Clone, Debug)]
-pub enum PackageSpecifier {
-    Plain(String),
-    WithSourceIdent((String, RangeIdent)),
-}
-
-impl PackageSpecifier {
-    // Return the package spec or filename string.
-    fn get_specifier(&self) -> &String {
-        match self {
-            PackageSpecifier::Plain(s) => s,
-            PackageSpecifier::WithSourceIdent((s, _)) => s,
-        }
-    }
-}
-
-impl std::str::FromStr for PackageSpecifier {
-    type Err = clap::Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        // On the command line, only `Plain` is possible.
-        Ok(PackageSpecifier::Plain(s.to_owned()))
-    }
-}
-
 /// Build a binary package from a spec file or source package.
 #[derive(Args)]
 #[clap(visible_aliases = &["mkbinary", "mkbin", "mkb"])]
@@ -72,9 +47,8 @@ pub struct MakeBinary {
     #[clap(long, short)]
     pub env: bool,
 
-    /// The local yaml spec files or published package/versions to build or rebuild
-    #[clap(name = "SPEC_FILE|PKG/VER")]
-    pub packages: Vec<PackageSpecifier>,
+    #[clap(flatten)]
+    pub packages: flags::Packages,
 
     /// Build only the specified variants
     #[clap(flatten)]
@@ -96,11 +70,7 @@ pub struct MakeBinary {
 impl CommandArgs for MakeBinary {
     // The important positional args for a make-binary are the packages
     fn get_positional_args(&self) -> Vec<String> {
-        self.packages
-            .iter()
-            .map(|ps| ps.get_specifier())
-            .cloned()
-            .collect()
+        self.packages.get_positional_args()
     }
 }
 
@@ -130,21 +100,12 @@ impl Run for MakeBinary {
             .map(|(_, r)| Arc::new(r))
             .collect::<Vec<_>>();
 
-        let mut packages: Vec<_> = self.packages.iter().cloned().map(Some).collect();
-        if packages.is_empty() {
-            packages.push(None)
-        }
-
         let opt_host_options =
             (!self.options.no_host).then(|| HOST_OPTIONS.get().unwrap_or_default());
 
-        for package in packages {
-            let (spec_data, filename) = flags::find_package_recipe_from_template_or_repo(
-                package.as_ref().map(|p| p.get_specifier()),
-                &options,
-                &repos,
-            )
-            .await?;
+        for (package, spec_data, filename) in
+            self.packages.find_all_recipes(&options, &repos).await?
+        {
             let recipe = spec_data.into_recipe().wrap_err_with(|| {
                 format!(
                     "{filename} was expected to contain a recipe",
@@ -220,7 +181,9 @@ impl Run for MakeBinary {
                         .into_diagnostic()
                         .wrap_err("Failed to get current directory")?;
                     builder.with_source(BuildSource::LocalPath(here));
-                } else if let Some(PackageSpecifier::WithSourceIdent((_, ref ident))) = package {
+                } else if let Some(flags::PackageSpecifier::WithSourceIdent((_, ref ident))) =
+                    package
+                {
                     // Use the source package `AnyIdent` if the caller supplied one.
                     builder.with_source(BuildSource::SourcePackage(ident.clone()));
                 }

crates/spk-cli/cmd-make-recipe/Cargo.toml

@@ -20,4 +20,5 @@ clap = { workspace = true, features = ["derive", "env"] }
 spk-cli-common = { workspace = true }
 spk-schema = { workspace = true }
 spk-schema-tera = { workspace = true }
+spk-workspace = { workspace = true }
 tracing = "0.1.35"

crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs

@@ -2,13 +2,11 @@
 // SPDX-License-Identifier: Apache-2.0
 // https://github.com/spkenv/spk
 
-use std::sync::Arc;
-
 use clap::Args;
 use miette::{Context, IntoDiagnostic, Result};
 use spk_cli_common::{flags, CommandArgs, Run};
 use spk_schema::foundation::format::FormatOptionMap;
-use spk_schema::{SpecFileData, SpecTemplate, Template, TemplateExt};
+use spk_schema::{SpecFileData, Template};
 
 /// Render a package spec template into a recipe
 ///
@@ -20,6 +18,9 @@ pub struct MakeRecipe {
     #[clap(flatten)]
     pub options: flags::Options,
 
+    #[clap(flatten)]
+    pub workspace: flags::Workspace,
+
     #[clap(short, long, global = true, action = clap::ArgAction::Count)]
     pub verbose: u8,
 
@@ -44,16 +45,13 @@ impl Run for MakeRecipe {
 
     async fn run(&mut self) -> Result<Self::Output> {
         let options = self.options.get_options()?;
+        let workspace = self.workspace.load_or_default()?;
 
-        let template = match flags::find_package_template(self.package.as_ref())? {
-            flags::FindPackageTemplateResult::NotFound(name) => {
-                Arc::new(SpecTemplate::from_file(name.as_ref())?)
-            }
-            res => {
-                let (_, template) = res.must_be_found();
-                template
-            }
-        };
+        let template = match self.package.as_ref() {
+            None => workspace.default_package_template(),
+            Some(p) => workspace.find_package_template(p),
+        }
+        .must_be_found();
 
         if let Some(name) = template.name() {
             tracing::info!("rendering template for {name}");