[pull] main from bacalhau-project:main

.circleci/trigger_pipeline.sh

@@ -1,7 +1,17 @@
 #!/usr/bin/env bash
-curl -X POST --header "Content-Type: application/json" --header "Circle-Token: ${CIRCLE_TOKEN}" -d "{
+set -xeuo pipefail
+IFS=$'\n\t'
+
+if [[ "${BRANCH}" =~ "refs/tags" ]]; then
+   TAG=$(echo "${BRANCH}" | sed 's:refs/tags/::')
+   TARGET="\"tag\": \"${TAG}\""
+else
+   TARGET="\"branch\": \"${BRANCH}\""
+fi
+
+curl --fail -X POST --header "Content-Type: application/json" --header "Circle-Token: ${CIRCLE_TOKEN}" -d "{
  \"parameters\": {
     \"GHA_Action\": \"trigger_pipeline\"
  },
- \"branch\": \"${BRANCH}\"
+ ${TARGET}
 }" https://circleci.com/api/v2/project/gh/bacalhau-project/bacalhau/pipeline

.github/workflows/circleci.yml

@@ -5,6 +5,8 @@ on:
   push:
     branches:
       - main
+    tags:
+      - v**
   workflow_run:
     workflows: ["Dependabot"]
     types:
@@ -54,4 +56,4 @@ jobs:
           ./.circleci/trigger_pipeline.sh
         env:
           CIRCLE_TOKEN: ${{ secrets.CIRCLE_CI_TOKEN }}
-          BRANCH: ${{ github.event.pull_request.head.ref }}
+          BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.event.ref }}

Makefile

@@ -174,15 +174,21 @@ build-webui:
 ################################################################################
 # Target: build-bacalhau
 ################################################################################
+${BINARY_PATH}: build-bacalhau build-plugins
+
 .PHONY: build-bacalhau
-build-bacalhau: ${BINARY_PATH}
+build-bacalhau: binary-web binary
 
 CMD_FILES := $(shell bash -c 'comm -23 <(git ls-files cmd | sort) <(git ls-files cmd --deleted | sort)')
 PKG_FILES := $(shell bash -c 'comm -23 <(git ls-files pkg | sort) <(git ls-files pkg --deleted | sort)')
 
-${BINARY_PATH}: ${CMD_FILES} ${PKG_FILES} build-webui ${WEB_GO_FILES} main.go
+.PHONY: binary
+
+binary: ${CMD_FILES} ${PKG_FILES} main.go
 	${GO} build -ldflags "${BUILD_FLAGS}" -trimpath -o ${BINARY_PATH} .
 
+binary-web: build-webui ${WEB_GO_FILES}
+
 ################################################################################
 # Target: build-docker-images
 ################################################################################

apps/job-info-consumer/consumer/go.mod

@@ -171,7 +171,7 @@ require (
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/labstack/echo/v4 v4.11.4 // indirect
@@ -231,9 +231,9 @@ require (
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/nats-io/jwt/v2 v2.5.3 // indirect
-	github.com/nats-io/nats-server/v2 v2.10.7 // indirect
-	github.com/nats-io/nats.go v1.31.0 // indirect
-	github.com/nats-io/nkeys v0.4.6 // indirect
+	github.com/nats-io/nats-server/v2 v2.10.11 // indirect
+	github.com/nats-io/nats.go v1.33.0 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/oklog/run v1.0.0 // indirect
 	github.com/onsi/ginkgo/v2 v2.13.0 // indirect

apps/job-info-consumer/consumer/go.sum

@@ -627,8 +627,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
@@ -849,12 +848,9 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo=
 github.com/nats-io/jwt/v2 v2.5.3/go.mod h1:iysuPemFcc7p4IoYots3IuELSI4EDe9Y0bQMe+I3Bf4=
-github.com/nats-io/nats-server/v2 v2.10.7 h1:f5VDy+GMu7JyuFA0Fef+6TfulfCs5nBTgq7MMkFJx5Y=
-github.com/nats-io/nats-server/v2 v2.10.7/go.mod h1:V2JHOvPiPdtfDXTuEUsthUnCvSDeFrK4Xn9hRo6du7c=
-github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
-github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
-github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
-github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts=
+github.com/nats-io/nats-server/v2 v2.10.11 h1:yKUiLVincZISpo3A4YljJQ+HfLltGAgoNNJl99KL8I0=
+github.com/nats-io/nats.go v1.33.0 h1:rRg0l2F29B30n6EPl0j50hl8eYp7rA2ecoJ74E62US8=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=

cmd/cli/create/create.go

@@ -19,7 +19,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags"
 	"github.com/bacalhau-project/bacalhau/cmd/util/hook"
 	"github.com/bacalhau-project/bacalhau/cmd/util/printer"
-	jobutils "github.com/bacalhau-project/bacalhau/pkg/job"
+	legacy_job "github.com/bacalhau-project/bacalhau/pkg/legacyjob"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
 	"github.com/bacalhau-project/bacalhau/pkg/userstrings"
 	"github.com/bacalhau-project/bacalhau/pkg/util/templates"
@@ -212,7 +212,7 @@ func create(cmd *cobra.Command, cmdArgs []string, OC *CreateOptions) error { //n
 		cmd.Printf("WARNING: The following fields have data in them and will be ignored on creation: %s\n", strings.Join(unusedFieldList, ", "))
 	}
 
-	err = jobutils.VerifyJob(ctx, j)
+	err = legacy_job.VerifyJob(ctx, j)
 	if err != nil {
 		return fmt.Errorf("error verifying job: %w", err)
 	}

cmd/cli/describe/describe_test.go

@@ -15,7 +15,7 @@ import (
 	cmdtesting "github.com/bacalhau-project/bacalhau/cmd/testing"
 	"github.com/bacalhau-project/bacalhau/cmd/util"
 	"github.com/bacalhau-project/bacalhau/pkg/bacerrors"
-	jobutils "github.com/bacalhau-project/bacalhau/pkg/job"
+	legacy_job "github.com/bacalhau-project/bacalhau/pkg/legacyjob"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
 	testutils "github.com/bacalhau-project/bacalhau/pkg/test/utils"
 )
@@ -54,7 +54,7 @@ func (s *DescribeSuite) TestDescribeJob() {
 				for i := 0; i < tc.numberOfAcceptNodes; i++ {
 					for k := 0; k < n.numOfJobs; k++ {
 						j := testutils.MakeJobWithOpts(s.T(),
-							jobutils.WithEngineSpec(
+							legacy_job.WithEngineSpec(
 								model.NewEngineBuilder().
 									WithType(strings.ToLower(model.EngineNoop.String())).
 									WithParam("Entrypoint-Unique-Array", uuid.NewString()).
@@ -206,7 +206,7 @@ func (s *DescribeSuite) TestDescribeJobEdgeCases() {
 
 				for i := 0; i < n.numOfJobs; i++ {
 					j := testutils.MakeJobWithOpts(s.T(),
-						jobutils.WithEngineSpec(
+						legacy_job.WithEngineSpec(
 							model.NewEngineBuilder().
 								WithType(strings.ToLower(model.EngineNoop.String())).
 								WithParam("Entrypoint-Unique-Array", uuid.NewString()).

cmd/cli/docker/docker_run.go

@@ -16,7 +16,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/cmd/util/parse"
 	"github.com/bacalhau-project/bacalhau/cmd/util/printer"
 	"github.com/bacalhau-project/bacalhau/pkg/bacerrors"
-	jobutils "github.com/bacalhau-project/bacalhau/pkg/job"
+	legacy_job "github.com/bacalhau-project/bacalhau/pkg/legacyjob"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
 	"github.com/bacalhau-project/bacalhau/pkg/util/templates"
 )
@@ -146,7 +146,7 @@ func dockerRun(cmd *cobra.Command, cmdArgs []string, opts *DockerRunOptions) err
 		return fmt.Errorf("creating job: %w", err)
 	}
 
-	if err := jobutils.VerifyJob(ctx, j); err != nil {
+	if err := legacy_job.VerifyJob(ctx, j); err != nil {
 		if _, ok := err.(*bacerrors.ImageNotFound); ok {
 			return fmt.Errorf("docker image '%s' not found in the registry, or needs authorization", image)
 		} else {
@@ -198,24 +198,24 @@ func CreateJob(ctx context.Context, image string, parameters []string, opts *Doc
 		return nil, err
 	}
 
-	spec, err := jobutils.MakeDockerSpec(
+	spec, err := legacy_job.MakeDockerSpec(
 		image, opts.WorkingDirectory, opts.Entrypoint, opts.SpecSettings.EnvVar, parameters,
-		jobutils.WithResources(
+		legacy_job.WithResources(
 			opts.ResourceSettings.CPU,
 			opts.ResourceSettings.Memory,
 			opts.ResourceSettings.Disk,
 			opts.ResourceSettings.GPU,
 		),
-		jobutils.WithNetwork(
+		legacy_job.WithNetwork(
 			opts.NetworkingSettings.Network,
 			opts.NetworkingSettings.Domains,
 		),
-		jobutils.WithTimeout(opts.SpecSettings.Timeout),
-		jobutils.WithInputs(opts.SpecSettings.Inputs.Values()...),
-		jobutils.WithOutputs(outputs...),
-		jobutils.WithAnnotations(labels...),
-		jobutils.WithNodeSelector(nodeSelectorRequirements),
-		jobutils.WithDeal(
+		legacy_job.WithTimeout(opts.SpecSettings.Timeout),
+		legacy_job.WithInputs(opts.SpecSettings.Inputs.Values()...),
+		legacy_job.WithOutputs(outputs...),
+		legacy_job.WithAnnotations(labels...),
+		legacy_job.WithNodeSelector(nodeSelectorRequirements),
+		legacy_job.WithDeal(
 			opts.DealSettings.TargetingMode,
 			opts.DealSettings.Concurrency,
 		),

cmd/cli/list/list.go

@@ -17,7 +17,7 @@ import (
 
 	"github.com/bacalhau-project/bacalhau/cmd/util"
 	"github.com/bacalhau-project/bacalhau/cmd/util/output"
-	"github.com/bacalhau-project/bacalhau/pkg/job"
+	legacy_job "github.com/bacalhau-project/bacalhau/pkg/legacyjob"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
 	"github.com/bacalhau-project/bacalhau/pkg/util/templates"
 )
@@ -165,11 +165,11 @@ var listColumns = []output.TableColumn[*model.JobWithInfo]{
 	},
 	{
 		ColumnConfig: table.ColumnConfig{Name: "state", WidthMax: 20, WidthMaxEnforcer: text.WrapText},
-		Value:        func(jwi *model.JobWithInfo) string { return job.ComputeStateSummary(jwi.State) },
+		Value:        func(jwi *model.JobWithInfo) string { return legacy_job.ComputeStateSummary(jwi.State) },
 	},
 	{
 		ColumnConfig: table.ColumnConfig{Name: "published"},
-		Value:        job.ComputeResultsSummary,
+		Value:        legacy_job.ComputeResultsSummary,
 	},
 }
 

cmd/cli/list/list_test.go

@@ -19,7 +19,7 @@ import (
 
 	"github.com/bacalhau-project/bacalhau/cmd/cli/list"
 	cmdtesting "github.com/bacalhau-project/bacalhau/cmd/testing"
-	jobutils "github.com/bacalhau-project/bacalhau/pkg/job"
+	legacy_job "github.com/bacalhau-project/bacalhau/pkg/legacyjob"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
 	"github.com/bacalhau-project/bacalhau/pkg/system"
 	testutils "github.com/bacalhau-project/bacalhau/pkg/test/utils"
@@ -173,7 +173,7 @@ func (suite *ListSuite) TestList_AnnotationFilter() {
 			suite.setupRun()
 
 			testJob := testutils.MakeJobWithOpts(suite.T(),
-				jobutils.WithAnnotations(tc.JobLabels...),
+				legacy_job.WithAnnotations(tc.JobLabels...),
 			)
 			j, err := suite.Client.Submit(ctx, &testJob)
 			require.NoError(suite.T(), err)

cmd/cli/serve/serve.go

@@ -400,16 +400,12 @@ func buildConnectCommand(ctx context.Context, nodeConfig *node.NodeConfig, ipfsC
 
 		switch nodeConfig.NetworkConfig.Type {
 		case models.NetworkTypeNATS:
-			advertisedAddr := nodeConfig.NetworkConfig.AdvertisedAddress
-			if advertisedAddr == "" {
-				advertisedAddr = fmt.Sprintf("127.0.0.1:%d", nodeConfig.NetworkConfig.Port)
-			}
+			advertisedAddr := getPublicNATSOrchestratorURL(nodeConfig)
 
 			headerB.WriteString("To connect a compute node to this orchestrator, run the following command in your shell:\n")
-
 			cmdB.WriteString(fmt.Sprintf("%s=%s ",
 				configflags.FlagNameForKey(types.NodeNetworkOrchestrators, configflags.NetworkFlags...),
-				advertisedAddr,
+				advertisedAddr.String(),
 			))
 
 		case models.NetworkTypeLibp2p:
@@ -471,20 +467,10 @@ func buildEnvVariables(ctx context.Context, nodeConfig *node.NodeConfig, ipfsCon
 
 		switch nodeConfig.NetworkConfig.Type {
 		case models.NetworkTypeNATS:
-			orchestrator := &url.URL{
-				Scheme: "nats",
-				Host:   nodeConfig.NetworkConfig.AdvertisedAddress,
-				User:   url.User(nodeConfig.NetworkConfig.AuthSecret),
-			}
-
-			if nodeConfig.NetworkConfig.AdvertisedAddress == "" {
-				orchestrator.Host = fmt.Sprintf("127.0.0.1:%d", nodeConfig.NetworkConfig.Port)
-			}
-
 			envVarBuilder.WriteString(fmt.Sprintf(
 				"export %s=%s\n",
 				config.KeyAsEnvVar(types.NodeNetworkOrchestrators),
-				orchestrator.String(),
+				getPublicNATSOrchestratorURL(nodeConfig).String(),
 			))
 		case models.NetworkTypeLibp2p:
 			p2pAddr, err := multiaddr.NewMultiaddr("/p2p/" + nodeConfig.NetworkConfig.Libp2pHost.ID().String())
@@ -517,6 +503,26 @@ func buildEnvVariables(ctx context.Context, nodeConfig *node.NodeConfig, ipfsCon
 	return envVarBuilder.String(), nil
 }
 
+func getPublicNATSOrchestratorURL(nodeConfig *node.NodeConfig) *url.URL {
+	orchestrator := &url.URL{
+		Scheme: "nats",
+		Host:   nodeConfig.NetworkConfig.AdvertisedAddress,
+	}
+
+	// Only display the secret if the user did not set it explicitly.
+	// Else, they should already know it!
+	secret, err := config.Get[string](types.NodeNetworkAuthSecret)
+	if err == nil && secret == "" && nodeConfig.NetworkConfig.AuthSecret != "" {
+		orchestrator.User = url.User(nodeConfig.NetworkConfig.AuthSecret)
+	}
+
+	if nodeConfig.NetworkConfig.AdvertisedAddress == "" {
+		orchestrator.Host = fmt.Sprintf("127.0.0.1:%d", nodeConfig.NetworkConfig.Port)
+	}
+
+	return orchestrator
+}
+
 // pickP2pAddress will aim to select a non-localhost IPv4 TCP address, or at least a non-localhost IPv6 one, from a list
 // of addresses.
 func pickP2pAddress(addresses []multiaddr.Multiaddr) multiaddr.Multiaddr {

cmd/cli/serve/timeout_test.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/bacalhau-project/bacalhau/pkg/docker"
-	"github.com/bacalhau-project/bacalhau/pkg/job"
+	legacy_job "github.com/bacalhau-project/bacalhau/pkg/legacyjob"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
 	"github.com/bacalhau-project/bacalhau/pkg/publicapi/client"
 	clientv2 "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2"
@@ -59,11 +59,11 @@ func (s *ServeSuite) TestNoTimeoutSetOrApplied() {
 			s.Require().NoError(apitest.WaitForAlive(s.ctx, clientV2))
 
 			testJob := model.NewJob()
-			specOpts := []job.SpecOpt{}
+			specOpts := []legacy_job.SpecOpt{}
 			if tc.timeoutSpecified != nil {
-				specOpts = append(specOpts, job.WithTimeout(int64(tc.timeoutSpecified.Seconds())))
+				specOpts = append(specOpts, legacy_job.WithTimeout(int64(tc.timeoutSpecified.Seconds())))
 			}
-			testJob.Spec, err = job.MakeSpec(specOpts...)
+			testJob.Spec, err = legacy_job.MakeSpec(specOpts...)
 			s.Require().NoError(err)
 
 			returnedJob, err := client.Submit(s.ctx, testJob)

cmd/cli/serve/util.go

@@ -2,8 +2,6 @@ package serve
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/base64"
 	"fmt"
 	"net/url"
 	"path/filepath"
@@ -26,6 +24,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/config"
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 	"github.com/bacalhau-project/bacalhau/pkg/ipfs"
+	"github.com/bacalhau-project/bacalhau/pkg/nats"
 	"github.com/bacalhau-project/bacalhau/pkg/node"
 	"github.com/bacalhau-project/bacalhau/pkg/system"
 )
@@ -199,13 +198,11 @@ func getNetworkConfig(nodeID string) (node.NetworkConfig, error) {
 		// Generate an auth token using the user's client key. It should not be
 		// possible to compute this value by anyone other than the NATS server, and
 		// should be stable across restarts of the node.
-		var keySig string
-		keySig, err := system.SignForClient([]byte(nodeID))
+		secret, err := nats.CreateAuthSecret(nodeID)
 		if err != nil {
 			return node.NetworkConfig{}, err
 		}
-		hash := sha256.Sum256([]byte(keySig))
-		networkCfg.AuthSecret = base64.RawURLEncoding.EncodeToString(hash[:])
+		networkCfg.AuthSecret = secret //nolint: gosec
 	} else {
 		// If the user supplied an auth secret and some orchestrator(s), assume
 		// they are passing a auth secret to be used as a client. Attach the
@@ -228,6 +225,7 @@ func getNetworkConfig(nodeID string) (node.NetworkConfig, error) {
 		Port:                     networkCfg.Port,
 		AdvertisedAddress:        networkCfg.AdvertisedAddress,
 		Orchestrators:            networkCfg.Orchestrators,
+		StoreDir:                 networkCfg.StoreDir,
 		AuthSecret:               networkCfg.AuthSecret,
 		ClusterName:              networkCfg.Cluster.Name,
 		ClusterPort:              networkCfg.Cluster.Port,