feat: serverless mode on AWS

This commit introduces a serverless module to run serverless operations on AWS, and as an example use case it introduces the timeout parameter for rhel, fedora and windows targets on AWS. When timeout parameter is set a serverless task is scheduled based on the timeout to self destroy resources created by the create operation

Signed-off-by: Adrian Riobo <[email protected]>

Makefile

@@ -2,13 +2,21 @@ VERSION ?= 0.8.0-dev
 CONTAINER_MANAGER ?= podman
 # Image URL to use all building/pushing image targets
 IMG ?= quay.io/redhat-developer/mapt:v${VERSION}
+# IMG ?= quay.io/rhqp/mapt:serverless-10-amd64
+
 TKN_IMG ?= quay.io/redhat-developer/mapt:v${VERSION}-tkn
 
 # Go and compilation related variables
 GOPATH ?= $(shell go env GOPATH)
 BUILD_DIR ?= out
 SOURCE_DIRS = cmd pkg
 SOURCES := $(shell find . -name "*.go" -not -path "./vendor/*")
+# repo
+ORG := github.com/redhat-developer
+MODULEPATH = $(ORG)/mapt
+# Linker flags
+VERSION_VARIABLES := -X $(MODULEPATH)/pkg/manager/context.OCI=$(IMG)
+
 # https://golang.org/cmd/link/
 # LDFLAGS := $(VERSION_VARIABLES) -extldflags='-static' ${GO_EXTRA_LDFLAGS}
 LDFLAGS := $(VERSION_VARIABLES) ${GO_EXTRA_LDFLAGS}
@@ -76,6 +84,7 @@ lint: $(TOOLS_BINDIR)/golangci-lint
 # Build the container image
 .PHONY: oci-build
 oci-build: clean
+	# ${CONTAINER_MANAGER} build -t $(IMG) -f oci/Containerfile .
 	${CONTAINER_MANAGER} build --platform linux/amd64 --manifest $(IMG)-amd64 -f oci/Containerfile .
 	${CONTAINER_MANAGER} build --platform linux/arm64 --manifest $(IMG)-arm64 -f oci/Containerfile .
 

cmd/mapt/cmd/aws/hosts/constans.go

@@ -1,10 +1,14 @@
 package hosts
 
 const (
-	spot       string = "spot"
-	spotDesc   string = "if this flag is set the host will be created only on the region set by the AWS Env (AWS_DEFAULT_REGION)"
-	airgap     string = "airgap"
-	airgapDesc string = "if this flag is set the host will be created as airgap machine. Access will done through a bastion"
+	spot           string = "spot"
+	spotDesc       string = "if this flag is set the host will be created only on the region set by the AWS Env (AWS_DEFAULT_REGION)"
+	airgap         string = "airgap"
+	airgapDesc     string = "if this flag is set the host will be created as airgap machine. Access will done through a bastion"
+	serverless     string = "serverless"
+	serverlessDesc string = "if serverless is set the command will be executed as a serverless action."
+	timeout        string = "timeout"
+	timeoutDesc    string = "if timeout is set a serverless destroy actions will be set on the time according to the timeout. The Timeout value is a duration conforming to Go ParseDuration format."
 
 	vmTypes            string = "vm-types"
 	vmTypesDescription string = "set an specific set of vm-types and ignore any CPUs, Memory, Arch parameters set. Note vm-type should match requested arch. Also if --spot flag is used set at least 3 types."

cmd/mapt/cmd/aws/hosts/fedora.go

@@ -19,7 +19,7 @@ const (
 
 	fedoraVersion        string = "version"
 	fedoraVersionDesc    string = "version for the Fedora Cloud OS"
-	fedoraVersionDefault string = "40"
+	fedoraVersionDefault string = "41"
 )
 
 func GetFedoraCmd() *cobra.Command {
@@ -81,6 +81,7 @@ func getFedoraCreate() *cobra.Command {
 					VMType:               viper.GetStringSlice(vmTypes),
 					InstanceRequest:      instanceRequest,
 					Spot:                 viper.IsSet(spot),
+					Timeout:              viper.GetString(timeout),
 					SetupGHActionsRunner: viper.IsSet(params.InstallGHActionsRunner),
 					Airgap:               viper.IsSet(airgap)}); err != nil {
 				logging.Error(err)
@@ -96,6 +97,7 @@ func getFedoraCreate() *cobra.Command {
 	flagSet.StringSliceP(vmTypes, "", []string{}, vmTypesDescription)
 	flagSet.Bool(airgap, false, airgapDesc)
 	flagSet.Bool(spot, false, spotDesc)
+	flagSet.StringP(timeout, "", "", timeout)
 	flagSet.AddFlagSet(params.GetGHActionsFlagset())
 	flagSet.AddFlagSet(params.GetCpusAndMemoryFlagset())
 	c.PersistentFlags().AddFlagSet(flagSet)
@@ -117,11 +119,16 @@ func getFedoraDestroy() *cobra.Command {
 				viper.IsSet(params.Debug),
 				viper.GetUint(params.DebugLevel))
 
-			if err := fedora.Destroy(); err != nil {
+			logging.Debug("Run fedora destroy")
+
+			if err := fedora.Destroy(viper.IsSet(serverless)); err != nil {
 				logging.Error(err)
 			}
 			return nil
 		},
 	}
+	flagSet := pflag.NewFlagSet(params.DestroyCmdName, pflag.ExitOnError)
+	flagSet.Bool(serverless, false, serverlessDesc)
+	c.PersistentFlags().AddFlagSet(flagSet)
 	return c
 }

cmd/mapt/cmd/aws/hosts/rhel.go

@@ -81,6 +81,7 @@ func getRHELCreate() *cobra.Command {
 					SubsUserpass:         viper.GetString(params.SubsUserpass),
 					ProfileSNC:           viper.IsSet(params.ProfileSNC),
 					Spot:                 viper.IsSet(spot),
+					Timeout:              viper.GetString(timeout),
 					Airgap:               viper.IsSet(airgap),
 					SetupGHActionsRunner: viper.IsSet(params.InstallGHActionsRunner),
 				}); err != nil {
@@ -99,18 +100,11 @@ func getRHELCreate() *cobra.Command {
 	flagSet.StringP(params.SubsUserpass, "", "", params.SubsUserpassDesc)
 	flagSet.Bool(airgap, false, airgapDesc)
 	flagSet.Bool(spot, false, spotDesc)
+	flagSet.StringP(timeout, "", "", timeout)
 	flagSet.Bool(params.ProfileSNC, false, params.ProfileSNCDesc)
 	flagSet.AddFlagSet(params.GetGHActionsFlagset())
 	flagSet.AddFlagSet(params.GetCpusAndMemoryFlagset())
 	c.PersistentFlags().AddFlagSet(flagSet)
-	// if err := c.MarkFlagRequired(subsUsername); err != nil {
-	// 	logging.Error(err)
-	// 	return nil
-	// }
-	// if err := c.MarkFlagRequired(subsUserpass); err != nil {
-	// 	logging.Error(err)
-	// 	return nil
-	// }
 	return c
 }
 
@@ -129,11 +123,14 @@ func getRHELDestroy() *cobra.Command {
 				viper.IsSet(params.Debug),
 				viper.GetUint(params.DebugLevel))
 
-			if err := rhel.Destroy(); err != nil {
+			if err := rhel.Destroy(viper.IsSet(serverless)); err != nil {
 				logging.Error(err)
 			}
 			return nil
 		},
 	}
+	flagSet := pflag.NewFlagSet(params.DestroyCmdName, pflag.ExitOnError)
+	flagSet.Bool(serverless, false, serverlessDesc)
+	c.PersistentFlags().AddFlagSet(flagSet)
 	return c
 }

cmd/mapt/cmd/aws/hosts/windows.go

@@ -86,6 +86,7 @@ func getWindowsCreate() *cobra.Command {
 					AMIKeepCopy:          viper.IsSet(amiKeepCopy),
 					Spot:                 viper.IsSet(spot),
 					Airgap:               viper.IsSet(airgap),
+					Timeout:              viper.GetString(timeout),
 					SetupGHActionsRunner: viper.IsSet(params.InstallGHActionsRunner),
 				}); err != nil {
 				logging.Error(err)
@@ -102,6 +103,7 @@ func getWindowsCreate() *cobra.Command {
 	flagSet.StringP(amiLang, "", amiLangDefault, amiLangDesc)
 	flagSet.Bool(airgap, false, airgapDesc)
 	flagSet.Bool(spot, false, spotDesc)
+	flagSet.StringP(timeout, "", "", timeout)
 	flagSet.Bool(amiKeepCopy, false, amiKeepCopyDesc)
 	flagSet.AddFlagSet(params.GetGHActionsFlagset())
 	c.PersistentFlags().AddFlagSet(flagSet)
@@ -123,11 +125,14 @@ func getWindowsDestroy() *cobra.Command {
 				viper.IsSet(params.Debug),
 				viper.GetUint(params.DebugLevel))
 
-			if err := windows.Destroy(); err != nil {
+			if err := windows.Destroy(viper.IsSet(serverless)); err != nil {
 				logging.Error(err)
 			}
 			return nil
 		},
 	}
+	flagSet := pflag.NewFlagSet(params.DestroyCmdName, pflag.ExitOnError)
+	flagSet.Bool(serverless, false, serverlessDesc)
+	c.PersistentFlags().AddFlagSet(flagSet)
 	return c
 }

docs/serverless-mode.md

@@ -0,0 +1,23 @@
+# Overview
+
+The serverless mode allows to run mapt as a serverless app on the provider the target will be provisioned (i.e. if mapt will create a rhel machine on AWS, it will be executed on Fargate).
+
+The serverless mode can be beneficial on certain circumstances like paralellize workloads, or release capacity on the system executing mapt. As an example running mapt on serverless-mode will not block a gh runner for the time it takes to provision the infrastructure.
+
+We can also take advantage of the serverless-mode to house keeping provisioned infrastructures, as an example on the create operation we can set a timeout which will create the matching destroy operation after the timeout. As so in case of any failure preventing actively run the destroy operation it will be executed anyway after the timeout is reached. 
+
+## Requirements
+
+Each cloud provider requires a specific setup for run serverless containers, it will be a requisite to have those setups in place with the specific values for mapt to facilitate this job mapt is providing a set of scripts to create those resoruces with the naming and values mathing its expectations.
+
+One of the main reasons to handle those resources outside of the action executed by mapt is that typically the resources will hadle sheduled tasks as so those resources are long time living resources (way beyond the mapt action time)
+
+## AWS - Fargate
+
+On AWS the serverless container service is named Fargate and it requires an ECS Cluster. 
+
+TBC
+
+## Azure - Azure Containers Instances
+
+TBC

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/coocood/freecache v1.2.4
 	github.com/pulumi/pulumi-command/sdk v1.0.1
 	github.com/pulumi/pulumi-random/sdk/v4 v4.16.7
-	github.com/pulumi/pulumi/sdk/v3 v3.142.0
+	github.com/pulumi/pulumi/sdk/v3 v3.143.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8
@@ -20,10 +20,12 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0
 	github.com/aws/amazon-ec2-instance-selector/v2 v2.4.2-0.20231006140257-d989c5d76f0e
-	github.com/aws/aws-sdk-go-v2 v1.27.2
+	github.com/aws/aws-sdk-go-v2 v1.32.7
 	github.com/aws/aws-sdk-go-v2/config v1.27.18
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.0
+	github.com/pulumi/pulumi-aws-native/sdk v1.17.0
 	github.com/pulumi/pulumi-aws/sdk/v6 v6.64.0
+	github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0
 	github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.76.0
 	github.com/pulumi/pulumi-azure-native-sdk/compute/v2 v2.76.0
 	github.com/pulumi/pulumi-azure-native-sdk/containerservice/v2 v2.76.0
@@ -35,7 +37,7 @@ require (
 
 require (
 	github.com/BurntSushi/toml v1.2.1 // indirect
-	github.com/aws/aws-sdk-go v1.45.6 // indirect
+	github.com/aws/aws-sdk-go v1.50.36 // indirect
 	github.com/aws/aws-sdk-go-v2/service/pricing v1.21.6 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/evertras/bubble-table v0.15.2 // indirect
@@ -44,6 +46,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pulumi/pulumi-docker/sdk/v4 v4.4.3 // indirect
 	github.com/sahilm/fuzzy v0.1.0 // indirect
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
@@ -59,17 +62,17 @@ require (
 	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.18 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.20.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.28.12 // indirect
-	github.com/aws/smithy-go v1.20.2 // indirect
+	github.com/aws/smithy-go v1.22.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/charmbracelet/bubbles v0.16.1 // indirect
 	github.com/charmbracelet/bubbletea v0.25.0 // indirect
 	github.com/charmbracelet/lipgloss v0.7.1 // indirect
@@ -80,7 +83,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	// github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/hashicorp/hcl/v2 v2.20.1 // indirect
+	github.com/hashicorp/hcl/v2 v2.22.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
@@ -98,7 +101,7 @@ require (
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
-	github.com/pulumi/esc v0.9.1 // indirect
+	github.com/pulumi/esc v0.10.0 // indirect
 	github.com/pulumi/pulumi-azure-native-sdk/v2 v2.76.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -110,8 +113,8 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/zclconf/go-cty v1.13.2 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
@@ -120,6 +123,8 @@ require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecs v1.53.0
+	github.com/aws/aws-sdk-go-v2/service/iam v1.38.3
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cheggaaa/pb v1.0.29 // indirect
 	github.com/djherbis/times v1.5.0 // indirect
@@ -129,7 +134,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/go-git/go-git/v5 v5.12.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/glog v1.2.1 // indirect
+	github.com/golang/glog v1.2.2 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -155,13 +160,13 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
-	golang.org/x/term v0.22.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	google.golang.org/grpc v1.63.2 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	google.golang.org/grpc v1.67.1 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	lukechampine.com/frand v1.4.2 // indirect