Day 97/100 commits - use http backend for plan and apply. Run apply b…

…ased on the plan and deploy resources. log outputs when error

README.md

@@ -57,12 +57,13 @@ Upps here should be demo video - no worry, I will be soon! - Now check the API o
   - [X] Add every method ctx to enrich logger
   - [X] Integrate handlers to use logger
 - [X] Handle scheduled plan in TerraformPlanner
-- [ ] Run plan using http backend 
+- [X] Run plan using http backend 
 - [ ] Access Token for Backend http
 - [ ] Clean solution to be more DDD
 - [ ] Create Plan changes during run, what was happened during the time
 - [ ] Apply Mechanism to handle the state
-  - [ ] Apply based on the Plan
+  - [X] Apply based on the Plan
+  - [ ] Save outputs as deployment, handle errors
   - [ ] Backup before apply using ApplyOptions.Backup
 - [ ] Integrate with the Git
 - [ ] Handle other version than version 4.0 of tf 

cmd/handlers/main.go

@@ -80,7 +80,7 @@ func main() {
 		log.Panic().Err(err)
 	}
 
-	handlerFactory := handlers.NewEventHandlerFactory(eventBus, eventBus, uow)
+	handlerFactory := handlers.NewEventHandlerFactory(eventBus, eventBus, uow, fmt.Sprintf("%s:%d", cfg.ServiceDiscovery, cfg.HttpPort))
 
 	allHandlers, err := handlerFactory.RegisterAllHandlers()
 	if err != nil {

cmd/monolith/main.go

@@ -95,7 +95,7 @@ func main() {
 	if err != nil {
 		log.Panic().Err(err)
 	}
-	handlerFactory := handlers.NewEventHandlerFactory(eventBus, eventBus, uow)
+	handlerFactory := handlers.NewEventHandlerFactory(eventBus, eventBus, uow, fmt.Sprintf("%s:%d", cfg.ServiceDiscovery, cfg.HttpPort))
 
 	allHandlers, err := handlerFactory.RegisterAllHandlers()
 	if err != nil {

config.go

@@ -1,12 +1,13 @@
 package labraboard
 
 type Config struct {
-	ConnectionString string `yaml:"connectionString" env:"CONNECTION_STRING" env-description:"Connection string to the database" env-required:"true"`
-	HttpPort         int    `yaml:"httpPort" env:"HTTP_PORT" env-description:"HTTP port to serve the application" env-default:"8080"`
-	RedisHost        string `yaml:"redisHost" env:"REDIS_HOST" env-description:"Redis host" env-default:"localhost"`
-	RedisPort        int    `yaml:"redisPort" env:"REDIS_PORT" env-description:"Redis port" env-default:"6379"`
-	RedisPassword    string `yaml:"redisPassword" env:"REDIS_PASSWORD" env-description:"Redis password" env-default:"eYVX7EwVmmxKPCDmwMtyKVge8oLd2t81"`
-	RedisDB          int    `yaml:"redisDB" env:"REDIS_DB" env-description:"Redis database" env-default:"0"`
-	LogLevel         int8   `yaml:"logLevel" env:"LOG_LEVEL" env-description:"Redis database" env-default:"1"`
-	UsePrettyLogs    bool   `yaml:"usePrettyLogs" env:"USE_PRETTY_LOGS" env-description:"use pretty logs instead of json. Logs are pushed to stdout" env-default:"false"`
+	ConnectionString string `yaml:"connectionString" env:"CONNECTION_STRING" env-description:"Connection string to the database" env-required:"true" json:"connectionString,omitempty"`
+	HttpPort         int    `yaml:"httpPort" env:"HTTP_PORT" env-description:"HTTP port to serve the application" env-default:"8080" json:"httpPort,omitempty"`
+	RedisHost        string `yaml:"redisHost" env:"REDIS_HOST" env-description:"Redis host" env-default:"localhost" json:"redisHost,omitempty"`
+	RedisPort        int    `yaml:"redisPort" env:"REDIS_PORT" env-description:"Redis port" env-default:"6379" json:"redisPort,omitempty"`
+	RedisPassword    string `yaml:"redisPassword" env:"REDIS_PASSWORD" env-description:"Redis password" env-default:"eYVX7EwVmmxKPCDmwMtyKVge8oLd2t81" json:"redisPassword,omitempty"`
+	RedisDB          int    `yaml:"redisDB" env:"REDIS_DB" env-description:"Redis database" env-default:"0" json:"redisDB,omitempty"`
+	LogLevel         int8   `yaml:"logLevel" env:"LOG_LEVEL" env-description:"Redis database" env-default:"1" json:"logLevel,omitempty"`
+	UsePrettyLogs    bool   `yaml:"usePrettyLogs" env:"USE_PRETTY_LOGS" env-description:"use pretty logs instead of json. Logs are pushed to stdout" env-default:"false" json:"usePrettyLogs,omitempty"`
+	ServiceDiscovery string `yaml:"serviceDiscovery" env:"SERVICE_DISCOVERY" env-default:"http://localhost" json:"serviceDiscovery,omitempty"`
 }

docs/docs.go

@@ -386,6 +386,15 @@ const docTemplate = `{
                         "name": "projectId",
                         "in": "path",
                         "required": true
+                    },
+                    {
+                        "description": "Plan to override",
+                        "name": "plan",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/dtos.CreatePlan"
+                        }
                     }
                 ],
                 "responses": {
@@ -599,6 +608,26 @@ const docTemplate = `{
                 }
             }
         },
+        "dtos.CreatePlan": {
+            "type": "object",
+            "properties": {
+                "repoCommit": {
+                    "type": "string"
+                },
+                "repoCommitType": {
+                    "type": "string"
+                },
+                "repoPath": {
+                    "type": "string"
+                },
+                "variables": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
         "dtos.CreateProjectDto": {
             "type": "object",
             "properties": {

docs/swagger.json

@@ -375,6 +375,15 @@
                         "name": "projectId",
                         "in": "path",
                         "required": true
+                    },
+                    {
+                        "description": "Plan to override",
+                        "name": "plan",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/dtos.CreatePlan"
+                        }
                     }
                 ],
                 "responses": {
@@ -588,6 +597,26 @@
                 }
             }
         },
+        "dtos.CreatePlan": {
+            "type": "object",
+            "properties": {
+                "repoCommit": {
+                    "type": "string"
+                },
+                "repoCommitType": {
+                    "type": "string"
+                },
+                "repoPath": {
+                    "type": "string"
+                },
+                "variables": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                }
+            }
+        },
         "dtos.CreateProjectDto": {
             "type": "object",
             "properties": {

docs/swagger.yaml

@@ -15,6 +15,19 @@ definitions:
       value:
         type: string
     type: object
+  dtos.CreatePlan:
+    properties:
+      repoCommit:
+        type: string
+      repoCommitType:
+        type: string
+      repoPath:
+        type: string
+      variables:
+        additionalProperties:
+          type: string
+        type: object
+    type: object
   dtos.CreateProjectDto:
     properties:
       'repositoryBranch:':
@@ -324,6 +337,12 @@ paths:
         name: projectId
         required: true
         type: string
+      - description: Plan to override
+        in: body
+        name: plan
+        required: true
+        schema:
+          $ref: '#/definitions/dtos.CreatePlan'
       produces:
       - application/json
       responses:

internal/handlers/event_handler_factory.go

@@ -14,18 +14,20 @@ var (
 )
 
 type EventHandlerFactory struct {
-	eventSubscriber eb.EventSubscriber
-	unitOfWork      *repositories.UnitOfWork
-	allowedEvents   []events.EventName
-	eventPublisher  eb.EventPublisher
+	eventSubscriber  eb.EventSubscriber
+	unitOfWork       *repositories.UnitOfWork
+	allowedEvents    []events.EventName
+	eventPublisher   eb.EventPublisher
+	serviceDiscovery string
 }
 
-func NewEventHandlerFactory(eventSubscriber eb.EventSubscriber, eventPublisher eb.EventPublisher, unitOfWork *repositories.UnitOfWork) *EventHandlerFactory {
+func NewEventHandlerFactory(eventSubscriber eb.EventSubscriber, eventPublisher eb.EventPublisher, unitOfWork *repositories.UnitOfWork, serviceDiscovery string) *EventHandlerFactory {
 	return &EventHandlerFactory{
-		eventSubscriber: eventSubscriber,
-		unitOfWork:      unitOfWork,
-		eventPublisher:  eventPublisher,
-		allowedEvents:   []events.EventName{events.LEASE_LOCK, events.TRIGGERED_PLAN, events.SCHEDULED_PLAN, events.IAC_APPLY_SCHEDULED},
+		eventSubscriber:  eventSubscriber,
+		unitOfWork:       unitOfWork,
+		eventPublisher:   eventPublisher,
+		serviceDiscovery: serviceDiscovery,
+		allowedEvents:    []events.EventName{events.LEASE_LOCK, events.TRIGGERED_PLAN, events.SCHEDULED_PLAN, events.IAC_APPLY_SCHEDULED},
 	}
 }
 
@@ -39,13 +41,13 @@ func (factory *EventHandlerFactory) RegisterHandler(event events.EventName) (Eve
 		return newTerraformStateLeaseLockHandler(factory.eventSubscriber, factory.unitOfWork)
 	case events.TRIGGERED_PLAN:
 		factory.allowedEvents = helpers.Remove(factory.allowedEvents, event)
-		return newTriggeredPlanHandler(factory.eventSubscriber, factory.unitOfWork)
+		return newTriggeredPlanHandler(factory.eventSubscriber, factory.unitOfWork, factory.serviceDiscovery)
 	case events.SCHEDULED_PLAN:
 		factory.allowedEvents = helpers.Remove(factory.allowedEvents, event)
 		return newScheduledPlanHandler(factory.eventSubscriber, factory.unitOfWork, factory.eventPublisher)
 	case events.IAC_APPLY_SCHEDULED:
 		factory.allowedEvents = helpers.Remove(factory.allowedEvents, event)
-		return newScheduledIaCApplyHandler(factory.eventSubscriber, factory.unitOfWork)
+		return newScheduledIaCApplyHandler(factory.eventSubscriber, factory.unitOfWork, factory.serviceDiscovery)
 	}
 	return nil, MissingHandlerImplementedFactory
 }

internal/handlers/event_handler_factory_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestEventHandlerFactory(t *testing.T) {
-	var factory = NewEventHandlerFactory(nil, nil, nil)
+	var factory = NewEventHandlerFactory(nil, nil, nil, "")
 	t.Run("Register Event that doesn't exist, should return null", func(t *testing.T) {
 		var Z events.EventName = "Z"
 		act, err := factory.RegisterHandler(Z)

internal/handlers/helpers.go

@@ -5,6 +5,33 @@ import (
 	"os"
 )
 
+func createLabraboardBackendFile(path string, endpoint string, projectId string) error {
+	content := `terraform {
+  backend "http" {
+    address = "%[1]s/api/v1/state/terraform/%[2]s"
+    lock_address = "%[1]s/api/v1/state/terraform/%[2]s/lock"
+    unlock_address = "%[1]s/api/v1/state/terraform/%[2]s/lock"
+  }
+}`
+	file, err := os.Create(fmt.Sprintf("%s/backend_override.tf", path))
+	if err != nil {
+		return err
+	}
+	defer func(file *os.File) {
+		err := file.Close()
+		if err != nil {
+
+		}
+	}(file)
+
+	_, err = fmt.Fprintf(file, content, endpoint, projectId)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func createBackendFile(path string, statePath string) error {
 	content := `terraform {
   backend "local" {

internal/handlers/plan_worker_handler.go

@@ -15,18 +15,20 @@ import (
 	"labraboard/internal/valueobjects/iacPlans"
 )
 
-// /todo redesing how to treat plan aggregate to keep whole runs history
+// todo redesing how to treat plan aggregate to keep whole runs history
 type triggeredPlanHandler struct {
-	eventSubscriber eb.EventSubscriber
-	unitOfWork      *repositories.UnitOfWork
-	assembler       *iacSvc.Assembler
+	eventSubscriber  eb.EventSubscriber
+	unitOfWork       *repositories.UnitOfWork
+	assembler        *iacSvc.Assembler
+	serviceDiscovery string
 }
 
-func newTriggeredPlanHandler(eventSubscriber eb.EventSubscriber, unitOfWork *repositories.UnitOfWork) (*triggeredPlanHandler, error) {
+func newTriggeredPlanHandler(eventSubscriber eb.EventSubscriber, unitOfWork *repositories.UnitOfWork, discovery string) (*triggeredPlanHandler, error) {
 	return &triggeredPlanHandler{
 		eventSubscriber,
 		unitOfWork,
 		iacSvc.NewAssembler(unitOfWork),
+		discovery,
 	}, nil
 }
 
@@ -85,7 +87,7 @@ func (handler *triggeredPlanHandler) handlePlanTriggered(obj events.PlanTriggere
 		}
 	}(git)
 
-	if err = createBackendFile(tofuFolderPath, "./.local-state"); err != nil {
+	if err = createLabraboardBackendFile(tofuFolderPath, handler.serviceDiscovery, assembly.ProjectId.String()); err != nil {
 		log.Error().Err(err)
 		return errors.Wrap(err, "Cannot create backend")
 	}

internal/handlers/plan_worker_handler_test.go

@@ -50,7 +50,7 @@ func TestPlanTriggerHandler(t *testing.T) {
 			Name: "2f5e1489476513212ae2f08c9a93beed7de47313",
 		},
 	}
-	handler, _ := newTriggeredPlanHandler(nil, uow)
+	handler, _ := newTriggeredPlanHandler(nil, uow, "")
 	handler.handlePlanTriggered(*obj, context.Background())
 	aggregate, _ = uow.IacRepository.Get(aggregate.GetID(), context.Background())
 	plan, err := aggregate.GetPlan(planId)

internal/handlers/scheduled_iac_apply_handler.go

@@ -15,14 +15,16 @@ import (
 )
 
 type scheduledIaCApplyHandler struct {
-	eventSubscriber eb.EventSubscriber
-	unitOfWork      *repositories.UnitOfWork
+	eventSubscriber  eb.EventSubscriber
+	unitOfWork       *repositories.UnitOfWork
+	serviceDiscovery string
 }
 
-func newScheduledIaCApplyHandler(eventSubscriber eb.EventSubscriber, unitOfWork *repositories.UnitOfWork) (*scheduledIaCApplyHandler, error) {
+func newScheduledIaCApplyHandler(eventSubscriber eb.EventSubscriber, unitOfWork *repositories.UnitOfWork, discovery string) (*scheduledIaCApplyHandler, error) {
 	return &scheduledIaCApplyHandler{
 		eventSubscriber,
 		unitOfWork,
+		discovery,
 	}, nil
 }
 
@@ -36,12 +38,15 @@ func (handler *scheduledIaCApplyHandler) Handle(ctx context.Context) {
 			log.Error().Err(fmt.Errorf("cannot handle message type %T", event))
 		}
 		log.Info().Msgf("Received message: %s", msg)
-		go handler.handle(event, log.WithContext(ctx))
+		err = handler.handle(event, log.WithContext(ctx))
+		if err != nil {
+			log.Error().Err(err).Msg("error to handle")
+			return
+		}
 	}
 }
 
 func (handler *scheduledIaCApplyHandler) handle(event events.IacApplyScheduled, ctx context.Context) error {
-	const tfPlanPath = "plan.tfplan"
 	log := logger.GetWitContext(ctx).
 		With().
 		Str("changeId", event.ChangeId.String()).
@@ -79,7 +84,7 @@ func (handler *scheduledIaCApplyHandler) handle(event events.IacApplyScheduled,
 	folderPath := fmt.Sprintf("/tmp/%s/apply", output.PlanId)
 	tofuFolderPath := fmt.Sprintf("%s/%s", folderPath, output.RepoPath)
 
-	planPath := fmt.Sprintf("%s/%s", tofuFolderPath, tfPlanPath)
+	planPath := fmt.Sprintf("%s/%s", tofuFolderPath, iac.PlanPath)
 	git, err := iac.GitClone(output.RepoUrl, folderPath, output.CommitName, output.CommitType)
 	if err != nil {
 		log.Error().Err(err).Msg(err.Error())
@@ -94,7 +99,7 @@ func (handler *scheduledIaCApplyHandler) handle(event events.IacApplyScheduled,
 		}
 	}(git)
 
-	if err = createBackendFile(tofuFolderPath, "./.local-state"); err != nil {
+	if err = createLabraboardBackendFile(tofuFolderPath, handler.serviceDiscovery, output.ProjectId.String()); err != nil {
 		log.Error().Err(err)
 		return nil
 	}
@@ -109,7 +114,7 @@ func (handler *scheduledIaCApplyHandler) handle(event events.IacApplyScheduled,
 		log.Error().Err(err)
 		return nil
 	}
-	_, err = tofu.Apply(output.PlanId, output.InlineEnvVariable(), output.InlineVariable(), planPath, ctx)
+	_, err = tofu.Apply(output.PlanId, output.InlineEnvVariable(), ctx) //todo handle output
 
 	return nil
 }

internal/managers/delay_task.go

@@ -98,7 +98,7 @@ func (dt *delayTask) Listen(ctx context.Context) {
 		log.Trace().Msg("nothing to publish")
 		return
 	}
-
+	///todo fix it: {"EventName":"lease_lock","Content":{"Id":"dfd5677a-7b62-4444-8f81-5dfad0ce1627","Type":"terraform","LeaseTime":"2024-06-05T19:59:38.160871Z"},"WaitTime":3600000000000}
 	for i, t := range resultSet {
 		err = json.Unmarshal([]byte(t), &tasks[i])
 		if err != nil {

internal/routers/api/terraform_controller.go

@@ -24,6 +24,7 @@ func NewTerraformPlanController(iac *services.IacService) (*TerraformPlanControl
 // @Summary Method to run Terraform Plan for a given project and return the plan id
 // @Schemes http
 // @Param projectId path string true "project id"
+// @Param plan body dtos.CreatePlan true "Plan to override"
 // @Description
 // @Tags terraform
 // @Accept json
@@ -48,6 +49,7 @@ func (c *TerraformPlanController) CreateTerraformPlan(g *gin.Context) {
 	if err = g.BindJSON(&dto); err != nil {
 		l.Warn().Err(err)
 		g.JSON(http.StatusBadRequest, gin.H{"message": "invalid payload"})
+		return
 	}
 
 	var planRunner = services.TerraformPlanRunner{