Better logging, more... testing.

cmd/bucket.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"gov.gsa.fac.cgov-util/internal/logging"
 	"gov.gsa.fac.cgov-util/internal/pipes"
+	"gov.gsa.fac.cgov-util/internal/structs"
 	"gov.gsa.fac.cgov-util/internal/util"
 
 	vcap "gov.gsa.fac.cgov-util/internal/vcap"
@@ -32,15 +33,15 @@ var backup_tag string
 // 	}
 // }
 
-func bucket_local_tables(source_creds *vcap.CredentialsRDS, up vcap.UserProvidedCredentials) {
+func bucket_local_tables(source_creds *structs.CredentialsRDS, up structs.UserProvidedCredentials) {
 	table_to_schema := util.Get_table_and_schema_names(source_creds)
 	for table, schema := range table_to_schema {
 		mc_pipe := pipes.Mc(
-			pipes.PG_Dump_Table(source_creds, schema, table, Debug),
+			pipes.PG_Dump_Table(source_creds, schema, table),
 			up,
 			backup_tag,
 			source_creds.DB_Name,
-			schema, table, Debug,
+			schema, table,
 		)
 		mc_pipe.Wait()
 		if err := mc_pipe.Error(); err != nil {
@@ -50,15 +51,15 @@ func bucket_local_tables(source_creds *vcap.CredentialsRDS, up vcap.UserProvided
 	}
 }
 
-func bucket_cgov_tables(source_creds *vcap.CredentialsRDS, up *vcap.CredentialsS3) {
+func bucket_cgov_tables(source_creds *structs.CredentialsRDS, up *structs.CredentialsS3) {
 	table_to_schema := util.Get_table_and_schema_names(source_creds)
 	for table, schema := range table_to_schema {
 		s3_pipe := pipes.S3(
-			pipes.PG_Dump_Table(source_creds, schema, table, Debug),
+			pipes.PG_Dump_Table(source_creds, schema, table),
 			up,
 			backup_tag,
 			source_creds.DB_Name,
-			schema, table, Debug,
+			schema, table,
 		)
 		s3_pipe.Wait()
 		if err := s3_pipe.Error(); err != nil {
@@ -89,7 +90,7 @@ to quickly create a Cobra application.`,
 				logging.Logger.Printf("BACKUPS could not get s3 credentials")
 				os.Exit(-1)
 			}
-			if Debug {
+			if util.IsDebugLevel("DEBUG") {
 				logging.Logger.Printf("BACKUPS s3 credentials %v\n", up)
 			}
 			bucket_cgov_tables(source_creds, up)
@@ -103,7 +104,6 @@ func init() {
 	bucketCmd.Flags().StringVarP(&SourceDB, "source-db", "", "", "source database (req)")
 	bucketCmd.Flags().StringVarP(&DestinationBucket, "destination-bucket", "", "", "destination database (req)")
 	bucketCmd.Flags().StringVarP(&backup_tag, "backup-tag", "", "", "SNAPSHOT, HOURLY-03, etc. (req)")
-	bucketCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "Log debug statements")
 	bucketCmd.MarkFlagRequired("source-db")
 	bucketCmd.MarkFlagRequired("destination-bucket")
 	bucketCmd.MarkFlagRequired("backup_tag")

cmd/check.go

@@ -10,10 +10,12 @@ import (
 
 	"github.com/spf13/cobra"
 	"gov.gsa.fac.cgov-util/internal/logging"
+	"gov.gsa.fac.cgov-util/internal/structs"
+
 	vcap "gov.gsa.fac.cgov-util/internal/vcap"
 )
 
-func get_row_count(creds *vcap.CredentialsRDS, table string) int {
+func get_row_count(creds *structs.CredentialsRDS, table string) int {
 	var count int
 	// FIXME: Not sure if `disable` is correct for RDS sslmode.
 	connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable",
@@ -30,7 +32,7 @@ func get_row_count(creds *vcap.CredentialsRDS, table string) int {
 	return count
 }
 
-func check_results(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS, tables []string) {
+func check_results(source *structs.CredentialsRDS, dest *structs.CredentialsRDS, tables []string) {
 	// FIXME: These won't exist in the VCAP_SERVICES version
 	// of the config. We'll have to always... load both?
 	// There needs to be a way to configure this in the remote env.

cmd/clone.go

@@ -9,25 +9,26 @@ import (
 	"github.com/spf13/cobra"
 	"gov.gsa.fac.cgov-util/internal/logging"
 	"gov.gsa.fac.cgov-util/internal/pipes"
+	"gov.gsa.fac.cgov-util/internal/structs"
 	"gov.gsa.fac.cgov-util/internal/util"
 	vcap "gov.gsa.fac.cgov-util/internal/vcap"
 
 	_ "github.com/lib/pq"
 )
 
-func clone(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) {
-	psql_pipe := pipes.Psql(pipes.PG_Dump(source, Debug), dest, Debug)
+func clone(source *structs.CredentialsRDS, dest *structs.CredentialsRDS) {
+	psql_pipe := pipes.Psql(pipes.PG_Dump(source), dest)
 	psql_pipe.Wait()
 	if err := psql_pipe.Error(); err != nil {
 		logging.Logger.Println("BACKUPS Pipe failed")
 		os.Exit(-1)
 	}
 }
 
-func clone_tables(source *vcap.CredentialsRDS, dest *vcap.CredentialsRDS) {
+func clone_tables(source *structs.CredentialsRDS, dest *structs.CredentialsRDS) {
 	table_to_schema := util.Get_table_and_schema_names(source)
 	for table, schema := range table_to_schema {
-		psql_pipe := pipes.Psql(pipes.PG_Dump_Table(source, schema, table, Debug), dest, Debug)
+		psql_pipe := pipes.Psql(pipes.PG_Dump_Table(source, schema, table), dest)
 		psql_pipe.Wait()
 		if err := psql_pipe.Error(); err != nil {
 			logging.Logger.Printf("BACKUPS Pipe failed for %s, %s\n", schema, table)
@@ -58,7 +59,6 @@ func init() {
 	rootCmd.AddCommand(cloneCmd)
 	cloneCmd.Flags().StringVarP(&SourceDB, "source-db", "", "", "source database (req)")
 	cloneCmd.Flags().StringVarP(&DestinationDB, "destination-db", "", "", "destination database (req)")
-	cloneCmd.Flags().BoolVarP(&Debug, "debug", "d", false, "Log debug statements")
 	cloneCmd.MarkFlagRequired("source-db")
 	cloneCmd.MarkFlagRequired("destination-db")
 

cmd/root.go

@@ -15,7 +15,6 @@ var (
 	DestinationDB     string
 	DestinationBucket string
 	SHA1              string
-	Debug             bool
 
 	// rootCmd represents the base command when called without any subcommands
 	rootCmd = &cobra.Command{

internal/pipes/mc.go

@@ -8,16 +8,17 @@ import (
 	"github.com/bitfield/script"
 	"github.com/google/uuid"
 	"gov.gsa.fac.cgov-util/internal/logging"
-	"gov.gsa.fac.cgov-util/internal/vcap"
+	"gov.gsa.fac.cgov-util/internal/structs"
+	"gov.gsa.fac.cgov-util/internal/util"
 )
 
 // https://bitfieldconsulting.com/golang/scripting
 func Mc(in_pipe *script.Pipe,
-	upc vcap.UserProvidedCredentials,
+	upc structs.UserProvidedCredentials,
 	prefix string,
 	source_db string,
 	schema string,
-	table string, debug bool) *script.Pipe {
+	table string) *script.Pipe {
 	// // mc pipe myminio/gsa-fac-private-s3/backups/${PREFIX}-${FROM_DATABASE}.dump
 	// Always set the alias first.
 	os.Setenv("AWS_PRIVATE_ACCESS_KEY_ID", upc["access_key_id"])
@@ -46,7 +47,7 @@ func Mc(in_pipe *script.Pipe,
 	}
 	// Combine the slice for printing and execution.
 	combined := strings.Join(cmd[:], " ")
-	if debug {
+	if util.IsDebugLevel("DEBUG") {
 		fmt.Printf("command: %s\n", combined)
 	}
 	logging.Logger.Printf("BACKUPS mc targeting %s", prefix)

internal/pipes/pg_dump.go

@@ -6,10 +6,11 @@ import (
 
 	"github.com/bitfield/script"
 	"gov.gsa.fac.cgov-util/internal/logging"
-	"gov.gsa.fac.cgov-util/internal/vcap"
+	"gov.gsa.fac.cgov-util/internal/structs"
+	"gov.gsa.fac.cgov-util/internal/util"
 )
 
-func PG_Dump_Table(creds *vcap.CredentialsRDS, schema string, table string, debug bool) *script.Pipe {
+func PG_Dump_Table(creds *structs.CredentialsRDS, schema string, table string) *script.Pipe {
 	// Compose the command as a slice
 	cmd := []string{
 		"pg_dump",
@@ -33,14 +34,14 @@ func PG_Dump_Table(creds *vcap.CredentialsRDS, schema string, table string, debu
 	// Combine the slice for printing and execution.
 	combined := strings.Join(cmd[:], " ")
 	logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name)
-	if debug {
+	if util.IsDebugLevel("DEBUG") {
 		fmt.Printf("command: %s\n", combined)
 	}
 	return script.Exec(combined)
 }
 
 // https://bitfieldconsulting.com/golang/scripting
-func PG_Dump(creds *vcap.CredentialsRDS, debug bool) *script.Pipe {
+func PG_Dump(creds *structs.CredentialsRDS) *script.Pipe {
 	// Compose the command as a slice
 	cmd := []string{
 		"pg_dump",
@@ -62,7 +63,7 @@ func PG_Dump(creds *vcap.CredentialsRDS, debug bool) *script.Pipe {
 	// Combine the slice for printing and execution.
 	combined := strings.Join(cmd[:], " ")
 	logging.Logger.Printf("BACKUPS pg_dump targeting %s\n", creds.DB_Name)
-	if debug {
+	if util.IsDebugLevel("DEBUG") {
 		fmt.Printf("command: %s\n", combined)
 	}
 	return script.Exec(combined)

internal/pipes/psql.go

@@ -6,10 +6,11 @@ import (
 
 	"github.com/bitfield/script"
 	"gov.gsa.fac.cgov-util/internal/logging"
-	"gov.gsa.fac.cgov-util/internal/vcap"
+	"gov.gsa.fac.cgov-util/internal/structs"
+	"gov.gsa.fac.cgov-util/internal/util"
 )
 
-func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS, debug bool) *script.Pipe {
+func Psql(in_pipe *script.Pipe, creds *structs.CredentialsRDS) *script.Pipe {
 	cmd := []string{
 		"psql",
 		"--no-password",
@@ -23,7 +24,7 @@ func Psql(in_pipe *script.Pipe, creds *vcap.CredentialsRDS, debug bool) *script.
 		),
 	}
 	combined := strings.Join(cmd[:], " ")
-	if debug {
+	if util.IsDebugLevel("DEBUG") {
 		logging.Logger.Printf("command: %s\n", combined)
 	}
 	logging.Logger.Printf("BACKUPS psql targeting %s\n", creds.DB_Name)

internal/pipes/s3.go

@@ -6,15 +6,16 @@ import (
 
 	"github.com/bitfield/script"
 	"gov.gsa.fac.cgov-util/internal/logging"
-	"gov.gsa.fac.cgov-util/internal/vcap"
+	"gov.gsa.fac.cgov-util/internal/structs"
+	"gov.gsa.fac.cgov-util/internal/util"
 )
 
 // https://bitfieldconsulting.com/golang/scripting
 func S3(in_pipe *script.Pipe,
-	up *vcap.CredentialsS3,
+	up *structs.CredentialsS3,
 	prefix string,
 	source_db string,
-	schema string, table string, debug bool) *script.Pipe {
+	schema string, table string) *script.Pipe {
 	// os.Setenv("AWS_ACCESS_KEY_ID", up.AccessKeyId)
 	// os.Setenv("AWS_SECRET_ACCESS_KEY", up.SecretAccessKey)
 	// os.Setenv("AWS_DEFAULT_REGION", up.Region)
@@ -37,7 +38,7 @@ func S3(in_pipe *script.Pipe,
 	// Combine the slice for printing and execution.
 	combined := strings.Join(cmd[:], " ")
 	logging.Logger.Printf("BACKUPS s3 targeting %s\n", prefix)
-	if debug {
+	if util.IsDebugLevel("DEBUG") {
 		fmt.Printf("command: %s\n", combined)
 	}
 	return in_pipe.Exec(combined)

internal/structs/vcap.go

@@ -0,0 +1,65 @@
+package structs
+
+type UserProvidedCredentials = map[string]string
+
+type CredentialsRDS struct {
+	DB_Name  string `json:"db_name"`
+	Host     string `json:"host"`
+	Name     string `json:"name"`
+	Password string `json:"password"`
+	Port     string `json:"port"`
+	Username string `json:"username"`
+	Uri      string `json:"uri"`
+}
+
+type CredentialsS3 struct {
+	Uri                string   `json:"uri"`
+	InsecureSkipVerify bool     `json:"insecure_skip_verify"`
+	AccessKeyId        string   `json:"access_key_id"`
+	SecretAccessKey    string   `json:"secret_access_key"`
+	Region             string   `json:"region"`
+	Bucket             string   `json:"bucket"`
+	Endpoint           string   `json:"endpoint"`
+	FipsEndpoint       string   `json:"fips_endpoint"`
+	AdditionalBuckets  []string `json:"additional_buckets"`
+	SyslogDrainUrl     string   `json:"syslog_drain_url"`
+	VolumeMounts       []string `json:"volume_mounts`
+}
+
+type InstanceS3 struct {
+	Label        string        `json:"label"`
+	Plan         string        `json:"plan"`
+	Name         string        `json:"name"`
+	Tags         []string      `json:"tags"`
+	InstanceGuid string        `json:"instance_guid"`
+	InstanceName string        `json:"instance_name"`
+	BindingGuid  string        `json:"binding_guid"`
+	BindingName  string        `json:"binding_name"`
+	Credentials  CredentialsS3 `json:"credentials"`
+}
+
+type InstanceRDS struct {
+	Label          string         `json:"label"`
+	Provider       string         `json:"provider"`
+	Plan           string         `json:"plan"`
+	Name           string         `json:"name"`
+	Tags           []string       `json:"tags"`
+	InstanceGuid   string         `json:"instance_guid"`
+	InstanceName   string         `json:"instance_name"`
+	BindingGuid    string         `json:"binding_guid"`
+	BindingName    string         `json:"binding_name"`
+	Credentials    CredentialsRDS `json:"credentials"`
+	SyslogDrainUrl string         `json:"syslog_drain_url"`
+	VolumeMounts   string         `json:"volume_mounts"`
+}
+
+type UserProvided struct {
+	Label        string            `json:"label"`
+	Name         string            `json:"name"`
+	Tags         []string          `json:"tags"`
+	InstanceGuid string            `json:"instance_guid"`
+	InstanceName string            `json:"instance_name"`
+	BindingGuid  string            `json:"binding_guid"`
+	BindingName  string            `json:"binding_name"`
+	Credentials  map[string]string `json:"credentials"`
+}

internal/util/db.go

@@ -5,10 +5,10 @@ import (
 	"os"
 
 	"gov.gsa.fac.cgov-util/internal/logging"
-	"gov.gsa.fac.cgov-util/internal/vcap"
+	"gov.gsa.fac.cgov-util/internal/structs"
 )
 
-func Get_table_and_schema_names(source_creds *vcap.CredentialsRDS) map[string]string {
+func Get_table_and_schema_names(source_creds *structs.CredentialsRDS) map[string]string {
 	// Do this table-by-table for RAM reasons.
 	db, err := sql.Open("postgres", source_creds.Uri)
 	if err != nil {

internal/util/debug.go

@@ -0,0 +1,11 @@
+package util
+
+import "os"
+
+func getDebugLevel() string {
+	return os.Getenv("CGOV_UTIL_DEBUG_LEVEL")
+}
+
+func IsDebugLevel(lvl string) bool {
+	return getDebugLevel() == lvl
+}