Allow tests to retry if request fails (#46)

* Add retry count and enable retry options

* Remove attempts http config

This config was not used and should be superseded by the retry
logic.

* Add dependency on hashicorp/go-retryablehttp

* Retry request if retries enabled

* Update default value for ENABLE_RETRIES to false

README.md

@@ -102,6 +102,13 @@ environment variables:
 
 - `TEST_VERBOSITY`: Increase logging output for tests. Default: `0`.
 
+- `ENABLE_RETRIES`: Enables retrying requests if a test does not succeed.
+   Defaults: `false`.
+
+- `DEFAULT_RETRY_COUNT`: Specify the number of times to retry a test request if
+   the initial request does not succeed. Only applied if `ENABLE_RETRIES` is set
+   to `true` Defaults: `2`.
+
 ### Environment variable substitution
 
 This program supports variable substitution from environment variables in YML

config.go

@@ -28,6 +28,8 @@ type Config struct {
 	PrintFailedTestsOnly bool
 	TestDirectory        string
 	Verbosity            int
+	EnableRetries        bool
+	RetryCount           int
 }
 
 // FromEnv returns config read from environment variables
@@ -54,13 +56,28 @@ func FromEnv() (*Config, error) {
 		printFailedOnly = true
 	}
 
+	enableRetries := true
+	if getEnv("ENABLE_RETRIES", "true") == "false" {
+		enableRetries = false
+	}
+
+	retryCount, err := strconv.Atoi(getEnv("DEFAULT_RETRY_COUNT", "2"))
+	if err != nil {
+		return nil, fmt.Errorf("invalid default retry count value: %s", err)
+	}
+	if retryCount < 0 {
+		return nil, fmt.Errorf("invalid default retry count value: %d", retryCount)
+	}
+
 	return &Config{
 		Concurrency:          concurrency,
 		Host:                 getEnv("TEST_HOST", ""),
 		DNSOverride:          getEnv("TEST_DNS_OVERRIDE", ""),
 		PrintFailedTestsOnly: printFailedOnly,
 		TestDirectory:        getEnv("TEST_DIRECTORY", "tests"),
 		Verbosity:            verbosity,
+		EnableRetries:        enableRetries,
+		RetryCount:           retryCount,
 	}, nil
 }
 

coordinator.go

@@ -33,7 +33,12 @@ func RunTests(tests []*Test, config *Config) bool {
 			// Release a slot when done
 			defer func() { <-sem }()
 
-			result := RunTest(t, config.Host)
+			maxRetries := 0
+			if config.EnableRetries {
+				maxRetries = config.RetryCount
+			}
+
+			result := RunTest(t, config.Host, maxRetries)
 
 			// Acquire lock before accessing shared variables and writing output.
 			// Code in critical section should not perform network I/O.

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/drone/envsubst v1.0.3
 	github.com/fatih/color v1.16.0
+	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/pretty v1.2.1
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
@@ -13,6 +14,7 @@ require (
 )
 
 require (
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/tidwall/match v1.1.1 // indirect

go.sum

@@ -1,16 +1,24 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g=
 github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
+github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
 github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=

http.go

@@ -16,11 +16,14 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
 	"time"
+
+	"github.com/hashicorp/go-retryablehttp"
 )
 
 // HTTPRequestConfig type
@@ -32,9 +35,10 @@ type HTTPRequestConfig struct {
 	BasicAuthUsername    string
 	BasicAuthPassword    string
 	Body                 io.Reader
-	Attempts             int
 	TimeoutSeconds       time.Duration
 	SkipCertVerification bool
+	MaxRetries           int
+	RetryCallback        func(ctx context.Context, resp *http.Response, err error) (bool, error)
 }
 
 // SendHTTPRequest sends an HTTP request and returns response body and status
@@ -45,23 +49,19 @@ func SendHTTPRequest(config *HTTPRequestConfig) (*http.Response, []byte, error)
 	}
 
 	if len(config.Method) <= 0 {
-		return nil, nil, fmt.Errorf("Method is required")
+		return nil, nil, fmt.Errorf("method is required")
 	}
 
 	if len(config.URL) <= 0 {
 		return nil, nil, fmt.Errorf("URL is required")
 	}
 
-	if config.Attempts == 0 {
-		config.Attempts = 1
-	}
-
 	if config.TimeoutSeconds == 0 {
 		config.TimeoutSeconds = 10
 	}
 
 	// Create request
-	req, err := http.NewRequest(
+	req, err := retryablehttp.NewRequest(
 		config.Method,
 		config.URL,
 		config.Body,
@@ -96,26 +96,30 @@ func SendHTTPRequest(config *HTTPRequestConfig) (*http.Response, []byte, error)
 		req.Header.Add(k, v)
 	}
 
-	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: config.SkipCertVerification},
+	client := retryablehttp.Client{
+		HTTPClient: &http.Client{
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: config.SkipCertVerification},
+			},
+			Timeout: time.Duration(config.TimeoutSeconds * time.Second),
+		},
 	}
 
-	client := http.Client{
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-		Transport: transport,
-		Timeout:   time.Duration(config.TimeoutSeconds * time.Second),
+	// Enable retries
+	if config.MaxRetries > 0 {
+		client.RetryMax = config.MaxRetries
+		client.CheckRetry = config.RetryCallback
+	} else {
+		// Don't retry requests
+		client.CheckRetry = func(ctx context.Context, resp *http.Response, inErr error) (bool, error) { return false, nil }
 	}
 
 	// Start sending request
-	var resp *http.Response
-	for a := config.Attempts; a > 0; a-- {
-		resp, err = client.Do(req)
-		if err == nil {
-			break
-		}
-	}
+	resp, err := client.Do(req)
+
 	if err != nil {
 		return nil, nil, err
 	}

tester.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -32,7 +33,7 @@ type TestResult struct {
 }
 
 // RunTest runs a single test
-func RunTest(test *Test, defaultHost string) *TestResult {
+func RunTest(test *Test, defaultHost string, maxRetries int) *TestResult {
 	result := &TestResult{}
 
 	// Validate test and assign default values
@@ -60,14 +61,31 @@ func RunTest(test *Test, defaultHost string) *TestResult {
 		body = strings.NewReader(test.Request.Body)
 	}
 
+	retryCallback := func(ctx context.Context, resp *http.Response, inErr error) (bool, error) {
+		if inErr != nil {
+			// retry is there is an error with the request
+			return true, nil
+		}
+
+		errs := validateResponseStatus(test, resp)
+		if len(errs) >= 1 {
+			// retry if there is an error
+			return true, nil
+		}
+
+		// stop retrying
+		return false, nil
+	}
+
 	reqConfig := &HTTPRequestConfig{
 		Method:               test.Request.Method,
 		URL:                  url,
 		Headers:              test.Request.Headers,
 		Body:                 body,
-		Attempts:             1,
 		TimeoutSeconds:       60,
 		SkipCertVerification: test.SkipCertVerification,
+		RetryCallback:        retryCallback,
+		MaxRetries:           maxRetries,
 	}
 
 	zap.L().Info("sending request",