diff --git a/README.md b/README.md index 4843222c..7b7c6b0c 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,13 @@ environment variables: - `TEST_VERBOSITY`: Increase logging output for tests. Default: `0`. +- `ENABLE_RETRIES`: Enables retrying requests if a test does not succeed. + Defaults: `false`. + +- `DEFAULT_RETRY_COUNT`: Specify the number of times to retry a test request if + the initial request does not succeed. Only applied if `ENABLE_RETRIES` is set + to `true` Defaults: `2`. + ### Environment variable substitution This program supports variable substitution from environment variables in YML diff --git a/config.go b/config.go index 91ea13c3..41bd6a71 100644 --- a/config.go +++ b/config.go @@ -28,6 +28,8 @@ type Config struct { PrintFailedTestsOnly bool TestDirectory string Verbosity int + EnableRetries bool + RetryCount int } // FromEnv returns config read from environment variables @@ -54,6 +56,19 @@ func FromEnv() (*Config, error) { printFailedOnly = true } + enableRetries := true + if getEnv("ENABLE_RETRIES", "true") == "false" { + enableRetries = false + } + + retryCount, err := strconv.Atoi(getEnv("DEFAULT_RETRY_COUNT", "2")) + if err != nil { + return nil, fmt.Errorf("invalid default retry count value: %s", err) + } + if retryCount < 0 { + return nil, fmt.Errorf("invalid default retry count value: %d", retryCount) + } + return &Config{ Concurrency: concurrency, Host: getEnv("TEST_HOST", ""), @@ -61,6 +76,8 @@ func FromEnv() (*Config, error) { PrintFailedTestsOnly: printFailedOnly, TestDirectory: getEnv("TEST_DIRECTORY", "tests"), Verbosity: verbosity, + EnableRetries: enableRetries, + RetryCount: retryCount, }, nil } diff --git a/coordinator.go b/coordinator.go index 8a35993a..1342edef 100644 --- a/coordinator.go +++ b/coordinator.go @@ -33,7 +33,12 @@ func RunTests(tests []*Test, config *Config) bool { // Release a slot when done defer func() { <-sem }() - result := RunTest(t, config.Host) + maxRetries := 0 + if config.EnableRetries { + maxRetries = config.RetryCount + } + + result := RunTest(t, config.Host, maxRetries) // Acquire lock before accessing shared variables and writing output. // Code in critical section should not perform network I/O. diff --git a/go.mod b/go.mod index 1e3ef27e..51ac92a2 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/drone/envsubst v1.0.3 github.com/fatih/color v1.16.0 + github.com/hashicorp/go-retryablehttp v0.7.5 github.com/tidwall/gjson v1.17.0 github.com/tidwall/pretty v1.2.1 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a @@ -13,6 +14,7 @@ require ( ) require ( + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index edf7fe08..a1a77552 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/http.go b/http.go index 0fa0e326..d6e6adc0 100644 --- a/http.go +++ b/http.go @@ -16,11 +16,14 @@ package main import ( "bytes" + "context" "crypto/tls" "fmt" "io" "net/http" "time" + + "github.com/hashicorp/go-retryablehttp" ) // HTTPRequestConfig type @@ -32,9 +35,10 @@ type HTTPRequestConfig struct { BasicAuthUsername string BasicAuthPassword string Body io.Reader - Attempts int TimeoutSeconds time.Duration SkipCertVerification bool + MaxRetries int + RetryCallback func(ctx context.Context, resp *http.Response, err error) (bool, error) } // SendHTTPRequest sends an HTTP request and returns response body and status @@ -45,23 +49,19 @@ func SendHTTPRequest(config *HTTPRequestConfig) (*http.Response, []byte, error) } if len(config.Method) <= 0 { - return nil, nil, fmt.Errorf("Method is required") + return nil, nil, fmt.Errorf("method is required") } if len(config.URL) <= 0 { return nil, nil, fmt.Errorf("URL is required") } - if config.Attempts == 0 { - config.Attempts = 1 - } - if config.TimeoutSeconds == 0 { config.TimeoutSeconds = 10 } // Create request - req, err := http.NewRequest( + req, err := retryablehttp.NewRequest( config.Method, config.URL, config.Body, @@ -96,26 +96,30 @@ func SendHTTPRequest(config *HTTPRequestConfig) (*http.Response, []byte, error) req.Header.Add(k, v) } - transport := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: config.SkipCertVerification}, + client := retryablehttp.Client{ + HTTPClient: &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: config.SkipCertVerification}, + }, + Timeout: time.Duration(config.TimeoutSeconds * time.Second), + }, } - client := http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - Transport: transport, - Timeout: time.Duration(config.TimeoutSeconds * time.Second), + // Enable retries + if config.MaxRetries > 0 { + client.RetryMax = config.MaxRetries + client.CheckRetry = config.RetryCallback + } else { + // Don't retry requests + client.CheckRetry = func(ctx context.Context, resp *http.Response, inErr error) (bool, error) { return false, nil } } // Start sending request - var resp *http.Response - for a := config.Attempts; a > 0; a-- { - resp, err = client.Do(req) - if err == nil { - break - } - } + resp, err := client.Do(req) + if err != nil { return nil, nil, err } diff --git a/tester.go b/tester.go index 8b819a19..fa03763a 100644 --- a/tester.go +++ b/tester.go @@ -15,6 +15,7 @@ package main import ( + "context" "fmt" "io" "net/http" @@ -32,7 +33,7 @@ type TestResult struct { } // RunTest runs a single test -func RunTest(test *Test, defaultHost string) *TestResult { +func RunTest(test *Test, defaultHost string, maxRetries int) *TestResult { result := &TestResult{} // Validate test and assign default values @@ -60,14 +61,31 @@ func RunTest(test *Test, defaultHost string) *TestResult { body = strings.NewReader(test.Request.Body) } + retryCallback := func(ctx context.Context, resp *http.Response, inErr error) (bool, error) { + if inErr != nil { + // retry is there is an error with the request + return true, nil + } + + errs := validateResponseStatus(test, resp) + if len(errs) >= 1 { + // retry if there is an error + return true, nil + } + + // stop retrying + return false, nil + } + reqConfig := &HTTPRequestConfig{ Method: test.Request.Method, URL: url, Headers: test.Request.Headers, Body: body, - Attempts: 1, TimeoutSeconds: 60, SkipCertVerification: test.SkipCertVerification, + RetryCallback: retryCallback, + MaxRetries: maxRetries, } zap.L().Info("sending request",