From f51849878c0a9aaf0bb62861d803ad5ecb4c10b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Fri, 2 Aug 2024 11:43:07 +0200 Subject: [PATCH 1/8] Add configurable retries for target writes Currently retrying [configuration](https://github.com/snowplow/snowbridge/blob/c8c94f8a13367260f9a1a280cdd5d86098f66521/cmd/cli/cli.go#L213) is hardcoded to 5 attempts with exponential delay (inital 1 second) for each type of an error. This commit brings a couple of improvements to retrying: * Make retrying settings configurable from HCL. Max attempts and delay can now be customized. * Split retrying strategies into 2 categories: transient and setup. Target can now signal what kind of an error has happened and what kind of retrying strategy should be applied. * HTTP target can now return setup errors by the new response rules configuration. These rules also allows the target to match invalid data. Previously it was either failure (retried) or oversized data. * Eventually setup errors will also trigger another actions like toggling application health status and sending monitoring alerts. For now behaviour for setup errors is basically the same as for transient errors, but extending behaviour for setup errors should be relatively easy with this structure. --- .github/workflows/ci.yml | 5 +- .../configuration/overview-full-example.hcl | 11 + assets/docs/configuration/retry-example.hcl | 9 + .../targets/http-full-example.hcl | 22 ++ cmd/cli/cli.go | 128 +++++-- cmd/cli/cli_test.go | 324 ++++++++++++++++++ config/component_test.go | 21 ++ config/config.go | 24 ++ config/config_test.go | 6 + docs/configuration_retry_docs_test.go | 32 ++ go.mod | 5 +- go.sum | 10 +- pkg/models/target_write_error.go | 21 ++ pkg/target/http.go | 110 +++++- pkg/target/http_oauth2_test.go | 4 +- pkg/target/http_response_rules_test.go | 78 +++++ pkg/target/http_test.go | 109 +++++- 17 files changed, 850 insertions(+), 69 deletions(-) create mode 100644 assets/docs/configuration/retry-example.hcl create mode 100644 cmd/cli/cli_test.go create mode 100644 docs/configuration_retry_docs_test.go create mode 100644 pkg/models/target_write_error.go create mode 100644 pkg/target/http_response_rules_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c396e91..e3deb9b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: check-latest: true cache: true + - name: Check if go mod tidy should be run + uses: katexochen/go-tidy-check@v2 + - name: Extract project version from file id: version run: | @@ -79,4 +82,4 @@ jobs: run: make e2e-up - name: Run e2e tests - run: make e2e-test + run: make e2e-test \ No newline at end of file diff --git a/assets/docs/configuration/overview-full-example.hcl b/assets/docs/configuration/overview-full-example.hcl index 1600371d..184f7d6b 100644 --- a/assets/docs/configuration/overview-full-example.hcl +++ b/assets/docs/configuration/overview-full-example.hcl @@ -93,6 +93,17 @@ stats_receiver { // log level configuration (default: "info") log_level = "info" +// Specifies how failed writes to the target should be retried, depending on an error type +retry { + transient { + delay_sec = 1 + max_attempts = 5 + } + setup { + delay_sec = 20 + } +} + license { accept = true } diff --git a/assets/docs/configuration/retry-example.hcl b/assets/docs/configuration/retry-example.hcl new file mode 100644 index 00000000..34e17e52 --- /dev/null +++ b/assets/docs/configuration/retry-example.hcl @@ -0,0 +1,9 @@ +retry { + transient { + delay_sec = 5 + max_attempts = 10 + } + setup { + delay_sec = 30 + } +} diff --git a/assets/docs/configuration/targets/http-full-example.hcl b/assets/docs/configuration/targets/http-full-example.hcl index bb43f720..f2b4e2b4 100644 --- a/assets/docs/configuration/targets/http-full-example.hcl +++ b/assets/docs/configuration/targets/http-full-example.hcl @@ -62,5 +62,27 @@ target { # Optional path to the file containing template which is used to build HTTP request based on a batch of input data template_file = "myTemplate.file" + + # 2 invalid + 1 setup error rules + response_rules { + # This one is a match when... + invalid { + # ...HTTP statuses match... + http_codes = [400] + # AND this string exists in a response body + body = "Invalid value for 'purchase' field" + } + # If no match yet, we can check the next one... + invalid { + # again 400 status... + http_codes = [400] + # BUT we expect different error message in the response body + body = "Invalid value for 'attributes' field" + } + # Same for 'setup' rules.. + setup { + http_codes = [401, 403] + } + } } } diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 04699b72..be44a5df 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -19,13 +19,13 @@ import ( "github.com/getsentry/sentry-go" log "github.com/sirupsen/logrus" - retry "github.com/snowplow-devops/go-retry" "github.com/urfave/cli" "net/http" // pprof imported for the side effect of registering its HTTP handlers _ "net/http/pprof" + retry "github.com/avast/retry-go/v4" "github.com/snowplow/snowbridge/cmd" "github.com/snowplow/snowbridge/config" "github.com/snowplow/snowbridge/pkg/failure/failureiface" @@ -171,7 +171,7 @@ func RunApp(cfg *config.Config, supportedSources []config.ConfigurationPair, sup // Callback functions for the source to leverage when writing data sf := sourceiface.SourceFunctions{ - WriteToTarget: sourceWriteFunc(t, ft, tr, o), + WriteToTarget: sourceWriteFunc(t, ft, tr, o, cfg), } // Read is a long running process and will only return when the source @@ -195,7 +195,7 @@ func RunApp(cfg *config.Config, supportedSources []config.ConfigurationPair, sup // 4. Observing these results // // All with retry logic baked in to remove any of this handling from the implementations -func sourceWriteFunc(t targetiface.Target, ft failureiface.Failure, tr transform.TransformationApplyFunction, o *observer.Observer) func(messages []*models.Message) error { +func sourceWriteFunc(t targetiface.Target, ft failureiface.Failure, tr transform.TransformationApplyFunction, o *observer.Observer, cfg *config.Config) func(messages []*models.Message) error { return func(messages []*models.Message) error { // Apply transformations @@ -215,65 +215,119 @@ func sourceWriteFunc(t targetiface.Target, ft failureiface.Failure, tr transform // Send message buffer messagesToSend := transformed.Result + invalid := transformed.Invalid + var oversized []*models.Message - res, err := retry.ExponentialWithInterface(5, time.Second, "target.Write", func() (interface{}, error) { - res, err := t.Write(messagesToSend) + write := func() error { + result, err := t.Write(messagesToSend) + + o.TargetWrite(result) + messagesToSend = result.Failed + oversized = append(oversized, result.Oversized...) + invalid = append(invalid, result.Invalid...) + return err + } + + err := handleWrite(cfg, write) - o.TargetWrite(res) - messagesToSend = res.Failed - return res, err - }) if err != nil { return err } - resCast := res.(*models.TargetWriteResult) // Send oversized message buffer - messagesToSend = resCast.Oversized - if len(messagesToSend) > 0 { - err2 := retry.Exponential(5, time.Second, "failureTarget.WriteOversized", func() error { - res, err := ft.WriteOversized(t.MaximumAllowedMessageSizeBytes(), messagesToSend) - if err != nil { - return err - } - if len(res.Oversized) != 0 || len(res.Invalid) != 0 { + if len(oversized) > 0 { + messagesToSend = oversized + writeOversized := func() error { + result, err := ft.WriteOversized(t.MaximumAllowedMessageSizeBytes(), messagesToSend) + if len(result.Oversized) != 0 || len(result.Invalid) != 0 { log.Fatal("Oversized message transformation resulted in new oversized / invalid messages") } - o.TargetWriteOversized(res) - messagesToSend = res.Failed + o.TargetWriteOversized(result) + messagesToSend = result.Failed + return err + } + + err := handleWrite(cfg, writeOversized) + + if err != nil { return err - }) - if err2 != nil { - return err2 } } // Send invalid message buffer - messagesToSend = append(resCast.Invalid, transformed.Invalid...) - if len(messagesToSend) > 0 { - err3 := retry.Exponential(5, time.Second, "failureTarget.WriteInvalid", func() error { - res, err := ft.WriteInvalid(messagesToSend) - if err != nil { - return err - } - if len(res.Oversized) != 0 || len(res.Invalid) != 0 { + if len(invalid) > 0 { + messagesToSend = invalid + writeInvalid := func() error { + result, err := ft.WriteInvalid(messagesToSend) + if len(result.Oversized) != 0 || len(result.Invalid) != 0 { log.Fatal("Invalid message transformation resulted in new invalid / oversized messages") } - o.TargetWriteInvalid(res) - messagesToSend = res.Failed + o.TargetWriteInvalid(result) + messagesToSend = result.Failed return err - }) - if err3 != nil { - return err3 } - } + err := handleWrite(cfg, writeInvalid) + + if err != nil { + return err + } + } return nil } } +// Wrap each target write operation with 2 kinds of retries: +// - setup errors: long delay, unlimited attempts, unhealthy state + alerts +// - transient errors: short delay, limited attempts +// If it's setup/transient error is decided based on a response returned by the target. +func handleWrite(cfg *config.Config, write func() error) error { + retryOnlySetupErrors := retry.RetryIf(func(err error) bool { + _, isSetup := err.(models.SetupWriteError) + return isSetup + }) + + onSetupError := retry.OnRetry(func(attempt uint, err error) { + log.Infof("Setup target write error. Attempt: %d, error: %s\n", attempt+1, err) + // Here we can set unhealthy status + send monitoring alerts in the future. Nothing happens here now. + }) + + //First try to handle error as setup... + err := retry.Do( + write, + retryOnlySetupErrors, + onSetupError, + retry.Delay(time.Duration(cfg.Data.Retry.Setup.Delay)*time.Second), + // for now let's limit attempts to 5 for setup errors, because we don't have health check which would allow app to be killed externally. Unlimited attempts don't make sense right now. + retry.Attempts(5), + retry.LastErrorOnly(true), + //enable when health check + monitoring implemented + // retry.Attempts(0), //unlimited + ) + + if err == nil { + return err + } + + //If no setup, then handle as transient. We already had at least 1 attempt from above 'setup' retrying section. + log.Infof("Transient target write error. Starting retrying. error: %s\n", err) + + onTransientError := retry.OnRetry(func(retry uint, err error) { + log.Infof("Retry failed with transient error. Retry counter: %d, error: %s\n", retry+1, err) + }) + + err = retry.Do( + write, + onTransientError, + retry.Delay(time.Duration(cfg.Data.Retry.Transient.Delay)*time.Second), + retry.Attempts(uint(cfg.Data.Retry.Transient.MaxAttempts)), + retry.LastErrorOnly(true), + ) + return err +} + // exitWithError will ensure we log the error and leave time for Sentry to flush func exitWithError(err error, flushSentry bool) { log.WithFields(log.Fields{"error": err}).Error(err) diff --git a/cmd/cli/cli_test.go b/cmd/cli/cli_test.go new file mode 100644 index 00000000..f1a3ac9d --- /dev/null +++ b/cmd/cli/cli_test.go @@ -0,0 +1,324 @@ +/** + * Copyright (c) 2020-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + */ + +package cli + +import ( + "errors" + "slices" + "testing" + "time" + + "github.com/snowplow/snowbridge/config" + "github.com/snowplow/snowbridge/pkg/failure" + "github.com/snowplow/snowbridge/pkg/models" + "github.com/snowplow/snowbridge/pkg/observer" + "github.com/stretchr/testify/assert" +) + +func TestWrite_AllOK(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {sent: []string{"m1", "m2"}}, //first write attempt, all good + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1", "m2"}, output.sentToGood) + assert.Empty(t, output.sentToFailed) + assert.Empty(t, output.err) +} + +func TestWrite_OKAfterFailed(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {failed: []string{"m1", "m2"}, err: "Error 1"}, //first write attempt fails + {sent: []string{"m1", "m2"}}, // but second is ok + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1", "m2"}, output.sentToGood) + assert.Empty(t, output.sentToFailed) + assert.Empty(t, output.err) +} + +func TestWrite_AllInvalid(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {invalid: []string{"m1", "m2"}}, //first write attempt signals that data is invalid + }, + failureTarget: []mockResult{ + {sent: []string{"m1", "m2"}}, // so it's later successfully sent to the failure target + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1", "m2"}, output.sentToFailed) + assert.Empty(t, output.sentToGood) + assert.Empty(t, output.err) +} + +func TestWrite_InvalidRetried(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {invalid: []string{"m1", "m2"}}, //first write attempt signals that data is invalid + }, + failureTarget: []mockResult{ + {failed: []string{"m1", "m2"}, err: "failure target error 1"}, // but first attempt to write invalid data to the failure target fails + {failed: []string{"m1", "m2"}, err: "failure target error 2"}, //the second one too + {sent: []string{"m1", "m2"}}, // but third one is ok + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1", "m2"}, output.sentToFailed) + assert.Empty(t, output.sentToGood) + assert.Empty(t, output.err) +} + +func TestWrite_SomeOKSomeInvalid(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {sent: []string{"m1"}, invalid: []string{"m2"}}, //first write attempt turns out to be a mix of valid and invalid data + }, + failureTarget: []mockResult{ + {sent: []string{"m2"}}, // so invalid part is later then sent to the failure target + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1"}, output.sentToGood) // but good data is in good target + assert.Equal(t, []string{"m2"}, output.sentToFailed) + assert.Empty(t, output.err) +} + +func TestWrite_OKAfterPartialFailure(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {sent: []string{"m1"}, failed: []string{"m2"}, err: "Error 1"}, // one message is ok, the second one fails + {failed: []string{"m2"}, err: "Error 2"}, // so the second one is retried and fails again + {sent: []string{"m2"}}, // but eventually is also successfull + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1", "m2"}, output.sentToGood) + assert.Empty(t, output.sentToFailed) + assert.Empty(t, output.err) +} + +func TestWrite_AllOversized(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {oversized: []string{"m1", "m2"}}, + }, + failureTarget: []mockResult{ + {sent: []string{"m1", "m2"}}, + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1", "m2"}, output.sentToFailed) + assert.Empty(t, output.sentToGood) + assert.Empty(t, output.err) +} + +func TestWrite_Combo(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + message("m3", "data 3"), + message("m4", "data 4"), + } + + // mix of everything - ok, retrying failures, invalid and oversized messages + mocks := targetMocks{ + //m1 and m2 are good but m2 fails at first + goodTarget: []mockResult{ + {sent: []string{"m1"}, failed: []string{"m2"}, oversized: []string{"m3"}, invalid: []string{"m4"}, err: "m2 failed!!!"}, + {failed: []string{"m2"}, err: "m2 failed again!!!"}, + {sent: []string{"m2"}}, + }, + //m3 and m4 are going to bad but with some retries + failureTarget: []mockResult{ + {failed: []string{"m3"}, err: "m3 (oversized) failed!!"}, + {failed: []string{"m3"}, err: "m3 (oversized) failed!!"}, + {sent: []string{"m3"}}, + {failed: []string{"m4"}, err: "m4 (invalid) failed!!"}, + {failed: []string{"m4"}, err: "m4 (invalid) failed!!"}, + {failed: []string{"m4"}, err: "m4 (invalid) failed!!"}, + {sent: []string{"m4"}}, + }, + } + + output := run(inputMessages, mocks) + assert.Equal(t, []string{"m1", "m2"}, output.sentToGood) + assert.Equal(t, []string{"m3", "m4"}, output.sentToFailed) + assert.Empty(t, output.err) +} + +func TestWrite_RunOutOfAttempts(t *testing.T) { + inputMessages := []*models.Message{ + message("m1", "data 1"), + message("m2", "data 2"), + } + + mocks := targetMocks{ + goodTarget: []mockResult{ + {failed: []string{"m1", "m2"}, err: "Error 1"}, + {failed: []string{"m1", "m2"}, err: "Error 2"}, + {failed: []string{"m1", "m2"}, err: "Error 3"}, + {failed: []string{"m1", "m2"}, err: "Error 4"}, + {failed: []string{"m1", "m2"}, err: "Error 5"}, + {failed: []string{"m1", "m2"}, err: "Error 6"}, + }, + } + + output := run(inputMessages, mocks) + assert.Empty(t, output.sentToGood) + assert.Empty(t, output.sentToFailed) + assert.Equal(t, "Error 6", output.err.Error()) +} + +func run(input []*models.Message, targetMocks targetMocks) testOutput { + config, _ := config.NewConfig() + goodTarget := testTarget{results: targetMocks.goodTarget} + failureTarget := testTarget{results: targetMocks.failureTarget} + failure, _ := failure.NewSnowplowFailure(&failureTarget, "test-processor", "test-version") + obs := observer.New(testStatsReceiver{}, time.Minute, time.Second) + trans := func(m []*models.Message) *models.TransformationResult { + return models.NewTransformationResult(m, nil, nil) + } + + f := sourceWriteFunc(&goodTarget, failure, trans, obs, config) + + err := f(input) + + return testOutput{ + sentToGood: goodTarget.sent, + sentToFailed: failureTarget.sent, + err: err, + } +} + +func (t *testTarget) Write(messages []*models.Message) (*models.TargetWriteResult, error) { + nextResponse := t.results[t.writesCounter] + t.writesCounter++ + + var err error + sent := findByKey(nextResponse.sent, messages) + failed := findByKey(nextResponse.failed, messages) + invalid := findByKey(nextResponse.invalid, messages) + oversized := findByKey(nextResponse.oversized, messages) + + if nextResponse.err != "" { + err = errors.New(nextResponse.err) + } + + for _, m := range sent { + t.sent = append(t.sent, m.PartitionKey) + } + + result := models.NewTargetWriteResult(sent, failed, oversized, invalid) + return result, err +} + +func message(key string, input string) *models.Message { + return &models.Message{PartitionKey: key, Data: []byte(input)} +} + +func findByKey(keys []string, messages []*models.Message) []*models.Message { + var out []*models.Message + + for _, msg := range messages { + if slices.Contains(keys, msg.PartitionKey) { + out = append(out, msg) + } + } + + return out +} + +type testOutput struct { + sentToGood []string + sentToFailed []string + err error +} + +type testTarget struct { + writesCounter int + results []mockResult + sent []string +} + +type targetMocks struct { + goodTarget []mockResult + failureTarget []mockResult +} + +type mockResult struct { + sent []string + failed []string + invalid []string + oversized []string + err string +} + +func (t *testTarget) Open() {} +func (t *testTarget) Close() {} +func (t *testTarget) MaximumAllowedMessageSizeBytes() int { + return 1000 +} +func (t *testTarget) GetID() string { + return "test target" +} + +type testStatsReceiver struct { + stats []*models.ObserverBuffer +} + +func (r testStatsReceiver) Send(buffer *models.ObserverBuffer) { + r.stats = append(r.stats, buffer) +} diff --git a/config/component_test.go b/config/component_test.go index 2cd0cd6b..964a3771 100644 --- a/config/component_test.go +++ b/config/component_test.go @@ -87,6 +87,10 @@ func TestCreateTargetComponentHCL(t *testing.T) { KeyFile: "", CaFile: "", SkipVerifyTLS: false, + ResponseRules: &target.ResponseRules{ + Invalid: []target.Rule{}, + SetupError: []target.Rule{}, + }, }, }, { @@ -112,6 +116,23 @@ func TestCreateTargetComponentHCL(t *testing.T) { SkipVerifyTLS: true, DynamicHeaders: true, TemplateFile: "myTemplate.file", + ResponseRules: &target.ResponseRules{ + Invalid: []target.Rule{ + { + MatchingHTTPCodes: []int{400}, + MatchingBodyPart: "Invalid value for 'purchase' field", + }, + { + MatchingHTTPCodes: []int{400}, + MatchingBodyPart: "Invalid value for 'attributes' field", + }, + }, + SetupError: []target.Rule{ + { + MatchingHTTPCodes: []int{401, 403}, + }, + }, + }, }, }, { diff --git a/config/config.go b/config/config.go index a4b321fd..558ccbfc 100644 --- a/config/config.go +++ b/config/config.go @@ -55,6 +55,7 @@ type configurationData struct { UserProvidedID string `hcl:"user_provided_id,optional"` DisableTelemetry bool `hcl:"disable_telemetry,optional"` License *licenseConfig `hcl:"license,block"` + Retry *retryConfig `hcl:"retry,block"` } // component is a type to abstract over configuration blocks. @@ -94,6 +95,20 @@ type licenseConfig struct { Accept bool `hcl:"accept,optional"` } +type retryConfig struct { + Transient *transientRetryConfig `hcl:"transient,block"` + Setup *setupRetryConfig `hcl:"setup,block"` +} + +type transientRetryConfig struct { + Delay int `hcl:"delay_sec,optional"` + MaxAttempts int `hcl:"max_attempts,optional"` +} + +type setupRetryConfig struct { + Delay int `hcl:"delay_sec,optional"` +} + // defaultConfigData returns the initial main configuration target. func defaultConfigData() *configurationData { return &configurationData{ @@ -118,6 +133,15 @@ func defaultConfigData() *configurationData { License: &licenseConfig{ Accept: false, }, + Retry: &retryConfig{ + Transient: &transientRetryConfig{ + Delay: 2, + MaxAttempts: 5, + }, + Setup: &setupRetryConfig{ + Delay: 20, + }, + }, } } diff --git a/config/config_test.go b/config/config_test.go index 02a79276..22e6e08e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -44,6 +44,9 @@ func TestNewConfig_NoConfig(t *testing.T) { assert.Equal(c.Data.LogLevel, "info") assert.Equal(c.Data.DisableTelemetry, false) assert.Equal(c.Data.License.Accept, false) + assert.Equal(2, c.Data.Retry.Transient.Delay) + assert.Equal(5, c.Data.Retry.Transient.MaxAttempts) + assert.Equal(20, c.Data.Retry.Setup.Delay) } func TestNewConfig_InvalidFailureFormat(t *testing.T) { @@ -173,6 +176,9 @@ func TestNewConfig_Hcl_defaults(t *testing.T) { assert.Equal(1, c.Data.StatsReceiver.TimeoutSec) assert.Equal(15, c.Data.StatsReceiver.BufferSec) assert.Equal("info", c.Data.LogLevel) + assert.Equal(2, c.Data.Retry.Transient.Delay) + assert.Equal(5, c.Data.Retry.Transient.MaxAttempts) + assert.Equal(20, c.Data.Retry.Setup.Delay) } func TestNewConfig_Hcl_sentry(t *testing.T) { diff --git a/docs/configuration_retry_docs_test.go b/docs/configuration_retry_docs_test.go new file mode 100644 index 00000000..a6c8561f --- /dev/null +++ b/docs/configuration_retry_docs_test.go @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2020-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION + * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. + */ + +package docs + +import ( + "path/filepath" + "testing" + + "github.com/snowplow/snowbridge/assets" + "github.com/stretchr/testify/assert" +) + +func TestRetryConfigDocumentation(t *testing.T) { + assert := assert.New(t) + retryFilePath := filepath.Join(assets.AssetsRootDir, "docs", "configuration", "retry-example.hcl") + c := getConfigFromFilepath(t, retryFilePath) + + retryConfig := c.Data.Retry + assert.NotNil(retryConfig) + assert.Equal(5, retryConfig.Transient.Delay) + assert.Equal(10, retryConfig.Transient.MaxAttempts) + assert.Equal(30, retryConfig.Setup.Delay) +} diff --git a/go.mod b/go.mod index 8e5f805f..4c146de5 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/smira/go-statsd v1.3.3 - github.com/snowplow-devops/go-retry v0.0.0-20210106090855-8989bbdbae1c github.com/snowplow-devops/go-sentryhook v0.0.0-20210106082031-21bf7f9dac2a github.com/snowplow/snowplow-golang-analytics-sdk v0.3.0 github.com/stretchr/testify v1.9.0 @@ -43,12 +42,14 @@ require ( ) require ( + github.com/avast/retry-go/v4 v4.6.0 github.com/davecgh/go-spew v1.1.1 github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 github.com/hashicorp/hcl/v2 v2.20.1 github.com/itchyny/gojq v0.12.16 github.com/json-iterator/go v1.1.12 github.com/snowplow/snowplow-golang-tracker/v2 v2.4.1 + github.com/twinj/uuid v1.0.0 github.com/zclconf/go-cty v1.14.4 ) @@ -96,6 +97,7 @@ require ( github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/myesui/uuid v1.0.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -116,6 +118,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/stretchr/testify.v1 v1.2.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8a680d1a..44ce218f 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= +github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= github.com/aws/aws-sdk-go v1.44.334 h1:h2bdbGb//fez6Sv6PaYv868s9liDeoYM6hYsAqTB4MU= github.com/aws/aws-sdk-go v1.44.334/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= @@ -299,6 +301,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/myesui/uuid v1.0.0 h1:xCBmH4l5KuvLYc5L7AS7SZg9/jKdIFubM7OVoLqaQUI= +github.com/myesui/uuid v1.0.0/go.mod h1:2CDfNgU0LR8mIdO8vdWd8i9gWWxLlcoIGGpSNgafq84= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= @@ -335,8 +339,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI= github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A= -github.com/snowplow-devops/go-retry v0.0.0-20210106090855-8989bbdbae1c h1:139vLp7J4q+GBcwDDINC8N5KboeG7ejt7j6r19xV5ZU= -github.com/snowplow-devops/go-retry v0.0.0-20210106090855-8989bbdbae1c/go.mod h1:PWBCMlOb8I7TfC0DQFH+4xFSfTcW3iPPcDLHkklJa3A= github.com/snowplow-devops/go-sentryhook v0.0.0-20210106082031-21bf7f9dac2a h1:9T2asgfkxijl85+wpKyCje4DgcjqAJH2czqEWk6+HI0= github.com/snowplow-devops/go-sentryhook v0.0.0-20210106082031-21bf7f9dac2a/go.mod h1:7/jMxl0yrvgiUlv5L37fw6pql71aNh55sKQc4kBFj5s= github.com/snowplow-devops/kinsumer v1.3.0 h1:uN8PPG8EffKjcfTcDqsHWnnsTFvYGMU39XlDPULIQcA= @@ -365,6 +367,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= +github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -552,6 +556,8 @@ gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/stretchr/testify.v1 v1.2.2 h1:yhQC6Uy5CqibAIlk1wlusa/MJ3iAN49/BsR/dCCKz3M= +gopkg.in/stretchr/testify.v1 v1.2.2/go.mod h1:QI5V/q6UbPmuhtm10CaFZxED9NreB8PnFYN9JcR6TxU= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/models/target_write_error.go b/pkg/models/target_write_error.go new file mode 100644 index 00000000..02aff900 --- /dev/null +++ b/pkg/models/target_write_error.go @@ -0,0 +1,21 @@ +/** + * Copyright (c) 2020-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION + * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. + */ + +package models + +// SetupWriteError is a wrapper for target write error. It is used by any target as a signal for a caller that this kind of error should be retried using 'setup-like' retry strategy. +type SetupWriteError struct { + Err error +} + +func (err SetupWriteError) Error() string { + return err.Err.Error() +} diff --git a/pkg/target/http.go b/pkg/target/http.go index 15e8bac4..6ef2b798 100644 --- a/pkg/target/http.go +++ b/pkg/target/http.go @@ -16,9 +16,11 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/url" "os" + "strings" "text/template" "time" @@ -55,7 +57,26 @@ type HTTPTargetConfig struct { RequestByteLimit int `hcl:"request_byte_limit,optional"` // note: breaking change here MessageByteLimit int `hcl:"message_byte_limit,optional"` - TemplateFile string `hcl:"template_file,optional"` + TemplateFile string `hcl:"template_file,optional"` + ResponseRules *ResponseRules `hcl:"response_rules,block"` +} + +// ResponseRules is part of HTTP target configuration. It provides rules how HTTP respones should be handled. Response can be categerized as 'invalid' (bad data), as setup error or (if none of the rules matches) as a transient error. +type ResponseRules struct { + Invalid []Rule `hcl:"invalid,block"` + SetupError []Rule `hcl:"setup,block"` +} + +// Rule configuration defines what kind of values are expected to exist in HTTP response, like status code or message in the body. +type Rule struct { + MatchingHTTPCodes []int `hcl:"http_codes,optional"` + MatchingBodyPart string `hcl:"body,optional"` +} + +// Helper struct storing response HTTP status code and parsed response body +type response struct { + Status int + Body string } // HTTPTarget holds a new client for writing messages to HTTP endpoints @@ -75,6 +96,7 @@ type HTTPTarget struct { requestTemplate *template.Template approxTmplSize int + responseRules *ResponseRules } func checkURL(str string) error { @@ -133,7 +155,8 @@ func newHTTPTarget( oAuth2ClientSecret string, oAuth2RefreshToken string, oAuth2TokenURL string, - templateFile string) (*HTTPTarget, error) { + templateFile string, + responseRules *ResponseRules) (*HTTPTarget, error) { err := checkURL(httpURL) if err != nil { return nil, err @@ -179,6 +202,7 @@ func newHTTPTarget( requestTemplate: requestTemplate, approxTmplSize: approxTmplSize, + responseRules: responseRules, }, nil } @@ -259,6 +283,7 @@ func HTTPTargetConfigFunction(c *HTTPTargetConfig) (*HTTPTarget, error) { c.OAuth2RefreshToken, c.OAuth2TokenURL, c.TemplateFile, + c.ResponseRules, ) } @@ -282,6 +307,10 @@ func (f HTTPTargetAdapter) ProvideDefault() (interface{}, error) { RequestTimeoutInSeconds: 5, ContentType: "application/json", + ResponseRules: &ResponseRules{ + Invalid: []Rule{}, + SetupError: []Rule{}, + }, } return cfg, nil @@ -313,6 +342,7 @@ func (ht *HTTPTarget) Write(messages []*models.Message) (*models.TargetWriteResu failed := []*models.Message{} invalid := []*models.Message{} var errResult error + var hitSetupError bool for _, chunk := range chunks { grouped := ht.groupByDynamicHeaders(chunk) @@ -336,11 +366,11 @@ func (ht *HTTPTarget) Write(messages []*models.Message) (*models.TargetWriteResu } request, err := http.NewRequest("POST", ht.httpURL, bytes.NewBuffer(reqBody)) + if err != nil { - failed = append(failed, goodMsgs...) - errResult = errors.Wrap(errResult, "Error creating request: "+err.Error()) - continue + panic(err) } + request.Header.Add("Content-Type", ht.contentType) // Add content type addHeadersToRequest(request, ht.headers, ht.retrieveHeaders(goodMsgs[0])) // Add headers if there are any - because they're grouped by header, we just need to pick the header from one message if ht.basicAuthUsername != "" && ht.basicAuthPassword != "" { // Add basic auth if set @@ -362,6 +392,7 @@ func (ht *HTTPTarget) Write(messages []*models.Message) (*models.TargetWriteResu continue } defer resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 300 { for _, msg := range goodMsgs { if msg.AckFunc != nil { // Ack successful messages @@ -369,18 +400,83 @@ func (ht *HTTPTarget) Write(messages []*models.Message) (*models.TargetWriteResu } sent = append(sent, msg) } - } else { - errResult = multierror.Append(errResult, errors.New("Got response status: "+resp.Status)) + continue + } + + responseBody, err := io.ReadAll(resp.Body) + + if err != nil { failed = append(failed, goodMsgs...) + errResult = multierror.Append(errResult, errors.New("Error reading response body: "+err.Error())) continue } + + response := response{Body: string(responseBody), Status: resp.StatusCode} + + if findMatchingRule(response, ht.responseRules.Invalid) != nil { + for _, msg := range goodMsgs { + // can we use response body as an error message for invalid data? + msg.SetError(errors.New(response.Body)) + } + + invalid = append(invalid, goodMsgs...) + continue + } + + var errorDetails error + if rule := findMatchingRule(response, ht.responseRules.SetupError); rule != nil { + hitSetupError = true + if rule.MatchingBodyPart != "" { + errorDetails = fmt.Errorf("Got setup error, response status: '%s' with error details: '%s'", resp.Status, rule.MatchingBodyPart) + } else { + errorDetails = fmt.Errorf("Got setup error, response status: '%s'", resp.Status) + } + } else { + errorDetails = fmt.Errorf("Got transient error, response status: '%s'", resp.Status) + } + errResult = multierror.Append(errResult, errorDetails) + failed = append(failed, goodMsgs...) } } + if hitSetupError { + errResult = models.SetupWriteError{Err: errResult} + } + ht.log.Debugf("Successfully wrote %d/%d messages", len(sent), len(messages)) return models.NewTargetWriteResult(sent, failed, oversized, invalid), errResult } +func findMatchingRule(res response, rules []Rule) *Rule { + for _, rule := range rules { + if ruleMatches(res, rule) { + return &rule + } + } + return nil +} + +func ruleMatches(res response, rule Rule) bool { + codeMatch := httpStatusMatches(res.Status, rule.MatchingHTTPCodes) + if rule.MatchingBodyPart != "" { + return codeMatch && responseBodyMatches(res.Body, rule.MatchingBodyPart) + } + return codeMatch +} + +func httpStatusMatches(actual int, expectedCodes []int) bool { + for _, expected := range expectedCodes { + if expected == actual { + return true + } + } + return false +} + +func responseBodyMatches(actual string, bodyPattern string) bool { + return strings.Contains(actual, bodyPattern) +} + // Open does nothing for this target func (ht *HTTPTarget) Open() {} diff --git a/pkg/target/http_oauth2_test.go b/pkg/target/http_oauth2_test.go index 1abeae33..bbc9b08a 100644 --- a/pkg/target/http_oauth2_test.go +++ b/pkg/target/http_oauth2_test.go @@ -102,7 +102,7 @@ func TestHTTP_OAuth2_CallTargetWithoutToken(t *testing.T) { writeResult, err := runTest(t, "", "", "") assert.NotNil(err) - assert.Contains(err.Error(), `Got response status: 403 Forbidden`) + assert.Contains(err.Error(), `Got transient error, response status: '403 Forbidden'`) assert.Equal(0, len(writeResult.Sent)) assert.Equal(1, len(writeResult.Failed)) } @@ -120,7 +120,7 @@ func runTest(t *testing.T, inputClientID string, inputClientSecret string, input } func oauth2Target(t *testing.T, targetURL string, inputClientID string, inputClientSecret string, inputRefreshToken string, tokenServerURL string) *HTTPTarget { - target, err := newHTTPTarget(targetURL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, inputClientID, inputClientSecret, inputRefreshToken, tokenServerURL, "") + target, err := newHTTPTarget(targetURL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, inputClientID, inputClientSecret, inputRefreshToken, tokenServerURL, "", defaultResponseRules()) if err != nil { t.Fatal(err) } diff --git a/pkg/target/http_response_rules_test.go b/pkg/target/http_response_rules_test.go new file mode 100644 index 00000000..82338a64 --- /dev/null +++ b/pkg/target/http_response_rules_test.go @@ -0,0 +1,78 @@ +/** + * Copyright (c) 2020-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION + * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. + */ + +package target + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHTTP_Rules_StatusMatch(t *testing.T) { + assert := assert.New(t) + + response := response{Status: 500, Body: "Invalid field 'attribute'"} + rules := []Rule{ + {MatchingHTTPCodes: []int{500, 503}}, + } + + matchingRule := findMatchingRule(response, rules) + assert.Equal(&rules[0], matchingRule) +} + +func TestHTTP_Rules_FullBodyMatch(t *testing.T) { + assert := assert.New(t) + + response := response{Status: 500, Body: "Invalid field 'attribute'"} + rules := []Rule{ + {MatchingHTTPCodes: []int{500, 503}, MatchingBodyPart: "Invalid field 'attribute'"}, + } + + matchingRule := findMatchingRule(response, rules) + assert.Equal(&rules[0], matchingRule) +} + +func TestHTTP_Rules_PartialBodyMatch(t *testing.T) { + assert := assert.New(t) + + response := response{Status: 500, Body: "Invalid field 'attribute'"} + rules := []Rule{ + {MatchingHTTPCodes: []int{500, 503}, MatchingBodyPart: "Invalid field"}, + } + + matchingRule := findMatchingRule(response, rules) + assert.Equal(&rules[0], matchingRule) +} + +func TestHTTP_Rules_StatusMatch_NoBodyMatch(t *testing.T) { + assert := assert.New(t) + + response := response{Status: 500, Body: "Invalid field 'attribute'"} + rules := []Rule{ + {MatchingHTTPCodes: []int{500, 503}, MatchingBodyPart: "Invalid field 'events'"}, + } + + matchingRule := findMatchingRule(response, rules) + assert.Nil(matchingRule) +} + +func TestHTTP_Rules_NoStatusMatch_BodyMatch(t *testing.T) { + assert := assert.New(t) + + response := response{Status: 500, Body: "Invalid field 'attribute'"} + rules := []Rule{ + {MatchingHTTPCodes: []int{503}, MatchingBodyPart: "Invalid field"}, + } + + matchingRule := findMatchingRule(response, rules) + assert.Nil(matchingRule) +} diff --git a/pkg/target/http_test.go b/pkg/target/http_test.go index 70217833..49fd82bc 100644 --- a/pkg/target/http_test.go +++ b/pkg/target/http_test.go @@ -31,7 +31,7 @@ import ( "github.com/snowplow/snowbridge/pkg/testutil" ) -func createTestServerWithResponseCode(results *[][]byte, responseCode int) *httptest.Server { +func createTestServerWithResponseCode(results *[][]byte, responseCode int, responseBody string) *httptest.Server { mutex := &sync.Mutex{} return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() @@ -42,12 +42,13 @@ func createTestServerWithResponseCode(results *[][]byte, responseCode int) *http mutex.Lock() *results = append(*results, data) w.WriteHeader(responseCode) + w.Write([]byte(responseBody)) mutex.Unlock() })) } func createTestServer(results *[][]byte) *httptest.Server { - return createTestServerWithResponseCode(results, 200) + return createTestServerWithResponseCode(results, 200, "") } func TestHTTP_GetHeaders(t *testing.T) { @@ -307,12 +308,12 @@ func TestHTTP_AddHeadersToRequest_WithDynamicHeaders(t *testing.T) { func TestHTTP_NewHTTPTarget(t *testing.T) { assert := assert.New(t) - httpTarget, err := newHTTPTarget("http://something", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + httpTarget, err := newHTTPTarget("http://something", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) assert.Nil(err) assert.NotNil(httpTarget) - failedHTTPTarget, err1 := newHTTPTarget("something", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + failedHTTPTarget, err1 := newHTTPTarget("something", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) assert.NotNil(err1) if err1 != nil { @@ -320,7 +321,7 @@ func TestHTTP_NewHTTPTarget(t *testing.T) { } assert.Nil(failedHTTPTarget) - failedHTTPTarget2, err2 := newHTTPTarget("", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + failedHTTPTarget2, err2 := newHTTPTarget("", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) assert.NotNil(err2) if err2 != nil { assert.Equal("Invalid url for HTTP target: ''", err2.Error()) @@ -345,10 +346,10 @@ func TestHTTP_Write_Simple(t *testing.T) { var results [][]byte wg := sync.WaitGroup{} - server := createTestServerWithResponseCode(&results, tt.ResponseCode) + server := createTestServerWithResponseCode(&results, tt.ResponseCode, "") defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -409,10 +410,10 @@ func TestHTTP_Write_Batched(t *testing.T) { var results [][]byte wg := sync.WaitGroup{} - server := createTestServerWithResponseCode(&results, 200) + server := createTestServerWithResponseCode(&results, 200, "") defer server.Close() - target, err := newHTTPTarget(server.URL, 5, tt.BatchSize, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + target, err := newHTTPTarget(server.URL, 5, tt.BatchSize, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -471,7 +472,7 @@ func TestHTTP_Write_Concurrent(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -515,7 +516,7 @@ func TestHTTP_Write_Failure(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget("http://NonexistentEndpoint", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + target, err := newHTTPTarget("http://NonexistentEndpoint", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -555,9 +556,9 @@ func TestHTTP_Write_InvalidResponseCode(t *testing.T) { assert := assert.New(t) var results [][]byte - server := createTestServerWithResponseCode(&results, tt.ResponseCode) + server := createTestServerWithResponseCode(&results, tt.ResponseCode, "") defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -592,7 +593,7 @@ func TestHTTP_Write_Oversized(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "") + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -635,7 +636,7 @@ func TestHTTP_Write_EnabledTemplating(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`)) + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -680,7 +681,67 @@ func TestHTTP_Write_EnabledTemplating(t *testing.T) { // openssl req -new -key localhost.key -out localhost.csr -subj "/CN=localhost" -addext "subjectAltName = DNS:localhost" // openssl x509 -req -in localhost.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -days 365 -out localhost.crt +func TestHTTP_Write_Invalid(t *testing.T) { + assert := assert.New(t) + + var results [][]byte + server := createTestServerWithResponseCode(&results, 400, "Request is invalid. Invalid value for field 'attribute'") + defer server.Close() + + responseRules := ResponseRules{ + Invalid: []Rule{ + {MatchingHTTPCodes: []int{400, 401}, MatchingBodyPart: "Invalid value for field 'attribute'"}, + }, + } + + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules) + if err != nil { + t.Fatal(err) + } + + input := []*models.Message{{Data: []byte(`{ "attribute": "value"}`)}} + + writeResult, err1 := target.Write(input) + + assert.Nil(err1) + assert.Equal(0, len(writeResult.Sent)) + assert.Equal(1, len(writeResult.Invalid)) + assert.Equal("Request is invalid. Invalid value for field 'attribute'", writeResult.Invalid[0].GetError().Error()) +} + +func TestHTTP_Write_Setup(t *testing.T) { + assert := assert.New(t) + + var results [][]byte + server := createTestServerWithResponseCode(&results, 401, "Authentication issue. Invalid token") + defer server.Close() + + responseRules := ResponseRules{ + SetupError: []Rule{ + {MatchingHTTPCodes: []int{401}, MatchingBodyPart: "Invalid token"}, + }, + } + + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules) + if err != nil { + t.Fatal(err) + } + + input := []*models.Message{{Data: []byte(`{ "attribute": "value"}`)}} + + writeResult, err := target.Write(input) + + assert.Equal(0, len(writeResult.Sent)) + assert.Equal(0, len(writeResult.Invalid)) + assert.Equal(1, len(writeResult.Failed)) + + _, isSetup := err.(models.SetupWriteError) + assert.True(isSetup) + assert.Regexp(".*Got setup error, response status: '401 Unauthorized' with error details: 'Invalid token'", err.Error()) +} + func TestHTTP_Write_TLS(t *testing.T) { + if testing.Short() { t.Skip("skipping integration test") } @@ -705,7 +766,8 @@ func TestHTTP_Write_TLS(t *testing.T) { "", "", "", - "") + "", + defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -745,7 +807,8 @@ func TestHTTP_Write_TLS(t *testing.T) { "", "", "", - "") + "", + defaultResponseRules()) if err2 != nil { t.Fatal(err2) } @@ -778,7 +841,8 @@ func TestHTTP_Write_TLS(t *testing.T) { "", "", "", - "") + "", + defaultResponseRules()) if err4 != nil { t.Fatal(err4) } @@ -933,7 +997,7 @@ func TestHTTP_Write_GroupedRequests(t *testing.T) { defer server.Close() //dynamicHeaders enabled - target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, true, "", "", "", "", "") + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, true, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -1010,6 +1074,13 @@ func getNgrokAddress() string { panic("no ngrok https endpoint found") } +func defaultResponseRules() *ResponseRules { + return &ResponseRules{ + Invalid: []Rule{}, + SetupError: []Rule{}, + } +} + func WaitForAcksWithTimeout(timeout time.Duration, wg *sync.WaitGroup) bool { c := make(chan struct{}) go func() { From b01ef54af6bd5c55cde965b71e00d838984df3ea Mon Sep 17 00:00:00 2001 From: colmsnowplow Date: Wed, 18 Sep 2024 12:31:25 +0100 Subject: [PATCH 2/8] Add boolean config to enable TLS (#370) * Add boolean config to enable TLS * Fix config test --- .../sources/kafka-full-example.hcl | 3 ++ .../targets/http-full-example.hcl | 3 ++ .../targets/kafka-full-example.hcl | 3 ++ config/component_test.go | 4 +++ pkg/source/kafka/kafka_source.go | 11 +++---- pkg/target/http.go | 18 ++++++++---- pkg/target/http_oauth2_test.go | 2 +- pkg/target/http_test.go | 29 ++++++++++--------- pkg/target/kafka.go | 9 +++--- 9 files changed, 53 insertions(+), 29 deletions(-) diff --git a/assets/docs/configuration/sources/kafka-full-example.hcl b/assets/docs/configuration/sources/kafka-full-example.hcl index 00ce3cfb..f13ef823 100644 --- a/assets/docs/configuration/sources/kafka-full-example.hcl +++ b/assets/docs/configuration/sources/kafka-full-example.hcl @@ -34,6 +34,9 @@ source { # The SASL Algorithm to use: "plaintext", "sha512" or "sha256" (default: "sha512") sasl_algorithm = "sha256" + # Whether to enable TLS + enable_tls = true + # The optional certificate file for client authentication cert_file = "myLocalhost.crt" diff --git a/assets/docs/configuration/targets/http-full-example.hcl b/assets/docs/configuration/targets/http-full-example.hcl index f2b4e2b4..9a8f8a64 100644 --- a/assets/docs/configuration/targets/http-full-example.hcl +++ b/assets/docs/configuration/targets/http-full-example.hcl @@ -32,6 +32,9 @@ target { # you could also reference an environment variable. basic_auth_password = env.MY_AUTH_PASSWORD + # Whether to enable TLS + enable_tls = true + # The optional certificate file for client authentication cert_file = "myLocalhost.crt" diff --git a/assets/docs/configuration/targets/kafka-full-example.hcl b/assets/docs/configuration/targets/kafka-full-example.hcl index 41df612d..61f42ead 100644 --- a/assets/docs/configuration/targets/kafka-full-example.hcl +++ b/assets/docs/configuration/targets/kafka-full-example.hcl @@ -38,6 +38,9 @@ target { # The SASL Algorithm to use: "plaintext", "sha512" or "sha256" (default: "sha512") sasl_algorithm = "sha256" + # Whether to enable TLS + enable_tls = true + # The optional certificate file for client authentication cert_file = "myLocalhost.crt" diff --git a/config/component_test.go b/config/component_test.go index 964a3771..d95bdb03 100644 --- a/config/component_test.go +++ b/config/component_test.go @@ -83,6 +83,7 @@ func TestCreateTargetComponentHCL(t *testing.T) { Headers: "", BasicAuthUsername: "", BasicAuthPassword: "", + EnableTLS: false, CertFile: "", KeyFile: "", CaFile: "", @@ -110,6 +111,7 @@ func TestCreateTargetComponentHCL(t *testing.T) { OAuth2ClientSecret: "myClientSecret", OAuth2RefreshToken: "myRefreshToken", OAuth2TokenURL: "https://my.auth.server/token", + EnableTLS: true, CertFile: "myLocalhost.crt", KeyFile: "myLocalhost.key", CaFile: "myRootCA.crt", @@ -151,6 +153,7 @@ func TestCreateTargetComponentHCL(t *testing.T) { SASLUsername: "", SASLPassword: "", SASLAlgorithm: "sha512", + EnableTLS: false, CertFile: "", KeyFile: "", CaFile: "", @@ -177,6 +180,7 @@ func TestCreateTargetComponentHCL(t *testing.T) { SASLUsername: "mySaslUsername", SASLPassword: "mySASLPassword", SASLAlgorithm: "sha256", + EnableTLS: true, CertFile: "myLocalhost.crt", KeyFile: "myLocalhost.key", CaFile: "myRootCA.crt", diff --git a/pkg/source/kafka/kafka_source.go b/pkg/source/kafka/kafka_source.go index cd63380e..b975873b 100644 --- a/pkg/source/kafka/kafka_source.go +++ b/pkg/source/kafka/kafka_source.go @@ -42,6 +42,7 @@ type Configuration struct { SASLUsername string `hcl:"sasl_username,optional" ` SASLPassword string `hcl:"sasl_password,optional"` SASLAlgorithm string `hcl:"sasl_algorithm,optional"` + EnableTLS bool `hcl:"enable_tls,optional"` CertFile string `hcl:"cert_file,optional"` KeyFile string `hcl:"key_file,optional"` CaFile string `hcl:"ca_file,optional"` @@ -50,7 +51,6 @@ type Configuration struct { // kafkaSource holds a new client for reading messages from Apache Kafka type kafkaSource struct { - config *sarama.Config concurrentWrites int topic string brokers string @@ -197,6 +197,7 @@ func (f adapter) ProvideDefault() (interface{}, error) { Assignor: "range", SASLAlgorithm: "sha512", ConcurrentWrites: 15, + EnableTLS: false, } return cfg, nil @@ -259,14 +260,14 @@ func newKafkaSource(cfg *Configuration) (*kafkaSource, error) { } } + // returns nil, nil if provided empty certs tlsConfig, err := common.CreateTLSConfiguration(cfg.CertFile, cfg.KeyFile, cfg.CaFile, cfg.SkipVerifyTLS) if err != nil { return nil, err } - if tlsConfig != nil { - saramaConfig.Net.TLS.Config = tlsConfig - saramaConfig.Net.TLS.Enable = true - } + + saramaConfig.Net.TLS.Enable = cfg.EnableTLS + saramaConfig.Net.TLS.Config = tlsConfig client, err := sarama.NewConsumerGroup(strings.Split(cfg.Brokers, ","), fmt.Sprintf(`%s-%s`, cfg.ConsumerName, cfg.TopicName), saramaConfig) if err != nil { diff --git a/pkg/target/http.go b/pkg/target/http.go index 6ef2b798..5ea521f2 100644 --- a/pkg/target/http.go +++ b/pkg/target/http.go @@ -42,11 +42,13 @@ type HTTPTargetConfig struct { Headers string `hcl:"headers,optional"` BasicAuthUsername string `hcl:"basic_auth_username,optional"` BasicAuthPassword string `hcl:"basic_auth_password,optional"` - CertFile string `hcl:"cert_file,optional"` - KeyFile string `hcl:"key_file,optional"` - CaFile string `hcl:"ca_file,optional"` - SkipVerifyTLS bool `hcl:"skip_verify_tls,optional"` // false - DynamicHeaders bool `hcl:"dynamic_headers,optional"` + + EnableTLS bool `hcl:"enable_tls,optional"` + CertFile string `hcl:"cert_file,optional"` + KeyFile string `hcl:"key_file,optional"` + CaFile string `hcl:"ca_file,optional"` + SkipVerifyTLS bool `hcl:"skip_verify_tls,optional"` // false + DynamicHeaders bool `hcl:"dynamic_headers,optional"` OAuth2ClientID string `hcl:"oauth2_client_id,optional"` OAuth2ClientSecret string `hcl:"oauth2_client_secret,optional"` @@ -146,6 +148,7 @@ func newHTTPTarget( headers string, basicAuthUsername string, basicAuthPassword string, + enableTLS bool, certFile string, keyFile string, caFile string, @@ -171,7 +174,8 @@ func newHTTPTarget( if err2 != nil { return nil, err2 } - if tlsConfig != nil { + + if enableTLS && tlsConfig != nil { transport.TLSClientConfig = tlsConfig } @@ -273,6 +277,7 @@ func HTTPTargetConfigFunction(c *HTTPTargetConfig) (*HTTPTarget, error) { c.Headers, c.BasicAuthUsername, c.BasicAuthPassword, + c.EnableTLS, c.CertFile, c.KeyFile, c.CaFile, @@ -304,6 +309,7 @@ func (f HTTPTargetAdapter) ProvideDefault() (interface{}, error) { RequestMaxMessages: 20, RequestByteLimit: 1048576, MessageByteLimit: 1048576, + EnableTLS: false, RequestTimeoutInSeconds: 5, ContentType: "application/json", diff --git a/pkg/target/http_oauth2_test.go b/pkg/target/http_oauth2_test.go index bbc9b08a..a96aa7c1 100644 --- a/pkg/target/http_oauth2_test.go +++ b/pkg/target/http_oauth2_test.go @@ -120,7 +120,7 @@ func runTest(t *testing.T, inputClientID string, inputClientSecret string, input } func oauth2Target(t *testing.T, targetURL string, inputClientID string, inputClientSecret string, inputRefreshToken string, tokenServerURL string) *HTTPTarget { - target, err := newHTTPTarget(targetURL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, inputClientID, inputClientSecret, inputRefreshToken, tokenServerURL, "", defaultResponseRules()) + target, err := newHTTPTarget(targetURL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, inputClientID, inputClientSecret, inputRefreshToken, tokenServerURL, "", defaultResponseRules()) if err != nil { t.Fatal(err) } diff --git a/pkg/target/http_test.go b/pkg/target/http_test.go index 49fd82bc..f7341f5c 100644 --- a/pkg/target/http_test.go +++ b/pkg/target/http_test.go @@ -308,12 +308,12 @@ func TestHTTP_AddHeadersToRequest_WithDynamicHeaders(t *testing.T) { func TestHTTP_NewHTTPTarget(t *testing.T) { assert := assert.New(t) - httpTarget, err := newHTTPTarget("http://something", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + httpTarget, err := newHTTPTarget("http://something", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) assert.Nil(err) assert.NotNil(httpTarget) - failedHTTPTarget, err1 := newHTTPTarget("something", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + failedHTTPTarget, err1 := newHTTPTarget("something", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) assert.NotNil(err1) if err1 != nil { @@ -321,7 +321,7 @@ func TestHTTP_NewHTTPTarget(t *testing.T) { } assert.Nil(failedHTTPTarget) - failedHTTPTarget2, err2 := newHTTPTarget("", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + failedHTTPTarget2, err2 := newHTTPTarget("", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) assert.NotNil(err2) if err2 != nil { assert.Equal("Invalid url for HTTP target: ''", err2.Error()) @@ -349,7 +349,7 @@ func TestHTTP_Write_Simple(t *testing.T) { server := createTestServerWithResponseCode(&results, tt.ResponseCode, "") defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -413,7 +413,7 @@ func TestHTTP_Write_Batched(t *testing.T) { server := createTestServerWithResponseCode(&results, 200, "") defer server.Close() - target, err := newHTTPTarget(server.URL, 5, tt.BatchSize, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + target, err := newHTTPTarget(server.URL, 5, tt.BatchSize, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -472,7 +472,7 @@ func TestHTTP_Write_Concurrent(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -516,7 +516,7 @@ func TestHTTP_Write_Failure(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget("http://NonexistentEndpoint", 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + target, err := newHTTPTarget("http://NonexistentEndpoint", 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -558,7 +558,7 @@ func TestHTTP_Write_InvalidResponseCode(t *testing.T) { var results [][]byte server := createTestServerWithResponseCode(&results, tt.ResponseCode, "") defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -593,7 +593,7 @@ func TestHTTP_Write_Oversized(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) + target, err := newHTTPTarget(server.URL, 5, 1, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -636,7 +636,7 @@ func TestHTTP_Write_EnabledTemplating(t *testing.T) { server := createTestServer(&results) defer server.Close() - target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), defaultResponseRules()) + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), defaultResponseRules()) if err != nil { t.Fatal(err) } @@ -694,7 +694,7 @@ func TestHTTP_Write_Invalid(t *testing.T) { }, } - target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules) + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules) if err != nil { t.Fatal(err) } @@ -722,7 +722,7 @@ func TestHTTP_Write_Setup(t *testing.T) { }, } - target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules) + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, false, "", "", "", "", string(`../../integration/http/template`), &responseRules) if err != nil { t.Fatal(err) } @@ -757,6 +757,7 @@ func TestHTTP_Write_TLS(t *testing.T) { "", "", "", + true, string(`../../integration/http/localhost.crt`), string(`../../integration/http/localhost.key`), string(`../../integration/http/rootCA.crt`), @@ -798,6 +799,7 @@ func TestHTTP_Write_TLS(t *testing.T) { "", "", "", + true, string(`../../integration/http/localhost.crt`), string(`../../integration/http/localhost.key`), string(`../../integration/http/rootCA.crt`), @@ -832,6 +834,7 @@ func TestHTTP_Write_TLS(t *testing.T) { "", "", "", + false, "", "", "", @@ -997,7 +1000,7 @@ func TestHTTP_Write_GroupedRequests(t *testing.T) { defer server.Close() //dynamicHeaders enabled - target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", "", "", "", true, true, "", "", "", "", "", defaultResponseRules()) + target, err := newHTTPTarget(server.URL, 5, 5, 1048576, 1048576, "application/json", "", "", "", false, "", "", "", true, true, "", "", "", "", "", defaultResponseRules()) if err != nil { t.Fatal(err) } diff --git a/pkg/target/kafka.go b/pkg/target/kafka.go index 720e9b64..a936b4ed 100644 --- a/pkg/target/kafka.go +++ b/pkg/target/kafka.go @@ -38,6 +38,7 @@ type KafkaConfig struct { SASLUsername string `hcl:"sasl_username,optional"` SASLPassword string `hcl:"sasl_password,optional"` SASLAlgorithm string `hcl:"sasl_algorithm,optional"` + EnableTLS bool `hcl:"enable_tls,optional"` CertFile string `hcl:"cert_file,optional"` KeyFile string `hcl:"key_file,optional"` CaFile string `hcl:"ca_file,optional"` @@ -111,14 +112,13 @@ func NewKafkaTarget(cfg *KafkaConfig) (*KafkaTarget, error) { } } + // returns nil if certs are empty tlsConfig, err := common.CreateTLSConfiguration(cfg.CertFile, cfg.KeyFile, cfg.CaFile, cfg.SkipVerifyTLS) if err != nil { return nil, err } - if tlsConfig != nil { - saramaConfig.Net.TLS.Config = tlsConfig - saramaConfig.Net.TLS.Enable = true - } + saramaConfig.Net.TLS.Enable = cfg.EnableTLS + saramaConfig.Net.TLS.Config = tlsConfig var asyncResults chan *saramaResult = nil var asyncProducer sarama.AsyncProducer = nil @@ -187,6 +187,7 @@ func (f KafkaTargetAdapter) ProvideDefault() (interface{}, error) { MaxRetries: 10, ByteLimit: 1048576, SASLAlgorithm: "sha512", + EnableTLS: false, } return cfg, nil From a99a13993e2fc4cd0be84222a3bdd17880e4ddcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Tue, 17 Sep 2024 12:55:25 +0200 Subject: [PATCH 3/8] Add expiry config + base64 validation to the GTM SS preview header transformation --- .../spGtmssPreview-full-example .hcl | 4 -- .../spGtmssPreview-full-example.hcl | 6 ++ ...configuration_transformations_docs_test.go | 2 +- pkg/transform/snowplow_gtmss_preview.go | 37 +++++++++--- pkg/transform/snowplow_gtmss_preview_test.go | 57 +++++++++++++++++-- 5 files changed, 88 insertions(+), 18 deletions(-) delete mode 100644 assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example .hcl create mode 100644 assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example.hcl diff --git a/assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example .hcl b/assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example .hcl deleted file mode 100644 index 63505ac6..00000000 --- a/assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example .hcl +++ /dev/null @@ -1,4 +0,0 @@ -transform { - use "spGtmssPreview" { - } -} diff --git a/assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example.hcl b/assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example.hcl new file mode 100644 index 00000000..a37c74d9 --- /dev/null +++ b/assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example.hcl @@ -0,0 +1,6 @@ +transform { + use "spGtmssPreview" { + # Message expiry time in seconds (comparing current time to the message's collector timestamp). If message is expired, it's sent to failure target. + expiry_seconds = 600 + } +} diff --git a/docs/configuration_transformations_docs_test.go b/docs/configuration_transformations_docs_test.go index 973aba67..6bb6af34 100644 --- a/docs/configuration_transformations_docs_test.go +++ b/docs/configuration_transformations_docs_test.go @@ -44,7 +44,7 @@ func TestBuiltinTransformationDocumentation(t *testing.T) { } func TestBuiltinSnowplowTransformationDocumentation(t *testing.T) { - transformationsToTest := []string{"spEnrichedFilter", "spEnrichedFilterContext", "spEnrichedFilterUnstructEvent", "spEnrichedSetPk", "spEnrichedToJson"} + transformationsToTest := []string{"spEnrichedFilter", "spEnrichedFilterContext", "spEnrichedFilterUnstructEvent", "spEnrichedSetPk", "spEnrichedToJson", "spGtmssPreview"} for _, tfm := range transformationsToTest { diff --git a/pkg/transform/snowplow_gtmss_preview.go b/pkg/transform/snowplow_gtmss_preview.go index 57b4db81..9eaf0e9b 100644 --- a/pkg/transform/snowplow_gtmss_preview.go +++ b/pkg/transform/snowplow_gtmss_preview.go @@ -1,7 +1,9 @@ package transform import ( + "encoding/base64" "errors" + "time" "github.com/snowplow/snowbridge/config" "github.com/snowplow/snowbridge/pkg/models" @@ -11,6 +13,7 @@ import ( // GTMSSPreviewConfig is a configuration object for the spEnrichedToJson transformation type GTMSSPreviewConfig struct { + Expiry int `hcl:"expiry_seconds,optional"` } // The gtmssPreviewAdapter implements the Pluggable interface @@ -18,7 +21,8 @@ type gtmssPreviewAdapter func(i interface{}) (interface{}, error) // ProvideDefault implements the ComponentConfigurable interface func (f gtmssPreviewAdapter) ProvideDefault() (interface{}, error) { - return nil, nil + cfg := >MSSPreviewConfig{Expiry: 300} // seconds -> 5 minutes + return cfg, nil } // Create implements the ComponentCreator interface. @@ -27,22 +31,24 @@ func (f gtmssPreviewAdapter) Create(i interface{}) (interface{}, error) { } // gtmssPreviewAdapterGenerator returns a gtmssPreviewAdapter -func gtmssPreviewAdapterGenerator(f func() (TransformationFunction, error)) gtmssPreviewAdapter { +func gtmssPreviewAdapterGenerator(f func(cfg *GTMSSPreviewConfig) (TransformationFunction, error)) gtmssPreviewAdapter { return func(i interface{}) (interface{}, error) { - if i != nil { + cfg, ok := i.(*GTMSSPreviewConfig) + if !ok { return nil, errors.New("unexpected configuration input for gtmssPreview transformation") } - return f() + return f(cfg) } } // gtmssPreviewConfigFunction returns a transformation function -func gtmssPreviewConfigFunction() (TransformationFunction, error) { +func gtmssPreviewConfigFunction(cfg *GTMSSPreviewConfig) (TransformationFunction, error) { ctx := "contexts_com_google_tag-manager_server-side_preview_mode_1" property := "x-gtm-server-preview" header := "x-gtm-server-preview" - return gtmssPreviewTransformation(ctx, property, header), nil + expiry := time.Duration(cfg.Expiry) * time.Second + return gtmssPreviewTransformation(ctx, property, header, expiry), nil } // GTMSSPreviewConfigPair is the configuration pair for the gtmss preview transformation @@ -52,7 +58,7 @@ var GTMSSPreviewConfigPair = config.ConfigurationPair{ } // gtmssPreviewTransformation returns a transformation function -func gtmssPreviewTransformation(ctx, property, headerKey string) TransformationFunction { +func gtmssPreviewTransformation(ctx, property, headerKey string, expiry time.Duration) TransformationFunction { return func(message *models.Message, interState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { parsedEvent, err := IntermediateAsSpEnrichedParsed(interState, message) if err != nil { @@ -60,6 +66,19 @@ func gtmssPreviewTransformation(ctx, property, headerKey string) TransformationF return nil, nil, message, nil } + tstamp, err := parsedEvent.GetValue("collector_tstamp") + if err != nil { + message.SetError(err) + return nil, nil, message, nil + } + + if collectorTstamp, ok := tstamp.(time.Time); ok { + if time.Now().UTC().After(collectorTstamp.Add(expiry)) { + message.SetError(errors.New("Message has expired")) + return nil, nil, message, nil + } + } + headerVal, err := extractHeaderValue(parsedEvent, ctx, property) if err != nil { message.SetError(err) @@ -96,6 +115,10 @@ func extractHeaderValue(parsedEvent analytics.ParsedEvent, ctx, prop string) (*s return nil, errors.New("invalid header value") } + _, err = base64.StdEncoding.DecodeString(headerVal) + if err != nil { + return nil, err + } return &headerVal, nil } diff --git a/pkg/transform/snowplow_gtmss_preview_test.go b/pkg/transform/snowplow_gtmss_preview_test.go index 76d78bd8..3def4d57 100644 --- a/pkg/transform/snowplow_gtmss_preview_test.go +++ b/pkg/transform/snowplow_gtmss_preview_test.go @@ -5,6 +5,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/davecgh/go-spew/spew" "github.com/stretchr/testify/assert" @@ -13,12 +14,15 @@ import ( "github.com/snowplow/snowplow-golang-analytics-sdk/analytics" ) +const fiftyYears = time.Hour * 24 * 365 * 50 + func TestGTMSSPreview(t *testing.T) { testCases := []struct { Scenario string Ctx string Property string HeaderKey string + Expiry time.Duration InputMsg *models.Message InputInterState interface{} Expected map[string]*models.Message @@ -30,6 +34,7 @@ func TestGTMSSPreview(t *testing.T) { Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", Property: "x-gtm-server-preview", HeaderKey: "x-gtm-server-preview", + Expiry: fiftyYears, InputMsg: &models.Message{ Data: spTsvWithGtmss, PartitionKey: "pk", @@ -54,6 +59,7 @@ func TestGTMSSPreview(t *testing.T) { Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", Property: "x-gtm-server-preview", HeaderKey: "x-gtm-server-preview", + Expiry: fiftyYears, InputMsg: &models.Message{ Data: spTsvNoGtmss, PartitionKey: "pk", @@ -76,6 +82,7 @@ func TestGTMSSPreview(t *testing.T) { Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", Property: "x-gtm-server-preview", HeaderKey: "x-gtm-server-preview", + Expiry: fiftyYears, InputMsg: &models.Message{ Data: []byte(`asdf`), PartitionKey: "pk", @@ -98,6 +105,7 @@ func TestGTMSSPreview(t *testing.T) { Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", Property: "x-gtm-server-preview", HeaderKey: "x-gtm-server-preview", + Expiry: fiftyYears, InputMsg: &models.Message{ Data: spTsvWithGtmss, PartitionKey: "pk", @@ -126,6 +134,7 @@ func TestGTMSSPreview(t *testing.T) { Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", Property: "x-gtm-server-preview", HeaderKey: "x-gtm-server-preview", + Expiry: fiftyYears, InputMsg: &models.Message{ Data: spTsvWithGtmss, PartitionKey: "pk", @@ -155,6 +164,7 @@ func TestGTMSSPreview(t *testing.T) { Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", Property: "x-gtm-server-preview", HeaderKey: "x-gtm-server-preview", + Expiry: fiftyYears, InputMsg: &models.Message{ Data: spTsvWithGtmss, PartitionKey: "pk", @@ -180,6 +190,7 @@ func TestGTMSSPreview(t *testing.T) { Ctx: "app_id", Property: "x-gtm-server-preview", HeaderKey: "x-gtm-server-preview", + Expiry: fiftyYears, InputMsg: &models.Message{ Data: spTsvWithGtmss, PartitionKey: "pk", @@ -200,11 +211,34 @@ func TestGTMSSPreview(t *testing.T) { ExpInterState: spTsvWithGtmssParsed, Error: nil, }, + { + Scenario: "expired_message", + Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", + Property: "x-gtm-server-preview", + HeaderKey: "x-gtm-server-preview", + Expiry: 1 * time.Hour, + InputMsg: &models.Message{ + Data: spTsvWithGtmss, + PartitionKey: "pk", + }, + InputInterState: nil, + Expected: map[string]*models.Message{ + "success": nil, + "filtered": nil, + "failed": { + Data: []byte(spTsvWithGtmss), + PartitionKey: "pk", + HTTPHeaders: nil, + }, + }, + ExpInterState: nil, + Error: errors.New("Message has expired"), + }, } for _, tt := range testCases { t.Run(tt.Scenario, func(t *testing.T) { - transFunction := gtmssPreviewTransformation(tt.Ctx, tt.Property, tt.HeaderKey) + transFunction := gtmssPreviewTransformation(tt.Ctx, tt.Property, tt.HeaderKey, tt.Expiry) s, f, e, i := transFunction(tt.InputMsg, tt.InputInterState) if !reflect.DeepEqual(i, tt.ExpInterState) { @@ -281,13 +315,21 @@ func TestExtractHeaderValue(t *testing.T) { Error: nil, }, { - Scenario: "invalid_header_value", + Scenario: "invalid_header_value (not a string type)", Event: fakeSpTsvParsed, Ctx: "contexts_com_snowplowanalytics_snowplow_web_page_1", Prop: "id", Expected: nil, Error: errors.New("invalid header value"), }, + { + Scenario: "invalid_header_value (not base64 encoding)", + Event: gtmssInvalidNoB64Parsed, + Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1", + Prop: "x-gtm-server-preview", + Expected: nil, + Error: errors.New("illegal base64 data at input"), + }, { Scenario: "event_without_contexts", Event: spTsvNoCtxParsed, @@ -343,7 +385,7 @@ func Benchmark_GTMSSPreview_With_Preview_Ctx_no_intermediate(b *testing.B) { prop := "x-gtm-server-preview" header := "x-gtm-server-preview" - transFunction := gtmssPreviewTransformation(ctx, prop, header) + transFunction := gtmssPreviewTransformation(ctx, prop, header, fiftyYears) for n := 0; n < b.N; n++ { transFunction(inputMsg, nil) @@ -362,7 +404,7 @@ func Benchmark_GTMSSPreview_With_Preview_Ctx_With_intermediate(b *testing.B) { prop := "x-gtm-server-preview" header := "x-gtm-server-preview" - transFunction := gtmssPreviewTransformation(ctx, prop, header) + transFunction := gtmssPreviewTransformation(ctx, prop, header, fiftyYears) for n := 0; n < b.N; n++ { transFunction(inputMsg, interState) @@ -380,7 +422,7 @@ func Benchmark_GTMSSPreview_No_Preview_Ctx_no_intermediate(b *testing.B) { prop := "x-gtm-server-preview" header := "x-gtm-server-preview" - transFunction := gtmssPreviewTransformation(ctx, prop, header) + transFunction := gtmssPreviewTransformation(ctx, prop, header, fiftyYears) for n := 0; n < b.N; n++ { transFunction(inputMsg, nil) @@ -399,7 +441,7 @@ func Benchmark_GTMSSPreview_No_Preview_Ctx_With_intermediate(b *testing.B) { prop := "x-gtm-server-preview" header := "x-gtm-server-preview" - transFunction := gtmssPreviewTransformation(ctx, prop, header) + transFunction := gtmssPreviewTransformation(ctx, prop, header, fiftyYears) for n := 0; n < b.N; n++ { transFunction(inputMsg, interState) @@ -449,6 +491,9 @@ var spTsvNoGtmssParsed, _ = analytics.ParseEvent(string(spTsvNoGtmss)) var spTsvWithGtmss = []byte(`media-test web 2024-03-12 04:27:01.760 2024-03-12 04:27:01.755 2024-03-12 04:27:01.743 unstruct 9be3afe8-8a62-41ac-93db-12f425d82ac9 spTest js-3.17.0 snowplow-micro-2.0.0-stdout$ snowplow-micro-2.0.0 media_tester 172.17.0.1 23a0eb65-83f6-4957-839e-f3044bfefb99 1 a2f53212-26a3-4781-81d6-f14aa8d4552b http://localhost:8000/?sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== http localhost 8000 / sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:org.whatwg/media_element/jsonschema/1-0-0","data":{"htmlId":"bunny-mp4","mediaType":"VIDEO","autoPlay":false,"buffered":[{"start":0,"end":1.291666}],"controls":true,"currentSrc":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","defaultMuted":false,"defaultPlaybackRate":1,"error":null,"networkState":"NETWORK_LOADING","preload":"","readyState":"HAVE_ENOUGH_DATA","seekable":[{"start":0,"end":596.503219}],"seeking":false,"src":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","textTracks":[],"fileExtension":"mp4","fullscreen":false,"pictureInPicture":false}},{"schema":"iglu:com.snowplowanalytics.snowplow/media_player/jsonschema/1-0-0","data":{"currentTime":0,"duration":596.503219,"ended":false,"loop":false,"muted":false,"paused":false,"playbackRate":1,"volume":100}},{"schema":"iglu:org.whatwg/video_element/jsonschema/1-0-0","data":{"poster":"","videoHeight":360,"videoWidth":640}},{"schema":"iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-0-0","data":{"id":"021c4d09-e502-4562-8182-5ac7247125ec"}},{"schema":"iglu:com.google.tag-manager.server-side/user_data/jsonschema/1-0-0","data":{"email_address":"foo@example.com","phone_number":"+15551234567","address":{"first_name":"Jane","last_name":"Doe","street":"123 Fake St","city":"San Francisco","region":"CA","postal_code":"94016","country":"US"}}},{"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-2","data":{"osType":"testOsType","osVersion":"testOsVersion","deviceManufacturer":"testDevMan","deviceModel":"testDevModel"}},{"schema":"iglu:com.google.tag-manager.server-side/preview_mode/jsonschema/1-0-0","data":{"x-gtm-server-preview":"ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw=="}},{"schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-2","data":{"userId":"23a0eb65-83f6-4957-839e-f3044bfefb99","sessionId":"73fcdaa3-0164-41ce-a336-fb00c4ebf68c","eventIndex":7,"sessionIndex":1,"previousSessionId":null,"storageMechanism":"COOKIE_1","firstEventId":"327b9ff9-ed5f-40cf-918a-1b1a775ae347","firstEventTimestamp":"2024-03-12T04:25:36.684Z"}}]} {"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/media_player_event/jsonschema/1-0-0","data":{"type":"play"}}} Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0 en-US 1 24 1920 935 Europe/Athens 1920 1080 windows-1252 1920 935 2024-03-12 04:27:01.745 73fcdaa3-0164-41ce-a336-fb00c4ebf68c 2024-03-12 04:27:01.753 com.snowplowanalytics.snowplow media_player_event jsonschema 1-0-0 `) var spTsvWithGtmssParsed, _ = analytics.ParseEvent(string(spTsvWithGtmss)) +var gtmssInvalidNoB64 = []byte(`media-test web 2024-03-12 04:27:01.760 2024-03-12 04:27:01.755 2024-03-12 04:27:01.743 unstruct 9be3afe8-8a62-41ac-93db-12f425d82ac9 spTest js-3.17.0 snowplow-micro-2.0.0-stdout$ snowplow-micro-2.0.0 media_tester 172.17.0.1 23a0eb65-83f6-4957-839e-f3044bfefb99 1 a2f53212-26a3-4781-81d6-f14aa8d4552b http://localhost:8000/?sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== http localhost 8000 / sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:org.whatwg/media_element/jsonschema/1-0-0","data":{"htmlId":"bunny-mp4","mediaType":"VIDEO","autoPlay":false,"buffered":[{"start":0,"end":1.291666}],"controls":true,"currentSrc":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","defaultMuted":false,"defaultPlaybackRate":1,"error":null,"networkState":"NETWORK_LOADING","preload":"","readyState":"HAVE_ENOUGH_DATA","seekable":[{"start":0,"end":596.503219}],"seeking":false,"src":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","textTracks":[],"fileExtension":"mp4","fullscreen":false,"pictureInPicture":false}},{"schema":"iglu:com.snowplowanalytics.snowplow/media_player/jsonschema/1-0-0","data":{"currentTime":0,"duration":596.503219,"ended":false,"loop":false,"muted":false,"paused":false,"playbackRate":1,"volume":100}},{"schema":"iglu:org.whatwg/video_element/jsonschema/1-0-0","data":{"poster":"","videoHeight":360,"videoWidth":640}},{"schema":"iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-0-0","data":{"id":"021c4d09-e502-4562-8182-5ac7247125ec"}},{"schema":"iglu:com.google.tag-manager.server-side/user_data/jsonschema/1-0-0","data":{"email_address":"foo@example.com","phone_number":"+15551234567","address":{"first_name":"Jane","last_name":"Doe","street":"123 Fake St","city":"San Francisco","region":"CA","postal_code":"94016","country":"US"}}},{"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-2","data":{"osType":"testOsType","osVersion":"testOsVersion","deviceManufacturer":"testDevMan","deviceModel":"testDevModel"}},{"schema":"iglu:com.google.tag-manager.server-side/preview_mode/jsonschema/1-0-0","data":{"x-gtm-server-preview":"this is not valid base64"}},{"schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-2","data":{"userId":"23a0eb65-83f6-4957-839e-f3044bfefb99","sessionId":"73fcdaa3-0164-41ce-a336-fb00c4ebf68c","eventIndex":7,"sessionIndex":1,"previousSessionId":null,"storageMechanism":"COOKIE_1","firstEventId":"327b9ff9-ed5f-40cf-918a-1b1a775ae347","firstEventTimestamp":"2024-03-12T04:25:36.684Z"}}]} {"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/media_player_event/jsonschema/1-0-0","data":{"type":"play"}}} Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0 en-US 1 24 1920 935 Europe/Athens 1920 1080 windows-1252 1920 935 2024-03-12 04:27:01.745 73fcdaa3-0164-41ce-a336-fb00c4ebf68c 2024-03-12 04:27:01.753 com.snowplowanalytics.snowplow media_player_event jsonschema 1-0-0 `) +var gtmssInvalidNoB64Parsed, _ = analytics.ParseEvent(string(gtmssInvalidNoB64)) + var fakeSpTsv = []byte(`media-test web 2024-03-12 04:25:40.277 2024-03-12 04:25:40.272 2024-03-12 04:25:36.685 page_view 1313411b-282f-4aa9-b37c-c60d4723cf47 spTest js-3.17.0 snowplow-micro-2.0.0-stdout$ snowplow-micro-2.0.0 media_tester 172.17.0.1 23a0eb65-83f6-4957-839e-f3044bfefb99 1 a2f53212-26a3-4781-81d6-f14aa8d4552b http://localhost:8000/ Test Media Tracking http localhost 8000 / {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-0-0","data":{"id":["FAILS"]}},{"schema":"iglu:com.google.tag-manager.server-side/user_data/jsonschema/1-0-0","data":{"email_address":"foo@example.com","phone_number":"+15551234567","address":{"first_name":"Jane","last_name":"Doe","street":"123 Fake St","city":"San Francisco","region":"CA","postal_code":"94016","country":"US"}}},{"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-2","data":{"osType":"testOsType","osVersion":"testOsVersion","deviceManufacturer":"testDevMan","deviceModel":"testDevModel"}},{"schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-2","data":{"userId":"23a0eb65-83f6-4957-839e-f3044bfefb99","sessionId":"73fcdaa3-0164-41ce-a336-fb00c4ebf68c","eventIndex":2,"sessionIndex":1,"previousSessionId":null,"storageMechanism":"COOKIE_1","firstEventId":"327b9ff9-ed5f-40cf-918a-1b1a775ae347","firstEventTimestamp":"2024-03-12T04:25:36.684Z"}}]} Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0 en-US 1 24 1920 935 Europe/Athens 1920 1080 windows-1252 1920 935 2024-03-12 04:25:40.268 73fcdaa3-0164-41ce-a336-fb00c4ebf68c 2024-03-12 04:25:36.689 com.snowplowanalytics.snowplow page_view jsonschema 1-0-0 `) var fakeSpTsvParsed, _ = analytics.ParseEvent(string(fakeSpTsv)) From 80853f51e74c09ae63644e08f7e1cb635880acab Mon Sep 17 00:00:00 2001 From: colmsnowplow Date: Thu, 19 Sep 2024 09:49:09 +0100 Subject: [PATCH 4/8] Update dependencies (#371) AWS sdk not updated as it broke e2e tests --- go.mod | 95 ++++++++++++------------ go.sum | 224 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 161 insertions(+), 158 deletions(-) diff --git a/go.mod b/go.mod index 4c146de5..d7b13f61 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,28 @@ module github.com/snowplow/snowbridge -go 1.22 +go 1.22.0 + +toolchain go1.23.0 require ( - cloud.google.com/go v0.112.2 // indirect - cloud.google.com/go/pubsub v1.37.0 + cloud.google.com/go v0.115.1 // indirect + cloud.google.com/go/pubsub v1.43.0 github.com/Azure/azure-event-hubs-go/v3 v3.6.2 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect - github.com/Azure/go-amqp v1.0.5 // indirect + github.com/Azure/go-amqp v1.1.0 // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect - github.com/IBM/sarama v1.43.1 + github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect + github.com/IBM/sarama v1.43.3 github.com/aws/aws-sdk-go v1.44.334 - github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect - github.com/getsentry/sentry-go v0.27.0 + github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect + github.com/getsentry/sentry-go v0.29.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 github.com/jpillora/backoff v1.0.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 @@ -29,34 +31,35 @@ require ( github.com/snowplow/snowplow-golang-analytics-sdk v0.3.0 github.com/stretchr/testify v1.9.0 github.com/twitchscience/kinsumer v0.0.0-20240315191529-9a48088063ec - github.com/urfave/cli v1.22.14 + github.com/urfave/cli v1.22.15 github.com/xdg/scram v1.0.5 - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.19.0 - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/api v0.172.0 // indirect - google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda - google.golang.org/grpc v1.63.2 + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.23.0 + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/api v0.197.0 // indirect + google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 + google.golang.org/grpc v1.66.2 ) require ( github.com/avast/retry-go/v4 v4.6.0 github.com/davecgh/go-spew v1.1.1 - github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 - github.com/hashicorp/hcl/v2 v2.20.1 + github.com/dop251/goja v0.0.0-20240828124009-016eb7256539 + github.com/hashicorp/hcl/v2 v2.22.0 github.com/itchyny/gojq v0.12.16 github.com/json-iterator/go v1.1.12 github.com/snowplow/snowplow-golang-tracker/v2 v2.4.1 github.com/twinj/uuid v1.0.0 - github.com/zclconf/go-cty v1.14.4 + github.com/zclconf/go-cty v1.15.0 ) require ( - cloud.google.com/go/compute v1.25.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.7 // indirect + cloud.google.com/go/auth v0.9.4 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.5.1 // indirect + cloud.google.com/go/iam v1.2.1 // indirect github.com/Azure/azure-amqp-common-go/v4 v4.2.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect @@ -67,21 +70,20 @@ require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/devigned/tab v0.1.1 // indirect - github.com/dlclark/regexp2 v1.11.0 // indirect - github.com/eapache/go-resiliency v1.6.0 // indirect + github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd // indirect - github.com/google/s2a-go v0.1.7 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-memdb v1.3.4 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -101,23 +103,24 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xdg/stringprep v1.0.3 // indirect - go.einride.tech/aip v0.66.0 // indirect + go.einride.tech/aip v0.68.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect - go.opentelemetry.io/otel v1.25.0 // indirect - go.opentelemetry.io/otel/metric v1.25.0 // indirect - go.opentelemetry.io/otel/sdk v1.25.0 // indirect - go.opentelemetry.io/otel/trace v1.25.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.20.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/protobuf v1.33.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/time v0.6.0 // indirect + golang.org/x/tools v0.25.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/stretchr/testify.v1 v1.2.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 44ce218f..bcc39aac 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= -cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= -cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= -cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= -cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= -cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs= -cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs= -cloud.google.com/go/pubsub v1.37.0 h1:0uEEfaB1VIJzabPpwpZf44zWAKAme3zwKKxHk7vJQxQ= -cloud.google.com/go/pubsub v1.37.0/go.mod h1:YQOQr1uiUM092EXwKs56OPT650nwnawc+8/IjoUeGzQ= +cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= +cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= +cloud.google.com/go/auth v0.9.4 h1:DxF7imbEbiFu9+zdKC6cKBko1e8XeJnipNqIbWZ+kDI= +cloud.google.com/go/auth v0.9.4/go.mod h1:SHia8n6//Ya940F1rLimhJCjjx7KE17t0ctFEci3HkA= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/compute/metadata v0.5.1 h1:NM6oZeZNlYjiwYje+sYFjEpP0Q0zCan1bmQW/KmIrGs= +cloud.google.com/go/compute/metadata v0.5.1/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= +cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= +cloud.google.com/go/kms v1.19.0 h1:x0OVJDl6UH1BSX4THKlMfdcFWoE4ruh90ZHuilZekrU= +cloud.google.com/go/kms v1.19.0/go.mod h1:e4imokuPJUc17Trz2s6lEXFDt8bgDmvpVynH39bdrHM= +cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI= +cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts= +cloud.google.com/go/pubsub v1.43.0 h1:s3Qx+F96J7Kwey/uVHdK3QxFLIlOvvw4SfMYw2jFjb4= +cloud.google.com/go/pubsub v1.43.0/go.mod h1:LNLfqItblovg7mHWgU5g84Vhza4J8kTxx0YqIeTzcXY= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-amqp-common-go/v4 v4.2.0 h1:q/jLx1KJ8xeI8XGfkOWMN9XrXzAfVTkyvCxPvHCjd2I= github.com/Azure/azure-amqp-common-go/v4 v4.2.0/go.mod h1:GD3m/WPPma+621UaU6KNjKEo5Hl09z86viKwQjTpV0Q= @@ -18,15 +22,15 @@ github.com/Azure/azure-event-hubs-go/v3 v3.6.2 h1:7rNj1/iqS/i3mUKokA2n2eMYO72TB7 github.com/Azure/azure-event-hubs-go/v3 v3.6.2/go.mod h1:n+ocYr9j2JCLYqUqz9eI+lx/TEAtL/g6rZzyTFSuIpc= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-amqp v1.0.5 h1:po5+ljlcNSU8xtapHTe8gIc8yHxCzC03E8afH2g1ftU= -github.com/Azure/go-amqp v1.0.5/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= +github.com/Azure/go-amqp v1.1.0 h1:XUhx5f4lZFVf6LQc5kBUFECW0iJW9VLxKCYrBeGwl0U= +github.com/Azure/go-amqp v1.1.0/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= -github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= +github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk= github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U= @@ -48,9 +52,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= -github.com/IBM/sarama v1.43.1 h1:Z5uz65Px7f4DhI/jQqEm/tV9t8aU+JUdTyW/K/fCXpA= -github.com/IBM/sarama v1.43.1/go.mod h1:GG5q1RURtDNPz8xxJs3mgX6Ytak8Z9eLhAkJPObe2xE= +github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= +github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H+FQ= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -64,9 +70,6 @@ github.com/aws/aws-sdk-go v1.44.334 h1:h2bdbGb//fez6Sv6PaYv868s9liDeoYM6hYsAqTB4 github.com/aws/aws-sdk-go v1.44.334/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= -github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= -github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -74,10 +77,9 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= +github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -88,18 +90,13 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= -github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 h1:O7I1iuzEA7SG+dK8ocOBSlYAA9jBUmCYl/Qa7ey7JAM= -github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= -github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= -github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dop251/goja v0.0.0-20240828124009-016eb7256539 h1:YIxvsQAoCLGScK2c9ag+4sFCgiQFpMzywJG6dQZFu9k= +github.com/dop251/goja v0.0.0-20240828124009-016eb7256539/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= -github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= +github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= @@ -119,8 +116,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.9.0/go.mod h1:kELm/9iCblqUYh+ZRML7PNdCvEuw24wBvJPYyi86cws= -github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= -github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA= +github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -128,12 +125,11 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= @@ -174,19 +170,18 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 h1:c5FlPPgxOn7kJz3VoPLkQYQXGBS3EklQ4Zfi57uOuqQ= +github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= -github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -213,10 +208,9 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M= +github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= @@ -263,17 +257,15 @@ github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7 github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -323,8 +315,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -356,6 +348,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -373,8 +366,8 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= -github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM= +github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= @@ -393,26 +386,26 @@ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FB github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= -github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= -go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8= -go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M= +github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ= +github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= +go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= +go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0 h1:zvpPXY7RfYAGSdYQLjp6zxdJNSYD/+FFoCTQN9IPxBs= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0/go.mod h1:BMn8NB1vsxTljvuorms2hyOs8IBuuBEq0pl7ltOfy30= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 h1:cEPbyTSEHlQR89XVlyo78gqluF8Y3oMeBkXGWzQsfXY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0/go.mod h1:DKdbWcT4GH1D0Y3Sqt/PFXt2naRKDWtU+eE6oLdFNA8= -go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k= -go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg= -go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA= -go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s= -go.opentelemetry.io/otel/sdk v1.25.0 h1:PDryEJPC8YJZQSyLY5eqLeafHtG+X7FWnf3aXMtxbqo= -go.opentelemetry.io/otel/sdk v1.25.0/go.mod h1:oFgzCM2zdsxKzz6zwpTZYLLQsFwc+K0daArPdIhuxkw= -go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM= -go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 h1:hCq2hNMwsegUvPzI7sPOvtO9cqyy5GbWt/Ybp2xrx8Q= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0/go.mod h1:LqaApwGx/oUmzsbqxkzuBvyoPpkxk3JQWnqfVrJ3wCA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -421,15 +414,17 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -450,18 +445,20 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= -golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -475,30 +472,34 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -509,32 +510,33 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= -google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= +google.golang.org/api v0.197.0 h1:x6CwqQLsFiA5JKAiGyGBjc2bNtHtLddhJCE2IKuhhcQ= +google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= -google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda h1:b6F6WIV4xHHD0FA4oIyzU6mHWg2WI2X1RBehwa5QN38= -google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda/go.mod h1:AHcE/gZH76Bk/ROZhQphlRoWo5xKDEtz3eVEO1LfA8c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU= +google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -544,13 +546,11 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= From 8c3fa06d46dc412d14f097d0acdebaaa10fd25bd Mon Sep 17 00:00:00 2001 From: colmsnowplow Date: Thu, 17 Oct 2024 14:11:44 +0100 Subject: [PATCH 5/8] Patch bug in avg request latency calculation (#376) --- pkg/models/observer_buffer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/models/observer_buffer.go b/pkg/models/observer_buffer.go index b7478891..eaf1d4e5 100644 --- a/pkg/models/observer_buffer.go +++ b/pkg/models/observer_buffer.go @@ -177,7 +177,7 @@ func (b *ObserverBuffer) GetAvgFilterLatency() time.Duration { // GetAvgRequestLatency calculates average request latency func (b *ObserverBuffer) GetAvgRequestLatency() time.Duration { - return common.GetAverageFromDuration(b.SumRequestLatency, b.MsgFiltered) + return common.GetAverageFromDuration(b.SumRequestLatency, b.MsgTotal) } func (b *ObserverBuffer) String() string { From 2d23c039415f195814efeaea009742c3914137c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Mon, 7 Oct 2024 16:06:31 +0200 Subject: [PATCH 6/8] Add JQ filter Adding JQ filter, which allows us to...filter messages based on a configured JQ command. Similar to already existing JQ mapper, but JQ filter requires output of a command to have boolean type. As there is some shared logic between the mapper and the new filter, I extracted common stuff to `jq_common.go` module. --- .../builtin/jqFilter-full-example.hcl | 7 + .../builtin/jqFilter-minimal-example.hcl | 5 + ...configuration_transformations_docs_test.go | 4 +- pkg/transform/filter/jq_filter.go | 79 ++++++++ pkg/transform/filter/jq_filter_test.go | 136 +++++++++++++ pkg/transform/jq.go | 187 ++++-------------- pkg/transform/jq_common.go | 132 +++++++++++++ pkg/transform/jq_test.go | 2 +- .../snowplow_enriched_to_json_test.go | 2 +- pkg/transform/transform_test.go | 4 +- pkg/transform/transform_test_variables.go | 4 +- .../transformconfig/transform_config.go | 1 + 12 files changed, 409 insertions(+), 154 deletions(-) create mode 100644 assets/docs/configuration/transformations/builtin/jqFilter-full-example.hcl create mode 100644 assets/docs/configuration/transformations/builtin/jqFilter-minimal-example.hcl create mode 100644 pkg/transform/filter/jq_filter.go create mode 100644 pkg/transform/filter/jq_filter_test.go create mode 100644 pkg/transform/jq_common.go diff --git a/assets/docs/configuration/transformations/builtin/jqFilter-full-example.hcl b/assets/docs/configuration/transformations/builtin/jqFilter-full-example.hcl new file mode 100644 index 00000000..f02e9042 --- /dev/null +++ b/assets/docs/configuration/transformations/builtin/jqFilter-full-example.hcl @@ -0,0 +1,7 @@ +transform { + use "jqFilter" { + jq_command = "has(\"app_id\")" + timeout_ms = 800 + snowplow_mode = true + } +} diff --git a/assets/docs/configuration/transformations/builtin/jqFilter-minimal-example.hcl b/assets/docs/configuration/transformations/builtin/jqFilter-minimal-example.hcl new file mode 100644 index 00000000..18a4b9cb --- /dev/null +++ b/assets/docs/configuration/transformations/builtin/jqFilter-minimal-example.hcl @@ -0,0 +1,5 @@ +transform { + use "jqFilter" { + jq_command = "has(\"app_id\")" + } +} diff --git a/docs/configuration_transformations_docs_test.go b/docs/configuration_transformations_docs_test.go index 6bb6af34..b98f9a71 100644 --- a/docs/configuration_transformations_docs_test.go +++ b/docs/configuration_transformations_docs_test.go @@ -29,7 +29,7 @@ import ( ) func TestBuiltinTransformationDocumentation(t *testing.T) { - transformationsToTest := []string{"base64Decode", "base64Encode", "jq"} + transformationsToTest := []string{"base64Decode", "base64Encode", "jq", "jqFilter"} for _, tfm := range transformationsToTest { @@ -161,6 +161,8 @@ func testTransformationConfig(t *testing.T, filepath string, fullExample bool) { configObject = &engine.JSEngineConfig{} case "jq": configObject = &transform.JQMapperConfig{} + case "jqFilter": + configObject = &filter.JQFilterConfig{} default: assert.Fail(fmt.Sprint("Source not recognised: ", use.Name)) } diff --git a/pkg/transform/filter/jq_filter.go b/pkg/transform/filter/jq_filter.go new file mode 100644 index 00000000..a9e2ff78 --- /dev/null +++ b/pkg/transform/filter/jq_filter.go @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2020-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION + * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. + */ + +package filter + +import ( + "errors" + + "github.com/snowplow/snowbridge/config" + "github.com/snowplow/snowbridge/pkg/models" + "github.com/snowplow/snowbridge/pkg/transform" +) + +// JQFilterConfig represents the configuration for the JQ filter transformation +type JQFilterConfig struct { + JQCommand string `hcl:"jq_command"` + RunTimeoutMs int `hcl:"timeout_ms,optional"` + SpMode bool `hcl:"snowplow_mode,optional"` +} + +// JQFilterConfigPair is a configuration pair for the jq filter transformation +var JQFilterConfigPair = config.ConfigurationPair{ + Name: "jqFilter", + Handle: jqFilterAdapterGenerator(jqFilterConfigFunction), +} + +func jqFilterConfigFunction(cfg *JQFilterConfig) (transform.TransformationFunction, error) { + return transform.GojqTransformationFunction(cfg.JQCommand, cfg.RunTimeoutMs, cfg.SpMode, filterOutput) +} + +func jqFilterAdapterGenerator(f func(*JQFilterConfig) (transform.TransformationFunction, error)) jqFilterAdapter { + return func(i interface{}) (interface{}, error) { + cfg, ok := i.(*JQFilterConfig) + if !ok { + return nil, errors.New("invalid input, expected JQFilterConfig") + } + + return f(cfg) + } +} + +// This is where actual filtering is implemented, based on a JQ command output. +func filterOutput(jqOutput transform.JqCommandOutput) transform.TransformationFunction { + return func(message *models.Message, interState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { + shouldKeepMessage, isBoolean := jqOutput.(bool) + + // maybe crash instead? + if !isBoolean { + message.SetError(errors.New("jq filter doesn't evaluate to boolean value")) + return nil, nil, message, nil + } + + if !shouldKeepMessage { + return nil, message, nil, nil + } + + return message, nil, nil, interState + } +} + +type jqFilterAdapter func(i interface{}) (interface{}, error) + +func (f jqFilterAdapter) ProvideDefault() (interface{}, error) { + return &JQFilterConfig{ + RunTimeoutMs: 100, + }, nil +} + +func (f jqFilterAdapter) Create(i interface{}) (interface{}, error) { + return f(i) +} diff --git a/pkg/transform/filter/jq_filter_test.go b/pkg/transform/filter/jq_filter_test.go new file mode 100644 index 00000000..5b703ec1 --- /dev/null +++ b/pkg/transform/filter/jq_filter_test.go @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2020-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION + * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. + */ + +package filter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/snowplow/snowbridge/pkg/models" + "github.com/snowplow/snowbridge/pkg/transform" +) + +func TestJQFilter_SpMode_true_keep(t *testing.T) { + assert := assert.New(t) + input := &models.Message{ + Data: transform.SnowplowTsv1, + PartitionKey: "some-key", + } + + config := &JQFilterConfig{JQCommand: `has("app_id")`, RunTimeoutMs: 100, SpMode: true} + filter := createFilter(t, config) + + kept, dropped, invalid, _ := filter(input, nil) + assert.Empty(dropped) + assert.Empty(invalid) + assert.Equal(string(transform.SnowplowTsv1), string(kept.Data)) +} + +func TestJQFilter_SpMode_true_drop(t *testing.T) { + assert := assert.New(t) + input := &models.Message{ + Data: transform.SnowplowTsv1, + PartitionKey: "some-key", + } + + config := &JQFilterConfig{JQCommand: `has("non_existent_key")`, RunTimeoutMs: 100, SpMode: true} + filter := createFilter(t, config) + + kept, dropped, invalid, _ := filter(input, nil) + assert.Empty(kept) + assert.Empty(invalid) + assert.Equal(string(transform.SnowplowTsv1), string(dropped.Data)) +} + +func TestJQFilter_SpMode_false_keep(t *testing.T) { + assert := assert.New(t) + input := &models.Message{ + Data: transform.SnowplowJSON1, + PartitionKey: "some-key", + } + + config := &JQFilterConfig{JQCommand: `has("app_id")`, RunTimeoutMs: 100, SpMode: false} + filter := createFilter(t, config) + + kept, dropped, invalid, _ := filter(input, nil) + assert.Empty(dropped) + assert.Empty(invalid) + assert.Equal(string(transform.SnowplowJSON1), string(kept.Data)) +} + +func TestJQFilter_SpMode_false_drop(t *testing.T) { + assert := assert.New(t) + input := &models.Message{ + Data: transform.SnowplowJSON1, + PartitionKey: "some-key", + } + + config := &JQFilterConfig{JQCommand: `has("non_existent_key")`, RunTimeoutMs: 100, SpMode: false} + filter := createFilter(t, config) + + kept, dropped, invalid, _ := filter(input, nil) + assert.Empty(kept) + assert.Empty(invalid) + assert.Equal(string(transform.SnowplowJSON1), string(dropped.Data)) +} + +func TestJQFilter_epoch(t *testing.T) { + assert := assert.New(t) + input := &models.Message{ + Data: transform.SnowplowTsv1, + PartitionKey: "some-key", + } + + config := &JQFilterConfig{JQCommand: `.collector_tstamp | epoch | . < 10`, RunTimeoutMs: 100, SpMode: true} + filter := createFilter(t, config) + + kept, dropped, invalid, _ := filter(input, nil) + assert.Empty(kept) + assert.Empty(invalid) + assert.Equal(string(transform.SnowplowTsv1), string(dropped.Data)) +} + +func TestJQFilter_non_boolean_output(t *testing.T) { + assert := assert.New(t) + input := &models.Message{ + Data: transform.SnowplowTsv1, + PartitionKey: "some-key", + } + + config := &JQFilterConfig{JQCommand: `.collector_tstamp | epoch`, RunTimeoutMs: 100, SpMode: true} + filter := createFilter(t, config) + + kept, dropped, invalid, _ := filter(input, nil) + + assert.Empty(kept) + assert.Empty(dropped) + assert.Equal("jq filter doesn't evaluate to boolean value", invalid.GetError().Error()) +} + +func TestJQFilter_invalid_jq_command(t *testing.T) { + assert := assert.New(t) + + config := &JQFilterConfig{JQCommand: `blabla`, RunTimeoutMs: 100, SpMode: true} + filter, err := jqFilterConfigFunction(config) + + assert.Nil(filter) + assert.Equal("error compiling jq query: function not defined: blabla/0", err.Error()) +} + +func createFilter(t *testing.T, config *JQFilterConfig) transform.TransformationFunction { + filter, err := jqFilterConfigFunction(config) + if err != nil { + t.Fatalf("failed to create transformation function with error: %q", err.Error()) + } + return filter +} diff --git a/pkg/transform/jq.go b/pkg/transform/jq.go index 228b983f..4182d8a3 100644 --- a/pkg/transform/jq.go +++ b/pkg/transform/jq.go @@ -12,13 +12,8 @@ package transform import ( - "context" "encoding/json" "errors" - "fmt" - "time" - - "github.com/itchyny/gojq" "github.com/snowplow/snowbridge/config" "github.com/snowplow/snowbridge/pkg/models" @@ -31,42 +26,23 @@ type JQMapperConfig struct { SpMode bool `hcl:"snowplow_mode,optional"` } -// JQMapper handles jq generic mapping as a transformation -type jqMapper struct { - JQCode *gojq.Code - RunTimeoutMs time.Duration - SpMode bool +// JQMapperConfigPair is a configuration pair for the jq mapper transformation +var JQMapperConfigPair = config.ConfigurationPair{ + Name: "jq", + Handle: jqMapperAdapterGenerator(jqMapperConfigFunction), } -// RunFunction runs a jq mapper transformation -func (jqm *jqMapper) RunFunction() TransformationFunction { - return func(message *models.Message, interState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { - input, err := mkJQInput(jqm, message, interState) - if err != nil { - message.SetError(err) - return nil, nil, message, nil - } - - ctx, cancel := context.WithTimeout(context.Background(), jqm.RunTimeoutMs) - defer cancel() - - iter := jqm.JQCode.RunWithContext(ctx, input) - // no looping since we only keep first value - v, ok := iter.Next() - if !ok { - message.SetError(errors.New("jq query got no output")) - return nil, nil, message, nil - } - - if err, ok := v.(error); ok { - message.SetError(err) - return nil, nil, message, nil - } +// jqMapperConfigFunction returns a jq mapper transformation function from a JQMapperConfig +func jqMapperConfigFunction(c *JQMapperConfig) (TransformationFunction, error) { + return GojqTransformationFunction(c.JQCommand, c.RunTimeoutMs, c.SpMode, transformOutput) +} - removeNullFields(v) +func transformOutput(jqOutput JqCommandOutput) TransformationFunction { + return func(message *models.Message, interState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { + removeNullFields(jqOutput) // here v is any, so we Marshal. alternative: gojq.Marshal - data, err := json.Marshal(v) + data, err := json.Marshal(jqOutput) if err != nil { message.SetError(errors.New("error encoding jq query output data")) return nil, nil, message, nil @@ -77,118 +53,6 @@ func (jqm *jqMapper) RunFunction() TransformationFunction { } } -// jqMapperAdapter implements the Pluggable interface -type jqMapperAdapter func(i interface{}) (interface{}, error) - -// ProvideDefault implements the ComponentConfigurable interface -func (f jqMapperAdapter) ProvideDefault() (interface{}, error) { - return &JQMapperConfig{ - RunTimeoutMs: 100, - }, nil -} - -// Create implements the ComponentCreator interface -func (f jqMapperAdapter) Create(i interface{}) (interface{}, error) { - return f(i) -} - -// jqMapperAdapterGenerator returns a jqAdapter -func jqMapperAdapterGenerator(f func(c *JQMapperConfig) (TransformationFunction, error)) jqMapperAdapter { - return func(i interface{}) (interface{}, error) { - cfg, ok := i.(*JQMapperConfig) - if !ok { - return nil, errors.New("invalid input, expected JQMapperConfig") - } - - return f(cfg) - } -} - -// jqMapperConfigFunction returns a jq mapper transformation function from a JQMapperConfig -func jqMapperConfigFunction(c *JQMapperConfig) (TransformationFunction, error) { - query, err := gojq.Parse(c.JQCommand) - if err != nil { - return nil, fmt.Errorf("error parsing jq command: %s", err) - } - - // epoch converts a time.Time to an epoch in seconds, as integer type. - // It must be an integer in order to chain with jq-native time functions - withEpochFunction := gojq.WithFunction("epoch", 0, 1, func(a1 any, a2 []any) any { - if a1 == nil { - return nil - } - - validTime, ok := a1.(time.Time) - - if !ok { - return errors.New("Not a valid time input to 'epoch' function") - } - - return int(validTime.Unix()) - }) - - // epochMillis converts a time.Time to an epoch in milliseconds - withEpochMillisFunction := gojq.WithFunction("epochMillis", 0, 1, func(a1 any, a2 []any) any { - if a1 == nil { - return nil - } - - validTime, ok := a1.(time.Time) - - if !ok { - return errors.New("Not a valid time input to 'epochMillis' function") - } - - return validTime.UnixMilli() - }) - - code, err := gojq.Compile(query, withEpochMillisFunction, withEpochFunction) - if err != nil { - return nil, fmt.Errorf("error compiling jq query: %s", err) - } - - jq := &jqMapper{ - JQCode: code, - RunTimeoutMs: time.Duration(c.RunTimeoutMs) * time.Millisecond, - SpMode: c.SpMode, - } - - return jq.RunFunction(), nil -} - -// JQMapperConfigPair is a configuration pair for the jq mapper transformation -var JQMapperConfigPair = config.ConfigurationPair{ - Name: "jq", - Handle: jqMapperAdapterGenerator(jqMapperConfigFunction), -} - -// mkJQInput ensures the input to JQ query is of expected type -func mkJQInput(jqm *jqMapper, message *models.Message, interState interface{}) (map[string]interface{}, error) { - if !jqm.SpMode { - // gojq input can only be map[string]any or []any - // here we only consider the first, but we could also expand - var input map[string]interface{} - err := json.Unmarshal(message.Data, &input) - if err != nil { - return nil, err - } - - return input, nil - } - - parsedEvent, err := IntermediateAsSpEnrichedParsed(interState, message) - if err != nil { - return nil, err - } - - spInput, err := parsedEvent.ToMap() - if err != nil { - return nil, err - } - - return spInput, nil -} - func removeNullFields(data any) { switch input := data.(type) { case map[string]any: @@ -216,3 +80,30 @@ func removeNullFromSlice(input []any) { removeNullFields(item) } } + +// jqMapperAdapterGenerator returns a jqAdapter +func jqMapperAdapterGenerator(f func(c *JQMapperConfig) (TransformationFunction, error)) jqMapperAdapter { + return func(i interface{}) (interface{}, error) { + cfg, ok := i.(*JQMapperConfig) + if !ok { + return nil, errors.New("invalid input, expected JQMapperConfig") + } + + return f(cfg) + } +} + +// jqMapperAdapter implements the Pluggable interface +type jqMapperAdapter func(i interface{}) (interface{}, error) + +// ProvideDefault implements the ComponentConfigurable interface +func (f jqMapperAdapter) ProvideDefault() (interface{}, error) { + return &JQMapperConfig{ + RunTimeoutMs: 100, + }, nil +} + +// Create implements the ComponentCreator interface +func (f jqMapperAdapter) Create(i interface{}) (interface{}, error) { + return f(i) +} diff --git a/pkg/transform/jq_common.go b/pkg/transform/jq_common.go new file mode 100644 index 00000000..467171fa --- /dev/null +++ b/pkg/transform/jq_common.go @@ -0,0 +1,132 @@ +/** + * Copyright (c) 2020-present Snowplow Analytics Ltd. + * All rights reserved. + * + * This software is made available by Snowplow Analytics, Ltd., + * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 + * located at https://docs.snowplow.io/limited-use-license-1.0 + * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION + * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. + */ + +package transform + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/itchyny/gojq" + + "github.com/snowplow/snowbridge/pkg/models" + "github.com/snowplow/snowplow-golang-analytics-sdk/analytics" +) + +// JqCommandOutput is a type representing output after executing JQ command. For filters for example we expect it to be boolean. +type JqCommandOutput = interface{} + +// JqOutputHandler is a function which accepts JqCommandOutput and is response for doing something with it. For filters for example that would be filtering message based on boolean output. +type JqOutputHandler func(JqCommandOutput) TransformationFunction + +// GojqTransformationFunction is a function returning another transformation function which allows us to do some GOJQ based mapping/filtering. Actual transformation happens in provided JqOutputHandler. +func GojqTransformationFunction(command string, timeoutMs int, spMode bool, jqOutputHandler JqOutputHandler) (TransformationFunction, error) { + query, err := gojq.Parse(command) + if err != nil { + return nil, fmt.Errorf("error parsing jq command: %s", err) + } + + // epoch converts a time.Time to an epoch in seconds, as integer type. + // It must be an integer in order to chain with jq-native time functions + withEpochFunction := gojq.WithFunction("epoch", 0, 1, func(a1 any, a2 []any) any { + if a1 == nil { + return nil + } + + validTime, ok := a1.(time.Time) + + if !ok { + return errors.New("Not a valid time input to 'epoch' function") + } + + return int(validTime.Unix()) + }) + + // epochMillis converts a time.Time to an epoch in milliseconds + withEpochMillisFunction := gojq.WithFunction("epochMillis", 0, 1, func(a1 any, a2 []any) any { + if a1 == nil { + return nil + } + + validTime, ok := a1.(time.Time) + + if !ok { + return errors.New("Not a valid time input to 'epochMillis' function") + } + + return validTime.UnixMilli() + }) + + code, err := gojq.Compile(query, withEpochMillisFunction, withEpochFunction) + if err != nil { + return nil, fmt.Errorf("error compiling jq query: %s", err) + } + + return runFunction(code, timeoutMs, spMode, jqOutputHandler), nil +} + +func runFunction(jqcode *gojq.Code, timeoutMs int, spMode bool, jqOutputHandler JqOutputHandler) TransformationFunction { + return func(message *models.Message, interState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { + input, parsedEvent, err := mkJQInput(message, interState, spMode) + if err != nil { + message.SetError(err) + return nil, nil, message, nil + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMs)*time.Millisecond) + defer cancel() + + iter := jqcode.RunWithContext(ctx, input) + // no looping since we only keep first value + jqOutput, ok := iter.Next() + if !ok { + message.SetError(errors.New("jq query got no output")) + return nil, nil, message, nil + } + + if err, ok := jqOutput.(error); ok { + message.SetError(err) + return nil, nil, message, nil + } + + return jqOutputHandler(jqOutput)(message, parsedEvent) + } +} + +// mkJQInput ensures the input to JQ query is of expected type +func mkJQInput(message *models.Message, interState interface{}, spMode bool) (map[string]interface{}, analytics.ParsedEvent, error) { + if !spMode { + // gojq input can only be map[string]any or []any + // here we only consider the first, but we could also expand + var input map[string]interface{} + err := json.Unmarshal(message.Data, &input) + if err != nil { + return nil, nil, err + } + + return input, nil, nil + } + + parsedEvent, err := IntermediateAsSpEnrichedParsed(interState, message) + if err != nil { + return nil, nil, err + } + + spInput, err := parsedEvent.ToMap() + if err != nil { + return nil, nil, err + } + + return spInput, parsedEvent, nil +} diff --git a/pkg/transform/jq_test.go b/pkg/transform/jq_test.go index f9178bfb..a99c53c4 100644 --- a/pkg/transform/jq_test.go +++ b/pkg/transform/jq_test.go @@ -212,7 +212,7 @@ func TestJQRunFunction_SpMode_false(t *testing.T) { Scenario: "happy_path", JQCommand: `{foo: .app_id}`, InputMsg: &models.Message{ - Data: snowplowJSON1, + Data: SnowplowJSON1, PartitionKey: "some-key", }, InputInterState: nil, diff --git a/pkg/transform/snowplow_enriched_to_json_test.go b/pkg/transform/snowplow_enriched_to_json_test.go index 64958837..08a38a33 100644 --- a/pkg/transform/snowplow_enriched_to_json_test.go +++ b/pkg/transform/snowplow_enriched_to_json_test.go @@ -33,7 +33,7 @@ func TestSpEnrichedToJson(t *testing.T) { } var expectedGood = models.Message{ - Data: snowplowJSON1, + Data: SnowplowJSON1, PartitionKey: "some-key", } diff --git a/pkg/transform/transform_test.go b/pkg/transform/transform_test.go index 5355252d..1062d5af 100644 --- a/pkg/transform/transform_test.go +++ b/pkg/transform/transform_test.go @@ -56,7 +56,7 @@ func TestNewTransformation_EnrichedToJson(t *testing.T) { var expectedGood = []*models.Message{ { - Data: snowplowJSON1, + Data: SnowplowJSON1, PartitionKey: "some-key", }, { @@ -117,7 +117,7 @@ func TestNewTransformation_Multiple(t *testing.T) { var expectedGood = []*models.Message{ { - Data: snowplowJSON1, + Data: SnowplowJSON1, PartitionKey: "test-data1", }, { diff --git a/pkg/transform/transform_test_variables.go b/pkg/transform/transform_test_variables.go index 841367dc..baf800b5 100644 --- a/pkg/transform/transform_test_variables.go +++ b/pkg/transform/transform_test_variables.go @@ -22,7 +22,9 @@ var SnowplowTsv1 = []byte(`test-data1 pc 2019-05-10 14:40:37.436 2019-05-10 14:4 // SpTsv1Parsed is test data var SpTsv1Parsed, _ = analytics.ParseEvent(string(SnowplowTsv1)) -var snowplowJSON1 = []byte(`{"app_id":"test-data1","collector_tstamp":"2019-05-10T14:40:35.972Z","contexts_com_acme_just_ints_1":[{"integerField":0},{"integerField":1},{"integerField":2}],"contexts_nl_basjes_yauaa_context_1":[{"agentClass":"Special","agentName":"python-requests","agentNameVersion":"python-requests 2.21.0","agentNameVersionMajor":"python-requests 2","agentVersion":"2.21.0","agentVersionMajor":"2","deviceBrand":"Unknown","deviceClass":"Unknown","deviceName":"Unknown","layoutEngineClass":"Unknown","layoutEngineName":"Unknown","layoutEngineVersion":"??","layoutEngineVersionMajor":"??","operatingSystemClass":"Unknown","operatingSystemName":"Unknown","operatingSystemVersion":"??"}],"derived_tstamp":"2019-05-10T14:40:35.972Z","dvce_created_tstamp":"2019-05-10T14:40:35.551Z","dvce_sent_tstamp":"2019-05-10T14:40:35Z","etl_tstamp":"2019-05-10T14:40:37.436Z","event":"unstruct","event_format":"jsonschema","event_id":"e9234345-f042-46ad-b1aa-424464066a33","event_name":"add_to_cart","event_vendor":"com.snowplowanalytics.snowplow","event_version":"1-0-0","network_userid":"d26822f5-52cc-4292-8f77-14ef6b7a27e2","platform":"pc","unstruct_event_com_snowplowanalytics_snowplow_add_to_cart_1":{"currency":"GBP","quantity":2,"sku":"item41","unitPrice":32.4},"user_id":"user\u003cbuilt-in function input\u003e","user_ipaddress":"18.194.133.57","useragent":"python-requests/2.21.0","v_collector":"ssc-0.15.0-googlepubsub","v_etl":"beam-enrich-0.2.0-common-0.36.0","v_tracker":"py-0.8.2"}`) + +// SnowplowJSON1 another test data +var SnowplowJSON1 = []byte(`{"app_id":"test-data1","collector_tstamp":"2019-05-10T14:40:35.972Z","contexts_com_acme_just_ints_1":[{"integerField":0},{"integerField":1},{"integerField":2}],"contexts_nl_basjes_yauaa_context_1":[{"agentClass":"Special","agentName":"python-requests","agentNameVersion":"python-requests 2.21.0","agentNameVersionMajor":"python-requests 2","agentVersion":"2.21.0","agentVersionMajor":"2","deviceBrand":"Unknown","deviceClass":"Unknown","deviceName":"Unknown","layoutEngineClass":"Unknown","layoutEngineName":"Unknown","layoutEngineVersion":"??","layoutEngineVersionMajor":"??","operatingSystemClass":"Unknown","operatingSystemName":"Unknown","operatingSystemVersion":"??"}],"derived_tstamp":"2019-05-10T14:40:35.972Z","dvce_created_tstamp":"2019-05-10T14:40:35.551Z","dvce_sent_tstamp":"2019-05-10T14:40:35Z","etl_tstamp":"2019-05-10T14:40:37.436Z","event":"unstruct","event_format":"jsonschema","event_id":"e9234345-f042-46ad-b1aa-424464066a33","event_name":"add_to_cart","event_vendor":"com.snowplowanalytics.snowplow","event_version":"1-0-0","network_userid":"d26822f5-52cc-4292-8f77-14ef6b7a27e2","platform":"pc","unstruct_event_com_snowplowanalytics_snowplow_add_to_cart_1":{"currency":"GBP","quantity":2,"sku":"item41","unitPrice":32.4},"user_id":"user\u003cbuilt-in function input\u003e","user_ipaddress":"18.194.133.57","useragent":"python-requests/2.21.0","v_collector":"ssc-0.15.0-googlepubsub","v_etl":"beam-enrich-0.2.0-common-0.36.0","v_tracker":"py-0.8.2"}`) // SnowplowTsv2 is test data var SnowplowTsv2 = []byte(`test-data2 pc 2019-05-10 14:40:32.392 2019-05-10 14:40:31.105 2019-05-10 14:40:30.218 transaction_item 5071169f-3050-473f-b03f-9748319b1ef2 py-0.8.2 ssc-0.15.0-googlepubsub beam-enrich-0.2.0-common-0.36.0 user 18.194.133.57 68220ade-307b-4898-8e25-c4c8ac92f1d7 transaction item58 35.87 1 python-requests/2.21.0 2019-05-10 14:40:30.000 {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-0","data":{"deviceBrand":"Unknown","deviceName":"Unknown","operatingSystemName":"Unknown","agentVersionMajor":"2","layoutEngineVersionMajor":"??","deviceClass":"Unknown","agentNameVersionMajor":"python-requests 2","operatingSystemClass":"Unknown","layoutEngineName":"Unknown","agentName":"python-requests","agentVersion":"2.21.0","layoutEngineClass":"Unknown","agentNameVersion":"python-requests 2.21.0","operatingSystemVersion":"??","agentClass":"Special","layoutEngineVersion":"??"}}]} 2019-05-10 14:40:31.105 com.snowplowanalytics.snowplow transaction_item jsonschema 1-0-0 `) diff --git a/pkg/transform/transformconfig/transform_config.go b/pkg/transform/transformconfig/transform_config.go index 580fedb3..ae6b76fc 100644 --- a/pkg/transform/transformconfig/transform_config.go +++ b/pkg/transform/transformconfig/transform_config.go @@ -25,6 +25,7 @@ var SupportedTransformations = []config.ConfigurationPair{ filter.AtomicFilterConfigPair, filter.UnstructFilterConfigPair, filter.ContextFilterConfigPair, + filter.JQFilterConfigPair, transform.SetPkConfigPair, transform.EnrichedToJSONConfigPair, transform.Base64DecodeConfigPair, From 8a082a502b18e0af17d700d8320a2e9e364bb4ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Mon, 30 Sep 2024 13:26:13 +0200 Subject: [PATCH 7/8] Bump to RC5 --- VERSION | 2 +- cmd/constants.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 8e8299dc..916e2438 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.2 +3.0.0-rc5 diff --git a/cmd/constants.go b/cmd/constants.go index 8c5f5869..b7174f42 100644 --- a/cmd/constants.go +++ b/cmd/constants.go @@ -13,7 +13,7 @@ package cmd const ( // AppVersion is the current version of the app - AppVersion = "2.4.2" + AppVersion = "3.0.0-rc5" // AppName is the name of the application to use in logging / places that require the artifact AppName = "snowbridge" From ab6070a6ce6c8060b9b4bf7ade9cfa93319fec6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Fri, 25 Oct 2024 13:39:45 +0200 Subject: [PATCH 8/8] More retrying changes --- .../docs/configuration/overview-full-example.hcl | 4 ++-- assets/docs/configuration/retry-example.hcl | 4 ++-- cmd/cli/cli.go | 16 ++++++++++------ config/config.go | 8 ++++---- config/config_test.go | 8 ++++---- docs/configuration_retry_docs_test.go | 4 ++-- 6 files changed, 24 insertions(+), 20 deletions(-) diff --git a/assets/docs/configuration/overview-full-example.hcl b/assets/docs/configuration/overview-full-example.hcl index 184f7d6b..a8f28749 100644 --- a/assets/docs/configuration/overview-full-example.hcl +++ b/assets/docs/configuration/overview-full-example.hcl @@ -96,11 +96,11 @@ log_level = "info" // Specifies how failed writes to the target should be retried, depending on an error type retry { transient { - delay_sec = 1 + delay_ms = 1000 max_attempts = 5 } setup { - delay_sec = 20 + delay_ms = 20000 } } diff --git a/assets/docs/configuration/retry-example.hcl b/assets/docs/configuration/retry-example.hcl index 34e17e52..2f0d5299 100644 --- a/assets/docs/configuration/retry-example.hcl +++ b/assets/docs/configuration/retry-example.hcl @@ -1,9 +1,9 @@ retry { transient { - delay_sec = 5 + delay_ms = 5000 max_attempts = 10 } setup { - delay_sec = 30 + delay_ms = 30000 } } diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index be44a5df..6a3d121e 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -290,7 +290,7 @@ func handleWrite(cfg *config.Config, write func() error) error { }) onSetupError := retry.OnRetry(func(attempt uint, err error) { - log.Infof("Setup target write error. Attempt: %d, error: %s\n", attempt+1, err) + log.Warnf("Setup target write error. Attempt: %d, error: %s\n", attempt+1, err) // Here we can set unhealthy status + send monitoring alerts in the future. Nothing happens here now. }) @@ -299,7 +299,7 @@ func handleWrite(cfg *config.Config, write func() error) error { write, retryOnlySetupErrors, onSetupError, - retry.Delay(time.Duration(cfg.Data.Retry.Setup.Delay)*time.Second), + retry.Delay(time.Duration(cfg.Data.Retry.Setup.Delay) * time.Millisecond), // for now let's limit attempts to 5 for setup errors, because we don't have health check which would allow app to be killed externally. Unlimited attempts don't make sense right now. retry.Attempts(5), retry.LastErrorOnly(true), @@ -311,17 +311,21 @@ func handleWrite(cfg *config.Config, write func() error) error { return err } - //If no setup, then handle as transient. We already had at least 1 attempt from above 'setup' retrying section. - log.Infof("Transient target write error. Starting retrying. error: %s\n", err) + // If no setup, then handle as transient. + log.Warnf("Transient target write error. Starting retrying. error: %s\n", err) + + // We already had at least 1 attempt from above 'setup' retrying section, so before we start transient retrying we need add 'manual' initial delay. + time.Sleep(time.Duration(cfg.Data.Retry.Transient.Delay) * time.Millisecond) onTransientError := retry.OnRetry(func(retry uint, err error) { - log.Infof("Retry failed with transient error. Retry counter: %d, error: %s\n", retry+1, err) + log.Warnf("Retry failed with transient error. Retry counter: %d, error: %s\n", retry+1, err) }) err = retry.Do( write, onTransientError, - retry.Delay(time.Duration(cfg.Data.Retry.Transient.Delay)*time.Second), + // * 2 because we have initial sleep above + retry.Delay(time.Duration(cfg.Data.Retry.Transient.Delay*2) * time.Millisecond), retry.Attempts(uint(cfg.Data.Retry.Transient.MaxAttempts)), retry.LastErrorOnly(true), ) diff --git a/config/config.go b/config/config.go index 558ccbfc..d4cd33c3 100644 --- a/config/config.go +++ b/config/config.go @@ -101,12 +101,12 @@ type retryConfig struct { } type transientRetryConfig struct { - Delay int `hcl:"delay_sec,optional"` + Delay int `hcl:"delay_ms,optional"` MaxAttempts int `hcl:"max_attempts,optional"` } type setupRetryConfig struct { - Delay int `hcl:"delay_sec,optional"` + Delay int `hcl:"delay_ms,optional"` } // defaultConfigData returns the initial main configuration target. @@ -135,11 +135,11 @@ func defaultConfigData() *configurationData { }, Retry: &retryConfig{ Transient: &transientRetryConfig{ - Delay: 2, + Delay: 1000, MaxAttempts: 5, }, Setup: &setupRetryConfig{ - Delay: 20, + Delay: 20000, }, }, } diff --git a/config/config_test.go b/config/config_test.go index 22e6e08e..63fccaf3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -44,9 +44,9 @@ func TestNewConfig_NoConfig(t *testing.T) { assert.Equal(c.Data.LogLevel, "info") assert.Equal(c.Data.DisableTelemetry, false) assert.Equal(c.Data.License.Accept, false) - assert.Equal(2, c.Data.Retry.Transient.Delay) + assert.Equal(1000, c.Data.Retry.Transient.Delay) assert.Equal(5, c.Data.Retry.Transient.MaxAttempts) - assert.Equal(20, c.Data.Retry.Setup.Delay) + assert.Equal(20000, c.Data.Retry.Setup.Delay) } func TestNewConfig_InvalidFailureFormat(t *testing.T) { @@ -176,9 +176,9 @@ func TestNewConfig_Hcl_defaults(t *testing.T) { assert.Equal(1, c.Data.StatsReceiver.TimeoutSec) assert.Equal(15, c.Data.StatsReceiver.BufferSec) assert.Equal("info", c.Data.LogLevel) - assert.Equal(2, c.Data.Retry.Transient.Delay) + assert.Equal(1000, c.Data.Retry.Transient.Delay) assert.Equal(5, c.Data.Retry.Transient.MaxAttempts) - assert.Equal(20, c.Data.Retry.Setup.Delay) + assert.Equal(20000, c.Data.Retry.Setup.Delay) } func TestNewConfig_Hcl_sentry(t *testing.T) { diff --git a/docs/configuration_retry_docs_test.go b/docs/configuration_retry_docs_test.go index a6c8561f..e67ed369 100644 --- a/docs/configuration_retry_docs_test.go +++ b/docs/configuration_retry_docs_test.go @@ -26,7 +26,7 @@ func TestRetryConfigDocumentation(t *testing.T) { retryConfig := c.Data.Retry assert.NotNil(retryConfig) - assert.Equal(5, retryConfig.Transient.Delay) + assert.Equal(5000, retryConfig.Transient.Delay) assert.Equal(10, retryConfig.Transient.MaxAttempts) - assert.Equal(30, retryConfig.Setup.Delay) + assert.Equal(30000, retryConfig.Setup.Delay) }