From f3d2f8a512beadb15081e511efc62189ff7c68c0 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 13:17:57 +0000 Subject: [PATCH 01/43] Added support for oauth2 tokens in opensearch output sink Signed-off-by: Casper Thygesen --- go.sum | 7 ++ internal/impl/opensearch/auth.go | 99 +++++++++++++++++++ internal/impl/opensearch/output.go | 79 ++++++++++----- website/docs/components/outputs/opensearch.md | 37 ++++++- 4 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 internal/impl/opensearch/auth.go diff --git a/go.sum b/go.sum index c9369f3585..b87e3165b3 100644 --- a/go.sum +++ b/go.sum @@ -437,6 +437,7 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -694,11 +695,14 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -717,6 +721,7 @@ github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -786,6 +791,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= @@ -796,6 +802,7 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a h1:lem6QCvxR0Y28gth9P+wV2K/zYUUAkJ+55U8cpS0p5I= github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= github.com/nats-io/nats-server/v2 v2.8.2/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4= diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go new file mode 100644 index 0000000000..825c3ee6ab --- /dev/null +++ b/internal/impl/opensearch/auth.go @@ -0,0 +1,99 @@ +package opensearch + +import ( + "context" + "fmt" + + "github.com/benthosdev/benthos/v4/public/service" +) + +const ( + aFieldOAuth = "oauth2" + + abFieldEnabled = "enabled" + + aFieldAuthStaticAccessToken = "access_token" + aFieldAuthTokenCache = "token_cache" + aFieldAuthTokenKey = "token_key" +) + +// OAuthAuthField returns a config field spec for basic authentication. +func OAuthAuthField() *service.ConfigField { + return service.NewObjectField(esoFieldOAuth, + service.NewStringField(aFieldAuthStaticAccessToken). + Description("A static access token to use for authentication."). + Advanced(). + Default(""), + service.NewStringField(aFieldAuthTokenCache). + Description("Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from."). + Advanced(). + Default(""), + service.NewStringField(aFieldAuthTokenKey). + Description("Required when using a `token_cache`, the key to query the cache with for tokens."). + Advanced(). + Default(""), + ).Description("Allows you to specify OAuth2 authentication."). + Advanced(). + Optional() +} + +func oAuthFromParsed(conf *service.ParsedConfig) (res OAuthConfig, err error) { + res = NewOAuth2Config() + if !conf.Contains(aFieldOAuth) { + return + } + conf = conf.Namespace(aFieldOAuth) + if res.Enabled, err = conf.FieldBool(abFieldEnabled); err != nil { + return + } + + if conf.Contains(aFieldAuthStaticAccessToken) { + if res.StaticAccessToken, err = conf.FieldString(aFieldAuthStaticAccessToken); err != nil { + return + } + } else { + if res.TokenCacheName, err = conf.FieldString(aFieldAuthTokenCache); err != nil { + return + } + if res.TokenCacheKey, err = conf.FieldString(aFieldAuthTokenKey); err != nil { + return + } + + } + + return +} + +type OAuthConfig struct { + Enabled bool + StaticAccessToken string + TokenCacheName string + TokenCacheKey string +} + +func (c *OAuthConfig) GetToken(mgr *service.Resources) (string, error) { + if c.StaticAccessToken != "" { + return c.StaticAccessToken, nil + } + + var tok []byte + var terr error + if err := mgr.AccessCache(context.Background(), c.TokenCacheName, func(cache service.Cache) { + tok, terr = cache.Get(context.Background(), c.TokenCacheKey) + }); err != nil { + return "", fmt.Errorf("failed to obtain cache resource '%v': %v", c.TokenCacheName, err) + } + if terr != nil { + return "", terr + } + return string(tok), nil +} + +func NewOAuth2Config() OAuthConfig { + return OAuthConfig{ + Enabled: false, + StaticAccessToken: "", + TokenCacheName: "", + TokenCacheKey: "", + } +} diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index 00bf53e779..3e76dc7678 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -13,6 +13,7 @@ import ( "github.com/opensearch-project/opensearch-go/v3/opensearchapi" "github.com/opensearch-project/opensearch-go/v3/opensearchutil" + "golang.org/x/oauth2" "github.com/benthosdev/benthos/v4/internal/component" "github.com/benthosdev/benthos/v4/internal/component/output" @@ -22,13 +23,14 @@ import ( ) const ( - esoFieldURLs = "urls" - esoFieldID = "id" - esoFieldAction = "action" - esoFieldIndex = "index" - esoFieldPipeline = "pipeline" - esoFieldRouting = "routing" - esoFieldTLS = "tls" + esoFieldURLs = "urls" + esoFieldID = "id" + esoFieldAction = "action" + esoFieldIndex = "index" + esoFieldPipeline = "pipeline" + esoFieldRouting = "routing" + esoFieldTLS = "tls" + esoFieldAuth = "basic_auth" esoFieldAuthEnabled = "enabled" esoFieldAuthUsername = "username" @@ -48,7 +50,7 @@ func notImportedAWSOptFn(conf *service.ParsedConfig, osconf *opensearchapi.Confi // AWSOptFn is populated with the child `aws` package when imported. var AWSOptFn = notImportedAWSOptFn -// AWSField represents the aws block within an elasticsearch field. This is +// AWSField represents the aws block within an opensearch field. This is // exported in order to make unit testing easier within the aws subpackage. func AWSField() *service.ConfigField { return service.NewObjectField(esoFieldAWS, @@ -71,7 +73,7 @@ type esoConfig struct { routingStr *service.InterpolatedString } -func esoConfigFromParsed(pConf *service.ParsedConfig) (conf esoConfig, err error) { +func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf *esoConfig, err error) { conf.clientOpts = opensearchapi.Config{} var tmpURLs []string @@ -86,25 +88,47 @@ func esoConfigFromParsed(pConf *service.ParsedConfig) (conf esoConfig, err error } } - { - authConf := pConf.Namespace(esoFieldAuth) - if enabled, _ := authConf.FieldBool(esoFieldAuthEnabled); enabled { - if conf.clientOpts.Client.Username, err = authConf.FieldString(esoFieldAuthUsername); err != nil { - return - } - if conf.clientOpts.Client.Password, err = authConf.FieldString(esoFieldAuthPassword); err != nil { - return - } + authConf := pConf.Namespace(esoFieldAuth) + if enabled, _ := authConf.FieldBool(esoFieldAuthEnabled); enabled { + + if conf.clientOpts.Client.Username, err = authConf.FieldString(esoFieldAuthUsername); err != nil { + return + } + if conf.clientOpts.Client.Password, err = authConf.FieldString(esoFieldAuthPassword); err != nil { + return + } + } + + oauth2conf, err := oAuthFromParsed(pConf) + if err != nil { + return + } + + if oauth2conf.Enabled { + token, err := oauth2conf.GetToken(mgr) + if err != nil { + return nil, err + } + conf.clientOpts.Client.Transport = &oauth2.Transport{ + Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), + Base: http.DefaultTransport, } } var tlsConf *tls.Config var tlsEnabled bool + + conf.clientOpts.Client.Transport = http.DefaultTransport + if tlsConf, tlsEnabled, err = pConf.FieldTLSToggled(esoFieldTLS); err != nil { return } else if tlsEnabled { - conf.clientOpts.Client.Transport = &http.Transport{ - TLSClientConfig: tlsConf, + if _, ok := conf.clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport); ok { + conf.clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport).TLSClientConfig = tlsConf + } else { + conf.clientOpts.Client.Transport = &http.Transport{ + TLSClientConfig: tlsConf, + } } } @@ -132,12 +156,12 @@ func esoConfigFromParsed(pConf *service.ParsedConfig) (conf esoConfig, err error //------------------------------------------------------------------------------ -// OutputSpec returns the config spec for an elasticsearch output writer. +// OutputSpec returns the config spec for an opensearch output writer. func OutputSpec() *service.ConfigSpec { return service.NewConfigSpec(). Stable(). Categories("Services"). - Summary(`Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.`). + Summary(`Publishes messages into an opensearch index. If the index does not exist then it is created with a dynamic mapping.`). Description(output.Description(true, true, ` Both the `+"`id` and `index`"+` fields can be dynamically set using function interpolations described [here](/docs/configuration/interpolation#bloblang-queries). When sending batched messages these interpolations are performed per message part.`)). Fields( @@ -166,6 +190,7 @@ Both the `+"`id` and `index`"+` fields can be dynamically set using function int httpclient.BasicAuthField(), service.NewBatchPolicyField(esoFieldBatching), AWSField(), + OAuthAuthField(), ). Example("Updating Documents", "When [updating documents](https://opensearch.org/docs/latest/api-reference/document-apis/update-document/) the request body should contain a combination of a `doc`, `upsert`, and/or `script` fields at the top level, this should be done via mapping processors.", ` output: @@ -198,17 +223,17 @@ func init() { } } -// Output implements service.BatchOutput for elasticsearch. +// Output implements service.BatchOutput for opensearch. type Output struct { log *service.Logger - conf esoConfig + conf *esoConfig client *opensearchapi.Client } -// OutputFromParsed returns an elasticsearch output writer from a parsed config. +// OutputFromParsed returns an opensearch output writer from a parsed config. func OutputFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (*Output, error) { - conf, err := esoConfigFromParsed(pConf) + conf, err := esoConfigFromParsed(pConf, mgr) if err != nil { return nil, err } @@ -231,7 +256,7 @@ func (e *Output) Connect(ctx context.Context) error { } e.client = client - e.log.Infof("Sending messages to Elasticsearch index at urls: %s\n", e.conf.clientOpts.Client.Addresses) + e.log.Infof("Sending messages to opensearch index at urls: %s\n", e.conf.clientOpts.Client.Addresses) return nil } diff --git a/website/docs/components/outputs/opensearch.md b/website/docs/components/outputs/opensearch.md index ce250947f4..7dd2f7d795 100644 --- a/website/docs/components/outputs/opensearch.md +++ b/website/docs/components/outputs/opensearch.md @@ -14,7 +14,7 @@ categories: ["Services"] import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping. +Publishes messages into an opensearch index. If the index does not exist then it is created with a dynamic mapping. @@ -568,4 +572,35 @@ An external ID to provide when assuming a role. Type: `string` Default: `""` +### `oauth2` + +Allows you to specify OAuth2 authentication. + + +Type: `object` + +### `oauth2.access_token` + +A static access token to use for authentication. + + +Type: `string` +Default: `""` + +### `oauth2.token_cache` + +Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from. + + +Type: `string` +Default: `""` + +### `oauth2.token_key` + +Required when using a `token_cache`, the key to query the cache with for tokens. + + +Type: `string` +Default: `""` + From 86a13042ac5494397cdd7829089541ef88e73826 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 08:12:00 +0000 Subject: [PATCH 02/43] Added direct support for oauth2 tokens in kafka using franz-lib Signed-off-by: Casper Thygesen --- internal/impl/kafka/sasl.go | 123 ++++++++++++++++++++++++++++++++---- 1 file changed, 110 insertions(+), 13 deletions(-) diff --git a/internal/impl/kafka/sasl.go b/internal/impl/kafka/sasl.go index 394c927adc..ef750716ca 100644 --- a/internal/impl/kafka/sasl.go +++ b/internal/impl/kafka/sasl.go @@ -2,8 +2,14 @@ package kafka import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" + "io" + "net/http" + "net/url" + "strings" "github.com/IBM/sarama" @@ -43,6 +49,18 @@ func saslField() *service.ConfigField { service.NewStringField("token"). Description("The token to use for a single session's OAUTHBEARER authentication."). Default(""), + service.NewStringField("tokenEndpoint"). + Description("The endpoint to use for OAUTHBEARER token acquisition."). + Default(""), + service.NewStringField("clientId"). + Description("The client ID to use for OAUTHBEARER token acquisition."). + Default(""), + service.NewStringField("clientSecret"). + Description("The client secret to use for OAUTHBEARER token acquisition."). + Default("").Secret(), + service.NewStringField("scope"). + Description("The scope to use for OAUTHBEARER token acquisition."). + Default(""), service.NewStringMapField("extensions"). Description("Key/value pairs to add to OAUTHBEARER authentication requests."). Optional(), @@ -129,22 +147,101 @@ func plainSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { } func oauthSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { - token, err := c.FieldString("token") - if err != nil { - return nil, err - } - var extensions map[string]string - if c.Contains("extensions") { - if extensions, err = c.FieldStringMap("extensions"); err != nil { + if c.Contains("token") { + token, err := c.FieldString("token") + if err != nil { return nil, err } + var extensions map[string]string + if c.Contains("extensions") { + if extensions, err = c.FieldStringMap("extensions"); err != nil { + return nil, err + } + } + return oauth.Oauth(func(c context.Context) (oauth.Auth, error) { + return oauth.Auth{ + Token: token, + Extensions: extensions, + }, nil + }), nil + } else if c.Contains("tokenEndpoint") { + return oauth.Oauth(func(ctx context.Context) (oauth.Auth, error) { + shortToken, err := acquireToken(ctx, c) + return oauth.Auth{Token: shortToken}, err + }), nil } - return oauth.Oauth(func(c context.Context) (oauth.Auth, error) { - return oauth.Auth{ - Token: token, - Extensions: extensions, - }, nil - }), nil + return nil, errors.New("field 'token' or 'tokenEndpoint' was not found in the config") +} + +func acquireToken(ctx context.Context, c *service.ParsedConfig) (string, error) { + + tokenEndpoint, err := c.FieldString("tokenEndpoint") + if err != nil { + return "", err + } + + clientId, err := c.FieldString("clientId") + if err != nil { + return "", err + } + + clientSecret, err := c.FieldString("clientSecret") + if err != nil { + return "", err + } + + scope, err := c.FieldString("scope") + if err != nil { + return "", err + } + + authHeaderValue := base64.StdEncoding.EncodeToString([]byte(clientId + ":" + clientSecret)) + + queryParams := url.Values{} + queryParams.Set("grant_type", "client_credentials") + queryParams.Set("scope", scope) + + req, err := http.NewRequestWithContext(ctx, "POST", tokenEndpoint, strings.NewReader(queryParams.Encode())) + if err != nil { + return "", err + } + + req.URL.RawQuery = queryParams.Encode() + + req.Header.Set("Authorization", "Basic "+authHeaderValue) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + if err := resp.Body.Close(); err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("token request failed with status code %d", resp.StatusCode) + } + + var tokenResponse map[string]interface{} + err = json.Unmarshal(body, &tokenResponse) + if err != nil { + return "", fmt.Errorf("failed to parse token response: %s", err) + } + + accessToken, ok := tokenResponse["access_token"].(string) + if !ok { + return "", fmt.Errorf("access_token not found in token response") + } + + return accessToken, nil } func scram256SaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { From 8f95960227c94e877b1bdf3db77b12ec89f19eb2 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 13:20:35 +0000 Subject: [PATCH 03/43] updated docs Signed-off-by: Casper Thygesen --- website/docs/components/inputs/kafka_franz.md | 35 +++++++++++++++++++ .../docs/components/outputs/kafka_franz.md | 35 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/website/docs/components/inputs/kafka_franz.md b/website/docs/components/inputs/kafka_franz.md index be2ed424d7..065f3eb84c 100644 --- a/website/docs/components/inputs/kafka_franz.md +++ b/website/docs/components/inputs/kafka_franz.md @@ -407,6 +407,41 @@ Default: `""` The token to use for a single session's OAUTHBEARER authentication. +Type: `string` +Default: `""` + +### `sasl[].tokenEndpoint` + +The endpoint to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientId` + +The client ID to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientSecret` + +The client secret to use for OAUTHBEARER token acquisition. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `sasl[].scope` + +The scope to use for OAUTHBEARER token acquisition. + + Type: `string` Default: `""` diff --git a/website/docs/components/outputs/kafka_franz.md b/website/docs/components/outputs/kafka_franz.md index a631f07814..110144aff5 100644 --- a/website/docs/components/outputs/kafka_franz.md +++ b/website/docs/components/outputs/kafka_franz.md @@ -568,6 +568,41 @@ Default: `""` The token to use for a single session's OAUTHBEARER authentication. +Type: `string` +Default: `""` + +### `sasl[].tokenEndpoint` + +The endpoint to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientId` + +The client ID to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientSecret` + +The client secret to use for OAUTHBEARER token acquisition. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `sasl[].scope` + +The scope to use for OAUTHBEARER token acquisition. + + Type: `string` Default: `""` From d33a1e6a53dda3b5bd090ec7b96c1e2a7d20315b Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 13:33:09 +0000 Subject: [PATCH 04/43] Typo Signed-off-by: Casper Thygesen --- internal/impl/opensearch/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 825c3ee6ab..159e9f81d8 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -19,7 +19,7 @@ const ( // OAuthAuthField returns a config field spec for basic authentication. func OAuthAuthField() *service.ConfigField { - return service.NewObjectField(esoFieldOAuth, + return service.NewObjectField(aFieldOAuth, service.NewStringField(aFieldAuthStaticAccessToken). Description("A static access token to use for authentication."). Advanced(). From 02c198e4d51dcc76bc4b649219da8e9855ddef32 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 14:19:06 +0000 Subject: [PATCH 05/43] Fix nullpointer issue Signed-off-by: Casper Thygesen --- internal/impl/opensearch/output.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index 3e76dc7678..ab380ed31f 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -73,7 +73,7 @@ type esoConfig struct { routingStr *service.InterpolatedString } -func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf *esoConfig, err error) { +func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf esoConfig, err error) { conf.clientOpts = opensearchapi.Config{} var tmpURLs []string @@ -107,7 +107,7 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c if oauth2conf.Enabled { token, err := oauth2conf.GetToken(mgr) if err != nil { - return nil, err + return conf, err } conf.clientOpts.Client.Transport = &oauth2.Transport{ Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), @@ -226,7 +226,7 @@ func init() { // Output implements service.BatchOutput for opensearch. type Output struct { log *service.Logger - conf *esoConfig + conf esoConfig client *opensearchapi.Client } From 6eccd2b5b05304c1bcbf01f7657f52ade1fcec52 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 14:22:40 +0000 Subject: [PATCH 06/43] mark static access token as secret Signed-off-by: Casper Thygesen --- internal/impl/opensearch/auth.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 159e9f81d8..5a1c07d7d5 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -23,6 +23,7 @@ func OAuthAuthField() *service.ConfigField { service.NewStringField(aFieldAuthStaticAccessToken). Description("A static access token to use for authentication."). Advanced(). + Secret(). Default(""), service.NewStringField(aFieldAuthTokenCache). Description("Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from."). From 670ff2152aa424585cc56869c157642c71407b8d Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 14:27:23 +0000 Subject: [PATCH 07/43] mark static token as secret Signed-off-by: Casper Thygesen --- internal/impl/kafka/sasl.go | 1 + internal/impl/kafka/sasl/sasl.go | 2 +- website/docs/components/inputs/kafka.md | 3 +++ website/docs/components/outputs/kafka.md | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/impl/kafka/sasl.go b/internal/impl/kafka/sasl.go index ef750716ca..f7521118f1 100644 --- a/internal/impl/kafka/sasl.go +++ b/internal/impl/kafka/sasl.go @@ -320,6 +320,7 @@ func SaramaSASLField() *service.ConfigField { Secret(), service.NewStringField(saramaFieldSASLAccessToken). Description("A static OAUTHBEARER access token"). + Secret(). Default(""), service.NewStringField(saramaFieldSASLTokenCache). Description("Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from"). diff --git a/internal/impl/kafka/sasl/sasl.go b/internal/impl/kafka/sasl/sasl.go index 67cffb9880..ac4f8e299c 100644 --- a/internal/impl/kafka/sasl/sasl.go +++ b/internal/impl/kafka/sasl/sasl.go @@ -33,7 +33,7 @@ func FieldSpec() docs.FieldSpec { ), docs.FieldString("user", "A PLAIN username. It is recommended that you use environment variables to populate this field.", "${USER}"), docs.FieldString("password", "A PLAIN password. It is recommended that you use environment variables to populate this field.", "${PASSWORD}").Secret(), - docs.FieldString("access_token", "A static OAUTHBEARER access token"), + docs.FieldString("access_token", "A static OAUTHBEARER access token").Secret(), docs.FieldString("token_cache", "Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from"), docs.FieldString("token_key", "Required when using a `token_cache`, the key to query the cache with for tokens."), ).Advanced() diff --git a/website/docs/components/inputs/kafka.md b/website/docs/components/inputs/kafka.md index fb92d6fba4..4dcd80d72c 100644 --- a/website/docs/components/inputs/kafka.md +++ b/website/docs/components/inputs/kafka.md @@ -391,6 +391,9 @@ password: ${PASSWORD} ### `sasl.access_token` A static OAUTHBEARER access token +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: Type: `string` diff --git a/website/docs/components/outputs/kafka.md b/website/docs/components/outputs/kafka.md index 7d8a51e508..a92c8cdefe 100644 --- a/website/docs/components/outputs/kafka.md +++ b/website/docs/components/outputs/kafka.md @@ -359,6 +359,9 @@ password: ${PASSWORD} ### `sasl.access_token` A static OAUTHBEARER access token +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: Type: `string` From 360bf9569081025f45d3fcb84281b06eb9c33185 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Fri, 1 Mar 2024 14:56:51 +0000 Subject: [PATCH 08/43] did not check if token is empty and not just non-existing Signed-off-by: Casper Thygesen --- internal/impl/kafka/sasl.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/impl/kafka/sasl.go b/internal/impl/kafka/sasl.go index f7521118f1..10882399ef 100644 --- a/internal/impl/kafka/sasl.go +++ b/internal/impl/kafka/sasl.go @@ -147,8 +147,9 @@ func plainSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { } func oauthSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { - if c.Contains("token") { - token, err := c.FieldString("token") + token, err := c.FieldString("token") + + if err != nil && token != "" { if err != nil { return nil, err } From 7b940cd8c07d212722bab1a409f7282bdd767bf5 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Sun, 3 Mar 2024 19:45:26 +0000 Subject: [PATCH 09/43] forgot to describe oauth2.enabled --- internal/impl/opensearch/auth.go | 12 +++++++++--- internal/impl/opensearch/output.go | 7 ++++--- website/docs/components/outputs/opensearch.md | 12 ++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 5a1c07d7d5..66b964289b 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -2,6 +2,7 @@ package opensearch import ( "context" + "errors" "fmt" "github.com/benthosdev/benthos/v4/public/service" @@ -20,6 +21,10 @@ const ( // OAuthAuthField returns a config field spec for basic authentication. func OAuthAuthField() *service.ConfigField { return service.NewObjectField(aFieldOAuth, + service.NewBoolField(abFieldEnabled). + Description("Whether to use OAuth2 authentication."). + Advanced(). + Default(false), service.NewStringField(aFieldAuthStaticAccessToken). Description("A static access token to use for authentication."). Advanced(). @@ -48,7 +53,8 @@ func oAuthFromParsed(conf *service.ParsedConfig) (res OAuthConfig, err error) { return } - if conf.Contains(aFieldAuthStaticAccessToken) { + staticToken, err := conf.FieldString(aFieldAuthStaticAccessToken) + if err != nil && staticToken != "" { if res.StaticAccessToken, err = conf.FieldString(aFieldAuthStaticAccessToken); err != nil { return } @@ -82,10 +88,10 @@ func (c *OAuthConfig) GetToken(mgr *service.Resources) (string, error) { if err := mgr.AccessCache(context.Background(), c.TokenCacheName, func(cache service.Cache) { tok, terr = cache.Get(context.Background(), c.TokenCacheKey) }); err != nil { - return "", fmt.Errorf("failed to obtain cache resource '%v': %v", c.TokenCacheName, err) + return "", fmt.Errorf("failed to obtain cache resource '%v' with key %v: %v", c.TokenCacheName, c.TokenCacheKey, err) } if terr != nil { - return "", terr + return "", errors.Join(terr, fmt.Errorf("failed to obtain token wih key %v from cache", c.TokenCacheKey)) } return string(tok), nil } diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index ab380ed31f..4e53de2d08 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -105,9 +105,10 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c } if oauth2conf.Enabled { - token, err := oauth2conf.GetToken(mgr) + token, _ := oauth2conf.GetToken(mgr) if err != nil { - return conf, err + return + //return conf, err } conf.clientOpts.Client.Transport = &oauth2.Transport{ Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), @@ -341,7 +342,7 @@ func (e *Output) WriteBatch(ctx context.Context, msg service.MessageBatch) error dur := time.Since(start) e.log.Debugf( - "Successfully dispatched [%s] documents in %s (%s docs/sec)", + "Successfully dispatched [%d] documents in %s (%d docs/sec)", biStats.NumFlushed, dur.Truncate(time.Millisecond), int64(1000.0/float64(dur/time.Millisecond)*float64(biStats.NumFlushed)), diff --git a/website/docs/components/outputs/opensearch.md b/website/docs/components/outputs/opensearch.md index 7dd2f7d795..4d776802b7 100644 --- a/website/docs/components/outputs/opensearch.md +++ b/website/docs/components/outputs/opensearch.md @@ -86,6 +86,7 @@ output: role: "" role_external_id: "" oauth2: + enabled: false access_token: "" token_cache: "" token_key: "" @@ -579,9 +580,20 @@ Allows you to specify OAuth2 authentication. Type: `object` +### `oauth2.enabled` + +Whether to use OAuth2 authentication. + + +Type: `bool` +Default: `false` + ### `oauth2.access_token` A static access token to use for authentication. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: Type: `string` From d8d1651488a5640fac78d9f18e14c5ab10b2e90c Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Mon, 4 Mar 2024 19:50:32 +0000 Subject: [PATCH 10/43] devcontainer support --- .devcontainer/Dockerfile | 33 ++++++++++++++++++ .devcontainer/devcontainer.json | 33 ++++++++++++++++++ resources/docker/Dockerfile | 12 +++++-- resources/docker/kafka/docker-compose.yaml | 21 ++++++++++++ resources/docker/kafka/in.yaml | 39 ++++++++++++++++++++++ resources/docker/kafka/out.yaml | 39 ++++++++++++++++++++++ 6 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 resources/docker/kafka/docker-compose.yaml create mode 100644 resources/docker/kafka/in.yaml create mode 100644 resources/docker/kafka/out.yaml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..7f435d4537 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,33 @@ +FROM mcr.microsoft.com/vscode/devcontainers/go:1.21 + +ARG NODE_VERSION="16" +ARG GOLANGCI_LINT_VERSION="1.46.2" + +# https://github.com/microsoft/vscode-dev-containers/blob/main/containers/go/.devcontainer/base.Dockerfile +ENV USERNAME=vscode +ENV LIBRARY_SCRIPTS_SRC="https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/containers/go/.devcontainer/library-scripts/node-debian.sh" +ENV NVM_DIR=/usr/local/share/nvm +ENV NVM_SYMLINK_CURRENT=true \ + PATH=${NVM_DIR}/current/bin:${PATH} +RUN mkdir /tmp/library-scripts \ + && curl -fsSL -o /tmp/library-scripts/node-debian.sh "${LIBRARY_SCRIPTS_SRC}" +RUN bash /tmp/library-scripts/node-debian.sh "${NVM_DIR}" "${NODE_VERSION}" "${USERNAME}" \ + && apt-get clean -y && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/library-scripts + +RUN echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' \ + | tee /etc/apt/sources.list.d/goreleaser.list +RUN apt-get update \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends goreleaser + +RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ + | sh -s -- -b $(go env GOPATH)/bin v$GOLANGCI_LINT_VERSION + +USER vscode +WORKDIR /home/vscode + +RUN mkdir -p .config/git \ + && echo ".vscode/*" >> .config/git/ignore \ + && echo "*.code-workspace" >> .config/git/ignore \ + && echo ".history/" >> .config/git/ignore diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9e780cc73c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,33 @@ +{ + "name": "Benthos Dev Container", + "build": { + "dockerfile": "Dockerfile", + "args": { + "NODE_VERSION": "16", + "GOLANGCI_LINT_VERSION": "1.46.2" + } + }, + "runArgs": ["--network=host", "--privileged"], + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + "extensions": [ + "golang.Go", + "dbaeumer.vscode-eslint", + "EditorConfig.EditorConfig", + "esbenp.prettier-vscode", + "github.vscode-github-actions", + "jebbs.plantuml", + "GitHub.copilot", + "github.vscode-github-actions", + "ms-vscode.makefile-tools", + "GeorgesHaidar.vsc-benthos" + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {} + }, + "postCreateCommand": "go mod download", + "remoteUser": "vscode" +} diff --git a/resources/docker/Dockerfile b/resources/docker/Dockerfile index f0fde55f1e..89309f1ffe 100644 --- a/resources/docker/Dockerfile +++ b/resources/docker/Dockerfile @@ -2,18 +2,24 @@ FROM golang:1.21 AS build ENV CGO_ENABLED=0 ENV GOOS=linux -RUN useradd -u 10001 benthos + +# Prepare the build environment +RUN go env -w GOCACHE=/go-cache; \ + go env -w GOMODCACHE=/gomod-cache; \ + useradd -u 10001 benthos WORKDIR /go/src/github.com/benthosdev/benthos/ # Update dependencies: On unchanged dependencies, cached layer will be reused COPY go.* /go/src/github.com/benthosdev/benthos/ -RUN go mod download +RUN --mount=type=cache,target=/gomod-cache \ + go mod download # Build COPY . /go/src/github.com/benthosdev/benthos/ # Tag timetzdata required for busybox base image: # https://github.com/benthosdev/benthos/issues/897 -RUN make TAGS="timetzdata" +RUN --mount=type=cache,target=/gomod-cache --mount=type=cache,target=/go-cache \ + make TAGS="timetzdata" # Pack FROM busybox AS package diff --git a/resources/docker/kafka/docker-compose.yaml b/resources/docker/kafka/docker-compose.yaml new file mode 100644 index 0000000000..086eb34e44 --- /dev/null +++ b/resources/docker/kafka/docker-compose.yaml @@ -0,0 +1,21 @@ +version: '3.3' +services: + benthos-in: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ '-w', '-c', '/benthos.yaml' ] + volumes: + - ./in.yaml:/benthos.yaml + network_mode: host + + benthos-out: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ '-w', '-c', '/benthos.yaml' ] + volumes: + - ./out.yaml:/benthos.yaml + network_mode: host \ No newline at end of file diff --git a/resources/docker/kafka/in.yaml b/resources/docker/kafka/in.yaml new file mode 100644 index 0000000000..92d73a8bab --- /dev/null +++ b/resources/docker/kafka/in.yaml @@ -0,0 +1,39 @@ +http: + enabled: false + +input: + generate: + interval: 5s + mapping: | + root.ID = uuid_v4() + root.Name = [ "frosty", "spot", "oodles" ].index(random_int() % 3) + root.Gooeyness = (random_int() % 100) / 100 + root.Bouncing = random_int() % 2 == 0 + +pipeline: + processors: + - bloblang: | + root = this + root.ID = this.ID + root.ReNamed = this.Name + root.Gooeyness = this.Gooeyness + root.Bouncing = this.Bouncing + + - catch: + - log: + level: ERROR + message: ${! error() } + - bloblang: root = deleted() + +output: + label: sink + kafka_franz: + seed_brokers: [localhost:9092] # No default (required) + topic: benthos_redpanda # No default (required) + key: "" # No default (optional) + sasl: + - mechanism: OAUTHBEARER + tokenEndpoint: http://localhost:1852/realms/local-development/protocol/openid-connect/token + scope: "kafka" + clientId: "default-access" + clientSecret: "default-access-secret" diff --git a/resources/docker/kafka/out.yaml b/resources/docker/kafka/out.yaml new file mode 100644 index 0000000000..afa38278d6 --- /dev/null +++ b/resources/docker/kafka/out.yaml @@ -0,0 +1,39 @@ +http: + address: 0.0.0.0:4195 + enabled: true + root_path: /benthos + debug_endpoints: false + +logger: + level: DEBUG + format: logfmt + add_timestamp: true + static_fields: + "@service": benthos + +input: + label: source + kafka_franz: + seed_brokers: [localhost:9092] # No default (required) + topics: [benthos_redpanda] # No default (required) + consumer_group: benthos + sasl: + - mechanism: OAUTHBEARER # No default (required) + tokenEndpoint: http://localhost:1852/realms/local-development/protocol/openid-connect/token + scope: "kafka" # No default (required) + clientId: "default-access" # No default (required) + clientSecret: "default-access-secret" # No default (required) + +pipeline: + processors: + - bloblang: | + root = this + + - catch: + - log: + level: ERROR + message: ${! error() } + - bloblang: root = deleted() + +output: + stdout: {} From c54fa47bf68488d5f87c2e4e3a572c8bea4c9167 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 09:40:36 +0000 Subject: [PATCH 11/43] oauth2 support with clientcredentials --- internal/impl/opensearch/auth.go | 170 ++++++++++++++++-- internal/impl/opensearch/output.go | 19 +- .../docker/opensearch/docker-compose.yaml | 11 ++ resources/docker/opensearch/oauth2.yaml | 28 +++ website/docs/components/outputs/opensearch.md | 39 ++++ 5 files changed, 242 insertions(+), 25 deletions(-) create mode 100644 resources/docker/opensearch/docker-compose.yaml create mode 100644 resources/docker/opensearch/oauth2.yaml diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 66b964289b..4d2e4ed87b 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -2,10 +2,17 @@ package opensearch import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" + "net/http" + "net/url" + "strings" + "time" "github.com/benthosdev/benthos/v4/public/service" + "golang.org/x/oauth2" ) const ( @@ -16,6 +23,10 @@ const ( aFieldAuthStaticAccessToken = "access_token" aFieldAuthTokenCache = "token_cache" aFieldAuthTokenKey = "token_key" + aFieldAuthTokenEndpoint = "tokenEndpoint" + aFieldAuthTokenClientId = "clientId" + aFieldAuthTokenClientSecret = "clientSecret" + aFieldAuthTokenScope = "scope" ) // OAuthAuthField returns a config field spec for basic authentication. @@ -30,6 +41,18 @@ func OAuthAuthField() *service.ConfigField { Advanced(). Secret(). Default(""), + service.NewStringField(aFieldAuthTokenEndpoint). + Description("The endpoint to use for OAUTHBEARER token acquisition."). + Default(""), + service.NewStringField(aFieldAuthTokenClientId). + Description("The client ID to use for OAUTHBEARER token acquisition."). + Default(""), + service.NewStringField(aFieldAuthTokenClientSecret). + Description("The client secret to use for OAUTHBEARER token acquisition."). + Default("").Secret(), + service.NewStringField(aFieldAuthTokenScope). + Description("The scope to use for OAUTHBEARER token acquisition."). + Default(""), service.NewStringField(aFieldAuthTokenCache). Description("Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from."). Advanced(). @@ -53,22 +76,25 @@ func oAuthFromParsed(conf *service.ParsedConfig) (res OAuthConfig, err error) { return } - staticToken, err := conf.FieldString(aFieldAuthStaticAccessToken) - if err != nil && staticToken != "" { - if res.StaticAccessToken, err = conf.FieldString(aFieldAuthStaticAccessToken); err != nil { + res.TokenCacheKey, _ = conf.FieldString(aFieldAuthTokenKey) + res.TokenCacheName, _ = conf.FieldString(aFieldAuthTokenCache) + res.StaticAccessToken, _ = conf.FieldString(aFieldAuthStaticAccessToken) + endpoint, err := conf.FieldString(aFieldAuthTokenEndpoint) + if err == nil && endpoint != "" { + if res.TokenEndpoint, err = conf.FieldString(aFieldAuthTokenEndpoint); err != nil { return } - } else { - if res.TokenCacheName, err = conf.FieldString(aFieldAuthTokenCache); err != nil { + if res.ClientId, err = conf.FieldString(aFieldAuthTokenClientId); err != nil { return } - if res.TokenCacheKey, err = conf.FieldString(aFieldAuthTokenKey); err != nil { + if res.ClientSecret, err = conf.FieldString(aFieldAuthTokenClientSecret); err != nil { + return + } + if res.Scope, err = conf.FieldString(aFieldAuthTokenScope); err != nil { return } - } - - return + return res, nil } type OAuthConfig struct { @@ -76,24 +102,83 @@ type OAuthConfig struct { StaticAccessToken string TokenCacheName string TokenCacheKey string + TokenEndpoint string + ClientId string + ClientSecret string + Scope string } -func (c *OAuthConfig) GetToken(mgr *service.Resources) (string, error) { - if c.StaticAccessToken != "" { - return c.StaticAccessToken, nil - } +func (c *OAuthConfig) GetCachedToken(mgr *service.Resources) (*oauth2.Token, error) { var tok []byte var terr error if err := mgr.AccessCache(context.Background(), c.TokenCacheName, func(cache service.Cache) { tok, terr = cache.Get(context.Background(), c.TokenCacheKey) }); err != nil { - return "", fmt.Errorf("failed to obtain cache resource '%v' with key %v: %v", c.TokenCacheName, c.TokenCacheKey, err) + return nil, fmt.Errorf("failed to obtain cache resource '%v': %v", c.TokenCacheName, err) } if terr != nil { - return "", errors.Join(terr, fmt.Errorf("failed to obtain token wih key %v from cache", c.TokenCacheKey)) + return nil, errors.Join(terr, fmt.Errorf("failed to obtain token wih key %v from cache", c.TokenCacheKey)) + } + + if tok == nil || string(tok) == "null" { + return nil, errors.New("token is empty") } - return string(tok), nil + + return &oauth2.Token{ + AccessToken: string(tok), + }, nil +} + +func (c *OAuthConfig) GetToken(ctx context.Context) (*oauth2.Token, error) { + + authHeaderValue := base64.StdEncoding.EncodeToString([]byte(c.ClientId + ":" + c.ClientSecret)) + + queryParams := url.Values{} + queryParams.Set("grant_type", "client_credentials") + queryParams.Set("scope", c.Scope) + + req, err := http.NewRequestWithContext(ctx, "POST", c.TokenEndpoint, strings.NewReader(queryParams.Encode())) + if err != nil { + return nil, err + } + + req.URL.RawQuery = queryParams.Encode() + + req.Header.Set("Authorization", "Basic "+authHeaderValue) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + // Decode the bodyBytes into the response object + decoder := json.NewDecoder(resp.Body) + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("token request failed with status code %d", resp.StatusCode) + } + + var token *tokenResponseBody + err = decoder.Decode(&token) + if err != nil { + panic("Unable to unmarshal response: " + err.Error()) + } + + if err := resp.Body.Close(); err != nil { + return nil, err + } + + duration := time.Duration(token.ExpiresIn) * time.Second + + return &oauth2.Token{ + AccessToken: token.AccessToken, + TokenType: token.TokenType, + Expiry: time.Now().Add(duration), + }, nil + } func NewOAuth2Config() OAuthConfig { @@ -102,5 +187,58 @@ func NewOAuth2Config() OAuthConfig { StaticAccessToken: "", TokenCacheName: "", TokenCacheKey: "", + TokenEndpoint: "", + ClientId: "", + ClientSecret: "", + Scope: "", } } + +// OsTokenProvider is an implementation of oauth2.TokenSource. +type OsTokenProvider struct { + Mgr *service.Resources + OAuth2Conf OAuthConfig + Logger *service.Logger +} + +// Token returns a token or an error. +func (s OsTokenProvider) Token() (*oauth2.Token, error) { + mgr := s.Mgr + + if s.OAuth2Conf.StaticAccessToken != "" { + mgr.Logger().Debug("Using static OAuth2 token for OpenSearch") + return &oauth2.Token{AccessToken: s.OAuth2Conf.StaticAccessToken}, nil + } + + if s.OAuth2Conf.TokenEndpoint != "" { + return s.OAuth2Conf.GetToken(context.Background()) + } + + if s.OAuth2Conf.TokenCacheKey != "" && s.OAuth2Conf.TokenCacheName != "" { + return s.OAuth2Conf.GetCachedToken(mgr) + } + + return nil, errors.New("no token source available") +} + +type tokenResponseBody struct { + // AccessToken is the token that authorizes and authenticates + // the requests. + AccessToken string `json:"access_token"` + + // TokenType is the type of token. + // The Type method returns either this or "Bearer", the default. + TokenType string `json:"token_type,omitempty"` + + // RefreshToken is a token that's used by the application + // (as opposed to the user) to refresh the access token + // if it expires. + RefreshToken string `json:"refresh_token,omitempty"` + + // ExpiresIn is the optional expiration time of the access token. + // + // If zero, TokenSource implementations will reuse the same + // token forever and RefreshToken or equivalent + // mechanisms for that TokenSource will not be used. + ExpiresIn int `json:"expires_in,omitempty"` +} diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index 4e53de2d08..4b0b490886 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -101,26 +101,27 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c oauth2conf, err := oAuthFromParsed(pConf) if err != nil { + mgr.Logger().Error("Failed to parse OAuth2 configuration") return } if oauth2conf.Enabled { - token, _ := oauth2conf.GetToken(mgr) - if err != nil { - return - //return conf, err - } + + mgr.Logger().Debug("Using OAuth2 authentication for OpenSearch") + conf.clientOpts.Client.Transport = &oauth2.Transport{ - Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), - Base: http.DefaultTransport, + Source: oauth2.ReuseTokenSource(nil, &OsTokenProvider{ + Mgr: mgr, + OAuth2Conf: oauth2conf, + Logger: mgr.Logger(), + }), + Base: http.DefaultTransport, } } var tlsConf *tls.Config var tlsEnabled bool - conf.clientOpts.Client.Transport = http.DefaultTransport - if tlsConf, tlsEnabled, err = pConf.FieldTLSToggled(esoFieldTLS); err != nil { return } else if tlsEnabled { diff --git a/resources/docker/opensearch/docker-compose.yaml b/resources/docker/opensearch/docker-compose.yaml new file mode 100644 index 0000000000..39a0b89c7e --- /dev/null +++ b/resources/docker/opensearch/docker-compose.yaml @@ -0,0 +1,11 @@ +version: '3.3' +services: + benthos-opensearch-oauth2: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ '-w', '-c', '/benthos.yaml' ] + volumes: + - ./oauth2.yaml:/benthos.yaml + network_mode: host diff --git a/resources/docker/opensearch/oauth2.yaml b/resources/docker/opensearch/oauth2.yaml new file mode 100644 index 0000000000..84f73d599f --- /dev/null +++ b/resources/docker/opensearch/oauth2.yaml @@ -0,0 +1,28 @@ +logger: + level: TRACE + +http: + enabled: false + +input: + generate: + interval: 5m + mapping: | + root.ID = uuid_v4() + root.Name = [ "frosty", "spot", "oodles" ].index(random_int() % 3) + root.Gooeyness = (random_int() % 100) / 100 + root.Bouncing = random_int() % 2 == 0 + +output: + label: sink + opensearch: + urls: ["http://localhost:9200"] + index: benthos_oauth2_test2 + action: "index" + id: ${!counter()}-${!timestamp_unix()} + oauth2: + enabled: true + tokenEndpoint: "http://localhost:1852/realms/local-development/protocol/openid-connect/token" + clientId: "default-access" + clientSecret: "default-access-secret" + scope: "opensearch" diff --git a/website/docs/components/outputs/opensearch.md b/website/docs/components/outputs/opensearch.md index 4d776802b7..cd84c0cc3b 100644 --- a/website/docs/components/outputs/opensearch.md +++ b/website/docs/components/outputs/opensearch.md @@ -88,6 +88,10 @@ output: oauth2: enabled: false access_token: "" + tokenEndpoint: "" + clientId: "" + clientSecret: "" + scope: "" token_cache: "" token_key: "" ``` @@ -596,6 +600,41 @@ This field contains sensitive information that usually shouldn't be added to a c ::: +Type: `string` +Default: `""` + +### `oauth2.tokenEndpoint` + +The endpoint to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `oauth2.clientId` + +The client ID to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `oauth2.clientSecret` + +The client secret to use for OAUTHBEARER token acquisition. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `oauth2.scope` + +The scope to use for OAUTHBEARER token acquisition. + + Type: `string` Default: `""` From 83ae73954c7ffe5879653bb4284f71967275fa3e Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 13:17:57 +0000 Subject: [PATCH 12/43] Added support for oauth2 tokens in opensearch output sink Signed-off-by: Casper Thygesen --- go.sum | 6 ++ internal/impl/opensearch/auth.go | 99 +++++++++++++++++++ internal/impl/opensearch/output.go | 78 ++++++++++----- website/docs/components/outputs/opensearch.md | 37 ++++++- 4 files changed, 193 insertions(+), 27 deletions(-) create mode 100644 internal/impl/opensearch/auth.go diff --git a/go.sum b/go.sum index cbe6a2c0a9..f75bd76ff1 100644 --- a/go.sum +++ b/go.sum @@ -448,6 +448,7 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -707,11 +708,14 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -730,6 +734,7 @@ github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -801,6 +806,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go new file mode 100644 index 0000000000..825c3ee6ab --- /dev/null +++ b/internal/impl/opensearch/auth.go @@ -0,0 +1,99 @@ +package opensearch + +import ( + "context" + "fmt" + + "github.com/benthosdev/benthos/v4/public/service" +) + +const ( + aFieldOAuth = "oauth2" + + abFieldEnabled = "enabled" + + aFieldAuthStaticAccessToken = "access_token" + aFieldAuthTokenCache = "token_cache" + aFieldAuthTokenKey = "token_key" +) + +// OAuthAuthField returns a config field spec for basic authentication. +func OAuthAuthField() *service.ConfigField { + return service.NewObjectField(esoFieldOAuth, + service.NewStringField(aFieldAuthStaticAccessToken). + Description("A static access token to use for authentication."). + Advanced(). + Default(""), + service.NewStringField(aFieldAuthTokenCache). + Description("Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from."). + Advanced(). + Default(""), + service.NewStringField(aFieldAuthTokenKey). + Description("Required when using a `token_cache`, the key to query the cache with for tokens."). + Advanced(). + Default(""), + ).Description("Allows you to specify OAuth2 authentication."). + Advanced(). + Optional() +} + +func oAuthFromParsed(conf *service.ParsedConfig) (res OAuthConfig, err error) { + res = NewOAuth2Config() + if !conf.Contains(aFieldOAuth) { + return + } + conf = conf.Namespace(aFieldOAuth) + if res.Enabled, err = conf.FieldBool(abFieldEnabled); err != nil { + return + } + + if conf.Contains(aFieldAuthStaticAccessToken) { + if res.StaticAccessToken, err = conf.FieldString(aFieldAuthStaticAccessToken); err != nil { + return + } + } else { + if res.TokenCacheName, err = conf.FieldString(aFieldAuthTokenCache); err != nil { + return + } + if res.TokenCacheKey, err = conf.FieldString(aFieldAuthTokenKey); err != nil { + return + } + + } + + return +} + +type OAuthConfig struct { + Enabled bool + StaticAccessToken string + TokenCacheName string + TokenCacheKey string +} + +func (c *OAuthConfig) GetToken(mgr *service.Resources) (string, error) { + if c.StaticAccessToken != "" { + return c.StaticAccessToken, nil + } + + var tok []byte + var terr error + if err := mgr.AccessCache(context.Background(), c.TokenCacheName, func(cache service.Cache) { + tok, terr = cache.Get(context.Background(), c.TokenCacheKey) + }); err != nil { + return "", fmt.Errorf("failed to obtain cache resource '%v': %v", c.TokenCacheName, err) + } + if terr != nil { + return "", terr + } + return string(tok), nil +} + +func NewOAuth2Config() OAuthConfig { + return OAuthConfig{ + Enabled: false, + StaticAccessToken: "", + TokenCacheName: "", + TokenCacheKey: "", + } +} diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index aaeb9aadb3..8139102346 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -13,6 +13,7 @@ import ( "github.com/opensearch-project/opensearch-go/v3/opensearchapi" "github.com/opensearch-project/opensearch-go/v3/opensearchutil" + "golang.org/x/oauth2" "github.com/benthosdev/benthos/v4/internal/component" "github.com/benthosdev/benthos/v4/internal/component/output" @@ -22,13 +23,14 @@ import ( ) const ( - esoFieldURLs = "urls" - esoFieldID = "id" - esoFieldAction = "action" - esoFieldIndex = "index" - esoFieldPipeline = "pipeline" - esoFieldRouting = "routing" - esoFieldTLS = "tls" + esoFieldURLs = "urls" + esoFieldID = "id" + esoFieldAction = "action" + esoFieldIndex = "index" + esoFieldPipeline = "pipeline" + esoFieldRouting = "routing" + esoFieldTLS = "tls" + esoFieldAuth = "basic_auth" esoFieldAuthEnabled = "enabled" esoFieldAuthUsername = "username" @@ -48,7 +50,7 @@ func notImportedAWSOptFn(conf *service.ParsedConfig, osconf *opensearchapi.Confi // AWSOptFn is populated with the child `aws` package when imported. var AWSOptFn = notImportedAWSOptFn -// AWSField represents the aws block within an elasticsearch field. This is +// AWSField represents the aws block within an opensearch field. This is // exported in order to make unit testing easier within the aws subpackage. func AWSField() *service.ConfigField { return service.NewObjectField(esoFieldAWS, @@ -71,7 +73,7 @@ type esoConfig struct { routingStr *service.InterpolatedString } -func esoConfigFromParsed(pConf *service.ParsedConfig) (conf esoConfig, err error) { +func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf *esoConfig, err error) { conf.clientOpts = opensearchapi.Config{} var tmpURLs []string @@ -86,25 +88,47 @@ func esoConfigFromParsed(pConf *service.ParsedConfig) (conf esoConfig, err error } } - { - authConf := pConf.Namespace(esoFieldAuth) - if enabled, _ := authConf.FieldBool(esoFieldAuthEnabled); enabled { - if conf.clientOpts.Client.Username, err = authConf.FieldString(esoFieldAuthUsername); err != nil { - return - } - if conf.clientOpts.Client.Password, err = authConf.FieldString(esoFieldAuthPassword); err != nil { - return - } + authConf := pConf.Namespace(esoFieldAuth) + if enabled, _ := authConf.FieldBool(esoFieldAuthEnabled); enabled { + + if conf.clientOpts.Client.Username, err = authConf.FieldString(esoFieldAuthUsername); err != nil { + return + } + if conf.clientOpts.Client.Password, err = authConf.FieldString(esoFieldAuthPassword); err != nil { + return + } + } + + oauth2conf, err := oAuthFromParsed(pConf) + if err != nil { + return + } + + if oauth2conf.Enabled { + token, err := oauth2conf.GetToken(mgr) + if err != nil { + return nil, err + } + conf.clientOpts.Client.Transport = &oauth2.Transport{ + Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), + Base: http.DefaultTransport, } } var tlsConf *tls.Config var tlsEnabled bool + + conf.clientOpts.Client.Transport = http.DefaultTransport + if tlsConf, tlsEnabled, err = pConf.FieldTLSToggled(esoFieldTLS); err != nil { return } else if tlsEnabled { - conf.clientOpts.Client.Transport = &http.Transport{ - TLSClientConfig: tlsConf, + if _, ok := conf.clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport); ok { + conf.clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport).TLSClientConfig = tlsConf + } else { + conf.clientOpts.Client.Transport = &http.Transport{ + TLSClientConfig: tlsConf, + } } } @@ -132,12 +156,12 @@ func esoConfigFromParsed(pConf *service.ParsedConfig) (conf esoConfig, err error //------------------------------------------------------------------------------ -// OutputSpec returns the config spec for an elasticsearch output writer. +// OutputSpec returns the config spec for an opensearch output writer. func OutputSpec() *service.ConfigSpec { return service.NewConfigSpec(). Stable(). Categories("Services"). - Summary(`Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping.`). + Summary(`Publishes messages into an opensearch index. If the index does not exist then it is created with a dynamic mapping.`). Description(output.Description(true, true, ` Both the `+"`id` and `index`"+` fields can be dynamically set using function interpolations described [here](/docs/configuration/interpolation#bloblang-queries). When sending batched messages these interpolations are performed per message part.`)). Fields( @@ -166,6 +190,7 @@ Both the `+"`id` and `index`"+` fields can be dynamically set using function int httpclient.BasicAuthField(), service.NewBatchPolicyField(esoFieldBatching), AWSField(), + OAuthAuthField(), ). Example("Updating Documents", "When [updating documents](https://opensearch.org/docs/latest/api-reference/document-apis/update-document/) the request body should contain a combination of a `doc`, `upsert`, and/or `script` fields at the top level, this should be done via mapping processors.", ` output: @@ -198,17 +223,17 @@ func init() { } } -// Output implements service.BatchOutput for elasticsearch. +// Output implements service.BatchOutput for opensearch. type Output struct { log *service.Logger - conf esoConfig + conf *esoConfig client *opensearchapi.Client } -// OutputFromParsed returns an elasticsearch output writer from a parsed config. +// OutputFromParsed returns an opensearch output writer from a parsed config. func OutputFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (*Output, error) { - conf, err := esoConfigFromParsed(pConf) + conf, err := esoConfigFromParsed(pConf, mgr) if err != nil { return nil, err } @@ -231,6 +256,7 @@ func (e *Output) Connect(ctx context.Context) error { } e.client = client + e.log.Infof("Sending messages to opensearch index at urls: %s\n", e.conf.clientOpts.Client.Addresses) return nil } diff --git a/website/docs/components/outputs/opensearch.md b/website/docs/components/outputs/opensearch.md index dbda3b1ade..058cf9c11d 100644 --- a/website/docs/components/outputs/opensearch.md +++ b/website/docs/components/outputs/opensearch.md @@ -15,7 +15,7 @@ categories: ["Services"] import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -Publishes messages into an Elasticsearch index. If the index does not exist then it is created with a dynamic mapping. +Publishes messages into an opensearch index. If the index does not exist then it is created with a dynamic mapping. @@ -569,4 +573,35 @@ An external ID to provide when assuming a role. Type: `string` Default: `""` +### `oauth2` + +Allows you to specify OAuth2 authentication. + + +Type: `object` + +### `oauth2.access_token` + +A static access token to use for authentication. + + +Type: `string` +Default: `""` + +### `oauth2.token_cache` + +Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from. + + +Type: `string` +Default: `""` + +### `oauth2.token_key` + +Required when using a `token_cache`, the key to query the cache with for tokens. + + +Type: `string` +Default: `""` + From 87b6b194d97ff45db83f419e6a41e8e79d0e7397 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 13:33:09 +0000 Subject: [PATCH 13/43] Typo Signed-off-by: Casper Thygesen --- internal/impl/opensearch/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 825c3ee6ab..159e9f81d8 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -19,7 +19,7 @@ const ( // OAuthAuthField returns a config field spec for basic authentication. func OAuthAuthField() *service.ConfigField { - return service.NewObjectField(esoFieldOAuth, + return service.NewObjectField(aFieldOAuth, service.NewStringField(aFieldAuthStaticAccessToken). Description("A static access token to use for authentication."). Advanced(). From f05f6b0b2e3c4721d7786e8e86793eacaa46f698 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 14:19:06 +0000 Subject: [PATCH 14/43] Fix nullpointer issue Signed-off-by: Casper Thygesen --- internal/impl/opensearch/output.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index 8139102346..6f831fc5e3 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -73,7 +73,7 @@ type esoConfig struct { routingStr *service.InterpolatedString } -func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf *esoConfig, err error) { +func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf esoConfig, err error) { conf.clientOpts = opensearchapi.Config{} var tmpURLs []string @@ -107,7 +107,7 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c if oauth2conf.Enabled { token, err := oauth2conf.GetToken(mgr) if err != nil { - return nil, err + return conf, err } conf.clientOpts.Client.Transport = &oauth2.Transport{ Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), @@ -226,7 +226,7 @@ func init() { // Output implements service.BatchOutput for opensearch. type Output struct { log *service.Logger - conf *esoConfig + conf esoConfig client *opensearchapi.Client } From 719301fed8351eb1c4a41483f9b3489e03803720 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 14:22:40 +0000 Subject: [PATCH 15/43] mark static access token as secret Signed-off-by: Casper Thygesen --- internal/impl/opensearch/auth.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 159e9f81d8..5a1c07d7d5 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -23,6 +23,7 @@ func OAuthAuthField() *service.ConfigField { service.NewStringField(aFieldAuthStaticAccessToken). Description("A static access token to use for authentication."). Advanced(). + Secret(). Default(""), service.NewStringField(aFieldAuthTokenCache). Description("Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from."). From 321a0301c7039bd956da28741f55d8ee1fc0f859 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 08:12:00 +0000 Subject: [PATCH 16/43] Added direct support for oauth2 tokens in kafka using franz-lib Signed-off-by: Casper Thygesen --- internal/impl/kafka/sasl.go | 123 ++++++++++++++++++++++++++++++++---- 1 file changed, 110 insertions(+), 13 deletions(-) diff --git a/internal/impl/kafka/sasl.go b/internal/impl/kafka/sasl.go index 394c927adc..ef750716ca 100644 --- a/internal/impl/kafka/sasl.go +++ b/internal/impl/kafka/sasl.go @@ -2,8 +2,14 @@ package kafka import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" + "io" + "net/http" + "net/url" + "strings" "github.com/IBM/sarama" @@ -43,6 +49,18 @@ func saslField() *service.ConfigField { service.NewStringField("token"). Description("The token to use for a single session's OAUTHBEARER authentication."). Default(""), + service.NewStringField("tokenEndpoint"). + Description("The endpoint to use for OAUTHBEARER token acquisition."). + Default(""), + service.NewStringField("clientId"). + Description("The client ID to use for OAUTHBEARER token acquisition."). + Default(""), + service.NewStringField("clientSecret"). + Description("The client secret to use for OAUTHBEARER token acquisition."). + Default("").Secret(), + service.NewStringField("scope"). + Description("The scope to use for OAUTHBEARER token acquisition."). + Default(""), service.NewStringMapField("extensions"). Description("Key/value pairs to add to OAUTHBEARER authentication requests."). Optional(), @@ -129,22 +147,101 @@ func plainSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { } func oauthSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { - token, err := c.FieldString("token") - if err != nil { - return nil, err - } - var extensions map[string]string - if c.Contains("extensions") { - if extensions, err = c.FieldStringMap("extensions"); err != nil { + if c.Contains("token") { + token, err := c.FieldString("token") + if err != nil { return nil, err } + var extensions map[string]string + if c.Contains("extensions") { + if extensions, err = c.FieldStringMap("extensions"); err != nil { + return nil, err + } + } + return oauth.Oauth(func(c context.Context) (oauth.Auth, error) { + return oauth.Auth{ + Token: token, + Extensions: extensions, + }, nil + }), nil + } else if c.Contains("tokenEndpoint") { + return oauth.Oauth(func(ctx context.Context) (oauth.Auth, error) { + shortToken, err := acquireToken(ctx, c) + return oauth.Auth{Token: shortToken}, err + }), nil } - return oauth.Oauth(func(c context.Context) (oauth.Auth, error) { - return oauth.Auth{ - Token: token, - Extensions: extensions, - }, nil - }), nil + return nil, errors.New("field 'token' or 'tokenEndpoint' was not found in the config") +} + +func acquireToken(ctx context.Context, c *service.ParsedConfig) (string, error) { + + tokenEndpoint, err := c.FieldString("tokenEndpoint") + if err != nil { + return "", err + } + + clientId, err := c.FieldString("clientId") + if err != nil { + return "", err + } + + clientSecret, err := c.FieldString("clientSecret") + if err != nil { + return "", err + } + + scope, err := c.FieldString("scope") + if err != nil { + return "", err + } + + authHeaderValue := base64.StdEncoding.EncodeToString([]byte(clientId + ":" + clientSecret)) + + queryParams := url.Values{} + queryParams.Set("grant_type", "client_credentials") + queryParams.Set("scope", scope) + + req, err := http.NewRequestWithContext(ctx, "POST", tokenEndpoint, strings.NewReader(queryParams.Encode())) + if err != nil { + return "", err + } + + req.URL.RawQuery = queryParams.Encode() + + req.Header.Set("Authorization", "Basic "+authHeaderValue) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + if err := resp.Body.Close(); err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("token request failed with status code %d", resp.StatusCode) + } + + var tokenResponse map[string]interface{} + err = json.Unmarshal(body, &tokenResponse) + if err != nil { + return "", fmt.Errorf("failed to parse token response: %s", err) + } + + accessToken, ok := tokenResponse["access_token"].(string) + if !ok { + return "", fmt.Errorf("access_token not found in token response") + } + + return accessToken, nil } func scram256SaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { From 0ca6c12d1caa7055c2d539b0fb8296786f5bf59c Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 13:20:35 +0000 Subject: [PATCH 17/43] updated docs Signed-off-by: Casper Thygesen --- website/docs/components/inputs/kafka_franz.md | 35 +++++++++++++++++++ .../docs/components/outputs/kafka_franz.md | 35 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/website/docs/components/inputs/kafka_franz.md b/website/docs/components/inputs/kafka_franz.md index 3ccd381d54..5598c2e577 100644 --- a/website/docs/components/inputs/kafka_franz.md +++ b/website/docs/components/inputs/kafka_franz.md @@ -418,6 +418,41 @@ Default: `""` The token to use for a single session's OAUTHBEARER authentication. +Type: `string` +Default: `""` + +### `sasl[].tokenEndpoint` + +The endpoint to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientId` + +The client ID to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientSecret` + +The client secret to use for OAUTHBEARER token acquisition. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `sasl[].scope` + +The scope to use for OAUTHBEARER token acquisition. + + Type: `string` Default: `""` diff --git a/website/docs/components/outputs/kafka_franz.md b/website/docs/components/outputs/kafka_franz.md index b979abf36f..57b2d385c1 100644 --- a/website/docs/components/outputs/kafka_franz.md +++ b/website/docs/components/outputs/kafka_franz.md @@ -569,6 +569,41 @@ Default: `""` The token to use for a single session's OAUTHBEARER authentication. +Type: `string` +Default: `""` + +### `sasl[].tokenEndpoint` + +The endpoint to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientId` + +The client ID to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `sasl[].clientSecret` + +The client secret to use for OAUTHBEARER token acquisition. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `sasl[].scope` + +The scope to use for OAUTHBEARER token acquisition. + + Type: `string` Default: `""` From 8b98e25098ce677d126afd379e45de4894031f9c Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 29 Feb 2024 14:27:23 +0000 Subject: [PATCH 18/43] mark static token as secret Signed-off-by: Casper Thygesen --- internal/impl/kafka/sasl.go | 1 + internal/impl/kafka/sasl/sasl.go | 2 +- website/docs/components/inputs/kafka.md | 3 +++ website/docs/components/outputs/kafka.md | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/impl/kafka/sasl.go b/internal/impl/kafka/sasl.go index ef750716ca..f7521118f1 100644 --- a/internal/impl/kafka/sasl.go +++ b/internal/impl/kafka/sasl.go @@ -320,6 +320,7 @@ func SaramaSASLField() *service.ConfigField { Secret(), service.NewStringField(saramaFieldSASLAccessToken). Description("A static OAUTHBEARER access token"). + Secret(). Default(""), service.NewStringField(saramaFieldSASLTokenCache). Description("Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from"). diff --git a/internal/impl/kafka/sasl/sasl.go b/internal/impl/kafka/sasl/sasl.go index 67cffb9880..ac4f8e299c 100644 --- a/internal/impl/kafka/sasl/sasl.go +++ b/internal/impl/kafka/sasl/sasl.go @@ -33,7 +33,7 @@ func FieldSpec() docs.FieldSpec { ), docs.FieldString("user", "A PLAIN username. It is recommended that you use environment variables to populate this field.", "${USER}"), docs.FieldString("password", "A PLAIN password. It is recommended that you use environment variables to populate this field.", "${PASSWORD}").Secret(), - docs.FieldString("access_token", "A static OAUTHBEARER access token"), + docs.FieldString("access_token", "A static OAUTHBEARER access token").Secret(), docs.FieldString("token_cache", "Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from"), docs.FieldString("token_key", "Required when using a `token_cache`, the key to query the cache with for tokens."), ).Advanced() diff --git a/website/docs/components/inputs/kafka.md b/website/docs/components/inputs/kafka.md index a85819d367..fe2c0eab2a 100644 --- a/website/docs/components/inputs/kafka.md +++ b/website/docs/components/inputs/kafka.md @@ -394,6 +394,9 @@ password: ${PASSWORD} ### `sasl.access_token` A static OAUTHBEARER access token +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: Type: `string` diff --git a/website/docs/components/outputs/kafka.md b/website/docs/components/outputs/kafka.md index 8d8a5901e1..3188d71cb9 100644 --- a/website/docs/components/outputs/kafka.md +++ b/website/docs/components/outputs/kafka.md @@ -361,6 +361,9 @@ password: ${PASSWORD} ### `sasl.access_token` A static OAUTHBEARER access token +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: Type: `string` From 1a0d3b0605a777faddf5556298352bafd020865e Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Fri, 1 Mar 2024 14:56:51 +0000 Subject: [PATCH 19/43] did not check if token is empty and not just non-existing Signed-off-by: Casper Thygesen --- internal/impl/kafka/sasl.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/impl/kafka/sasl.go b/internal/impl/kafka/sasl.go index f7521118f1..10882399ef 100644 --- a/internal/impl/kafka/sasl.go +++ b/internal/impl/kafka/sasl.go @@ -147,8 +147,9 @@ func plainSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { } func oauthSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { - if c.Contains("token") { - token, err := c.FieldString("token") + token, err := c.FieldString("token") + + if err != nil && token != "" { if err != nil { return nil, err } From 21df213b9908cf358120954aa2f363e6725253e1 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Sun, 3 Mar 2024 19:45:26 +0000 Subject: [PATCH 20/43] forgot to describe oauth2.enabled --- internal/impl/opensearch/auth.go | 12 +++++++++--- internal/impl/opensearch/output.go | 7 ++++--- website/docs/components/outputs/opensearch.md | 12 ++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 5a1c07d7d5..66b964289b 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -2,6 +2,7 @@ package opensearch import ( "context" + "errors" "fmt" "github.com/benthosdev/benthos/v4/public/service" @@ -20,6 +21,10 @@ const ( // OAuthAuthField returns a config field spec for basic authentication. func OAuthAuthField() *service.ConfigField { return service.NewObjectField(aFieldOAuth, + service.NewBoolField(abFieldEnabled). + Description("Whether to use OAuth2 authentication."). + Advanced(). + Default(false), service.NewStringField(aFieldAuthStaticAccessToken). Description("A static access token to use for authentication."). Advanced(). @@ -48,7 +53,8 @@ func oAuthFromParsed(conf *service.ParsedConfig) (res OAuthConfig, err error) { return } - if conf.Contains(aFieldAuthStaticAccessToken) { + staticToken, err := conf.FieldString(aFieldAuthStaticAccessToken) + if err != nil && staticToken != "" { if res.StaticAccessToken, err = conf.FieldString(aFieldAuthStaticAccessToken); err != nil { return } @@ -82,10 +88,10 @@ func (c *OAuthConfig) GetToken(mgr *service.Resources) (string, error) { if err := mgr.AccessCache(context.Background(), c.TokenCacheName, func(cache service.Cache) { tok, terr = cache.Get(context.Background(), c.TokenCacheKey) }); err != nil { - return "", fmt.Errorf("failed to obtain cache resource '%v': %v", c.TokenCacheName, err) + return "", fmt.Errorf("failed to obtain cache resource '%v' with key %v: %v", c.TokenCacheName, c.TokenCacheKey, err) } if terr != nil { - return "", terr + return "", errors.Join(terr, fmt.Errorf("failed to obtain token wih key %v from cache", c.TokenCacheKey)) } return string(tok), nil } diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index 6f831fc5e3..f10ce53389 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -105,9 +105,10 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c } if oauth2conf.Enabled { - token, err := oauth2conf.GetToken(mgr) + token, _ := oauth2conf.GetToken(mgr) if err != nil { - return conf, err + return + //return conf, err } conf.clientOpts.Client.Transport = &oauth2.Transport{ Source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}), @@ -341,7 +342,7 @@ func (e *Output) WriteBatch(ctx context.Context, msg service.MessageBatch) error dur := time.Since(start) e.log.Debugf( - "Successfully dispatched [%s] documents in %s (%s docs/sec)", + "Successfully dispatched [%d] documents in %s (%d docs/sec)", biStats.NumFlushed, dur.Truncate(time.Millisecond), int64(1000.0/float64(dur/time.Millisecond)*float64(biStats.NumFlushed)), diff --git a/website/docs/components/outputs/opensearch.md b/website/docs/components/outputs/opensearch.md index 058cf9c11d..a9b0e307d9 100644 --- a/website/docs/components/outputs/opensearch.md +++ b/website/docs/components/outputs/opensearch.md @@ -87,6 +87,7 @@ output: role: "" role_external_id: "" oauth2: + enabled: false access_token: "" token_cache: "" token_key: "" @@ -580,9 +581,20 @@ Allows you to specify OAuth2 authentication. Type: `object` +### `oauth2.enabled` + +Whether to use OAuth2 authentication. + + +Type: `bool` +Default: `false` + ### `oauth2.access_token` A static access token to use for authentication. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: Type: `string` From bee5c5344a873640bbd6e2deff3f13c8679f3601 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Mon, 4 Mar 2024 19:50:32 +0000 Subject: [PATCH 21/43] devcontainer support --- .devcontainer/Dockerfile | 33 ++++++++++++++++++ .devcontainer/devcontainer.json | 33 ++++++++++++++++++ resources/docker/Dockerfile | 12 +++++-- resources/docker/kafka/docker-compose.yaml | 21 ++++++++++++ resources/docker/kafka/in.yaml | 39 ++++++++++++++++++++++ resources/docker/kafka/out.yaml | 39 ++++++++++++++++++++++ 6 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 resources/docker/kafka/docker-compose.yaml create mode 100644 resources/docker/kafka/in.yaml create mode 100644 resources/docker/kafka/out.yaml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..7f435d4537 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,33 @@ +FROM mcr.microsoft.com/vscode/devcontainers/go:1.21 + +ARG NODE_VERSION="16" +ARG GOLANGCI_LINT_VERSION="1.46.2" + +# https://github.com/microsoft/vscode-dev-containers/blob/main/containers/go/.devcontainer/base.Dockerfile +ENV USERNAME=vscode +ENV LIBRARY_SCRIPTS_SRC="https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/containers/go/.devcontainer/library-scripts/node-debian.sh" +ENV NVM_DIR=/usr/local/share/nvm +ENV NVM_SYMLINK_CURRENT=true \ + PATH=${NVM_DIR}/current/bin:${PATH} +RUN mkdir /tmp/library-scripts \ + && curl -fsSL -o /tmp/library-scripts/node-debian.sh "${LIBRARY_SCRIPTS_SRC}" +RUN bash /tmp/library-scripts/node-debian.sh "${NVM_DIR}" "${NODE_VERSION}" "${USERNAME}" \ + && apt-get clean -y && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/library-scripts + +RUN echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' \ + | tee /etc/apt/sources.list.d/goreleaser.list +RUN apt-get update \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends goreleaser + +RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ + | sh -s -- -b $(go env GOPATH)/bin v$GOLANGCI_LINT_VERSION + +USER vscode +WORKDIR /home/vscode + +RUN mkdir -p .config/git \ + && echo ".vscode/*" >> .config/git/ignore \ + && echo "*.code-workspace" >> .config/git/ignore \ + && echo ".history/" >> .config/git/ignore diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9e780cc73c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,33 @@ +{ + "name": "Benthos Dev Container", + "build": { + "dockerfile": "Dockerfile", + "args": { + "NODE_VERSION": "16", + "GOLANGCI_LINT_VERSION": "1.46.2" + } + }, + "runArgs": ["--network=host", "--privileged"], + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + "extensions": [ + "golang.Go", + "dbaeumer.vscode-eslint", + "EditorConfig.EditorConfig", + "esbenp.prettier-vscode", + "github.vscode-github-actions", + "jebbs.plantuml", + "GitHub.copilot", + "github.vscode-github-actions", + "ms-vscode.makefile-tools", + "GeorgesHaidar.vsc-benthos" + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {} + }, + "postCreateCommand": "go mod download", + "remoteUser": "vscode" +} diff --git a/resources/docker/Dockerfile b/resources/docker/Dockerfile index f0fde55f1e..89309f1ffe 100644 --- a/resources/docker/Dockerfile +++ b/resources/docker/Dockerfile @@ -2,18 +2,24 @@ FROM golang:1.21 AS build ENV CGO_ENABLED=0 ENV GOOS=linux -RUN useradd -u 10001 benthos + +# Prepare the build environment +RUN go env -w GOCACHE=/go-cache; \ + go env -w GOMODCACHE=/gomod-cache; \ + useradd -u 10001 benthos WORKDIR /go/src/github.com/benthosdev/benthos/ # Update dependencies: On unchanged dependencies, cached layer will be reused COPY go.* /go/src/github.com/benthosdev/benthos/ -RUN go mod download +RUN --mount=type=cache,target=/gomod-cache \ + go mod download # Build COPY . /go/src/github.com/benthosdev/benthos/ # Tag timetzdata required for busybox base image: # https://github.com/benthosdev/benthos/issues/897 -RUN make TAGS="timetzdata" +RUN --mount=type=cache,target=/gomod-cache --mount=type=cache,target=/go-cache \ + make TAGS="timetzdata" # Pack FROM busybox AS package diff --git a/resources/docker/kafka/docker-compose.yaml b/resources/docker/kafka/docker-compose.yaml new file mode 100644 index 0000000000..086eb34e44 --- /dev/null +++ b/resources/docker/kafka/docker-compose.yaml @@ -0,0 +1,21 @@ +version: '3.3' +services: + benthos-in: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ '-w', '-c', '/benthos.yaml' ] + volumes: + - ./in.yaml:/benthos.yaml + network_mode: host + + benthos-out: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ '-w', '-c', '/benthos.yaml' ] + volumes: + - ./out.yaml:/benthos.yaml + network_mode: host \ No newline at end of file diff --git a/resources/docker/kafka/in.yaml b/resources/docker/kafka/in.yaml new file mode 100644 index 0000000000..92d73a8bab --- /dev/null +++ b/resources/docker/kafka/in.yaml @@ -0,0 +1,39 @@ +http: + enabled: false + +input: + generate: + interval: 5s + mapping: | + root.ID = uuid_v4() + root.Name = [ "frosty", "spot", "oodles" ].index(random_int() % 3) + root.Gooeyness = (random_int() % 100) / 100 + root.Bouncing = random_int() % 2 == 0 + +pipeline: + processors: + - bloblang: | + root = this + root.ID = this.ID + root.ReNamed = this.Name + root.Gooeyness = this.Gooeyness + root.Bouncing = this.Bouncing + + - catch: + - log: + level: ERROR + message: ${! error() } + - bloblang: root = deleted() + +output: + label: sink + kafka_franz: + seed_brokers: [localhost:9092] # No default (required) + topic: benthos_redpanda # No default (required) + key: "" # No default (optional) + sasl: + - mechanism: OAUTHBEARER + tokenEndpoint: http://localhost:1852/realms/local-development/protocol/openid-connect/token + scope: "kafka" + clientId: "default-access" + clientSecret: "default-access-secret" diff --git a/resources/docker/kafka/out.yaml b/resources/docker/kafka/out.yaml new file mode 100644 index 0000000000..afa38278d6 --- /dev/null +++ b/resources/docker/kafka/out.yaml @@ -0,0 +1,39 @@ +http: + address: 0.0.0.0:4195 + enabled: true + root_path: /benthos + debug_endpoints: false + +logger: + level: DEBUG + format: logfmt + add_timestamp: true + static_fields: + "@service": benthos + +input: + label: source + kafka_franz: + seed_brokers: [localhost:9092] # No default (required) + topics: [benthos_redpanda] # No default (required) + consumer_group: benthos + sasl: + - mechanism: OAUTHBEARER # No default (required) + tokenEndpoint: http://localhost:1852/realms/local-development/protocol/openid-connect/token + scope: "kafka" # No default (required) + clientId: "default-access" # No default (required) + clientSecret: "default-access-secret" # No default (required) + +pipeline: + processors: + - bloblang: | + root = this + + - catch: + - log: + level: ERROR + message: ${! error() } + - bloblang: root = deleted() + +output: + stdout: {} From ddfd5d693f6b4fe890a9386ea6029e0ce8ed7dfe Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 09:50:03 +0000 Subject: [PATCH 22/43] prepare release flow --- .github/workflows/cheetah_release.yaml | 128 +++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 .github/workflows/cheetah_release.yaml diff --git a/.github/workflows/cheetah_release.yaml b/.github/workflows/cheetah_release.yaml new file mode 100644 index 0000000000..c3b6014c13 --- /dev/null +++ b/.github/workflows/cheetah_release.yaml @@ -0,0 +1,128 @@ +name: Release Cheetah cheetah-benthos + +on: + workflow_dispatch: + + push: + branches: ["main"] + tags: + - "v*" + pull_request: + branches: ["main"] + +env: + IMAGE_NAME: cheetah-benthos + DOCKERFILE_PATH: ./resources/docker/Dockerfile + CONTEXT: ./ + +permissions: + contents: read + packages: write + +jobs: + push_plugins: + name: "Build and push backend plugin" + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + + - name: Build and push + uses: trifork/cheetah-infrastructure-utils-workflows/.github/actions/build-image/default@main + with: + read_package_pat: ${{ secrets.PACKAGE_PAT }} # we need this, as GITHUB_TOKEN only have permission to its own repo + context: ${{ env.CONTEXT }} + image_name: ${{ env.IMAGE_NAME }} + github_run_id: ${{ github.run_id }} + dockerfile_path: ${{ env.DOCKERFILE_PATH }} + push_image: ${{ (github.ref_type == 'tag' || github.event_name == 'workflow_dispatch') && 'true' || 'false' }} + upload_image: ${{ (github.event_name == 'pull_request' || github.ref_name == 'main' || github.event_name == 'workflow_dispatch') && 'true' || 'false' }} + + trivy-container-scan-pr: + name: trivy (PR report) + needs: [push_plugins] + runs-on: ubuntu-latest + timeout-minutes: 5 + if: ${{ github.event_name == 'pull_request' }} + permissions: + contents: read + packages: read + pull-requests: write + issues: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download Docker image artifact + uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4 + with: + name: dockerimage + path: ${{ github.workspace }}/tmp + + - name: Update pull request + uses: trifork/cheetah-infrastructure-utils/.github/actions/trivy-report@main + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + WORKSPACE_PATH: ${{ github.workspace }} + with: + image_name: ${{ env.IMAGE_NAME }} + base-tag: "latest" # "pr-${{ github.event.pull_request.number }}" + orga: "trifork" + repo: ${{ github.repository }} + pr-nb: ${{ github.event.pull_request.number }} + topic: ${{ env.IMAGE_NAME }} + + trivy-container-scan: + name: trivy (container scan) + needs: [push_plugins] + runs-on: ubuntu-latest + timeout-minutes: 5 + if: github.ref_name == 'main' && github.event_name == 'push' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download Docker image artifact + uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4 + with: + name: dockerimage + path: ${{ github.workspace }}/tmp + + - name: Run Trivy vulnerability scanner for containers + uses: aquasecurity/trivy-action@master + with: + input: /github/workspace/tmp/image.tar # -v "/home/runner/work/cheetah-backstage/cheetah-backstage":"/github/workspace" + format: "sarif" + #exit-code: "0" + output: "trivy-container-results.sarif" + github-pat: ${{ secrets.GITHUB_TOKEN }} + + #- name: Upload Trivy scan results to GitHub Security tab + # uses: github/codeql-action/upload-sarif@v2 + # if: never() + # with: + # sarif_file: 'trivy-results.sarif' + + - name: Publish container report + id: import-container-scan + uses: trifork/cheetah-infrastructure-utils/.github/actions/defectdojo-import-scan@main + if: github.ref_name == 'main' && github.event_name == 'push' + with: + token: ${{ secrets.DEFECTDOJO_TOKEN }} + defectdojo_url: "https://defectdojo.cheetah.trifork.dev" + defectdojo_endpoint: "/api/v2/reimport-scan/" + file: "trivy-container-results.sarif" + scan_type: SARIF # Trivy Scan + engagement_name: import-container-scan + product_name: ${{ env.IMAGE_NAME }} + branch_tag: ${{ github.ref_name }} + #commit_hash: ${{ github.sha }} + #minimum_severity: Info, Low, Medium, High, Critical. Default value: Info + + - name: Show response + if: github.ref_name == 'main' && github.event_name == 'push' + run: | + set -e + printf '%s\n' '${{ steps.import-container-scan.outputs.response }}' From 0fbb75e608e95bd76111fd0ff450b8fda3c15098 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 10:00:54 +0000 Subject: [PATCH 23/43] try fix workflow --- .github/workflows/cheetah_release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cheetah_release.yaml b/.github/workflows/cheetah_release.yaml index c3b6014c13..792c8ae3db 100644 --- a/.github/workflows/cheetah_release.yaml +++ b/.github/workflows/cheetah_release.yaml @@ -1,14 +1,14 @@ -name: Release Cheetah cheetah-benthos +name: Release cheetah-benthos on: workflow_dispatch: push: - branches: ["main"] + branches: ["cheetah-main"] tags: - "v*" pull_request: - branches: ["main"] + branches: ["cheetah-main"] env: IMAGE_NAME: cheetah-benthos @@ -20,7 +20,7 @@ permissions: packages: write jobs: - push_plugins: + push_image: name: "Build and push backend plugin" runs-on: ubuntu-latest timeout-minutes: 15 From 23620906bd6491fef8d0024de3ca56b3b76bb538 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 10:02:24 +0000 Subject: [PATCH 24/43] fixes --- .github/workflows/cheetah_release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cheetah_release.yaml b/.github/workflows/cheetah_release.yaml index 792c8ae3db..5972247623 100644 --- a/.github/workflows/cheetah_release.yaml +++ b/.github/workflows/cheetah_release.yaml @@ -42,7 +42,7 @@ jobs: trivy-container-scan-pr: name: trivy (PR report) - needs: [push_plugins] + needs: [push_image] runs-on: ubuntu-latest timeout-minutes: 5 if: ${{ github.event_name == 'pull_request' }} @@ -76,7 +76,7 @@ jobs: trivy-container-scan: name: trivy (container scan) - needs: [push_plugins] + needs: [push_image] runs-on: ubuntu-latest timeout-minutes: 5 if: github.ref_name == 'main' && github.event_name == 'push' From db6e4dc9b87d9c9a00405b7928978329fd0fbde6 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 10:09:54 +0000 Subject: [PATCH 25/43] fix IMAGE_NAME --- .github/workflows/cheetah_release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cheetah_release.yaml b/.github/workflows/cheetah_release.yaml index 5972247623..81f7964e0a 100644 --- a/.github/workflows/cheetah_release.yaml +++ b/.github/workflows/cheetah_release.yaml @@ -11,7 +11,7 @@ on: branches: ["cheetah-main"] env: - IMAGE_NAME: cheetah-benthos + IMAGE_NAME: ${{ github.repository }} DOCKERFILE_PATH: ./resources/docker/Dockerfile CONTEXT: ./ @@ -21,7 +21,7 @@ permissions: jobs: push_image: - name: "Build and push backend plugin" + name: "Build and push image" runs-on: ubuntu-latest timeout-minutes: 15 From 7bc10732b9ade8f7354ba3d3cf146a65c988994f Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 10:37:43 +0000 Subject: [PATCH 26/43] go mod tidy --- go.sum | 8 -------- 1 file changed, 8 deletions(-) diff --git a/go.sum b/go.sum index b13a4026b7..cbe6a2c0a9 100644 --- a/go.sum +++ b/go.sum @@ -448,8 +448,6 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -709,14 +707,11 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -735,7 +730,6 @@ github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -807,7 +801,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= @@ -817,7 +810,6 @@ github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuH github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a h1:lem6QCvxR0Y28gth9P+wV2K/zYUUAkJ+55U8cpS0p5I= github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= From c25a321a3ecb195c7a8c8a722291bfe9b0d5530d Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 10:40:27 +0000 Subject: [PATCH 27/43] GOLANGCI_LINT_VERSION="1.57.1" --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7f435d4537..261001f430 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/go:1.21 ARG NODE_VERSION="16" -ARG GOLANGCI_LINT_VERSION="1.46.2" +ARG GOLANGCI_LINT_VERSION="1.57.1" # https://github.com/microsoft/vscode-dev-containers/blob/main/containers/go/.devcontainer/base.Dockerfile ENV USERNAME=vscode From 7743a61b8fed201839f4e0077075cf6f8fc36c54 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 10:56:26 +0000 Subject: [PATCH 28/43] linting --- internal/impl/kafka/sasl.go | 9 +++------ internal/impl/opensearch/auth.go | 12 ++++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/internal/impl/kafka/sasl.go b/internal/impl/kafka/sasl.go index 10882399ef..f4315c3583 100644 --- a/internal/impl/kafka/sasl.go +++ b/internal/impl/kafka/sasl.go @@ -150,9 +150,6 @@ func oauthSaslFromConfig(c *service.ParsedConfig) (sasl.Mechanism, error) { token, err := c.FieldString("token") if err != nil && token != "" { - if err != nil { - return nil, err - } var extensions map[string]string if c.Contains("extensions") { if extensions, err = c.FieldStringMap("extensions"); err != nil { @@ -181,7 +178,7 @@ func acquireToken(ctx context.Context, c *service.ParsedConfig) (string, error) return "", err } - clientId, err := c.FieldString("clientId") + clientID, err := c.FieldString("clientId") if err != nil { return "", err } @@ -196,7 +193,7 @@ func acquireToken(ctx context.Context, c *service.ParsedConfig) (string, error) return "", err } - authHeaderValue := base64.StdEncoding.EncodeToString([]byte(clientId + ":" + clientSecret)) + authHeaderValue := base64.StdEncoding.EncodeToString([]byte(clientID + ":" + clientSecret)) queryParams := url.Values{} queryParams.Set("grant_type", "client_credentials") @@ -239,7 +236,7 @@ func acquireToken(ctx context.Context, c *service.ParsedConfig) (string, error) accessToken, ok := tokenResponse["access_token"].(string) if !ok { - return "", fmt.Errorf("access_token not found in token response") + return "", errors.New("access_token not found in token response") } return accessToken, nil diff --git a/internal/impl/opensearch/auth.go b/internal/impl/opensearch/auth.go index 4d2e4ed87b..e0425d618f 100644 --- a/internal/impl/opensearch/auth.go +++ b/internal/impl/opensearch/auth.go @@ -24,7 +24,7 @@ const ( aFieldAuthTokenCache = "token_cache" aFieldAuthTokenKey = "token_key" aFieldAuthTokenEndpoint = "tokenEndpoint" - aFieldAuthTokenClientId = "clientId" + aFieldAuthTokenClientID = "clientId" aFieldAuthTokenClientSecret = "clientSecret" aFieldAuthTokenScope = "scope" ) @@ -44,7 +44,7 @@ func OAuthAuthField() *service.ConfigField { service.NewStringField(aFieldAuthTokenEndpoint). Description("The endpoint to use for OAUTHBEARER token acquisition."). Default(""), - service.NewStringField(aFieldAuthTokenClientId). + service.NewStringField(aFieldAuthTokenClientID). Description("The client ID to use for OAUTHBEARER token acquisition."). Default(""), service.NewStringField(aFieldAuthTokenClientSecret). @@ -84,7 +84,7 @@ func oAuthFromParsed(conf *service.ParsedConfig) (res OAuthConfig, err error) { if res.TokenEndpoint, err = conf.FieldString(aFieldAuthTokenEndpoint); err != nil { return } - if res.ClientId, err = conf.FieldString(aFieldAuthTokenClientId); err != nil { + if res.ClientID, err = conf.FieldString(aFieldAuthTokenClientID); err != nil { return } if res.ClientSecret, err = conf.FieldString(aFieldAuthTokenClientSecret); err != nil { @@ -103,7 +103,7 @@ type OAuthConfig struct { TokenCacheName string TokenCacheKey string TokenEndpoint string - ClientId string + ClientID string ClientSecret string Scope string } @@ -132,7 +132,7 @@ func (c *OAuthConfig) GetCachedToken(mgr *service.Resources) (*oauth2.Token, err func (c *OAuthConfig) GetToken(ctx context.Context) (*oauth2.Token, error) { - authHeaderValue := base64.StdEncoding.EncodeToString([]byte(c.ClientId + ":" + c.ClientSecret)) + authHeaderValue := base64.StdEncoding.EncodeToString([]byte(c.ClientID + ":" + c.ClientSecret)) queryParams := url.Values{} queryParams.Set("grant_type", "client_credentials") @@ -188,7 +188,7 @@ func NewOAuth2Config() OAuthConfig { TokenCacheName: "", TokenCacheKey: "", TokenEndpoint: "", - ClientId: "", + ClientID: "", ClientSecret: "", Scope: "", } From abdaa537f74762d9dd7a871cf222775b1085df99 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 10 Apr 2024 12:56:24 +0000 Subject: [PATCH 29/43] increase timeout --- .github/workflows/cheetah_release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cheetah_release.yaml b/.github/workflows/cheetah_release.yaml index 81f7964e0a..7df9db341c 100644 --- a/.github/workflows/cheetah_release.yaml +++ b/.github/workflows/cheetah_release.yaml @@ -23,7 +23,7 @@ jobs: push_image: name: "Build and push image" runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 steps: - name: Checkout repository From 27ea1711461a20a1f338e351d16222c3b1a2a780 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Tue, 16 Apr 2024 15:40:30 +0000 Subject: [PATCH 30/43] initial work with GET support Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 152 ++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 internal/impl/opensearch/cache.go diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go new file mode 100644 index 0000000000..a49ed1bbc2 --- /dev/null +++ b/internal/impl/opensearch/cache.go @@ -0,0 +1,152 @@ +package opensearch + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/benthosdev/benthos/v4/internal/httpclient" + "github.com/benthosdev/benthos/v4/public/service" + "github.com/opensearch-project/opensearch-go/v3/opensearchapi" +) + +func opensearchCacheConfig() *service.ConfigSpec { + return service.NewConfigSpec(). + Beta(). + Version("1.0.0"). + Summary(`Use a OpenSearch instance as a cache. Only GET is supported`). + Fields(service.NewStringListField(esoFieldURLs)). + Fields(httpclient.BasicAuthField()). + Fields(OAuthAuthField()). + Fields( + service.NewStringField("collection"). + Description("The name of the target collection."), + service.NewStringField("key_field"). + Description("The field in the document that is used as the key."), + service.NewStringField("value_field"). + Description("The field in the document that is used as the value.").Optional(), + ) +} + +func init() { + err := service.RegisterCache( + "opensearch", opensearchCacheConfig(), + func(conf *service.ParsedConfig, mgr *service.Resources) (service.Cache, error) { + return newOpensearchCacheFromConfig(conf, mgr) + }) + if err != nil { + panic(err) + } +} + +func newOpensearchCacheFromConfig(parsedConf *service.ParsedConfig, mgr *service.Resources) (*opensearchCache, error) { + conf, err := esoConfigFromParsed(parsedConf, mgr) + if err != nil { + return nil, err + } + + collectionName, err := parsedConf.FieldString("collection") + if err != nil { + return nil, err + } + + keyField, err := parsedConf.FieldString("key_field") + if err != nil { + return nil, err + } + + valueField, err := parsedConf.FieldString("value_field") + if err != nil { + return nil, err + } + + return newOpensearchCache(collectionName, keyField, valueField, conf) +} + +//------------------------------------------------------------------------------ + +type opensearchCache struct { + client *opensearchapi.Client + collection string + + keyField string + valueField string +} + +func newOpensearchCache(collectionName, keyField, valueField string, conf esoConfig) (*opensearchCache, error) { + + client, err := opensearchapi.NewClient(conf.clientOpts) + if err != nil { + return nil, err + } + + return &opensearchCache{ + client: client, + collection: collectionName, + keyField: keyField, + valueField: valueField, + }, nil +} + +func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { + query := fmt.Sprintf(`{ + "query": { + "term": { + "%s": "%s" + } + } + }`, m.keyField, key) + + search := &opensearchapi.SearchReq{ + Indices: []string{m.collection}, + Body: strings.NewReader(query), + Params: opensearchapi.SearchParams{ + Size: opensearchapi.ToPointer(1), + }, + } + + searchResponse, err := m.client.Search(ctx, search) + + if err != nil { + return nil, fmt.Errorf("error searching for key %s: %v", key, err) + } + + if searchResponse.Hits.Total.Value == 0 { + return nil, service.ErrKeyNotFound + } + + if m.valueField != "" { + var message map[string]interface{} + err = json.Unmarshal(searchResponse.Hits.Hits[0].Source, &message) + if err != nil { + return nil, fmt.Errorf("error getting field from document %s: %v", m.valueField, err) + } + var val, ok = message[m.valueField].(string) + if ok { + return []byte(val), nil + } + return nil, fmt.Errorf("error getting field from document %s: %v", m.valueField, val) + } else { + return searchResponse.Hits.Hits[0].Source, nil // return the entire document + } + +} + +func (m *opensearchCache) Set(ctx context.Context, key string, value []byte, _ *time.Duration) error { + return errors.New("not implemented") +} + +func (m *opensearchCache) Add(ctx context.Context, key string, value []byte, _ *time.Duration) error { + return errors.New("not implemented") +} + +func (m *opensearchCache) Delete(ctx context.Context, key string) error { + return errors.New("not implemented") +} + +func (m *opensearchCache) Close(ctx context.Context) error { + return nil +} From d4c561075f6e0bb27a1a380e79ced01c17bdfba9 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Tue, 16 Apr 2024 15:41:50 +0000 Subject: [PATCH 31/43] docs Signed-off-by: Casper Thygesen --- website/docs/components/caches/opensearch.md | 212 +++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 website/docs/components/caches/opensearch.md diff --git a/website/docs/components/caches/opensearch.md b/website/docs/components/caches/opensearch.md new file mode 100644 index 0000000000..571b6f569c --- /dev/null +++ b/website/docs/components/caches/opensearch.md @@ -0,0 +1,212 @@ +--- +title: opensearch +slug: opensearch +type: cache +status: beta +--- + + + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +:::caution BETA +This component is mostly stable but breaking changes could still be made outside of major version releases if a fundamental problem with the component is found. +::: +Use a OpenSearch instance as a cache. Only GET is supported + +Introduced in version 1.0.0. + + + + + + +```yml +# Common config fields, showing default values +label: "" +opensearch: + urls: [] # No default (required) + collection: "" # No default (required) + key_field: "" # No default (required) + value_field: "" # No default (optional) +``` + + + + +```yml +# All config fields, showing default values +label: "" +opensearch: + urls: [] # No default (required) + basic_auth: + enabled: false + username: "" + password: "" + oauth2: + enabled: false + access_token: "" + tokenEndpoint: "" + clientId: "" + clientSecret: "" + scope: "" + token_cache: "" + token_key: "" + collection: "" # No default (required) + key_field: "" # No default (required) + value_field: "" # No default (optional) +``` + + + + +## Fields + +### `urls` + +Sorry! This field is missing documentation. + + +Type: `array` + +### `basic_auth` + +Allows you to specify basic authentication. + + +Type: `object` + +### `basic_auth.enabled` + +Whether to use basic authentication in requests. + + +Type: `bool` +Default: `false` + +### `basic_auth.username` + +A username to authenticate as. + + +Type: `string` +Default: `""` + +### `basic_auth.password` + +A password to authenticate with. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `oauth2` + +Allows you to specify OAuth2 authentication. + + +Type: `object` + +### `oauth2.enabled` + +Whether to use OAuth2 authentication. + + +Type: `bool` +Default: `false` + +### `oauth2.access_token` + +A static access token to use for authentication. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `oauth2.tokenEndpoint` + +The endpoint to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `oauth2.clientId` + +The client ID to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `oauth2.clientSecret` + +The client secret to use for OAUTHBEARER token acquisition. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `oauth2.scope` + +The scope to use for OAUTHBEARER token acquisition. + + +Type: `string` +Default: `""` + +### `oauth2.token_cache` + +Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch tokens from. + + +Type: `string` +Default: `""` + +### `oauth2.token_key` + +Required when using a `token_cache`, the key to query the cache with for tokens. + + +Type: `string` +Default: `""` + +### `collection` + +The name of the target collection. + + +Type: `string` + +### `key_field` + +The field in the document that is used as the key. + + +Type: `string` + +### `value_field` + +The field in the document that is used as the value. + + +Type: `string` + + From 0c4342b2ce8dd0489b945944c86c1b3d51c648ca Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Tue, 16 Apr 2024 15:50:50 +0000 Subject: [PATCH 32/43] wrote down some thoughts Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go index a49ed1bbc2..b4fab157b2 100644 --- a/internal/impl/opensearch/cache.go +++ b/internal/impl/opensearch/cache.go @@ -137,6 +137,16 @@ func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { func (m *opensearchCache) Set(ctx context.Context, key string, value []byte, _ *time.Duration) error { return errors.New("not implemented") + // document := map[string]interface{}{ + // m.keyField: key, + // m.valueField: string(value), + // } + + // req := &opensearchapi.IndexReq{ + // Index: m.collection, + // Body: bytes.NewReader(encodeJSON(document)), + // } + // m.client.Index(ctx, req) } func (m *opensearchCache) Add(ctx context.Context, key string, value []byte, _ *time.Duration) error { From f7a5a35eb09bf0c2f0d8aea57f750cbbe063653e Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Tue, 16 Apr 2024 16:11:57 +0000 Subject: [PATCH 33/43] rename collection to index Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 32 +++++++++++--------- website/docs/components/caches/opensearch.md | 8 ++--- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go index b4fab157b2..6981a883ad 100644 --- a/internal/impl/opensearch/cache.go +++ b/internal/impl/opensearch/cache.go @@ -22,8 +22,8 @@ func opensearchCacheConfig() *service.ConfigSpec { Fields(httpclient.BasicAuthField()). Fields(OAuthAuthField()). Fields( - service.NewStringField("collection"). - Description("The name of the target collection."), + service.NewStringField("index"). + Description("The name of the target index."), service.NewStringField("key_field"). Description("The field in the document that is used as the key."), service.NewStringField("value_field"). @@ -48,7 +48,7 @@ func newOpensearchCacheFromConfig(parsedConf *service.ParsedConfig, mgr *service return nil, err } - collectionName, err := parsedConf.FieldString("collection") + indexName, err := parsedConf.FieldString("index") if err != nil { return nil, err } @@ -63,20 +63,20 @@ func newOpensearchCacheFromConfig(parsedConf *service.ParsedConfig, mgr *service return nil, err } - return newOpensearchCache(collectionName, keyField, valueField, conf) + return newOpensearchCache(indexName, keyField, valueField, conf) } //------------------------------------------------------------------------------ type opensearchCache struct { - client *opensearchapi.Client - collection string + client *opensearchapi.Client + indexName string keyField string valueField string } -func newOpensearchCache(collectionName, keyField, valueField string, conf esoConfig) (*opensearchCache, error) { +func newOpensearchCache(indexName, keyField, valueField string, conf esoConfig) (*opensearchCache, error) { client, err := opensearchapi.NewClient(conf.clientOpts) if err != nil { @@ -85,7 +85,7 @@ func newOpensearchCache(collectionName, keyField, valueField string, conf esoCon return &opensearchCache{ client: client, - collection: collectionName, + indexName: indexName, keyField: keyField, valueField: valueField, }, nil @@ -93,15 +93,17 @@ func newOpensearchCache(collectionName, keyField, valueField string, conf esoCon func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { query := fmt.Sprintf(`{ - "query": { - "term": { - "%s": "%s" - } - } - }`, m.keyField, key) + "query": { + "term": { + "%s": { + "value": "%s" + } + } + } + }`, m.keyField, key) search := &opensearchapi.SearchReq{ - Indices: []string{m.collection}, + Indices: []string{m.indexName}, Body: strings.NewReader(query), Params: opensearchapi.SearchParams{ Size: opensearchapi.ToPointer(1), diff --git a/website/docs/components/caches/opensearch.md b/website/docs/components/caches/opensearch.md index 571b6f569c..8886361035 100644 --- a/website/docs/components/caches/opensearch.md +++ b/website/docs/components/caches/opensearch.md @@ -34,7 +34,7 @@ Introduced in version 1.0.0. label: "" opensearch: urls: [] # No default (required) - collection: "" # No default (required) + index: "" # No default (required) key_field: "" # No default (required) value_field: "" # No default (optional) ``` @@ -60,7 +60,7 @@ opensearch: scope: "" token_cache: "" token_key: "" - collection: "" # No default (required) + index: "" # No default (required) key_field: "" # No default (required) value_field: "" # No default (optional) ``` @@ -188,9 +188,9 @@ Required when using a `token_cache`, the key to query the cache with for tokens. Type: `string` Default: `""` -### `collection` +### `index` -The name of the target collection. +The name of the target index. Type: `string` From 442cbb500dc77d9f2b5be4db0d35ea8b9fcbbfa1 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 17 Apr 2024 18:17:36 +0000 Subject: [PATCH 34/43] move stuff around Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 7 +++-- internal/impl/opensearch/output.go | 21 ++++++++++---- resources/docker/opensearch/cache.yaml | 28 +++++++++++++++++++ .../docker/opensearch/docker-compose.yaml | 11 +++++++- 4 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 resources/docker/opensearch/cache.yaml diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go index 6981a883ad..e0f535f985 100644 --- a/internal/impl/opensearch/cache.go +++ b/internal/impl/opensearch/cache.go @@ -21,6 +21,7 @@ func opensearchCacheConfig() *service.ConfigSpec { Fields(service.NewStringListField(esoFieldURLs)). Fields(httpclient.BasicAuthField()). Fields(OAuthAuthField()). + Fields(service.NewTLSToggledField(esoFieldTLS)). Fields( service.NewStringField("index"). Description("The name of the target index."), @@ -43,7 +44,7 @@ func init() { } func newOpensearchCacheFromConfig(parsedConf *service.ParsedConfig, mgr *service.Resources) (*opensearchCache, error) { - conf, err := esoConfigFromParsed(parsedConf, mgr) + conf, err := esoClientConfigFromParsed(parsedConf, mgr) if err != nil { return nil, err } @@ -76,9 +77,9 @@ type opensearchCache struct { valueField string } -func newOpensearchCache(indexName, keyField, valueField string, conf esoConfig) (*opensearchCache, error) { +func newOpensearchCache(indexName, keyField, valueField string, clientOpts opensearchapi.Config) (*opensearchCache, error) { - client, err := opensearchapi.NewClient(conf.clientOpts) + client, err := opensearchapi.NewClient(clientOpts) if err != nil { return nil, err } diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index d97212a544..7ad7cc558b 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -73,8 +73,8 @@ type esoConfig struct { routingStr *service.InterpolatedString } -func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf esoConfig, err error) { - conf.clientOpts = opensearchapi.Config{} +func esoClientConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (opensearchapi.Config, error) { + clientOpts := opensearchapi.Config{} var tmpURLs []string if tmpURLs, err = pConf.FieldStringList(esoFieldURLs); err != nil { @@ -83,7 +83,7 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c for _, u := range tmpURLs { for _, splitURL := range strings.Split(u, ",") { if splitURL != "" { - conf.clientOpts.Client.Addresses = append(conf.clientOpts.Client.Addresses, splitURL) + clientOpts.Client.Addresses = append(conf.clientOpts.Client.Addresses, splitURL) } } } @@ -91,10 +91,10 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c authConf := pConf.Namespace(esoFieldAuth) if enabled, _ := authConf.FieldBool(esoFieldAuthEnabled); enabled { - if conf.clientOpts.Client.Username, err = authConf.FieldString(esoFieldAuthUsername); err != nil { + if clientOpts.Client.Username, err = authConf.FieldString(esoFieldAuthUsername); err != nil { return } - if conf.clientOpts.Client.Password, err = authConf.FieldString(esoFieldAuthPassword); err != nil { + if clientOpts.Client.Password, err = authConf.FieldString(esoFieldAuthPassword); err != nil { return } } @@ -109,7 +109,7 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c mgr.Logger().Debug("Using OAuth2 authentication for OpenSearch") - conf.clientOpts.Client.Transport = &oauth2.Transport{ + clientOpts.Client.Transport = &oauth2.Transport{ Source: oauth2.ReuseTokenSource(nil, &OsTokenProvider{ Mgr: mgr, OAuth2Conf: oauth2conf, @@ -134,6 +134,15 @@ func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (c } } + return clientOpts +} + +func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf esoConfig, err error) { + conf.clientOpts, err = esoClientConfigFromParsed(pConf, mgr) + if err != nil { + return + } + if conf.actionStr, err = pConf.FieldInterpolatedString(esoFieldAction); err != nil { return } diff --git a/resources/docker/opensearch/cache.yaml b/resources/docker/opensearch/cache.yaml new file mode 100644 index 0000000000..3064b395bf --- /dev/null +++ b/resources/docker/opensearch/cache.yaml @@ -0,0 +1,28 @@ +pipeline: + processors: + - label: fieldvalue_cache + cache: + resource: opensearch + operator: get + key: "1" + +cache_resources: + - label: opensearch + opensearch: + urls: ["https://osproxy-saptest.cheetah.trifork.dev"] + index: sap-salesorders_s4hanacloud + key_field: SalesOrder + value_field: CreatedByUser + basic_auth: + enabled: true + username: "apikey" + password: "Bb4Lf4ZLS3uN4bRE^2DGSJ5Ju4CMCB5x6m2pujo5" + +tests: + - name: Example test case fieldvalue_cache + environment: {} + target_processors: fieldvalue_cache #/pipeline/processors + input_batch: + - content: 'ignored value' + output_batches: + - - content_equals: 'CB9980000027' \ No newline at end of file diff --git a/resources/docker/opensearch/docker-compose.yaml b/resources/docker/opensearch/docker-compose.yaml index 39a0b89c7e..04fc9b130e 100644 --- a/resources/docker/opensearch/docker-compose.yaml +++ b/resources/docker/opensearch/docker-compose.yaml @@ -1,4 +1,3 @@ -version: '3.3' services: benthos-opensearch-oauth2: image: jeffail/benthos:local @@ -9,3 +8,13 @@ services: volumes: - ./oauth2.yaml:/benthos.yaml network_mode: host + + benthos-opensearch-cachetest: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ 'test', '/benthos.yaml' ] + volumes: + - ./cache.yaml:/benthos.yaml + network_mode: host From 9548fbc9321476bed1f2085f44d0ea6372bb3f27 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 17 Apr 2024 18:35:58 +0000 Subject: [PATCH 35/43] fix compilation issues Signed-off-by: Casper Thygesen --- internal/impl/opensearch/output.go | 21 +-- website/docs/components/caches/opensearch.md | 147 +++++++++++++++++++ 2 files changed, 158 insertions(+), 10 deletions(-) diff --git a/internal/impl/opensearch/output.go b/internal/impl/opensearch/output.go index 7ad7cc558b..8a9621d6f9 100644 --- a/internal/impl/opensearch/output.go +++ b/internal/impl/opensearch/output.go @@ -77,13 +77,14 @@ func esoClientConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resourc clientOpts := opensearchapi.Config{} var tmpURLs []string + var err error if tmpURLs, err = pConf.FieldStringList(esoFieldURLs); err != nil { - return + return clientOpts, nil } for _, u := range tmpURLs { for _, splitURL := range strings.Split(u, ",") { if splitURL != "" { - clientOpts.Client.Addresses = append(conf.clientOpts.Client.Addresses, splitURL) + clientOpts.Client.Addresses = append(clientOpts.Client.Addresses, splitURL) } } } @@ -92,17 +93,17 @@ func esoClientConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resourc if enabled, _ := authConf.FieldBool(esoFieldAuthEnabled); enabled { if clientOpts.Client.Username, err = authConf.FieldString(esoFieldAuthUsername); err != nil { - return + return clientOpts, nil } if clientOpts.Client.Password, err = authConf.FieldString(esoFieldAuthPassword); err != nil { - return + return clientOpts, nil } } oauth2conf, err := oAuthFromParsed(pConf) if err != nil { mgr.Logger().Error("Failed to parse OAuth2 configuration") - return + return clientOpts, err } if oauth2conf.Enabled { @@ -123,18 +124,18 @@ func esoClientConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resourc var tlsEnabled bool if tlsConf, tlsEnabled, err = pConf.FieldTLSToggled(esoFieldTLS); err != nil { - return + return clientOpts, nil } else if tlsEnabled { - if _, ok := conf.clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport); ok { - conf.clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport).TLSClientConfig = tlsConf + if _, ok := clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport); ok { + clientOpts.Client.Transport.(*oauth2.Transport).Base.(*http.Transport).TLSClientConfig = tlsConf } else { - conf.clientOpts.Client.Transport = &http.Transport{ + clientOpts.Client.Transport = &http.Transport{ TLSClientConfig: tlsConf, } } } - return clientOpts + return clientOpts, nil } func esoConfigFromParsed(pConf *service.ParsedConfig, mgr *service.Resources) (conf esoConfig, err error) { diff --git a/website/docs/components/caches/opensearch.md b/website/docs/components/caches/opensearch.md index 8886361035..995e9c4167 100644 --- a/website/docs/components/caches/opensearch.md +++ b/website/docs/components/caches/opensearch.md @@ -60,6 +60,13 @@ opensearch: scope: "" token_cache: "" token_key: "" + tls: + enabled: false + skip_cert_verify: false + enable_renegotiation: false + root_cas: "" + root_cas_file: "" + client_certs: [] index: "" # No default (required) key_field: "" # No default (required) value_field: "" # No default (optional) @@ -188,6 +195,146 @@ Required when using a `token_cache`, the key to query the cache with for tokens. Type: `string` Default: `""` +### `tls` + +Custom TLS settings can be used to override system defaults. + + +Type: `object` + +### `tls.enabled` + +Whether custom TLS settings are enabled. + + +Type: `bool` +Default: `false` + +### `tls.skip_cert_verify` + +Whether to skip server side certificate verification. + + +Type: `bool` +Default: `false` + +### `tls.enable_renegotiation` + +Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. + + +Type: `bool` +Default: `false` +Requires version 3.45.0 or newer + +### `tls.root_cas` + +An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +### `tls.root_cas_file` + +An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas_file: ./root_cas.pem +``` + +### `tls.client_certs` + +A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +client_certs: + - cert: foo + key: bar + +client_certs: + - cert_file: ./example.pem + key_file: ./example.key +``` + +### `tls.client_certs[].cert` + +A plain text certificate to use. + + +Type: `string` +Default: `""` + +### `tls.client_certs[].key` + +A plain text certificate key to use. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +### `tls.client_certs[].cert_file` + +The path of a certificate to use. + + +Type: `string` +Default: `""` + +### `tls.client_certs[].key_file` + +The path of a certificate key to use. + + +Type: `string` +Default: `""` + +### `tls.client_certs[].password` + +A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format. Warning: Since it does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext. +:::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: + + +Type: `string` +Default: `""` + +```yml +# Examples + +password: foo + +password: ${KEY_PASSWORD} +``` + ### `index` The name of the target index. From 04cf0c80d77dcd68f74dcc3e698041522721b232 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 17 Apr 2024 18:55:00 +0000 Subject: [PATCH 36/43] made the cache more flexible Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 86 +++++++++++++++----------- resources/docker/opensearch/cache.yaml | 5 +- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go index e0f535f985..b81a2c40a3 100644 --- a/internal/impl/opensearch/cache.go +++ b/internal/impl/opensearch/cache.go @@ -26,9 +26,11 @@ func opensearchCacheConfig() *service.ConfigSpec { service.NewStringField("index"). Description("The name of the target index."), service.NewStringField("key_field"). - Description("The field in the document that is used as the key."), + Description("The field in the document that is used as the key. If not set, it will use the _id on the document."). + Optional(), service.NewStringField("value_field"). - Description("The field in the document that is used as the value.").Optional(), + Description("The field in the document that is used as the value. If not set, it will retrieve the entire document"). + Optional(), ) } @@ -54,15 +56,9 @@ func newOpensearchCacheFromConfig(parsedConf *service.ParsedConfig, mgr *service return nil, err } - keyField, err := parsedConf.FieldString("key_field") - if err != nil { - return nil, err - } + keyField, _ := parsedConf.FieldString("key_field") - valueField, err := parsedConf.FieldString("value_field") - if err != nil { - return nil, err - } + valueField, _ := parsedConf.FieldString("value_field") return newOpensearchCache(indexName, keyField, valueField, conf) } @@ -93,7 +89,23 @@ func newOpensearchCache(indexName, keyField, valueField string, clientOpts opens } func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { - query := fmt.Sprintf(`{ + var searchHit json.RawMessage + if m.keyField == "" { + documentResponse, err := m.client.Document.Get(ctx, opensearchapi.DocumentGetReq{ + Index: m.indexName, + DocumentID: key, + }) + + if !documentResponse.Found { + return nil, service.ErrKeyNotFound + } + if err != nil { + return nil, fmt.Errorf("error getting document %s: %v", key, err) + } + searchHit = documentResponse.Source + } else { + + query := fmt.Sprintf(`{ "query": { "term": { "%s": { @@ -103,38 +115,40 @@ func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { } }`, m.keyField, key) - search := &opensearchapi.SearchReq{ - Indices: []string{m.indexName}, - Body: strings.NewReader(query), - Params: opensearchapi.SearchParams{ - Size: opensearchapi.ToPointer(1), - }, - } + search := &opensearchapi.SearchReq{ + Indices: []string{m.indexName}, + Body: strings.NewReader(query), + Params: opensearchapi.SearchParams{ + Size: opensearchapi.ToPointer(1), + }, + } - searchResponse, err := m.client.Search(ctx, search) + searchResponse, err := m.client.Search(ctx, search) - if err != nil { - return nil, fmt.Errorf("error searching for key %s: %v", key, err) + if err != nil { + return nil, fmt.Errorf("error searching for key %s: %v", key, err) + } + + if searchResponse.Hits.Total.Value == 0 { + return nil, service.ErrKeyNotFound + } + searchHit = searchResponse.Hits.Hits[0].Source } - if searchResponse.Hits.Total.Value == 0 { - return nil, service.ErrKeyNotFound + if m.valueField == "" { + return searchHit, nil // return the entire document } - if m.valueField != "" { - var message map[string]interface{} - err = json.Unmarshal(searchResponse.Hits.Hits[0].Source, &message) - if err != nil { - return nil, fmt.Errorf("error getting field from document %s: %v", m.valueField, err) - } - var val, ok = message[m.valueField].(string) - if ok { - return []byte(val), nil - } - return nil, fmt.Errorf("error getting field from document %s: %v", m.valueField, val) - } else { - return searchResponse.Hits.Hits[0].Source, nil // return the entire document + var message map[string]interface{} + err := json.Unmarshal(searchHit, &message) + if err != nil { + return nil, fmt.Errorf("error getting field from document %s: %v", m.valueField, err) + } + var val, ok = message[m.valueField].(string) + if ok { + return []byte(val), nil } + return nil, fmt.Errorf("error getting field from document %s: %v", m.valueField, val) } diff --git a/resources/docker/opensearch/cache.yaml b/resources/docker/opensearch/cache.yaml index 3064b395bf..f83051ded2 100644 --- a/resources/docker/opensearch/cache.yaml +++ b/resources/docker/opensearch/cache.yaml @@ -12,7 +12,7 @@ cache_resources: urls: ["https://osproxy-saptest.cheetah.trifork.dev"] index: sap-salesorders_s4hanacloud key_field: SalesOrder - value_field: CreatedByUser + #value_field: CreatedByUser basic_auth: enabled: true username: "apikey" @@ -25,4 +25,5 @@ tests: input_batch: - content: 'ignored value' output_batches: - - - content_equals: 'CB9980000027' \ No newline at end of file + - - json_contains: + CreatedByUser: 'CB9980000027' \ No newline at end of file From 4b1cb0b932b0a44fb1ec31f50b063ea227da6b63 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 17 Apr 2024 19:20:11 +0000 Subject: [PATCH 37/43] supported all operators Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 79 ++++++++++++++------ website/docs/components/caches/opensearch.md | 14 ++-- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go index b81a2c40a3..175c6d9b53 100644 --- a/internal/impl/opensearch/cache.go +++ b/internal/impl/opensearch/cache.go @@ -1,9 +1,9 @@ package opensearch import ( + "bytes" "context" "encoding/json" - "errors" "fmt" "strings" "time" @@ -17,7 +17,7 @@ func opensearchCacheConfig() *service.ConfigSpec { return service.NewConfigSpec(). Beta(). Version("1.0.0"). - Summary(`Use a OpenSearch instance as a cache. Only GET is supported`). + Summary(`Use a OpenSearch instance as a cache. The get operator can be used to look into any existing document in Opensearch`). Fields(service.NewStringListField(esoFieldURLs)). Fields(httpclient.BasicAuthField()). Fields(OAuthAuthField()). @@ -26,10 +26,12 @@ func opensearchCacheConfig() *service.ConfigSpec { service.NewStringField("index"). Description("The name of the target index."), service.NewStringField("key_field"). - Description("The field in the document that is used as the key. If not set, it will use the _id on the document."). + Description("Not used together with get,set and delete operatior. The field in the document that is used as the key. If not set, it will use the _id on the document."). + Advanced(). Optional(), service.NewStringField("value_field"). - Description("The field in the document that is used as the value. If not set, it will retrieve the entire document"). + Description("The field in the document that is used as the value. If set to empty, it will retrieve the entire document"). + Default("value"). Optional(), ) } @@ -63,8 +65,6 @@ func newOpensearchCacheFromConfig(parsedConf *service.ParsedConfig, mgr *service return newOpensearchCache(indexName, keyField, valueField, conf) } -//------------------------------------------------------------------------------ - type opensearchCache struct { client *opensearchapi.Client indexName string @@ -96,12 +96,14 @@ func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { DocumentID: key, }) - if !documentResponse.Found { - return nil, service.ErrKeyNotFound - } if err != nil { return nil, fmt.Errorf("error getting document %s: %v", key, err) } + + if !documentResponse.Found { + return nil, service.ErrKeyNotFound + } + searchHit = documentResponse.Source } else { @@ -153,27 +155,60 @@ func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { } func (m *opensearchCache) Set(ctx context.Context, key string, value []byte, _ *time.Duration) error { - return errors.New("not implemented") - // document := map[string]interface{}{ - // m.keyField: key, - // m.valueField: string(value), - // } - - // req := &opensearchapi.IndexReq{ - // Index: m.collection, - // Body: bytes.NewReader(encodeJSON(document)), - // } - // m.client.Index(ctx, req) + if m.keyField != "" { + return fmt.Errorf("key_field is used, cannot be used with set operator. key_field is only supported for get") + } + return index(ctx, m, value, key, "index") + } func (m *opensearchCache) Add(ctx context.Context, key string, value []byte, _ *time.Duration) error { - return errors.New("not implemented") + if m.keyField != "" { + return fmt.Errorf("key_field is used, cannot be used with set operator. key_field is only supported for get") + } + return index(ctx, m, value, key, "create") } func (m *opensearchCache) Delete(ctx context.Context, key string) error { - return errors.New("not implemented") + if m.keyField != "" { + return fmt.Errorf("key_field is used, cannot be used with set operator. key_field is only supported for get") + } + _, err := m.client.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{ + Index: m.indexName, + DocumentID: key, + }) + return err } func (m *opensearchCache) Close(ctx context.Context) error { return nil } + +func index(ctx context.Context, m *opensearchCache, value []byte, key string, optype string) error { + if m.keyField != "" { + return fmt.Errorf("key_field is used, cannot be used with set operator. key_field is only supported for get") + } + value_field := m.valueField + if m.valueField == "" { + value_field = "value" + } + document := map[string]interface{}{ + value_field: string(value), + } + + data, err := json.Marshal(document) + if err != nil { + return err + } + + req := opensearchapi.IndexReq{ + Index: m.indexName, + DocumentID: key, + Body: bytes.NewReader(data), + Params: opensearchapi.IndexParams{ + OpType: optype, + }, + } + _, err = m.client.Index(ctx, req) + return err +} diff --git a/website/docs/components/caches/opensearch.md b/website/docs/components/caches/opensearch.md index 995e9c4167..63cf15c02a 100644 --- a/website/docs/components/caches/opensearch.md +++ b/website/docs/components/caches/opensearch.md @@ -17,7 +17,7 @@ import TabItem from '@theme/TabItem'; :::caution BETA This component is mostly stable but breaking changes could still be made outside of major version releases if a fundamental problem with the component is found. ::: -Use a OpenSearch instance as a cache. Only GET is supported +Use a OpenSearch instance as a cache. The get operator can be used to look into any existing document in Opensearch Introduced in version 1.0.0. @@ -35,8 +35,7 @@ label: "" opensearch: urls: [] # No default (required) index: "" # No default (required) - key_field: "" # No default (required) - value_field: "" # No default (optional) + value_field: value ``` @@ -68,8 +67,8 @@ opensearch: root_cas_file: "" client_certs: [] index: "" # No default (required) - key_field: "" # No default (required) - value_field: "" # No default (optional) + key_field: "" # No default (optional) + value_field: value ``` @@ -344,16 +343,17 @@ Type: `string` ### `key_field` -The field in the document that is used as the key. +Not used together with get,set and delete operatior. The field in the document that is used as the key. If not set, it will use the _id on the document. Type: `string` ### `value_field` -The field in the document that is used as the value. +The field in the document that is used as the value. If set to empty, it will retrieve the entire document Type: `string` +Default: `"value"` From 2beb25bd2ca27206696a427188568e9278e288b3 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 17 Apr 2024 19:28:46 +0000 Subject: [PATCH 38/43] simplify release flow Signed-off-by: Casper Thygesen --- .github/workflows/cheetah_release.yaml | 91 +------------------------- 1 file changed, 2 insertions(+), 89 deletions(-) diff --git a/.github/workflows/cheetah_release.yaml b/.github/workflows/cheetah_release.yaml index 7df9db341c..60ee18b838 100644 --- a/.github/workflows/cheetah_release.yaml +++ b/.github/workflows/cheetah_release.yaml @@ -37,92 +37,5 @@ jobs: image_name: ${{ env.IMAGE_NAME }} github_run_id: ${{ github.run_id }} dockerfile_path: ${{ env.DOCKERFILE_PATH }} - push_image: ${{ (github.ref_type == 'tag' || github.event_name == 'workflow_dispatch') && 'true' || 'false' }} - upload_image: ${{ (github.event_name == 'pull_request' || github.ref_name == 'main' || github.event_name == 'workflow_dispatch') && 'true' || 'false' }} - - trivy-container-scan-pr: - name: trivy (PR report) - needs: [push_image] - runs-on: ubuntu-latest - timeout-minutes: 5 - if: ${{ github.event_name == 'pull_request' }} - permissions: - contents: read - packages: read - pull-requests: write - issues: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download Docker image artifact - uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4 - with: - name: dockerimage - path: ${{ github.workspace }}/tmp - - - name: Update pull request - uses: trifork/cheetah-infrastructure-utils/.github/actions/trivy-report@main - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - WORKSPACE_PATH: ${{ github.workspace }} - with: - image_name: ${{ env.IMAGE_NAME }} - base-tag: "latest" # "pr-${{ github.event.pull_request.number }}" - orga: "trifork" - repo: ${{ github.repository }} - pr-nb: ${{ github.event.pull_request.number }} - topic: ${{ env.IMAGE_NAME }} - - trivy-container-scan: - name: trivy (container scan) - needs: [push_image] - runs-on: ubuntu-latest - timeout-minutes: 5 - if: github.ref_name == 'main' && github.event_name == 'push' - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download Docker image artifact - uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4 - with: - name: dockerimage - path: ${{ github.workspace }}/tmp - - - name: Run Trivy vulnerability scanner for containers - uses: aquasecurity/trivy-action@master - with: - input: /github/workspace/tmp/image.tar # -v "/home/runner/work/cheetah-backstage/cheetah-backstage":"/github/workspace" - format: "sarif" - #exit-code: "0" - output: "trivy-container-results.sarif" - github-pat: ${{ secrets.GITHUB_TOKEN }} - - #- name: Upload Trivy scan results to GitHub Security tab - # uses: github/codeql-action/upload-sarif@v2 - # if: never() - # with: - # sarif_file: 'trivy-results.sarif' - - - name: Publish container report - id: import-container-scan - uses: trifork/cheetah-infrastructure-utils/.github/actions/defectdojo-import-scan@main - if: github.ref_name == 'main' && github.event_name == 'push' - with: - token: ${{ secrets.DEFECTDOJO_TOKEN }} - defectdojo_url: "https://defectdojo.cheetah.trifork.dev" - defectdojo_endpoint: "/api/v2/reimport-scan/" - file: "trivy-container-results.sarif" - scan_type: SARIF # Trivy Scan - engagement_name: import-container-scan - product_name: ${{ env.IMAGE_NAME }} - branch_tag: ${{ github.ref_name }} - #commit_hash: ${{ github.sha }} - #minimum_severity: Info, Low, Medium, High, Critical. Default value: Info - - - name: Show response - if: github.ref_name == 'main' && github.event_name == 'push' - run: | - set -e - printf '%s\n' '${{ steps.import-container-scan.outputs.response }}' + push_image: ${{ (github.event_name == 'pull_request' || github.ref_type == 'tag' || github.event_name == 'workflow_dispatch') && 'true' || 'false' }} + upload_image: "false" From c38417fbfa35317b2712f7655d7739f885c0baba Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 17 Apr 2024 19:30:29 +0000 Subject: [PATCH 39/43] remove tmp secret Signed-off-by: Casper Thygesen --- resources/docker/opensearch/cache.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/docker/opensearch/cache.yaml b/resources/docker/opensearch/cache.yaml index f83051ded2..08707153bf 100644 --- a/resources/docker/opensearch/cache.yaml +++ b/resources/docker/opensearch/cache.yaml @@ -16,7 +16,7 @@ cache_resources: basic_auth: enabled: true username: "apikey" - password: "Bb4Lf4ZLS3uN4bRE^2DGSJ5Ju4CMCB5x6m2pujo5" + password: "${SAPHANA_APIKEY}" tests: - name: Example test case fieldvalue_cache From ddeba9eec861a59cfe42b2a94c3521894013bbda Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Wed, 17 Apr 2024 20:05:33 +0000 Subject: [PATCH 40/43] MarshalJSON Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 3 ++- resources/docker/opensearch/cache.yaml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go index 175c6d9b53..1305299b0e 100644 --- a/internal/impl/opensearch/cache.go +++ b/internal/impl/opensearch/cache.go @@ -138,7 +138,8 @@ func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { } if m.valueField == "" { - return searchHit, nil // return the entire document + json, _ := searchHit.MarshalJSON() + return json, nil } var message map[string]interface{} diff --git a/resources/docker/opensearch/cache.yaml b/resources/docker/opensearch/cache.yaml index 08707153bf..02132c9618 100644 --- a/resources/docker/opensearch/cache.yaml +++ b/resources/docker/opensearch/cache.yaml @@ -12,11 +12,11 @@ cache_resources: urls: ["https://osproxy-saptest.cheetah.trifork.dev"] index: sap-salesorders_s4hanacloud key_field: SalesOrder - #value_field: CreatedByUser + value_field: "" # CreatedByUser basic_auth: enabled: true username: "apikey" - password: "${SAPHANA_APIKEY}" + password: "${OS_API_KEY}" tests: - name: Example test case fieldvalue_cache From 2f4c7573b9e2b4a2f3ba1aa1042a5d3391114000 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 18 Apr 2024 15:33:46 +0000 Subject: [PATCH 41/43] added opensearch ping Signed-off-by: Casper Thygesen --- internal/impl/opensearch/cache.go | 8 ++- resources/docker/opensearch/cache.yaml | 62 ++++++++++++++++--- .../docker/opensearch/docker-compose.yaml | 16 +++++ 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/internal/impl/opensearch/cache.go b/internal/impl/opensearch/cache.go index 1305299b0e..dc1430ae8f 100644 --- a/internal/impl/opensearch/cache.go +++ b/internal/impl/opensearch/cache.go @@ -73,13 +73,18 @@ type opensearchCache struct { valueField string } -func newOpensearchCache(indexName, keyField, valueField string, clientOpts opensearchapi.Config) (*opensearchCache, error) { +func newOpensearchCache(indexName string, keyField string, valueField string, clientOpts opensearchapi.Config) (*opensearchCache, error) { client, err := opensearchapi.NewClient(clientOpts) if err != nil { return nil, err } + _, err = client.Ping(context.Background(), &opensearchapi.PingReq{}) + if err != nil { + return nil, fmt.Errorf("error pinging opensearch: %v", err) + } + return &opensearchCache{ client: client, indexName: indexName, @@ -126,7 +131,6 @@ func (m *opensearchCache) Get(ctx context.Context, key string) ([]byte, error) { } searchResponse, err := m.client.Search(ctx, search) - if err != nil { return nil, fmt.Errorf("error searching for key %s: %v", key, err) } diff --git a/resources/docker/opensearch/cache.yaml b/resources/docker/opensearch/cache.yaml index 02132c9618..41cba246d6 100644 --- a/resources/docker/opensearch/cache.yaml +++ b/resources/docker/opensearch/cache.yaml @@ -1,29 +1,75 @@ +logger: + level: VERBOSE + pipeline: processors: + - label: doclevel_cache + cache: + resource: opensearch_entiredoc + operator: get + key: "1" - label: fieldvalue_cache cache: - resource: opensearch + resource: opensearch_fieldvalue + operator: get + key: "1" + - label: leveled_cache + cache: + resource: opensearch_entiredoc operator: get key: "1" cache_resources: - - label: opensearch - opensearch: + - label: opensearch_entiredoc + opensearch: urls: ["https://osproxy-saptest.cheetah.trifork.dev"] index: sap-salesorders_s4hanacloud key_field: SalesOrder value_field: "" # CreatedByUser basic_auth: enabled: true - username: "apikey" - password: "${OS_API_KEY}" + username: ${USERNAME} + password: ${PASSWORD} + - label: opensearch_fieldvalue + opensearch: + urls: ["https://osproxy-saptest.cheetah.trifork.dev"] + index: sap-salesorders_s4hanacloud + key_field: SalesOrder + value_field: "CreatedByUser" + basic_auth: + enabled: true + username: ${USERNAME} + password: ${PASSWORD} + - label: leveled + multilevel: [hot, opensearch_entiredoc] + - label: hot + memory: + default_ttl: 60s tests: + - name: Example test case doclevel_cache + environment: {} + target_processors: doclevel_cache #/pipeline/processors + input_batch: + - content: "ignored value" + output_batches: + - - json_contains: + CreatedByUser: "CB9980000027" - name: Example test case fieldvalue_cache environment: {} target_processors: fieldvalue_cache #/pipeline/processors input_batch: - - content: 'ignored value' + - content: "ignored value" + output_batches: + - - content_equals: "CB9980000027" + - name: Example test case leveled_cache + environment: {} + target_processors: leveled_cache #/pipeline/processors + input_batch: + - content: "ignored value" + - content: "ignored value" output_batches: - - - json_contains: - CreatedByUser: 'CB9980000027' \ No newline at end of file + - - json_contains: + CreatedByUser: "CB9980000027" + - json_contains: + CreatedByUser: "CB9980000027" diff --git a/resources/docker/opensearch/docker-compose.yaml b/resources/docker/opensearch/docker-compose.yaml index 04fc9b130e..26152e7b82 100644 --- a/resources/docker/opensearch/docker-compose.yaml +++ b/resources/docker/opensearch/docker-compose.yaml @@ -18,3 +18,19 @@ services: volumes: - ./cache.yaml:/benthos.yaml network_mode: host + environment: + - USERNAME=apikey + - PASSWORD=${OS_API_KEY} + + benthos-opensearch-cache: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile.cgo + command: [ '-c', '/benthos.yaml' ] + volumes: + - ./cache.yaml:/benthos.yaml + network_mode: host + environment: + - USERNAME=apikey + - PASSWORD=${OS_API_KEY} From fa70b9cd5dbce5cff71f4f7ed9b1d639cf23b009 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Thu, 18 Apr 2024 16:59:40 +0000 Subject: [PATCH 42/43] tested multilevel better Signed-off-by: Casper Thygesen --- resources/docker/opensearch/cache.yaml | 18 ++++++++++++++---- .../docker/opensearch/docker-compose.yaml | 13 ------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/resources/docker/opensearch/cache.yaml b/resources/docker/opensearch/cache.yaml index 41cba246d6..f4a062242f 100644 --- a/resources/docker/opensearch/cache.yaml +++ b/resources/docker/opensearch/cache.yaml @@ -14,10 +14,20 @@ pipeline: operator: get key: "1" - label: leveled_cache - cache: - resource: opensearch_entiredoc - operator: get - key: "1" + processors: + - cache: + resource: opensearch_entiredoc + operator: get + key: "1" + - cache: + resource: hot + operator: get + key: "1" + - cached: + key: "1" + cache: hot + processors: + - mapping: throw("hot cache not found") cache_resources: - label: opensearch_entiredoc diff --git a/resources/docker/opensearch/docker-compose.yaml b/resources/docker/opensearch/docker-compose.yaml index 26152e7b82..5127a6a013 100644 --- a/resources/docker/opensearch/docker-compose.yaml +++ b/resources/docker/opensearch/docker-compose.yaml @@ -21,16 +21,3 @@ services: environment: - USERNAME=apikey - PASSWORD=${OS_API_KEY} - - benthos-opensearch-cache: - image: jeffail/benthos:local - build: - context: ../../../ - dockerfile: ./resources/docker/Dockerfile.cgo - command: [ '-c', '/benthos.yaml' ] - volumes: - - ./cache.yaml:/benthos.yaml - network_mode: host - environment: - - USERNAME=apikey - - PASSWORD=${OS_API_KEY} From c5e436747680c0d52b5d7e211c32892e243ae8c9 Mon Sep 17 00:00:00 2001 From: Casper Thygesen Date: Mon, 27 May 2024 15:13:04 +0000 Subject: [PATCH 43/43] save work Signed-off-by: Casper Thygesen --- resources/docker/amqp/docker-compose.yaml | 27 ++++++++++++++++ resources/docker/amqp/in.yaml | 36 +++++++++++++++++++++ resources/docker/amqp/out.yaml | 38 +++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 resources/docker/amqp/docker-compose.yaml create mode 100644 resources/docker/amqp/in.yaml create mode 100644 resources/docker/amqp/out.yaml diff --git a/resources/docker/amqp/docker-compose.yaml b/resources/docker/amqp/docker-compose.yaml new file mode 100644 index 0000000000..02580d4ac1 --- /dev/null +++ b/resources/docker/amqp/docker-compose.yaml @@ -0,0 +1,27 @@ +version: '3.3' +services: + benthos-in: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ '-w', '-c', '/in.yaml' ] + volumes: + - ./in.yaml:/in.yaml + network_mode: host + profiles: + - in + - full + + benthos-out: + image: jeffail/benthos:local + build: + context: ../../../ + dockerfile: ./resources/docker/Dockerfile + command: [ '-w', '-c', '/out.yaml' ] + volumes: + - ./out.yaml:/out.yaml + network_mode: host + profiles: + - out + - full \ No newline at end of file diff --git a/resources/docker/amqp/in.yaml b/resources/docker/amqp/in.yaml new file mode 100644 index 0000000000..c3974f6e8f --- /dev/null +++ b/resources/docker/amqp/in.yaml @@ -0,0 +1,36 @@ +http: + enabled: false + +input: + label: source + generate: + interval: 5s + mapping: | + root.ID = uuid_v4() + root.Name = [ "frosty", "spot", "oodles" ].index(random_int() % 3) + root.Gooeyness = (random_int() % 100) / 100 + root.Bouncing = random_int() % 2 == 0 + +pipeline: + processors: + - bloblang: | + root = this + root.ID = this.ID + root.ReNamed = this.Name + root.Gooeyness = this.Gooeyness + root.Bouncing = this.Bouncing + + +output: + label: sink + type: amqp_1 + amqp_1: + #target_address: "/iothub-ehub-iothubdemo-58177125-3887de637f/queue-name" + target_address: "/devices/1/messages/events" #send device-to-cloud telemetry data + urls: + - "amqps://ihsuprodamres031dednamespace.servicebus.windows.net/iothub-ehub-iothubdemo-58177125-3887de637f" + #- "sb://ihsuprodamres031dednamespace.servicebus.windows.net/;SharedAccessKeyName=iothubowner;SharedAccessKey=7nqmfjCWvpPY9KpijN1CiVIcJ3b3N4NM4AIoTPV/Rts=;EntityPath=iothub-ehub-iothubdemo-58177125-3887de637f" + sasl: + mechanism: plain + user: "iothubowner" + password: "7nqmfjCWvpPY9KpijN1CiVIcJ3b3N4NM4AIoTPV/Rts=" diff --git a/resources/docker/amqp/out.yaml b/resources/docker/amqp/out.yaml new file mode 100644 index 0000000000..07b323dec6 --- /dev/null +++ b/resources/docker/amqp/out.yaml @@ -0,0 +1,38 @@ +http: + address: 0.0.0.0:4195 + enabled: true + root_path: /benthos + debug_endpoints: false + +logger: + level: DEBUG + format: logfmt + add_timestamp: true + static_fields: + "@service": benthos + +input: + label: source + type: amqp_1 + amqp_1: + source_address: "/iothub-ehub-iothubdemo-58177125-3887de637f/ConsumerGroups/$Default/Partitions/0" + url: "amqps://ihsuprodamres031dednamespace.servicebus.windows.net" + #url: Endpoint=sb://ihsuprodamres031dednamespace.servicebus.windows.net/;SharedAccessKeyName=iothubowner;SharedAccessKey=7nqmfjCWvpPY9KpijN1CiVIcJ3b3N4NM4AIoTPV/Rts=;EntityPath=iothub-ehub-iothubdemo-58177125-3887de637f + sasl: + mechanism: plain + user: "iothubowner" + password: "7nqmfjCWvpPY9KpijN1CiVIcJ3b3N4NM4AIoTPV/Rts=" + +pipeline: + processors: + - bloblang: | + root = this + + - catch: + - log: + level: ERROR + message: ${! error() } + - bloblang: root = deleted() + +output: + stdout: {}