From 630ae1d382908575796ab82fae2fd9723a091f73 Mon Sep 17 00:00:00 2001 From: Rostyslav Syz'ko Date: Tue, 7 Feb 2023 15:55:59 +0200 Subject: [PATCH] Facts from RBAC service was added --- Dockerfile | 2 +- assets/apis/example_MultipleDefinitions.json | 93 -------------------- assets/apis/example_SingleDefinition.json | 31 ------- assets/auth/auth.json | 27 ------ assets/config/config.yaml | 4 + assets/docker-compose.yml | 51 +++-------- assets/janus.toml | 22 +++++ go.mod | 82 ++++++++++++++--- go.sum | 81 ++++++++++++----- pkg/config/config.go | 23 +++++ pkg/jwt/jwt.go | 9 ++ pkg/kafka/consumer.go | 44 +++++++++ pkg/kafka/factConsumer.go | 62 +++++++++++++ pkg/models/fact.go | 14 +++ pkg/models/rbac.go | 17 ++++ pkg/plugin/oauth2/jwt_manager.go | 31 +++++++ pkg/plugin/oauth2/middleware.go | 13 +++ pkg/plugin/oauth2/setup.go | 11 ++- 18 files changed, 387 insertions(+), 230 deletions(-) delete mode 100644 assets/apis/example_MultipleDefinitions.json delete mode 100644 assets/apis/example_SingleDefinition.json delete mode 100644 assets/auth/auth.json create mode 100644 assets/config/config.yaml create mode 100644 assets/janus.toml create mode 100644 pkg/config/config.go create mode 100644 pkg/jwt/jwt.go create mode 100644 pkg/kafka/consumer.go create mode 100644 pkg/kafka/factConsumer.go create mode 100644 pkg/models/fact.go create mode 100644 pkg/models/rbac.go diff --git a/Dockerfile b/Dockerfile index 330e11019..5c180b90a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ####### Start from a golang base image ############### -FROM golang:1.13.6-buster as builder +FROM golang:1.17-buster as builder LABEL maintainer="Motiv Labs " WORKDIR /app COPY ./ ./ diff --git a/assets/apis/example_MultipleDefinitions.json b/assets/apis/example_MultipleDefinitions.json deleted file mode 100644 index e61489f05..000000000 --- a/assets/apis/example_MultipleDefinitions.json +++ /dev/null @@ -1,93 +0,0 @@ -[{ - "name" : "exampleFirst", - "active" : true, - "proxy" : { - "preserve_host" : false, - "listen_path" : "/exampleFirst/*", - "upstreams" : { - "balancing": "roundrobin", - "targets": [ - {"target": "http://localhost:9089/hello-world"} - ] - }, - "strip_path" : false, - "append_path" : false, - "methods" : ["GET"] - }, - "plugins": [ - { - "name": "rate_limit", - "enabled": true, - "config": { - "limit": "5-M", - "policy": "local" - } - } - ], - "health_check": { - "url": "http://localhost:9089/status", - "timeout": 3 - } -}, -{ - "name" : "exampleSecond", - "active" : true, - "proxy" : { - "preserve_host" : false, - "listen_path" : "/exampleSecond/*", - "upstreams" : { - "balancing": "roundrobin", - "targets": [ - {"target": "http://localhost:9089/hello-world"} - ] - }, - "strip_path" : false, - "append_path" : false, - "methods" : ["GET"] - }, - "plugins": [ - { - "name": "rate_limit", - "enabled": true, - "config": { - "limit": "5-M", - "policy": "local" - } - } - ], - "health_check": { - "url": "http://localhost:9089/status", - "timeout": 3 - } -}, -{ - "name" : "exampleSecond", - "active" : true, - "proxy" : { - "preserve_host" : false, - "listen_path" : "/exampleRepeatedSecond/*", - "upstreams" : { - "balancing": "roundrobin", - "targets": [ - {"target": "http://localhost:9089/hello-world"} - ] - }, - "strip_path" : false, - "append_path" : false, - "methods" : ["GET"] - }, - "plugins": [ - { - "name": "rate_limit", - "enabled": true, - "config": { - "limit": "5-M", - "policy": "local" - } - } - ], - "health_check": { - "url": "http://localhost:9089/status", - "timeout": 3 - } -}] diff --git a/assets/apis/example_SingleDefinition.json b/assets/apis/example_SingleDefinition.json deleted file mode 100644 index 508e73d6a..000000000 --- a/assets/apis/example_SingleDefinition.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name" : "example", - "active" : true, - "proxy" : { - "preserve_host" : false, - "listen_path" : "/example/*", - "upstreams" : { - "balancing": "roundrobin", - "targets": [ - {"target": "http://localhost:9089/hello-world"} - ] - }, - "strip_path" : false, - "append_path" : false, - "methods" : ["GET"] - }, - "plugins": [ - { - "name": "rate_limit", - "enabled": true, - "config": { - "limit": "5-M", - "policy": "local" - } - } - ], - "health_check": { - "url": "http://localhost:9089/status", - "timeout": 3 - } -} diff --git a/assets/auth/auth.json b/assets/auth/auth.json deleted file mode 100644 index f778ea232..000000000 --- a/assets/auth/auth.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name" : "local", - "oauth_endpoints" : { - "token" : { - "preserve_host" : false, - "listen_path" : "/auth/token", - "upstreams" : { - "balancing": "roundrobin", - "targets": [ - {"target": "http://localhost:9088/token"} - ] - }, - "strip_path" : true, - "append_path" : false, - "methods" : [ - "GET", - "POST" - ] - } - }, - "token_strategy" : { - "name" : "jwt", - "settings" : [ - {"alg": "HS256", "key" : "secret"} - ] - } -} diff --git a/assets/config/config.yaml b/assets/config/config.yaml new file mode 100644 index 000000000..d445c99b9 --- /dev/null +++ b/assets/config/config.yaml @@ -0,0 +1,4 @@ +kafkaAddr: "kafka:9092" +kafkaFactTopic: "rbac-facts" +kafkaDLQTopic: "DLQRBAC" +kafkaConsumerGroup: "RABAC" diff --git a/assets/docker-compose.yml b/assets/docker-compose.yml index ac5cb3e6d..b569d4d27 100644 --- a/assets/docker-compose.yml +++ b/assets/docker-compose.yml @@ -1,43 +1,16 @@ # This file is used to run integration tests on travis version: '3' services: - - mongo: - image: mongo:3 + gateway: ports: - - "27017:27017" - healthcheck: - test: "mongo localhost:27017/test --quiet --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)'" - interval: 10s - timeout: 5s - retries: 5 - - upstreams: - image: rodolpheche/wiremock:2.27.1-alpine - ports: - - '9089:8080' - - auth-service: - image: rodolpheche/wiremock:2.27.1-alpine - ports: - - '9088:8080' - - jaeger: - image: jaegertracing/all-in-one - environment: - COLLECTOR_ZIPKIN_HTTP_PORT: 9411 - ports: - - "5775:5775/udp" - - "6831:6831/udp" - - "6832:6832/udp" - - "5778:5778" - - "16686:16686" - - "14268:14268" - - zipkin: - image: openzipkin/zipkin - environment: - STORAGE_TYPE: mem - JAVA_OPTS: -Dlogging.level.zipkin=DEBUG - ports: - - 9411:9411 + - "8080:8080" + - "8081:8081" + container_name: "gateway" + build: + context: ../ + dockerfile: Dockerfile + volumes: + - ${PWD}/config/config.yaml:/config/config.yaml + - .janus.toml:/etc/janus/janus.toml + - ./apis:/etc/janus/apis + - ./auth:/etc/janus/auth diff --git a/assets/janus.toml b/assets/janus.toml new file mode 100644 index 000000000..7e9613a23 --- /dev/null +++ b/assets/janus.toml @@ -0,0 +1,22 @@ +################################################################ +# Global configuration +################################################################ +port = 8080 + +[log] + level = "debug" + +################################################################ +# API configuration backend +################################################################ +[web] + port = 8081 + + [web.credentials] + secret = "secret" + + [web.credentials.basic] + users = {admin = "admin"} + +[database] + dsn = "file:///etc/janus" diff --git a/go.mod b/go.mod index 5d0036c9c..89d06ea48 100644 --- a/go.mod +++ b/go.mod @@ -1,59 +1,113 @@ module github.com/hellofresh/janus -go 1.15 +go 1.17 require ( code.cloudfoundry.org/bytefmt v0.0.0-20180108190415-b31f603f5e1e contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.3.0 - github.com/DataDog/datadog-go v0.0.0-20180330214955-e67964b4021a // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 github.com/asaskevich/govalidator v0.0.0-20171111151018-521b25f4b05f - github.com/bshuster-repo/logrus-logstash-hook v0.4.1 // indirect - github.com/cactus/go-statsd-client v3.1.1+incompatible // indirect github.com/cucumber/godog v0.10.0 github.com/cucumber/messages-go/v10 v10.0.3 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/felixge/httpsnoop v1.0.0 - github.com/fiam/gounidecode v0.0.0-20150629112515-8deddbd03fec // indirect github.com/fsnotify/fsnotify v1.4.9 github.com/go-chi/chi v3.3.2+incompatible github.com/go-redis/redis/v7 v7.4.0 github.com/gocql/gocql v0.0.0-20200624222514-34081eda590e github.com/gofrs/uuid v3.3.0+incompatible + github.com/golang-jwt/jwt/v4 v4.4.3 github.com/google/go-github v17.0.0+incompatible - github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect github.com/hellofresh/health-go/v3 v3.2.0 github.com/hellofresh/logging-go v0.1.6 github.com/hellofresh/opencensus-go-extras v0.0.0-20191004131501-7bd94f603dcf github.com/hellofresh/stats-go v0.8.0 github.com/kelseyhightower/envconfig v1.3.0 - github.com/klauspost/compress v1.10.10 // indirect github.com/magiconair/properties v1.8.1 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.1.2 - github.com/onsi/ginkgo v1.13.0 // indirect github.com/opentracing/opentracing-go v1.2.0 github.com/rafaeljesus/retry-go v0.0.0-20171214204623-5981a380a879 github.com/rs/cors v1.4.0 + github.com/segmentio/kafka-go v0.4.38 github.com/sirupsen/logrus v1.7.0 github.com/spf13/cobra v1.0.0 - github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.7.1 - github.com/stretchr/testify v1.6.1 + github.com/stretchr/testify v1.8.1 github.com/tidwall/gjson v1.1.0 - github.com/tidwall/match v1.0.0 // indirect github.com/ulule/limiter/v3 v3.5.0 go.mongodb.org/mongo-driver v1.4.1 go.opencensus.io v0.23.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/net v0.0.0-20201110031124-69a78807bb2b + golang.org/x/crypto v0.5.0 + golang.org/x/net v0.5.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d +) + +require ( + github.com/DataDog/datadog-go v0.0.0-20180330214955-e67964b4021a // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/aws/aws-sdk-go v1.29.15 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bshuster-repo/logrus-logstash-hook v0.4.1 // indirect + github.com/cactus/go-statsd-client v3.1.1+incompatible // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cucumber/gherkin-go/v11 v11.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fiam/gounidecode v0.0.0-20150629112515-8deddbd03fec // indirect + github.com/go-stack/stack v1.8.0 // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + github.com/hashicorp/go-immutable-radix v1.2.0 // indirect + github.com/hashicorp/go-memdb v1.2.1 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect + github.com/klauspost/compress v1.15.15 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/onsi/ginkgo v1.13.0 // indirect + github.com/onsi/gomega v1.10.1 // indirect + github.com/pelletier/go-toml v1.4.0 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.9.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.15.0 // indirect + github.com/prometheus/procfs v0.2.0 // indirect + github.com/prometheus/statsd_exporter v0.20.0 // indirect + github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect + github.com/spf13/afero v1.1.2 // indirect + github.com/spf13/cast v1.3.0 // indirect + github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tidwall/match v1.0.0 // indirect + github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect + github.com/xdg/scram v1.0.5 // indirect + github.com/xdg/stringprep v1.0.3 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/api v0.29.0 // indirect + google.golang.org/appengine v1.6.5 // indirect + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/gemnasium/logrus-graylog-hook.v2 v2.0.6 // indirect - gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.51.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace git.apache.org/thrift.git => github.com/apache/thrift v0.12.0 diff --git a/go.sum b/go.sum index 425c14854..cb4f67306 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -221,7 +223,6 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= @@ -342,16 +343,18 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.6/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= -github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -398,7 +401,6 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -436,7 +438,11 @@ github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfS github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -497,6 +503,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/kafka-go v0.4.38 h1:iQdOBbUSdfuYlFpvjuALgj7N6DrdPA0HfB4AhREOdtg= +github.com/segmentio/kafka-go v0.4.38/go.mod h1:ikyuGon/60MN/vXFgykf7Zm8P5Be49gJU6vezwjnnhU= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -533,14 +541,18 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/gjson v1.1.0 h1:/7OBSUzFP8NhuzLlHg0vETJrRL02C++0ql5uSY3DITs= @@ -563,13 +575,16 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= +github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= +github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -579,7 +594,6 @@ go.opencensus.io v0.10.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -603,8 +617,11 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -634,6 +651,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -664,13 +682,16 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= @@ -683,8 +704,9 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -730,14 +752,26 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -784,6 +818,7 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -806,7 +841,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= @@ -852,7 +886,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= @@ -865,8 +898,9 @@ gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -887,11 +921,12 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 000000000..b5bd65bfc --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,23 @@ +package config + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +type Config struct { + KafkaAddr string `yaml:"kafkaAddr"` + KafkaFactTopic string `yaml:"kafkaFactTopic"` + KafkaDLQTopic string `yaml:"kafkaDLQTopic"` + KafkaConsumerGroup string `yaml:"kafkaConsumerGroup"` +} + +func UnmarshalYAML(path string, dest *Config) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + + return yaml.Unmarshal(b, dest) +} diff --git a/pkg/jwt/jwt.go b/pkg/jwt/jwt.go new file mode 100644 index 000000000..7e1ad9854 --- /dev/null +++ b/pkg/jwt/jwt.go @@ -0,0 +1,9 @@ +package jwt + +import "github.com/golang-jwt/jwt/v4" + +type Claims struct { + UserID int + FullUserName string + jwt.StandardClaims +} diff --git a/pkg/kafka/consumer.go b/pkg/kafka/consumer.go new file mode 100644 index 000000000..172c89f2a --- /dev/null +++ b/pkg/kafka/consumer.go @@ -0,0 +1,44 @@ +package kafka + +import ( + "context" + + "github.com/segmentio/kafka-go" + + log "github.com/sirupsen/logrus" +) + +type Message = kafka.Message +type Header = kafka.Header +type KafkaMessageHandler func(Message) error +type FaultyKafkaMessageHandler func(Message, error) + +func ConsumeMessages(kafkaAddr, topic, consumerGroup string, msgHandler KafkaMessageHandler, DLQHandler FaultyKafkaMessageHandler) { + // Minimum amount of bytes in a message batch + const minBytes = 10e3 + // Maximum amount of bytes in a message batch + const maxBytes = 10e6 + + r := kafka.NewReader(kafka.ReaderConfig{ + Brokers: []string{kafkaAddr}, + Topic: topic, + GroupID: consumerGroup, + MinBytes: minBytes, + MaxBytes: maxBytes, + }) + defer r.Close() + + for { + m, err := r.ReadMessage(context.Background()) + if err != nil { + log.Printf("Cound not read kafka message (Err: %s)\n", err.Error()) + DLQHandler(m, err) + continue + } + err = msgHandler(m) + if err != nil { + log.Printf("Cound not handle kafka message (Err: %s)\n", err.Error()) + DLQHandler(m, err) + } + } +} diff --git a/pkg/kafka/factConsumer.go b/pkg/kafka/factConsumer.go new file mode 100644 index 000000000..6c47d6306 --- /dev/null +++ b/pkg/kafka/factConsumer.go @@ -0,0 +1,62 @@ +package kafka + +import ( + "context" + "encoding/json" + "github.com/hellofresh/janus/pkg/models" + "github.com/segmentio/kafka-go" + log "github.com/sirupsen/logrus" +) + +func StartFactConsumer(kafkaAddr, topic, dlqtopic, consumerGroup string) { + ConsumeMessages(kafkaAddr, topic, consumerGroup, + func(msg Message) error { + var fact models.Fact + err := json.Unmarshal(msg.Value, &fact) + if err != nil { + return err + } + var role models.Role + + err = json.Unmarshal(*fact.Object, &role) + if err != nil { + log.Println(err) + } + + return nil + }, + func(msg Message, inerr error) { + msg.Headers = []kafka.Header{{Key: "Error", Value: []byte(inerr.Error())}} + producer := NewKafkaProducer(kafkaAddr, dlqtopic) + defer producer.Close() + err := producer.ProduceMadeMessage(msg) + if err != nil { + log.Println(err) + } + return + }, + ) + return +} + +type KafkaProducer struct { + kafkaWriter *kafka.Writer +} + +func NewKafkaProducer(kafkaAddr string, topic string) *KafkaProducer { + return &KafkaProducer{kafkaWriter: &kafka.Writer{ + Addr: kafka.TCP(kafkaAddr), + Topic: topic, + Balancer: &kafka.LeastBytes{}, + }} +} + +func (producer *KafkaProducer) ProduceMadeMessage(msg kafka.Message) error { + err := producer.kafkaWriter.WriteMessages(context.Background(), msg) + return err +} + +func (producer *KafkaProducer) Close() error { + err := producer.kafkaWriter.Close() + return err +} diff --git a/pkg/models/fact.go b/pkg/models/fact.go new file mode 100644 index 000000000..cbb19db66 --- /dev/null +++ b/pkg/models/fact.go @@ -0,0 +1,14 @@ +package models + +import ( + "encoding/json" + "github.com/hellofresh/janus/pkg/jwt" +) + +type Fact struct { + ID uint64 `json:"id"` + ObjectType string `json:"objectType"` + ActionType string `json:"actionType"` + Object *json.RawMessage `json:"object"` + Claims *jwt.Claims `json:"claims"` +} diff --git a/pkg/models/rbac.go b/pkg/models/rbac.go new file mode 100644 index 000000000..07546184a --- /dev/null +++ b/pkg/models/rbac.go @@ -0,0 +1,17 @@ +package models + +type Role struct { + Name string + Features []*Feature +} + +type Feature struct { + Name string + Endpoints []*Endpoint +} + +type Endpoint struct { + Name string + Path string + Method string +} diff --git a/pkg/plugin/oauth2/jwt_manager.go b/pkg/plugin/oauth2/jwt_manager.go index 7e9f4006d..25fbba781 100644 --- a/pkg/plugin/oauth2/jwt_manager.go +++ b/pkg/plugin/oauth2/jwt_manager.go @@ -2,8 +2,13 @@ package oauth2 import ( "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" jwtBase "github.com/dgrijalva/jwt-go" + jwtUser "github.com/golang-jwt/jwt/v4" "github.com/hellofresh/janus/pkg/jwt" "github.com/hellofresh/janus/pkg/metrics" obs "github.com/hellofresh/janus/pkg/observability" @@ -14,6 +19,32 @@ import ( "go.opencensus.io/tag" ) +type UserClaims struct { + UserID int + FullUserName string + Roles []string + jwtUser.StandardClaims +} + +func ExtractClaims(jwtToken string) (*UserClaims, error) { + parts := strings.Split(jwtToken, ".") + if len(parts) < 2 { + return nil, fmt.Errorf("JWT token invalid") + } + + decoded, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return nil, err + } + + claims := &UserClaims{} + err = json.Unmarshal([]byte(decoded), claims) + if err != nil { + return nil, err + } + return claims, nil +} + // JWTManager is responsible for managing the JWT tokens type JWTManager struct { parser *jwt.Parser diff --git a/pkg/plugin/oauth2/middleware.go b/pkg/plugin/oauth2/middleware.go index 1870acf00..e6f907c4d 100644 --- a/pkg/plugin/oauth2/middleware.go +++ b/pkg/plugin/oauth2/middleware.go @@ -2,6 +2,7 @@ package oauth2 import ( "context" + "fmt" "net/http" "strings" @@ -71,6 +72,18 @@ func NewKeyExistsMiddleware(manager Manager) func(http.Handler) http.Handler { statsClient.TrackOperation(tokensSection, bucket.MetricOperation{"key-exists", "malformed"}, nil, true) accessToken := parts[1] + + claims, err := ExtractClaims(accessToken) + if err != nil { + errors.Handler(w, r, err) + return + } + + if len(claims.Roles) <= 0 { + errors.Handler(w, r, fmt.Errorf("No roles have been set")) + return + } + keyExists := manager.IsKeyAuthorized(r.Context(), accessToken) statsClient.TrackOperation(tokensSection, bucket.MetricOperation{"key-exists", "authorized"}, nil, keyExists) if keyExists { diff --git a/pkg/plugin/oauth2/setup.go b/pkg/plugin/oauth2/setup.go index f916ceff8..bd9baf5c0 100644 --- a/pkg/plugin/oauth2/setup.go +++ b/pkg/plugin/oauth2/setup.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/hellofresh/janus/pkg/kafka" "net/url" "time" @@ -21,8 +22,8 @@ import ( ) const ( - mongodb = "mongodb" - file = "file" + mongodb = "mongodb" + file = "file" cassandra = "cassandra" mongoIdxTimeout = 10 * time.Second @@ -35,6 +36,12 @@ var ( ) func init() { + var kafkaConfig config.Config + + _ = config.UnmarshalYAML("config/config.yaml", &kafkaConfig) + + go kafka.StartFactConsumer(kafkaConfig.KafkaAddr, kafkaConfig.KafkaFactTopic, kafkaConfig.KafkaDLQTopic, kafkaConfig.KafkaConsumerGroup) + plugin.RegisterEventHook(plugin.StartupEvent, onStartup) plugin.RegisterEventHook(plugin.ReloadEvent, onReload) plugin.RegisterEventHook(plugin.AdminAPIStartupEvent, onAdminAPIStartup)