diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6ef7e18c1..33aed3889 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -72,7 +72,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 - + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Get current version id: ver run: | @@ -162,6 +163,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Get current version id: ver run: | diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 000000000..1bd0dc17c --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,3 @@ +examples/config.hocon.sample:hashicorp-tf-password:365 +.github/workflows/integration_tests/telemetry/sender_config/config_disabled.hocon:hashicorp-tf-password:234 +.github/workflows/integration_tests/telemetry/sender_config/config.hocon:hashicorp-tf-password:235 diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a8ea688..eadd2a482 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed ### Security +## [3.3.0] - 2024-12-18 +### Changed +- Kafka sink use a dedicated thread for potentially blocking send [snowplow/stream-collector-private#3] + +### Security +- Add limit for payload allowed sizes [snowplow/stream-collector-private#1] +- Security and license updates [snowplow/stream-collector-private#1] + ## [3.2.1] - 2024-11-25 ### Changed - Kafka sink to open fewer threads [#431] @@ -564,7 +572,8 @@ Ensure docker image has latest libfreetype6 version [#247] - Scala Stream Collector, Scala Kinesis Enrich: replaced stream list with describe to tighten permissions, thanks @pkallos! [snowplow/snowplow#535] -[Unreleased]: https://github.com/snowplow/stream-collector/compare/3.2.1...HEAD +[Unreleased]: https://github.com/snowplow/stream-collector/compare/3.3.0...HEAD +[3.3.0]: https://github.com/snowplow/stream-collector/compare/3.2.1...3.3.0 [3.2.1]: https://github.com/snowplow/stream-collector/compare/3.2.0...3.2.1 [3.2.0]: https://github.com/snowplow/stream-collector/compare/3.1.2...3.2.0 [3.1.2]: https://github.com/snowplow/stream-collector/compare/3.1.1...3.1.2 diff --git a/LICENSE.md b/LICENSE.md index 6abbe69c7..0a7d3cc95 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,18 +1,20 @@ # Snowplow Limited Use License Agreement -_Version 1.0, January 2024_ +_Version 1.1, November, 2024_ -This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software. +This Snowplow Limited Use License Agreement, Version 1.1 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software. -## LICENSE GRANT AND CONDITIONS +## 1. LICENSE GRANT AND CONDITIONS -**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Excluded Purpose. +**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Competing Use, and Licensee may exercise the License only for Non-Production Use or Non-Commercial Use. -**1.2** For purposes of this Agreement, an “Excluded Purpose” is any use that is either a Competing Use or a Highly-Available Production Use, or both of them. +**1.2 Definitions.** For purposes of this Agreement: -* **1.2.1** A “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software. +* **1.2.1** “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software. -* **1.2.2** Highly-Available Production Use is any highly-available use, including without limitation any use where multiple instances of any Software component run concurrently to avoid a single point of failure, in a production environment, where production means use on live data. +* **1.2.2** “Non-Production Use” means any use of the Software to process test or synthetic data to evaluate the sufficiency of the Software for use by Licensee. + +* **1.2.3** “Non-Commercial Use” is only: (a) personal use for research, experiment, personal study, or hobby projects, without any anticipated commercial application, or (b) use for teaching purposes by lecturers of a school or university. **1.3 Conditions.** In consideration of the License, Licensee’s use of the Software is subject to the following conditions: @@ -22,8 +24,8 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s ``` This software is made available by Snowplow Analytics, Ltd., - under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - located at https://docs.snowplow.io/limited-use-license-1.0 + under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + located at https://docs.snowplow.io/limited-use-license-1.1 BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. ``` @@ -32,23 +34,23 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s **1.5 No Sublicensing.** The License does not include the right to sublicense the Software, however, each recipient to which Licensee provides the Software may exercise the Licenses so long as such recipient agrees to the terms and conditions of this Agreement. -## TERM AND TERMINATION +## 2. TERM AND TERMINATION This Agreement will continue unless and until earlier terminated as set forth herein. If Licensee breaches any of its conditions or obligations under this Agreement, this Agreement will terminate automatically and the License will terminate automatically and permanently. -## INTELLECTUAL PROPERTY +## 3. INTELLECTUAL PROPERTY As between the parties, Snowplow will retain all right, title, and interest in the Software, and all intellectual property rights therein. Snowplow hereby reserves all rights not expressly granted to Licensee in this Agreement. Snowplow hereby reserves all rights in its trademarks and service marks, and no licenses therein are granted in this Agreement. -## DISCLAIMER +## 4. DISCLAIMER SNOWPLOW HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, WITH RESPECT TO THE SOFTWARE. -## LIMITATION OF LIABILITY +## 5. LIMITATION OF LIABILITY SNOWPLOW WILL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO LOST PROFITS OR ANY CONSEQUENTIAL, SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, ARISING OUT OF THIS AGREEMENT. THE FOREGOING SHALL APPLY TO THE EXTENT PERMITTED BY APPLICABLE LAW. -## GENERAL +## 6. GENERAL **6.1 Governing Law.** This Agreement will be governed by and interpreted in accordance with the laws of the state of Delaware, without reference to its conflict of laws principles. If Licensee is located within the United States, all disputes arising out of this Agreement are subject to the exclusive jurisdiction of courts located in Delaware, USA. If Licensee is located outside of the United States, any dispute, controversy or claim arising out of or relating to this Agreement will be referred to and finally determined by arbitration in accordance with the JAMS International Arbitration Rules. The tribunal will consist of one arbitrator. The place of arbitration will be in the State of Delaware, USA. The language to be used in the arbitral proceedings will be English. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction thereof. diff --git a/README.md b/README.md index 1ff7a1bb1..bf6cae57b 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Licensed under the [Snowplow Limited Use License Agreement][license]. _(If you a [release-image]: https://img.shields.io/github/v/release/snowplow/stream-collector?sort=semver&style=flat [releases]: https://github.com/snowplow/stream-collector -[license]: https://docs.snowplow.io/limited-use-license-1.0 -[license-image]: https://img.shields.io/badge/license-Snowplow--Limited-Use-blue.svg?style=flat +[license]: https://docs.snowplow.io/limited-use-license-1.1 +[license-image]: https://img.shields.io/badge/license-Snowplow--Limited--Use-blue.svg?style=flat [faq]: https://docs.snowplow.io/docs/contributing/limited-use-license-faq/ diff --git a/build.sbt b/build.sbt index 7aad81ed9..50b7ade5b 100644 --- a/build.sbt +++ b/build.sbt @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala index 344391426..f303b9b57 100644 --- a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala +++ b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorOutput.scala b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorOutput.scala index a14ea04af..5b02f8d86 100644 --- a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorOutput.scala +++ b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorOutput.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/EventGenerator.scala b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/EventGenerator.scala index e25dd11ad..398a3bee8 100644 --- a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/EventGenerator.scala +++ b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/EventGenerator.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/Http.scala b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/Http.scala index e7d1d613a..698ffb3b8 100644 --- a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/Http.scala +++ b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/Http.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/utils.scala b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/utils.scala index 485836c1e..617ab4fc0 100644 --- a/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/utils.scala +++ b/core/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/utils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index fb5ef5d1f..85fce8395 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -97,6 +97,8 @@ responseHeaderTimeout = 30 seconds maxRequestLineLength = 20480 maxHeadersLength = 40960 + maxPayloadSize = 1048576 # 1MB + dropPayloadSize = 2097152 # 2MB } enableDefaultRedirect = false diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/App.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/App.scala index 9d7d8855b..f4ad90225 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/App.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/App.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/AppInfo.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/AppInfo.scala index dddcff6f1..eadcd23b4 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/AppInfo.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/AppInfo.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Config.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Config.scala index 8e878a454..39d9873ea 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Config.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Config.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -161,7 +161,9 @@ object Config { idleTimeout: FiniteDuration, responseHeaderTimeout: FiniteDuration, maxRequestLineLength: Int, - maxHeadersLength: Int + maxHeadersLength: Int, + maxPayloadSize: Long, + dropPayloadSize: Long ) case class License( diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/ConfigParser.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/ConfigParser.scala index 8167bf257..92609b9f1 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/ConfigParser.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/ConfigParser.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/HttpServer.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/HttpServer.scala index 32f2894d4..a3cb94aa2 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/HttpServer.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/HttpServer.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -19,7 +19,7 @@ import org.http4s.{HttpApp, HttpRoutes} import org.http4s.blaze.server.BlazeServerBuilder import org.http4s.headers.`Strict-Transport-Security` import org.http4s.server.Server -import org.http4s.server.middleware.{HSTS, Metrics, Timeout} +import org.http4s.server.middleware.{EntityLimiter, HSTS, Metrics, Timeout} import org.typelevel.log4cats.Logger import org.typelevel.log4cats.slf4j.Slf4jLogger @@ -76,7 +76,7 @@ object HttpServer { networking: Config.Networking ): HttpApp[F] = hstsApp( hsts, - timeoutMiddleware(routes, networking) <+> healthRoutes + timeoutMiddleware(entityLimiter(routes, networking.dropPayloadSize), networking) <+> healthRoutes ) private def createMetricsMiddleware[F[_]: Async]( @@ -103,6 +103,12 @@ object HttpServer { HSTS(routes.orNotFound, `Strict-Transport-Security`.unsafeFromDuration(hsts.maxAge)) else routes.orNotFound + private def entityLimiter[F[_]: Async](routes: HttpRoutes[F], dropPayloadSize: Long): HttpRoutes[F] = + EntityLimiter.httpRoutes[F](routes, dropPayloadSize).recover { + case _: EntityLimiter.EntityTooLarge => + Response[F](Status.PayloadTooLarge) + } + private def timeoutMiddleware[F[_]: Async](routes: HttpRoutes[F], networking: Config.Networking): HttpRoutes[F] = Timeout.httpRoutes[F](timeout = networking.responseHeaderTimeout)(routes).map { case Response(Status.ServiceUnavailable, httpVersion, headers, body, attributes) => diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Rfc6265Cookie.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Rfc6265Cookie.scala index 983f2087d..3e915c884 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Rfc6265Cookie.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Rfc6265Cookie.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Routes.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Routes.scala index b340113f6..65744bcac 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Routes.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Routes.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Run.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Run.scala index 194124168..d7e8534c2 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Run.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Run.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Service.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Service.scala index 78121d16d..7a1c8b424 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Service.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Service.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -323,7 +323,9 @@ class Service[F[_]: Sync]( ): F[Unit] = for { // Split events into Good and Bad - eventSplit <- Sync[F].delay(splitBatch.splitAndSerializePayload(event, sinks.good.maxBytes)) + eventSplit <- Sync[F].delay( + splitBatch.splitAndSerializePayload(event, sinks.good.maxBytes, config.networking.maxPayloadSize) + ) // Send events to respective sinks _ <- sinks.good.storeRawEvents(eventSplit.good, partitionKey) _ <- sinks.bad.storeRawEvents(eventSplit.bad, partitionKey) diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Sink.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Sink.scala index 089ab72e4..b83a096a7 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Sink.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Sink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatch.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatch.scala index a35d76626..16cb6dd4a 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatch.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatch.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -77,32 +77,45 @@ case class SplitBatch(appInfo: AppInfo) { * @param event Incoming CollectorPayload * @return a List of Good and Bad events */ - def splitAndSerializePayload(event: CollectorPayload, maxBytes: Int): EventSerializeResult = { + def splitAndSerializePayload(payload: CollectorPayload, maxBytes: Int, maxPayloadSize: Long): EventSerializeResult = { val serializer = ThriftSerializer.get() - val everythingSerialized = serializer.serialize(event) - val wholeEventBytes = getSize(everythingSerialized) + val everythingSerialized = serializer.serialize(payload) + val wholePayloadBytes = getSize(everythingSerialized) // If the event is below the size limit, no splitting is necessary - if (wholeEventBytes < maxBytes) { + if (wholePayloadBytes < maxBytes) { EventSerializeResult(List(everythingSerialized), Nil) + // If the event is above max payload size it is turned into SizeViolation + } else if (wholePayloadBytes > maxPayloadSize) { + EventSerializeResult( + Nil, + List( + oversizedPayload( + payload, + wholePayloadBytes, + maxPayloadSize.toInt, + s"Payload exceeds max size of $maxPayloadSize. Actual length: $wholePayloadBytes" + ) + ) + ) } else { (for { - body <- Option(event.getBody).toRight("GET requests cannot be split") + body <- Option(payload.getBody).toRight("GET requests cannot be split") children <- splitBody(body) initialBodyDataBytes = getSize(Json.arr(children._2: _*).noSpaces) _ <- Either.cond[String, Unit]( - wholeEventBytes - initialBodyDataBytes < maxBytes, + wholePayloadBytes - initialBodyDataBytes < maxBytes, (), "cannot split this POST request because event without \"data\" field is still too big" ) - splitted = split(children._2, maxBytes - wholeEventBytes + initialBodyDataBytes) - goodSerialized = serializeBatch(serializer, event, splitted.goodBatches, children._1) + splitted = split(children._2, maxBytes - wholePayloadBytes + initialBodyDataBytes) + goodSerialized = serializeBatch(serializer, payload, splitted.goodBatches, children._1) badList = splitted.failedBigEvents.map { e => val msg = "this POST request split is still too large" - oversizedPayload(event, getSize(e), maxBytes, msg) + oversizedPayload(payload, getSize(e), maxBytes, msg) } } yield EventSerializeResult(goodSerialized, badList)).fold({ msg => - val tooBigPayload = oversizedPayload(event, wholeEventBytes, maxBytes, msg) + val tooBigPayload = oversizedPayload(payload, wholePayloadBytes, maxBytes, msg) EventSerializeResult(Nil, List(tooBigPayload)) }, identity) } diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Telemetry.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Telemetry.scala index 9f0f10828..aa895e683 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Telemetry.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/Telemetry.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/model.scala b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/model.scala index 0b1d6e8d0..cf1d4b21c 100644 --- a/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/model.scala +++ b/core/src/main/scala/com.snowplowanalytics.snowplow.collector.core/model.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/HttpServerSpec.scala b/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/HttpServerSpec.scala index 44d1939b5..21830fbc2 100644 --- a/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/HttpServerSpec.scala +++ b/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/HttpServerSpec.scala @@ -6,12 +6,15 @@ import cats.effect.IO import org.http4s.client.Client import org.http4s._ import org.http4s.dsl.io._ +import cats.implicits._ import org.http4s.implicits._ import scala.concurrent.duration._ import cats.effect.testing.specs2._ class HttpServerSpec extends Specification with CatsEffect { val routes = HttpRoutes.of[IO] { + case r if r.pathInfo == path"/large" => + r.decode[String](Response[IO](Ok).withEntity(_).pure[IO]) case _ -> Root / "fast" => Ok("Fast") case _ -> Root / "never" => @@ -29,23 +32,58 @@ class HttpServerSpec extends Specification with CatsEffect { TestUtils .testConfig .copy(networking = TestUtils.testConfig.networking.copy(responseHeaderTimeout = 100.millis)) - val httpApp = HttpServer.httpApp( - routes, - healthRoutes, - config.hsts, - config.networking - ) - val client: Client[IO] = Client.fromHttpApp(httpApp) + val request: Request[IO] = Request(method = Method.GET, uri = uri"/never") - val res: IO[String] = client.expect[String](request) - res - .attempt - .map(_ must beLeft[Throwable].which { + check(config, request)( + _ must beLeft[Throwable].which { case org.http4s.client.UnexpectedStatus(Status.RequestTimeout, _, _) => true case _ => false - }) + } + ) + } + } + "manage request size" should { + "drop requests larger than `networking.dropPayloadSize`" in { + val config = + TestUtils + .testConfig + .copy(networking = TestUtils.testConfig.networking.copy(maxPayloadSize = 5L, dropPayloadSize = 10L)) + val request: Request[IO] = Request( + Method.POST, + uri"/large" + ).withEntity("s" * 1000) + + check(config, request)( + _ must beLeft[Throwable].which { + case org.http4s.client.UnexpectedStatus(Status.PayloadTooLarge, _, _) => true + case _ => false + } + ) + } + "allow request that's smaller than `networking.dropPayloadSize`" in { + val config = + TestUtils.testConfig.copy(networking = TestUtils.testConfig.networking.copy(dropPayloadSize = 1002L)) + val body = "s" * 1000 + val request: Request[IO] = Request( + Method.POST, + uri"/large" + ).withEntity(body) + + check(config, request)(_ must beRight(body)) } } } + + private[this] def check(config: Config[Any], request: Request[IO])(assert: Either[Throwable, _] => Boolean) = { + val httpApp = HttpServer.httpApp( + routes, + healthRoutes, + config.hsts, + config.networking + ) + + Client.fromHttpApp(httpApp).expect[String](request).attempt.map(assert) + } + } diff --git a/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatchSpec.scala b/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatchSpec.scala index ef734ec1b..e30ab8d18 100644 --- a/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatchSpec.scala +++ b/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/SplitBatchSpec.scala @@ -54,7 +54,7 @@ class SplitBatchSpec extends Specification { "SplitBatch.splitAndSerializePayload" should { "Serialize an empty CollectorPayload" in { - val actual = splitBatch.splitAndSerializePayload(new CollectorPayload(), 100) + val actual = splitBatch.splitAndSerializePayload(new CollectorPayload(), 100, 101L) val target = new CollectorPayload() new TDeserializer().deserialize(target, actual.good.head) target must_== new CollectorPayload() @@ -63,7 +63,7 @@ class SplitBatchSpec extends Specification { "Reject an oversized GET CollectorPayload" in { val payload = new CollectorPayload() payload.setQuerystring("x" * 1000) - val actual = splitBatch.splitAndSerializePayload(payload, 100) + val actual = splitBatch.splitAndSerializePayload(payload, 100, 1020L) val res = parse(new String(actual.bad.head)).toOption.get val selfDesc = SelfDescribingData.parse(res).toOption.get val badRow = selfDesc.data.as[BadRow].toOption.get @@ -77,10 +77,30 @@ class SplitBatchSpec extends Specification { actual.good must_== Nil } + "Reject an oversized POST CollectorPayload when exceeds max payload size" in { + val payload = new CollectorPayload() + payload.setBody("s" * 1010) + val actual = splitBatch.splitAndSerializePayload(payload, 1000, 1000L) + val res = parse(new String(actual.bad.head)).toOption.get + val selfDesc = SelfDescribingData.parse(res).toOption.get + val badRow = selfDesc.data.as[BadRow].toOption.get + badRow must beAnInstanceOf[BadRow.SizeViolation] + val sizeViolation = badRow.asInstanceOf[BadRow.SizeViolation] + sizeViolation.failure.maximumAllowedSizeBytes must_== 1000 + sizeViolation.failure.actualSizeBytes must_== 1029 + sizeViolation + .failure + .expectation must_== "oversized collector payload: Payload exceeds max size of 1000. Actual length: 1029" + sizeViolation + .payload + .event must_== "CollectorPayload(schema:null, ipAddress:null, timestamp:0, encoding:null, collector:null, body:sssss" + sizeViolation.processor shouldEqual Processor(TestUtils.appName, TestUtils.appVersion) + } + "Reject an oversized POST CollectorPayload with an unparseable body" in { val payload = new CollectorPayload() payload.setBody("s" * 1000) - val actual = splitBatch.splitAndSerializePayload(payload, 100) + val actual = splitBatch.splitAndSerializePayload(payload, 100, 1020L) val res = parse(new String(actual.bad.head)).toOption.get val selfDesc = SelfDescribingData.parse(res).toOption.get val badRow = selfDesc.data.as[BadRow].toOption.get @@ -106,7 +126,7 @@ class SplitBatchSpec extends Specification { ) payload.setBody(data.noSpaces) payload.setPath("p" * 1000) - val actual = splitBatch.splitAndSerializePayload(payload, 1000) + val actual = splitBatch.splitAndSerializePayload(payload, 1000, 2000L) actual.bad.size must_== 1 val res = parse(new String(actual.bad.head)).toOption.get val selfDesc = SelfDescribingData.parse(res).toOption.get @@ -139,7 +159,7 @@ class SplitBatchSpec extends Specification { ) ) payload.setBody(data.noSpaces) - val actual = splitBatch.splitAndSerializePayload(payload, 1000) + val actual = splitBatch.splitAndSerializePayload(payload, 1000, 10000) actual.bad.size must_== 4 actual.good.size must_== 2 } diff --git a/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/TestUtils.scala b/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/TestUtils.scala index a79802aae..6134b1f4a 100644 --- a/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/TestUtils.scala +++ b/core/src/test/scala/com.snowplowanalytics.snowplow.collector.core/TestUtils.scala @@ -122,7 +122,9 @@ object TestUtils { 610.seconds, 30.seconds, 20480, - 40960 + 40960, + 1048576, + 2097152 ), enableDefaultRedirect = false, redirectDomains = Set.empty[String], diff --git a/examples/config.kafka.extended.hocon b/examples/config.kafka.extended.hocon index 095118c22..1e7c75763 100644 --- a/examples/config.kafka.extended.hocon +++ b/examples/config.kafka.extended.hocon @@ -2,8 +2,8 @@ # All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/examples/config.kinesis.extended.hocon b/examples/config.kinesis.extended.hocon index 0efd17d72..df83a6d30 100644 --- a/examples/config.kinesis.extended.hocon +++ b/examples/config.kinesis.extended.hocon @@ -2,8 +2,8 @@ # All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/examples/config.nsq.extended.hocon b/examples/config.nsq.extended.hocon index f2925ad6d..4927289e6 100644 --- a/examples/config.nsq.extended.hocon +++ b/examples/config.nsq.extended.hocon @@ -2,8 +2,8 @@ # All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/examples/config.pubsub.extended.hocon b/examples/config.pubsub.extended.hocon index 0d4fcc439..4927daae9 100644 --- a/examples/config.pubsub.extended.hocon +++ b/examples/config.pubsub.extended.hocon @@ -2,8 +2,8 @@ # All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/examples/config.sqs.extended.hocon b/examples/config.sqs.extended.hocon index 65a58db79..fe9c47ee8 100644 --- a/examples/config.sqs.extended.hocon +++ b/examples/config.sqs.extended.hocon @@ -2,8 +2,8 @@ # All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/examples/config.stdout.extended.hocon b/examples/config.stdout.extended.hocon index 42593a3d7..bdad4c172 100644 --- a/examples/config.stdout.extended.hocon +++ b/examples/config.stdout.extended.hocon @@ -2,8 +2,8 @@ # All rights reserved. # # This software is made available by Snowplow Analytics, Ltd., -# under the terms of the Snowplow Limited Use License Agreement, Version 1.0 -# located at https://docs.snowplow.io/limited-use-license-1.0 +# under the terms of the Snowplow Limited Use License Agreement, Version 1.1 +# located at https://docs.snowplow.io/limited-use-license-1.1 # BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION # OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. diff --git a/flake.nix b/flake.nix index 7258c456f..69a3b8974 100644 --- a/flake.nix +++ b/flake.nix @@ -35,13 +35,19 @@ jre metals sbt - # pkgs.nodePackages.snyk pkgs.kubernetes-helm # (pkgs.wrapHelm pkgs.kubernetes-helm {plugins = [pkgs.kubernetes-helmPlugins.helm-diff];}) # pkgs.google-cloud-sdk.withExtraComponents( with pkgs.google-cloud-sdk.components [ gke-gcloud-auth-plugin ]); (pkgs.google-cloud-sdk.withExtraComponents [pkgs.google-cloud-sdk.components.gke-gcloud-auth-plugin]) # pkgs.google-cloud-sdk-gce + pkgs.snyk + pkgs.gitleaks ]; + scripts = { + snyk-check.exec = '' + for p in kinesis pubsub kafka nsq; do sbt "project ''${p}Distroless; set version := \"latest\"; Docker / publishLocal"; snyk container test --platform=linux/arm64 --app-vulns snowplow/scala-stream-collector-''${p}:latest-distroless; done + ''; + }; languages.nix.enable = true; pre-commit.hooks = { alejandra.enable = true; diff --git a/http4s/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala b/http4s/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala index a0a7c886c..5035c2925 100644 --- a/http4s/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala +++ b/http4s/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/CollectorContainer.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/Containers.scala b/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/Containers.scala index 6ff92eaba..5d0e6c334 100644 --- a/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/Containers.scala +++ b/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/Containers.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaCollectorSpec.scala b/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaCollectorSpec.scala index 63bdece82..54e57d493 100644 --- a/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaCollectorSpec.scala +++ b/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaCollectorSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaUtils.scala b/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaUtils.scala index 588619c97..caae533d4 100644 --- a/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaUtils.scala +++ b/kafka/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kafka/KafkaUtils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaCollector.scala b/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaCollector.scala index 77bdba300..4cd8bd82e 100644 --- a/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaCollector.scala +++ b/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaCollector.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/AzureAuthenticationCallbackHandler.scala b/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/AzureAuthenticationCallbackHandler.scala index 15feaf8e1..8becc1e3d 100644 --- a/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/AzureAuthenticationCallbackHandler.scala +++ b/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/AzureAuthenticationCallbackHandler.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KafkaSink.scala b/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KafkaSink.scala index c909f8e7d..b5b08e837 100644 --- a/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KafkaSink.scala +++ b/kafka/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KafkaSink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -12,6 +12,7 @@ package com.snowplowanalytics.snowplow.collectors.scalastream package sinks import cats.implicits._ +import cats.effect.implicits._ import cats.effect._ import org.typelevel.log4cats.Logger import org.typelevel.log4cats.slf4j.Slf4jLogger @@ -20,6 +21,8 @@ import org.apache.kafka.clients.producer.{Callback, KafkaProducer, ProducerRecor import com.snowplowanalytics.snowplow.collector.core.{Config, Sink} import scala.jdk.CollectionConverters._ +import scala.concurrent.ExecutionContext +import java.util.concurrent.Executors /** * Kafka Sink for the Scala Stream Collector @@ -28,7 +31,8 @@ class KafkaSink[F[_]: Async: Logger]( val maxBytes: Int, isHealthyState: Ref[F, Boolean], kafkaProducer: KafkaProducer[String, Array[Byte]], - topicName: String + topicName: String, + ec: ExecutionContext ) extends Sink[F] { override def isHealthy: F[Boolean] = isHealthyState.get @@ -40,14 +44,20 @@ class KafkaSink[F[_]: Async: Logger]( * @param key The partition key to use */ override def storeRawEvents(events: List[Array[Byte]], key: String): F[Unit] = + storeRawEventsAndWait(events, key).start.void + + private def storeRawEventsAndWait(events: List[Array[Byte]], key: String): F[Unit] = Logger[F].debug(s"Writing ${events.size} Thrift records to Kafka topic $topicName at key $key") *> - events.traverse_ { e => + events.parTraverse_ { e => def go: F[Unit] = Async[F] - .async_[Unit] { cb => - val record = new ProducerRecord(topicName, key, e) - kafkaProducer.send(record, callback(cb)) - () + .async[Unit] { cb => + val blockingSend = Sync[F].delay { + val record = new ProducerRecord(topicName, key, e) + kafkaProducer.send(record, callback(cb)) + Option.empty[F[Unit]] + } + Async[F].startOn(blockingSend, ec).map(f => Some(f.cancel)) } .handleErrorWith { e => handlePublishError(e) >> go @@ -80,11 +90,13 @@ object KafkaSink { for { isHealthyState <- Resource.eval(Ref.of[F, Boolean](false)) kafkaProducer <- createProducer(sinkConfig.config, sinkConfig.buffer, authCallbackClass) + ec <- createExecutionContext } yield new KafkaSink( sinkConfig.config.maxBytes, isHealthyState, kafkaProducer, - sinkConfig.name + sinkConfig.name, + ec ) /** @@ -113,4 +125,12 @@ object KafkaSink { } Resource.make(make)(p => Sync[F].blocking(p.close)) } + + def createExecutionContext[F[_]: Sync]: Resource[F, ExecutionContext] = { + val make = Sync[F].delay { + Executors.newSingleThreadExecutor + } + Resource.make(make)(e => Sync[F].blocking(e.shutdown)).map(ExecutionContext.fromExecutorService(_)) + } + } diff --git a/kafka/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaConfigSpec.scala b/kafka/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaConfigSpec.scala index 7ea51fa5b..084921e65 100644 --- a/kafka/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaConfigSpec.scala +++ b/kafka/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KafkaConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -173,7 +173,9 @@ object KafkaConfigSpec { idleTimeout = 610.seconds, responseHeaderTimeout = 30.seconds, maxRequestLineLength = 20480, - maxHeadersLength = 40960 + maxHeadersLength = 40960, + maxPayloadSize = 1048576, + dropPayloadSize = 2097152 ), license = Config.License(accept = true) ) diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CookieSpec.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CookieSpec.scala index 3d6f4599c..663c70946 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CookieSpec.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CookieSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CustomPathsSpec.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CustomPathsSpec.scala index b13fba8f3..b0d8973f0 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CustomPathsSpec.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/CustomPathsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/DoNotTrackCookieSpec.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/DoNotTrackCookieSpec.scala index fe4e3309e..f9bf9d73e 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/DoNotTrackCookieSpec.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/DoNotTrackCookieSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/HealthEndpointSpec.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/HealthEndpointSpec.scala index 27afc2e9c..d426e684d 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/HealthEndpointSpec.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/HealthEndpointSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/RobotsSpec.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/RobotsSpec.scala index 6d9368c1b..0fc4dc086 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/RobotsSpec.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/RobotsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/XForwardedForSpec.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/XForwardedForSpec.scala index 302e907cc..50a77ed8c 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/XForwardedForSpec.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/core/XForwardedForSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/Kinesis.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/Kinesis.scala index 5d2dda5d0..f6963f646 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/Kinesis.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/Kinesis.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/KinesisCollectorSpec.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/KinesisCollectorSpec.scala index 1a5f2c3d2..520d114ab 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/KinesisCollectorSpec.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/KinesisCollectorSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Collector.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Collector.scala index 64f2e601b..46af9bc67 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Collector.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Collector.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Localstack.scala b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Localstack.scala index c421753ba..42a3853bc 100644 --- a/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Localstack.scala +++ b/kinesis/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/kinesis/containers/Localstack.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KinesisCollector.scala b/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KinesisCollector.scala index 99c63e779..0a7879ab3 100644 --- a/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KinesisCollector.scala +++ b/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/KinesisCollector.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSink.scala b/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSink.scala index f555aeaaa..a229e99eb 100644 --- a/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSink.scala +++ b/kinesis/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisConfigSpec.scala b/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisConfigSpec.scala index c3c962712..532883eca 100644 --- a/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisConfigSpec.scala +++ b/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -127,7 +127,9 @@ object KinesisConfigSpec { idleTimeout = 610.seconds, responseHeaderTimeout = 30.seconds, maxRequestLineLength = 20480, - maxHeadersLength = 40960 + maxHeadersLength = 40960, + maxPayloadSize = 1048576, + dropPayloadSize = 2097152 ), streams = Config.Streams( useIpAddressAsPartitionKey = false, diff --git a/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSinkSpec.scala b/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSinkSpec.scala index 02e1a3c0a..4099e074c 100644 --- a/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSinkSpec.scala +++ b/kinesis/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/KinesisSinkSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqCollector.scala b/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqCollector.scala index 4b95f1367..6bad0d411 100644 --- a/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqCollector.scala +++ b/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqCollector.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSink.scala b/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSink.scala index 0653d3244..bf00b2ae1 100644 --- a/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSink.scala +++ b/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSinkConfig.scala b/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSinkConfig.scala index 2a6c13bd7..b258cc35f 100644 --- a/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSinkConfig.scala +++ b/nsq/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/NsqSinkConfig.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/nsq/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqConfigSpec.scala b/nsq/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqConfigSpec.scala index f57902f30..48596a88a 100644 --- a/nsq/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqConfigSpec.scala +++ b/nsq/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/NsqConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -160,7 +160,9 @@ object NsqConfigSpec { idleTimeout = 610.seconds, responseHeaderTimeout = 30.seconds, maxRequestLineLength = 20480, - maxHeadersLength = 40960 + maxHeadersLength = 40960, + maxPayloadSize = 1048576, + dropPayloadSize = 2097152 ), license = Config.License(accept = true) ) diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 73389c784..35fc605c0 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -90,6 +90,7 @@ object BuildSettings { Dependencies.Libraries.kafka, Dependencies.Libraries.mskAuth, Dependencies.Libraries.azureIdentity, + Dependencies.Libraries.jacksonCbor, // integration tests dependencies Dependencies.Libraries.IntegrationTests.specs2, diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 77f4a2002..951ce98ba 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -24,14 +24,15 @@ object Dependencies { val decline = "2.4.1" val fs2PubSub = "0.22.0" val http4s = "0.23.23" - val jackson = "2.12.7" // force this version to mitigate security vulnerabilities - val kafka = "3.8.1" + val jackson = "2.15.2" + val jacksonCbor = "2.12.7" // force this version to mitigate security vulnerabilities + val kafka = "3.9.0" val log4cats = "2.6.0" val log4j = "2.17.2" // CVE-2021-44228 val mskAuth = "1.1.1" - val nettyAll = "4.1.95.Final" // to fix nsq dependency + val nettyAll = "4.1.115.Final" // to fix nsq dependency val nsqClient = "1.3.0" - val pubsub = "1.125.11" // force this version to mitigate security vulnerabilities + val pubsub = "1.134.2" // force this version to mitigate security vulnerabilities val rabbitMQ = "5.15.0" val slf4j = "1.7.32" val specs2 = "4.11.0" @@ -40,7 +41,7 @@ object Dependencies { val thrift = "0.15.0" // force this version to mitigate security vulnerabilities val tracker = "2.0.0" val dataDog4s = "0.32.0" - val jnrPosix = "3.1.8" // force this version to mitigate security vulnerabilities + val jnrPosix = "3.1.20" // force this version to mitigate security vulnerabilities val azureIdentity = "1.13.2" } @@ -66,12 +67,13 @@ object Dependencies { val jnrPosix = "com.github.jnr" % "jnr-posix" % V.jnrPosix //sinks - val fs2PubSub = "com.permutive" %% "fs2-google-pubsub-grpc" % V.fs2PubSub - val jackson = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson - val kafka = "org.apache.kafka" % "kafka-clients" % V.kafka - val kinesis = "com.amazonaws" % "aws-java-sdk-kinesis" % V.awsSdk - val log4j = "org.apache.logging.log4j" % "log4j-core" % V.log4j - val mskAuth = "software.amazon.msk" % "aws-msk-iam-auth" % V.mskAuth % Runtime // Enables AWS MSK IAM authentication https://github.com/snowplow/stream-collector/pull/214 + val fs2PubSub = "com.permutive" %% "fs2-google-pubsub-grpc" % V.fs2PubSub + val jackson = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson + val jacksonCbor = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % V.jackson + val kafka = "org.apache.kafka" % "kafka-clients" % V.kafka + val kinesis = "com.amazonaws" % "aws-java-sdk-kinesis" % V.awsSdk + val log4j = "org.apache.logging.log4j" % "log4j-core" % V.log4j + val mskAuth = "software.amazon.msk" % "aws-msk-iam-auth" % V.mskAuth % Runtime // Enables AWS MSK IAM authentication https://github.com/snowplow/stream-collector/pull/214 val nettyAll = "io.netty" % "netty-all" % V.nettyAll val nsqClient = "com.snowplowanalytics" % "nsq-java-client" % V.nsqClient val pubsub = "com.google.cloud" % "google-cloud-pubsub" % V.pubsub diff --git a/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/Containers.scala b/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/Containers.scala index 92ccbb577..fcc9f979e 100644 --- a/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/Containers.scala +++ b/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/Containers.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/GooglePubSubCollectorSpec.scala b/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/GooglePubSubCollectorSpec.scala index 88cfaf98c..5ac610e94 100644 --- a/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/GooglePubSubCollectorSpec.scala +++ b/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/GooglePubSubCollectorSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/PubSub.scala b/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/PubSub.scala index f23da17fa..3a6db9958 100644 --- a/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/PubSub.scala +++ b/pubsub/src/it/scala/com/snowplowanalytics/snowplow/collectors/scalastream/it/pubsub/PubSub.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/pubsub/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/PubSubSink.scala b/pubsub/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/PubSubSink.scala index 3c607ad54..96f8e62eb 100644 --- a/pubsub/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/PubSubSink.scala +++ b/pubsub/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/PubSubSink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/ConfigSpec.scala b/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/ConfigSpec.scala index bfd039b33..d4868886e 100644 --- a/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/ConfigSpec.scala +++ b/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/ConfigSpec.scala @@ -2,8 +2,8 @@ * Copyright (c) 2012-present Snowplow Analytics Ltd. All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -116,7 +116,9 @@ object ConfigSpec { idleTimeout = 610.seconds, responseHeaderTimeout = 30.seconds, maxRequestLineLength = 20480, - maxHeadersLength = 40960 + maxHeadersLength = 40960, + maxPayloadSize = 1048576, + dropPayloadSize = 2097152 ), streams = Config.Streams( useIpAddressAsPartitionKey = false, diff --git a/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/GcpUserAgentSpec.scala b/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/GcpUserAgentSpec.scala index c1e464c1d..7126d6273 100644 --- a/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/GcpUserAgentSpec.scala +++ b/pubsub/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/GcpUserAgentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsCollector.scala b/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsCollector.scala index 019986d5a..786b033a8 100644 --- a/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsCollector.scala +++ b/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsCollector.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/SqsSink.scala b/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/SqsSink.scala index c3d3fa4a5..896e68b53 100644 --- a/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/SqsSink.scala +++ b/sqs/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/sinks/SqsSink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/sqs/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsConfigSpec.scala b/sqs/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsConfigSpec.scala index df6fe1a08..a58ffb946 100644 --- a/sqs/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsConfigSpec.scala +++ b/sqs/src/test/scala/com.snowplowanalytics.snowplow.collectors.scalastream/SqsConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -117,7 +117,9 @@ object SqsConfigSpec { idleTimeout = 610.seconds, responseHeaderTimeout = 30.seconds, maxRequestLineLength = 20480, - maxHeadersLength = 40960 + maxHeadersLength = 40960, + maxPayloadSize = 1048576, + dropPayloadSize = 2097152 ), streams = Config.Streams( useIpAddressAsPartitionKey = false,