feat(dataflow-kafka-to-bigquery): Support dynamic proto definition JARs

* Upgrade the pipeline to use Java21

Change-Id: I8dbf1c93af5b4bafdc64287c1d3c55449ecfc81a
GitOrigin-RevId: dd53e31b214e123df6addd5c32e8361a932ab9d1

projects/dataflow-kafka-to-bigquery/buildSrc/build.gradle.kts

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+    // Support convention plugins written in Kotlin. Convention plugins are build scripts
+    // in 'src/main' that automatically become available as plugins in the main build.
+    `kotlin-dsl`
+}
+
+repositories {
+    // Use the plugin portal to apply community plugins in convention plugins.
+    mavenCentral()
+    gradlePluginPortal()
+    maven {
+        url = uri("https://plugins.gradle.org/m2/")
+    }
+}
+
+val libs = extensions.getByType<VersionCatalogsExtension>().named("libs")
+
+dependencies {
+    implementation("com.diffplug.spotless:spotless-plugin-gradle:${libs.findVersion("spotless-plugin").get()}")
+    implementation("com.google.protobuf:protobuf-gradle-plugin:${libs.findVersion("protobuf-plugin").get()}")
+}

projects/dataflow-kafka-to-bigquery/buildSrc/settings.gradle.kts

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+dependencyResolutionManagement {
+    // Reuse version catalog from the main build.
+    versionCatalogs {
+        create("libs") { from(files("../gradle/libs.versions.toml")) }
+    }
+}
+
+rootProject.name = "buildSrc"

projects/dataflow-kafka-to-bigquery/buildSrc/src/main/kotlin/buildlogic.java-common-conventions.gradle.kts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+    java
+    jacoco
+    id("com.diffplug.spotless")
+    idea
+}
+
+group = "com.google.cloud.solutions.dataflow.kafka2bigquery"
+
+val libs = extensions.getByType<VersionCatalogsExtension>().named("libs")
+
+repositories {
+    // Use Maven Central for resolving dependencies.
+    mavenLocal()
+    mavenCentral()
+    gradlePluginPortal()
+    maven(url = "https://packages.confluent.io/maven/")
+}
+
+dependencies {
+    testImplementation(libs.findBundle("test-bundle").get())
+}
+
+tasks.test {
+    finalizedBy(tasks.jacocoTestReport)
+    testLogging.events("passed", "skipped", "failed", "standardOut", "standardError")
+    testLogging.showStandardStreams = true
+}
+
+tasks.withType<JavaCompile> {
+    options.encoding = "UTF-8"
+}
+
+tasks.jacocoTestReport {
+    dependsOn(tasks.test)
+    reports {
+        xml.required = true
+        csv.required = false
+    }
+}
+
+jacoco {
+    toolVersion = "0.8.12"
+}
+
+spotless {
+    java {
+        targetExclude("**/generated/**/*.java")
+        googleJavaFormat("1.24.0").reflowLongStrings().reorderImports(true)
+        trimTrailingWhitespace()
+        endWithNewline()
+    }
+}

projects/dataflow-kafka-to-bigquery/buildSrc/src/main/kotlin/buildlogic.java-library-conventions.gradle.kts

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+    // Apply the common convention plugin for shared build configuration
+    // between library and application projects.
+    id("buildlogic.java-common-conventions")
+
+    // Apply the java-library plugin for API and implementation separation.
+    `java-library`
+}

projects/dataflow-kafka-to-bigquery/buildSrc/src/main/kotlin/buildlogic.java-proto-library-conventions.gradle.kts

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins {
+    // Apply the common convention plugin for shared build configuration
+    // between library and application projects.
+    id("buildlogic.java-common-conventions")
+
+    // Apply the java-library plugin for API and implementation separation.
+    `java-library`
+    id("com.google.protobuf")
+}
+
+val libs = extensions.getByType<VersionCatalogsExtension>().named("libs")
+
+dependencies {
+    compileOnly(libs.findBundle("proto-libs").get())
+}
+
+protobuf {
+    protoc {
+        artifact = "com.google.protobuf:protoc:${libs.findVersion("protobuf").get()}"
+    }
+}

projects/dataflow-kafka-to-bigquery/buildtest.dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM gradle:8-jdk21
+
+ARG PROJECT_SUBDIRECTORY
+WORKDIR "${PROJECT_SUBDIRECTORY}"
+
+CMD ["gradle", "build"]

projects/dataflow-kafka-to-bigquery/gradle.properties

@@ -0,0 +1,21 @@
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.gradle.caching=true
+org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED

projects/dataflow-kafka-to-bigquery/gradle/libs.versions.toml

@@ -0,0 +1,74 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format
+
+[versions]
+apache-beam = "2.59.0"
+autovalue = "1.11.0"
+flogger = "0.8"
+protobuf = "3.25.3"
+truth = "1.4.2"
+spotless-plugin = "6.25.0"
+protobuf-plugin = "0.9.4"
+
+[plugins]
+shadow-jar = { id = "com.github.johnrengelman.shadow", version = "8.1.1" }
+
+[libraries]
+# Protobuf Core
+protobuf-java = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protobuf" }
+protobuf-java-util = { group = "com.google.protobuf", name = "protobuf-java-util", version.ref="protobuf"}
+
+# Autovalue annotations
+auto-value = { group = "com.google.auto.value", name = "auto-value", version.ref = "autovalue"}
+auto-value-annotations = { group = "com.google.auto.value", name = "auto-value-annotations", version.ref = "autovalue"}
+
+# Kafka
+kafka = { group = "org.apache.kafka", name = "kafka_2.13", version ="3.8.1" }
+kafka-client = { group = "org.apache.kafka", name = "kafka-clients", version ="3.8.1" }
+testcontainers-kafka = { group = "org.testcontainers", name = "kafka", version = "1.20.3" }
+
+# Commons
+guava = { group = "com.google.guava", name = "guava", version.ref = "autovalue"}
+checker-qual = { group = "org.checkerframework", name = "checker-qual", version = "3.42.0"}
+
+# Logging
+log4j-core = { group = "org.apache.logging.log4j", name = "log4j-core", version = "2.17.2"}
+flogger = { group = "com.google.flogger", name = "flogger", version.ref = "flogger"}
+flogger-google-extensions = { group = "com.google.flogger", name = "google-extensions", version.ref = "flogger"}
+flogger-system-backend = { group = "com.google.flogger", name = "flogger-system-backend", version.ref = "flogger"}
+
+# Apache Beam
+beam-sdks-java-core = { group = "org.apache.beam", name = "beam-sdks-java-core", version.ref = "apache-beam"}
+beam-sdks-java-io-kafka = { group = "org.apache.beam", name = "beam-sdks-java-io-kafka", version.ref = "apache-beam"}
+beam-sdks-java-io-google-cloud-platform = { group = "org.apache.beam", name = "beam-sdks-java-io-google-cloud-platform", version.ref = "apache-beam"}
+beam-runners-direct-java = { group = "org.apache.beam", name = "beam-runners-direct-java", version.ref = "apache-beam"}
+beam-runners-google-cloud-dataflow-java = { group = "org.apache.beam", name = "beam-runners-google-cloud-dataflow-java", version.ref = "apache-beam"}
+
+# Testing
+junit = { group = "junit", name = "junit", version = "4.13.2"}
+hamcrest-all = { group = "org.hamcrest", name = "hamcrest-all", version = "1.3"}
+mockito-all = { group = "org.mockito", name = "mockito-all", version = "1.10.19"}
+truth = { group = "com.google.truth", name = "truth", version.ref = "truth"}
+truth-java8-extension = { group = "com.google.truth.extensions", name = "truth-java8-extension", version.ref = "truth"}
+truth-proto-extension = { group = "com.google.truth.extensions", name = "truth-proto-extension", version.ref = "truth"}
+jsonassert = { group = "org.skyscreamer", name = "jsonassert", version = "1.5.0" }
+
+[bundles]
+proto-libs = ["protobuf-java", "protobuf-java-util"]
+apache-beam = ["beam-sdks-java-core", "beam-sdks-java-io-kafka", "beam-sdks-java-io-google-cloud-platform"]
+common = ["guava", "checker-qual"]
+flogger = ["flogger", "flogger-google-extensions"]
+logger-runtime = ["log4j-core", "flogger-system-backend"]
+test-bundle = ["junit", "hamcrest-all", "mockito-all", "truth", "truth-java8-extension", "truth-proto-extension"]

projects/dataflow-kafka-to-bigquery/kafka-utils/build.gradle.kts

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+    id("buildlogic.java-library-conventions")
+    alias(libs.plugins.shadow.jar)
+}
+
+dependencies {
+    implementation("commons-cli:commons-cli:1.9.0")
+    implementation(project(":proto-serde"))
+    implementation(libs.kafka.client)
+    implementation(libs.bundles.common)
+    implementation(libs.bundles.flogger)
+    implementation(libs.bundles.proto.libs)
+    implementation(project(":sample-proto"))
+
+    // This module is a testing library
+    implementation(libs.bundles.test.bundle)
+
+    testImplementation(libs.testcontainers.kafka)
+    testImplementation("com.google.cloud.solutions.satools.common:testing")
+}
+
+tasks.withType<ShadowJar> {
+    isZip64 = true
+    mergeServiceFiles()
+    manifest {
+        attributes(mapOf("Main-Class" to "com.google.cloud.solutions.dataflow.kafka2bigquery.kafkautils.KafkaProtoProducerApp"))
+    }
+}