[duckdb] Changed DuckDB dep to be a snapshot

.github/workflows/VeniceCI-E2ETests.yml

@@ -108,6 +108,101 @@ jobs:
           key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
           secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
 
+  IntegrationTests_1001:
+    name: IntegrationTests_1001
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [17]
+    runs-on: ubuntu-latest
+    permissions:
+     id-token: write
+     contents: read
+     checks: write
+     pull-requests: write
+     issues: write
+    timeout-minutes: 120
+    concurrency:
+     group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk${{ matrix.jdk }}-IntegrationTests_1001
+     cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.jdk }}
+          distribution: 'temurin'
+          cache: 'gradle'
+      - shell: bash
+        run: |
+          git remote set-head origin --auto
+          git remote add upstream https://github.com/linkedin/venice
+          git fetch upstream
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          add-job-summary: never
+      - name: Run Integration Tests
+        run: ./gradlew --continue --no-daemon -DforkEvery=1 -DmaxParallelForks=1 integrationTests_1001
+      - name: Package Build Artifacts
+        if: success() || failure()
+        shell: bash
+        run: |
+          mkdir ${{ github.job }}-artifacts
+          echo "Repository owner: ${{ github.repository_owner }}"
+          echo "Repository name: ${{ github.repository }}"
+          echo "event name: ${{ github.event_name }}"
+          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
+          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
+          tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
+      - name: Generate Fork Repo Test Reports
+        if: ${{ (github.repository_owner != 'linkedin') && (success() || failure()) }}
+        uses: dorny/[email protected]
+        env:
+         NODE_OPTIONS: --max-old-space-size=9182
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: ${{ github.job }} Test Reports       # Name where it report the test results
+          path: '**/TEST-*.xml'
+          fail-on-error: 'false'
+          max-annotations: '10'
+          list-tests: 'all'
+          list-suites: 'all'
+          reporter: java-junit
+      - name: Publish Test Report
+        continue-on-error: true
+        env:
+          NODE_OPTIONS: "--max_old_space_size=8192"
+        uses: mikepenz/action-junit-report@v5
+        if: always()
+        with:
+          check_name: ${{ github.job }}-jdk${{ matrix.jdk }} Report
+          comment: false
+          annotate_only: true
+          flaky_summary: true
+          commit: ${{github.event.workflow_run.head_sha}}
+          detailed_summary: true
+          report_paths: '**/build/test-results/test/TEST-*.xml'
+      - name: Upload Build Artifacts
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}
+          path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
+          retention-days: 30
+      - name: Upload test results to BuildPulse for flaky test detection
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && !cancelled()
+        uses: buildpulse/buildpulse-action@main
+        with:
+          account: 100582612927
+          repository: 100441445875
+          path: |
+            **/TEST-*.xml
+          key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
+          secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
+
   IntegrationTests_1010:
     name: IntegrationTests_1010
     strategy:
@@ -2964,7 +3059,7 @@ jobs:
       matrix:
         jdk: [17]
     runs-on: ubuntu-latest
-    needs: [IntegrationTests_1000, IntegrationTests_1010, IntegrationTests_1020, IntegrationTests_1030, IntegrationTests_1040, IntegrationTests_1050, IntegrationTests_1060, IntegrationTests_1070, IntegrationTests_1080, IntegrationTests_1090, IntegrationTests_1100, IntegrationTests_1110, IntegrationTests_1120, IntegrationTests_1130, IntegrationTests_1200, IntegrationTests_1210, IntegrationTests_1220, IntegrationTests_1230, IntegrationTests_1240, IntegrationTests_1250, IntegrationTests_1260, IntegrationTests_1270, IntegrationTests_1280, IntegrationTests_1400, IntegrationTests_1410, IntegrationTests_1420, IntegrationTests_1430, IntegrationTests_1440, IntegrationTests_1500, IntegrationTests_1550, IntegrationTests_9999]
+    needs: [IntegrationTests_1000, IntegrationTests_1001, IntegrationTests_1010, IntegrationTests_1020, IntegrationTests_1030, IntegrationTests_1040, IntegrationTests_1050, IntegrationTests_1060, IntegrationTests_1070, IntegrationTests_1080, IntegrationTests_1090, IntegrationTests_1100, IntegrationTests_1110, IntegrationTests_1120, IntegrationTests_1130, IntegrationTests_1200, IntegrationTests_1210, IntegrationTests_1220, IntegrationTests_1230, IntegrationTests_1240, IntegrationTests_1250, IntegrationTests_1260, IntegrationTests_1270, IntegrationTests_1280, IntegrationTests_1400, IntegrationTests_1410, IntegrationTests_1420, IntegrationTests_1430, IntegrationTests_1440, IntegrationTests_1500, IntegrationTests_1550, IntegrationTests_9999]
     timeout-minutes: 20
     steps:
     - name: NoOp

build.gradle

@@ -82,7 +82,7 @@ ext.libraries = [
     commonsLang: 'commons-lang:commons-lang:2.6',
     conscrypt: 'org.conscrypt:conscrypt-openjdk-uber:2.5.2',
     d2: "com.linkedin.pegasus:d2:${pegasusVersion}",
-    duckdbJdbc: "org.duckdb:duckdb_jdbc:1.1.3",
+    duckdbJdbc: "org.duckdb:duckdb_jdbc:1.2.0-20250116.012809-118", // TODO: Remove SNAPSHOT when the real release is published!
     failsafe: 'net.jodah:failsafe:2.4.0',
     fastUtil: 'it.unimi.dsi:fastutil:8.3.0',
     grpcNettyShaded: "io.grpc:grpc-netty-shaded:${grpcVersion}",

internal/venice-test-common/build.gradle

@@ -138,7 +138,10 @@ def integrationTestBuckets = [
   "1000": [
       "com.linkedin.davinci.*",
       "com.linkedin.venice.endToEnd.DaVinciClientDiskFullTest",
-      "com.linkedin.venice.endToEnd.DaVinciClientMemoryLimitTest"],
+      "com.linkedin.venice.endToEnd.DaVinciClientMemoryLimitTest",
+      "com.linkedin.venice.endToEnd.DaVinciClientRecordTransformerTest"],
+  "1001": [
+      "com.linkedin.venice.endToEnd.DuckDBDaVinciRecordTransformerIntegrationTest"],
   "1010": [
        "com.linkedin.venice.endToEnd.DaVinciClientTest"],
   "1020": [
@@ -179,8 +182,7 @@ def integrationTestBuckets = [
       "com.linkedin.venice.endToEnd.TestVson*",
       "com.linkedin.venice.endToEnd.Push*"],
   "1210": [
-      "com.linkedin.venice.hadoop.*",
-      "com.linkedin.venice.endToEnd.DaVinciClientRecordTransformerTest"],
+      "com.linkedin.venice.hadoop.*"],
   "1220": [
       "com.linkedin.venice.endToEnd.TestPushJob*"],
   "1230": [

internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DuckDBDaVinciRecordTransformerIntegrationTest.java

@@ -43,6 +43,7 @@
 import com.linkedin.venice.utils.PushInputSchemaBuilder;
 import com.linkedin.venice.utils.TestUtils;
 import com.linkedin.venice.utils.TestWriteUtils;
+import com.linkedin.venice.utils.Time;
 import com.linkedin.venice.utils.Utils;
 import com.linkedin.venice.utils.VeniceProperties;
 import io.tehuti.metrics.MetricsRepository;
@@ -68,7 +69,7 @@
 
 public class DuckDBDaVinciRecordTransformerIntegrationTest {
   private static final Logger LOGGER = LogManager.getLogger(DaVinciClientRecordTransformerTest.class);
-  private static final int TEST_TIMEOUT = 120_000;
+  private static final int TEST_TIMEOUT = 3 * Time.MS_PER_MINUTE;
   private VeniceClusterWrapper cluster;
   private D2Client d2Client;
 
@@ -112,7 +113,7 @@ public void deleteClassHash() {
    *
    * TODO: Re-enable once we can depend on a clean release.
    */
-  @Test(timeOut = TEST_TIMEOUT, enabled = false)
+  @Test(timeOut = TEST_TIMEOUT)
   public void testRecordTransformer() throws Exception {
     DaVinciConfig clientConfig = new DaVinciConfig();
 

settings.gradle

@@ -6,6 +6,10 @@ pluginManagement {
       url  "https://linkedin.jfrog.io/artifactory/open-source"
     }
     gradlePluginPortal()
+    maven {
+      // Needed for DuckDB SNAPSHOT. TODO: Remove when the real release is published!
+      url = uri('https://oss.sonatype.org/content/repositories/snapshots/')
+    }
   }
 }