From 324b3b63ddfdc037167832809a1b121fb0d9b670 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:11:21 +0800 Subject: [PATCH 01/12] fix spotbugs check failed --- .../main/java/org/apache/bookkeeper/client/PendingAddOp.java | 2 ++ buildtools/src/main/resources/bookkeeper/findbugsExclude.xml | 5 +++++ .../org/apache/bookkeeper/bookie/GroupSortBenchmark.java | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java index 05f740d33a3..394c961cbc9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java @@ -23,6 +23,7 @@ import static org.apache.bookkeeper.proto.BookieProtocol.FLAG_RECOVERY_ADD; import com.google.common.collect.ImmutableMap; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; @@ -80,6 +81,7 @@ class PendingAddOp implements WriteCallback { boolean allowFailFast = false; List ensemble; + @SuppressFBWarnings("IS2_INCONSISTENT_SYNC") static PendingAddOp create(LedgerHandle lh, ClientContext clientCtx, List ensemble, ByteBuf payload, EnumSet writeFlags, diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml index 09daf196569..60a3a3c3285 100644 --- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml +++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml @@ -317,6 +317,11 @@ + + + + + diff --git a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java index 8df5691ae70..94699a26197 100644 --- a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java +++ b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java @@ -21,8 +21,10 @@ package org.apache.bookkeeper.bookie; +import java.security.SecureRandom; import java.util.Arrays; import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.bookie.storage.ldb.ArrayGroupSort; import org.openjdk.jmh.annotations.Benchmark; @@ -55,7 +57,7 @@ public static class TestState { private long[] items; public TestState() { - Random r = new Random(); + Random r = ThreadLocalRandom.current(); for (int i = 0; i < (N * 4); i++) { randomItems[i] = r.nextLong(); } From f9911defe826dbe9d96402f17b7dd66b2a78628d Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:13:56 +0800 Subject: [PATCH 02/12] format code --- buildtools/src/main/resources/bookkeeper/findbugsExclude.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml index 60a3a3c3285..dec4118b58a 100644 --- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml +++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml @@ -317,7 +317,7 @@ - + From f7ce9d0905d8bc8e93bdeab2cfbdeb2dc84155fa Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:20:10 +0800 Subject: [PATCH 03/12] install the package first --- .github/workflows/bk-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index c38a2680ac1..d989ba7bc94 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -81,7 +81,9 @@ jobs: - name: Validate pull request if: steps.check_changes.outputs.docs_only != 'true' - run: mvn clean -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + run: | + mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: Check license files if: steps.check_changes.outputs.docs_only != 'true' From 44cebc0244d750e0b4606ced491204820ea21513 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:43:24 +0800 Subject: [PATCH 04/12] upgrade spotbugs version to 4.7.3.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 43e1217e9ff..e8227092bb3 100644 --- a/pom.xml +++ b/pom.xml @@ -205,7 +205,7 @@ 1.4.1.Final 0.6.1 9.3 - 4.6.0.0 + 4.7.3.2 1 4.0.0 3.0.1 From b6e2b309445d324d267325dd4474ee55c20688d3 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 15:27:55 +0800 Subject: [PATCH 05/12] upgrade javadoc plugin to 3.5.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e8227092bb3..3f171606298 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ 2.7 2.5.1 3.2.2 - 3.1.1 + 3.5.0 3.2.0 3.2.1 3.0.0-M5 From f71d9c1fd1e1bbbc2731ed5e3b827c8a47f52ee0 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 15:36:51 +0800 Subject: [PATCH 06/12] upgrade javadoc plugin to 3.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3f171606298..aeb3d95add4 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ 2.7 2.5.1 3.2.2 - 3.5.0 + 3.2.0 3.2.0 3.2.1 3.0.0-M5 From 2b3397a52fd275c1bd955ed5eb484c3aa1d72364 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 15:37:32 +0800 Subject: [PATCH 07/12] format code --- .../java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java | 1 - 1 file changed, 1 deletion(-) diff --git a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java index 94699a26197..b51fc758b6d 100644 --- a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java +++ b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java @@ -21,7 +21,6 @@ package org.apache.bookkeeper.bookie; -import java.security.SecureRandom; import java.util.Arrays; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; From 9ce71cdc98c28252fc49ae6ee02440d614535f87 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 18:29:43 +0800 Subject: [PATCH 08/12] set maven version 3.8.7 --- .github/workflows/bk-ci.yml | 5 +++++ pom.xml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index d989ba7bc94..deae0a0204f 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -79,6 +79,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Validate pull request if: steps.check_changes.outputs.docs_only != 'true' run: | diff --git a/pom.xml b/pom.xml index aeb3d95add4..e8227092bb3 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ 2.7 2.5.1 3.2.2 - 3.2.0 + 3.1.1 3.2.0 3.2.1 3.0.0-M5 From 5ef84fec0fce27fac85696c0119892b87eb91229 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 23:38:47 +0800 Subject: [PATCH 09/12] set all test maven version to 3.8.7 --- .github/workflows/bk-ci.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index deae0a0204f..a274f3a01ef 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -159,6 +159,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build run: | projects_list= @@ -230,6 +235,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build with Maven run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO @@ -272,6 +282,11 @@ jobs: distribution: 'temurin' java-version: 8 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build with Maven run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO @@ -318,6 +333,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: mvn package run: mvn -B -nsu clean package -DskipTests @@ -350,6 +370,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: mvn package run: mvn -B -nsu clean package -DskipTests @@ -394,6 +419,11 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk_version }} + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build with Maven run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO @@ -429,6 +459,11 @@ jobs: with: java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: run "clean install verify" to trigger dependency check # excluding dlfs because it includes hadoop lib with # CVEs that we cannot patch up anyways From 8185356cdf3b8df035d69fa918c38c5304d897d0 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 10:36:09 +0800 Subject: [PATCH 10/12] fix failed CI --- .../src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java index 5dc1a5d8cce..08bf9cfb385 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java @@ -95,7 +95,7 @@ public static void bestEffortRemoveFromPageCache(int fd, long offset, long len) } try { NATIVE_IO.posix_fadvise(fd, offset, len, POSIX_FADV_DONTNEED); - } catch (Exception e) { + } catch (Throwable e) { log.warn("Failed to perform posix_fadvise: {}", e.getMessage()); fadvisePossible = false; } From 06695f74b8cdcb32489323aa52c2946dd26f7917 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 16:13:02 +0800 Subject: [PATCH 11/12] add test failure --- .github/workflows/bk-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index a274f3a01ef..a373830a81f 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -206,6 +206,10 @@ jobs: path: surefire-reports retention-days: 7 + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + integration-tests: name: Integration Tests runs-on: ubuntu-latest From bf98fe45ce55359a63c86f7cbd7341ea9126ea0e Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 19:19:04 +0800 Subject: [PATCH 12/12] add test failure --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e8227092bb3..37870611fa4 100644 --- a/pom.xml +++ b/pom.xml @@ -1118,7 +1118,7 @@ ${maven-surefire-plugin.version} - @{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid + -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} ${redirectTestOutputToFile} ${forkCount.variable} false