diff --git a/.github/ISSUE_TEMPLATE/bp.md b/.github/ISSUE_TEMPLATE/bp.md index 9f1566a2fee..e71f4d7a417 100644 --- a/.github/ISSUE_TEMPLATE/bp.md +++ b/.github/ISSUE_TEMPLATE/bp.md @@ -9,7 +9,7 @@ assignees: '' **BP** -> Follow the instructions at http://bookkeeper.apache.org/community/bookkeeper_proposals/ to create a proposal. +> Follow the instructions at https://bookkeeper.apache.org/community/bookkeeper-proposals/ to create a proposal. This is the master ticket for tracking BP-xyz : diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml new file mode 100644 index 00000000000..d74c3f25fc6 --- /dev/null +++ b/.github/actions/clean-disk/action.yml @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: clean disk +description: makes some more space available on the disk by removing files +inputs: + mode: + description: "Use 'full' to clean as much as possible" + required: false +runs: + using: composite + steps: + - run: | + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + directories=(/usr/local/lib/android /opt/ghc) + if [[ "${{ inputs.mode }}" == "full" ]]; then + # remove these directories only when mode is 'full' + directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL) + fi + emptydir=/tmp/empty$$/ + mkdir $emptydir + echo "::group::Available diskspace" + time df -BM / /mnt + echo "::endgroup::" + for directory in "${directories[@]}"; do + echo "::group::Removing $directory" + # fast way to delete a lot of files on linux + time sudo eatmydata rsync -a --delete $emptydir ${directory}/ + time sudo eatmydata rm -rf ${directory} + time df -BM / /mnt + echo "::endgroup::" + done + echo "::group::Cleaning apt state" + time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" + time df -BM / /mnt + echo "::endgroup::" + fi + echo "::group::Available diskspace" + time df -BM / /mnt + echo "::endgroup::" + shell: bash diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml index 59f977c3150..402b9201dc2 100644 --- a/.github/actions/tune-runner-vm/action.yml +++ b/.github/actions/tune-runner-vm/action.yml @@ -24,8 +24,79 @@ runs: steps: - run: | if [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "::group::Configure and tune OS" # Ensure that reverse lookups for current hostname are handled properly # Add the current IP address, long hostname and short hostname record to /etc/hosts file echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + + # The default vm.swappiness setting is 60 which has a tendency to start swapping when memory + # consumption is high. + # Set vm.swappiness=1 to avoid swapping and allow high RAM usage + echo 1 | sudo tee /proc/sys/vm/swappiness + ( + shopt -s nullglob + # Set swappiness to 1 for all cgroups and sub-groups + for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do + echo 1 | sudo tee $swappiness_file > /dev/null + done + ) || true + + # use "madvise" Linux Transparent HugePages (THP) setting + # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html + # "madvise" is generally a better option than the default "always" setting + # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp + echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled + echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled + echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag + echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag + + # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt + # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds) + # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes) + sudo mount -o remount,nodiscard,commit=999999,barrier=0 / + sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt + # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective + # https://www.spinics.net/lists/linux-ide/msg52562.html + for i in /sys/block/sd*/queue/discard_max_bytes; do + echo 0 | sudo tee $i + done + # disable any background jobs that run SSD discard/trim + sudo systemctl disable fstrim.timer || true + sudo systemctl stop fstrim.timer || true + sudo systemctl disable fstrim.service || true + sudo systemctl stop fstrim.service || true + + # stop php-fpm + sudo systemctl stop php8.0-fpm.service || true + sudo systemctl stop php7.4-fpm.service || true + # stop mono-xsp4 + sudo systemctl disable mono-xsp4.service || true + sudo systemctl stop mono-xsp4.service || true + sudo killall mono || true + + # stop Azure Linux agent to save RAM + sudo systemctl stop walinuxagent.service || true + + # enable docker experimental mode which is + # required for using "docker build --squash" / "-Ddocker.squash=true" + daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')" + echo "$daemon_json" | sudo tee /etc/docker/daemon.json + # restart docker daemon + sudo systemctl restart docker + echo '::endgroup::' + + # show memory + echo "::group::Available Memory" + free -m + echo '::endgroup::' + # show disk + echo "::group::Available diskspace" + df -BM + echo "::endgroup::" + # show cggroup + echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP" + CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }') + sudo cgget -a $CURRENT_CGGROUP || true + echo '::endgroup::' fi - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml new file mode 100644 index 00000000000..676ba76724d --- /dev/null +++ b/.github/changes-filter.yaml @@ -0,0 +1,16 @@ +# contains pattern definitions used in workflows "changes" step +# pattern syntax: https://github.com/micromatch/picomatch +all: + - '**' +docs: + - 'site3/**' + - '.asf.yaml' + - '*.md' + - '**/*.md' + - '.github/changes-filter.yaml' + - '.github/ISSUE_TEMPLATE/**' +need_owasp: + - 'pom.xml' + - '**/pom.xml' + - 'src/owasp-dependency-check-false-positives.xml' + - 'src/owasp-dependency-check-suppressions.xml' \ No newline at end of file diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000000..b0c71c6deff --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,60 @@ + + +## GitHub Workflows + +This directory contains all BookKeeper CI checks. + +### Required Workflows + +When adding new CI workflows, please update the [.asf.yaml](../../.asf.yaml) if the workflow is required to pass before +a PR can be merged. Instructions on how to update the file are below. + +This project uses the [.asf.yaml](../../.asf.yaml) to configure which workflows are required to pass before a PR can +be merged. In the `.asf.yaml`, the required contexts are defined in the `github.protected_branches.*.required_status_checks.contexts.[]` +where * is any key in the `protected_branches` map. + +You can view the currently required status checks by running the following command: + +```shell +curl -s -H 'Accept: application/vnd.github.v3+json' https://api.github.com/repos/apache/bookkeeper/branches/master | \ +jq .protection +``` + +These contexts get their names in one of two ways depending on how the workflow file is written in this directory. The +following command will print out the names of each file and the associated with the check. If the `name` field is `null`, +the context will be named by the `id`. + +```shell +for f in .github/workflows/*.yaml .github/workflows/*.yml; \ +do FILE=$f yq eval -o j '.jobs | to_entries | {"file": env(FILE),"id":.[].key, "name":.[].value.name}' $f; \ +done +``` + +Duplicate names are allowed, and all checks with the same name will be treated the same (required or not required). + +When working on workflow changes, one way to find out the names of the status checks is to retrieve the names +from the PR build run. The "check-runs" can be found by commit id. Here's an example: + +```shell +curl -s "https://api.github.com/repos/apache/bookkeeper/commits/$(git rev-parse HEAD)/check-runs" | \ + jq -r '.check_runs | .[] | .name' |sort +``` diff --git a/.github/workflows/backward-compat-tests.yml b/.github/workflows/backward-compat-tests.yml deleted file mode 100644 index 65c2812de36..00000000000 --- a/.github/workflows/backward-compat-tests.yml +++ /dev/null @@ -1,60 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Backward compatibility tests - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Install Module - run: mvn -B -nsu clean install -DskipTests - - name: Test current server with old clients - run: mvn -B -nsu -DintegrationTests -pl :backward-compat-current-server-old-clients test - - name: Test progressive upgrade - run: mvn -B -nsu -DintegrationTests -pl :upgrade test - - name: Other tests - run: | - mvn -B -nsu -DintegrationTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct,:yahoo-custom-version test diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml new file mode 100644 index 00000000000..b064c4cbc5c --- /dev/null +++ b/.github/workflows/bk-ci.yml @@ -0,0 +1,490 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: BookKeeper CI + +on: + pull_request: + branches: + - master + - branch-* + push: + branches: + - master + - branch-* + workflow_dispatch: + +env: + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-license-check: + name: PR Validation + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Detect changed files + id: changes + uses: apache/pulsar-test-infra/paths-filter@master + with: + filters: .github/changes-filter.yaml + list-files: csv + + - name: Check changed files + id: check_changes + run: | + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT + + - name: Cache local Maven repository + if: steps.check_changes.outputs.docs_only != 'true' + id: cache + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/bookkeeper + !~/.m2/repository/org/apache/distributedlog + key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} + + - name: Set up JDK 11 + if: steps.check_changes.outputs.docs_only != 'true' + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Validate pull request + if: steps.check_changes.outputs.docs_only != 'true' + run: | + mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Check license files + if: steps.check_changes.outputs.docs_only != 'true' + run: dev/check-all-licenses + + - name: Generate Javadoc + if: steps.check_changes.outputs.docs_only != 'true' + run: mvn -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider javadoc:aggregate -DskipTests -Pdelombok -Dchesktyle.skip -Dspotbugs.skip + + unit-tests: + name: ${{ matrix.step_name }} + runs-on: ubuntu-latest + timeout-minutes: ${{ matrix.timeout || 60 }} + needs: [ 'build-and-license-check' ] + if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} + strategy: + fail-fast: false + matrix: + include: + - step_name: Bookie Tests + module: bookkeeper-server + flag: bookie + test_args: "-Dtest='org.apache.bookkeeper.bookie.**'" + - step_name: Client Tests + module: bookkeeper-server + flag: client + test_args: "-Dtest='org.apache.bookkeeper.client.**'" + timeout: 75 + - step_name: Replication Tests + module: bookkeeper-server + flag: replication + test_args: "-Dtest='org.apache.bookkeeper.replication.**'" + - step_name: Remaining Tests + module: bookkeeper-server + flag: remaining + test_args: "-Dtest='!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**'" + - step_name: TLS Tests + module: bookkeeper-server + flag: tls + test_args: "-Dtest='org.apache.bookkeeper.tls.**'" + - step_name: StreamStorage Tests + test_args: "-f stream/pom.xml -DstreamTests" + flag: stream + - step_name: Shell tests + module: tests/scripts + flag: shell + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/bookkeeper + !~/.m2/repository/org/apache/distributedlog + key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Build + run: | + projects_list= + if [[ ! -z "${{ matrix.module }}" ]]; then + projects_list="-pl ${{ matrix.module }}" + fi + mvn -q -T 1C -B -nsu $projects_list install -am -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Drat.skip -Dmaven.javadoc.skip + + - name: Test - ${{ matrix.step_name }} + run: | + projects_list= + if [[ ! -z "${{ matrix.module }}" ]]; then + projects_list="-pl ${{ matrix.module }}" + fi + mvn -B -nsu $projects_list verify ${{ matrix.test_args }} + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'surefire-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: failure() + continue-on-error: true + with: + name: unit-${{ matrix.step_name }}-reports + path: surefire-reports + retention-days: 7 + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: ./dev/ci-tool print_thread_dumps + + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + timeout-minutes: 75 + needs: [ 'build-and-license-check' ] + if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Clean Disk + uses: ./.github/actions/clean-disk + with: + mode: full + + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/bookkeeper + !~/.m2/repository/org/apache/distributedlog + key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: Pick ubuntu mirror for the docker image build + run: | + # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable + $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror + + - name: Build with Maven + run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Run metadata driver tests + run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Run all integration tests (except backward compatibility tests) + run: | + mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO -DredirectTestOutputToFile=false -DtestRetryCount=0 + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: ./dev/ci-tool print_thread_dumps + + - name: Upload container logs on failure + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + continue-on-error: true + with: + retention-days: 7 + name: integration-tests-container-logs + if-no-files-found: ignore + path: | + **/docker.log + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'surefire-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: failure() + continue-on-error: true + with: + name: integration-tests-reports + path: surefire-reports + if-no-files-found: ignore + retention-days: 7 + + backward-compatibility-tests: + name: Backward compatibility tests + runs-on: ubuntu-latest + timeout-minutes: 75 + needs: [ 'build-and-license-check' ] + if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Clean Disk + uses: ./.github/actions/clean-disk + with: + mode: full + + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/bookkeeper + !~/.m2/repository/org/apache/distributedlog + key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} + + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 8 + + - name: Pick ubuntu mirror for the docker image build + run: | + # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable + $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror + + - name: Build with Maven + run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: Test current server with old clients + run: mvn -B -nsu -DbackwardCompatTests -pl :backward-compat-current-server-old-clients test + + - name: Test progressive upgrade + run: mvn -B -nsu -DbackwardCompatTests -pl :upgrade test + + - name: Other tests + run: | + mvn -B -nsu -DbackwardCompatTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct,:yahoo-custom-version test + + - name: Upload container logs on failure + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + continue-on-error: true + with: + retention-days: 7 + name: backward-compatibility-tests-container-logs + if-no-files-found: ignore + path: | + **/docker.log + + - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories + if: ${{ always() }} + uses: ./.github/actions/copy-test-reports + + - name: Publish Test Report + uses: apache/pulsar-test-infra/action-junit-report@master + if: ${{ always() }} + with: + report_paths: 'surefire-reports/TEST-*.xml' + annotate_only: 'true' + + - name: Upload Surefire reports + uses: actions/upload-artifact@v4 + if: failure() + continue-on-error: true + with: + name: backward-compatibility-tests-reports + path: surefire-reports + if-no-files-found: ignore + retention-days: 7 + + macos-build: + name: Build with macos on JDK 11 + runs-on: macos-latest + timeout-minutes: 30 + needs: [ 'build-and-license-check' ] + if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/bookkeeper + !~/.m2/repository/org/apache/distributedlog + key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + + - name: mvn package + run: mvn -B -nsu clean package -DskipTests + + jdk-compatibility-checks: + name: ${{ matrix.step_name }} + runs-on: ubuntu-latest + timeout-minutes: ${{ matrix.timeout || 60 }} + needs: [ 'build-and-license-check' ] + if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }} + strategy: + fail-fast: false + matrix: + include: + - step_name: Compatibility Check Java8 + jdk_version: 8 + - step_name: Compatibility Check Java11 + jdk_version: 11 + - step_name: Compatibility Check Java17 + jdk_version: 17 + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/bookkeeper + !~/.m2/repository/org/apache/distributedlog + key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} + + - name: Set up JDK ${{ matrix.jdk_version }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk_version }} + + - name: Build with Maven + run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + + - name: print JVM thread dumps when cancelled + if: cancelled() + run: ./dev/ci-tool print_thread_dumps + + owasp-dependency-check: + name: OWASP Dependency Check + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: [ 'build-and-license-check' ] + if: ${{ needs.build-and-license-check.outputs.need_owasp == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/bookkeeper + !~/.m2/repository/org/apache/distributedlog + key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }} + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + + - name: run "clean install verify" to trigger dependency check + # excluding dlfs because it includes hadoop lib with + # CVEs that we cannot patch up anyways + run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs' + + - name: Upload report + uses: actions/upload-artifact@v4 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: dependency report + path: target/dependency-check-report.html + retention-days: 7 diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml deleted file mode 100644 index f2ae745eb0b..00000000000 --- a/.github/workflows/bookie-tests.yml +++ /dev/null @@ -1,78 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Bookie Tests - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: Maven build bookkeeper-server - run: mvn -B -nsu -am -pl bookkeeper-server clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Run bookie tests - run: mvn -B -nsu -pl bookkeeper-server test -Dtest="org.apache.bookkeeper.bookie.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Run shell tests - run: mvn -B -nsu -pl tests/scripts install -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure - if: failure() - continue-on-error: true - uses: ./.github/actions/copy-test-reports - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: bookie-tests-reports - path: surefire-reports - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml deleted file mode 100644 index 5feb1468337..00000000000 --- a/.github/workflows/client-tests.yml +++ /dev/null @@ -1,71 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Client Tests - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - name: Run client tests - run: mvn -B -am -nsu -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.client.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure - if: failure() - continue-on-error: true - uses: ./.github/actions/copy-test-reports - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: client-tests-reports - path: surefire-reports - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml deleted file mode 100644 index 3ce6a888967..00000000000 --- a/.github/workflows/compatibility-check-java11.yml +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Compatibility Check Java11 - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - check: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - name: Build with Maven - run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml deleted file mode 100644 index 3e18450e621..00000000000 --- a/.github/workflows/compatibility-check-java8.yml +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Compatibility Check Java8 - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - check: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build with Maven - run: mvn clean package spotbugs:check -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 8fb3d5c517c..00000000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Integration Tests - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: Build with Maven - run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Run metadata driver tests - run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Run all integration tests - run: mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/owasp-dep-check.yml b/.github/workflows/owasp-dep-check.yml deleted file mode 100644 index d61b019ae92..00000000000 --- a/.github/workflows/owasp-dep-check.yml +++ /dev/null @@ -1,75 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: OWASP Dependency Check - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - check: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed pom files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: | - poms: - - 'pom.xml' - - '**/pom.xml' - - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - if: ${{ steps.changes.outputs.poms == 'true' }} - with: - java-version: 1.8 - - - name: run "clean install verify" to trigger dependency check - if: ${{ steps.changes.outputs.poms == 'true' }} - # excluding dlfs because it includes hadoop lib with - # CVEs that we cannot patch up anyways - run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs' - - - name: Upload report - uses: actions/upload-artifact@v3 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml deleted file mode 100644 index 56c9a8e92fa..00000000000 --- a/.github/workflows/pr-validation.yml +++ /dev/null @@ -1,60 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: PR Validation - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - check: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - name: Validate pull request - run: mvn clean -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Check license files - run: dev/check-all-licenses - - - name: Generate Javadoc - run: mvn clean -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider install javadoc:aggregate -DskipTests -Pdelombok diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml deleted file mode 100644 index d7eb6562a0b..00000000000 --- a/.github/workflows/remaining-tests.yml +++ /dev/null @@ -1,71 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Remaining Tests - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - name: Run remaining tests - run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure - if: failure() - continue-on-error: true - uses: ./.github/actions/copy-test-reports - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: remaining-tests-reports - path: surefire-reports - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml deleted file mode 100644 index e8997bc33c6..00000000000 --- a/.github/workflows/replication-tests.yml +++ /dev/null @@ -1,71 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Replication Tests - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - name: Run replication tests - run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.replication.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure - if: failure() - continue-on-error: true - uses: ./.github/actions/copy-test-reports - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: replication-tests-reports - path: surefire-reports - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml deleted file mode 100644 index d8bbe3b49fc..00000000000 --- a/.github/workflows/stream-tests.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: StreamStorage Tests - -on: - push: - pull_request: - branches: - - master - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - name: Build with Maven - run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - name: Run StreamStorage tests - run: mvn -B -nsu -f stream/pom.xml verify -DstreamTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure - if: failure() - continue-on-error: true - uses: ./.github/actions/copy-test-reports - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: stream-tests-reports - path: surefire-reports - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps - diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml deleted file mode 100644 index 203e5ce0d00..00000000000 --- a/.github/workflows/tls-tests.yml +++ /dev/null @@ -1,71 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: TLS Tests - -on: - push: - pull_request: - branches: - - master - - branch-* - paths-ignore: - - 'site/**' - - 'site3/**' - workflow_dispatch: - -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - test: - - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - name: Run tls tests - run: mvn -B -am -nsu -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure - if: failure() - continue-on-error: true - uses: ./.github/actions/copy-test-reports - - - name: Upload Surefire reports - uses: actions/upload-artifact@v3 - if: failure() - continue-on-error: true - with: - name: tls-tests-reports - path: surefire-reports - - - name: print JVM thread dumps when cancelled - if: cancelled() - run: ./dev/ci-tool print_thread_dumps diff --git a/.github/workflows/website-deploy.yaml b/.github/workflows/website-deploy.yaml deleted file mode 100644 index 6506550c799..00000000000 --- a/.github/workflows/website-deploy.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Website deploy -on: - workflow_dispatch: - push: - branches: - - master - paths: - - 'site3/**' - - '.github/workflows/website-deploy.yaml' - -env: - DEPLOY_URL: "https://bookkeeper.apache.org/" - -jobs: - build-website: - name: Build and deploy the website - if: ${{ github.repository == 'apache/bookkeeper' }} - runs-on: ubuntu-latest - timeout-minutes: 180 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: Setup NodeJS - uses: actions/setup-node@v2 - with: - node-version: '14' - - - name: Setup yarn - run: npm install -g yarn - - - name: Publish - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - run: | - ./site3/website/scripts/build-website.sh - ./site3/website/scripts/publish-website.sh \ No newline at end of file diff --git a/.github/workflows/website-pr-validation.yml b/.github/workflows/website-pr-validation.yml deleted file mode 100644 index 1f19cfcc3ed..00000000000 --- a/.github/workflows/website-pr-validation.yml +++ /dev/null @@ -1,56 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Website PR validation - -on: - pull_request: - branches: - - master - - branch-* - paths: - - 'site3/**' - - '.github/workflows/website-pr-validation.yaml' - workflow_dispatch: - -jobs: - website-pull-validation: - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: Setup NodeJS - uses: actions/setup-node@v2 - with: - node-version: '14' - - - name: Setup yarn - run: npm install -g yarn - - - name: Build website - run: | - ./site3/website/scripts/build-website.sh diff --git a/bin/bookkeeper-daemon.sh b/bin/bookkeeper-daemon.sh index 621aa18e6a8..4e85a50ed64 100755 --- a/bin/bookkeeper-daemon.sh +++ b/bin/bookkeeper-daemon.sh @@ -108,75 +108,111 @@ rotate_out_log () mkdir -p "$BOOKIE_LOG_DIR" -case $startStop in - (start) - if [ -f $pid_file ]; then - PREVIOUS_PID=$(cat $pid_file) - if kill -0 $PREVIOUS_PID > /dev/null 2>&1; then - echo $command running as process $PREVIOUS_PID. Stop it first. - exit 1 - fi - fi - - rotate_out_log $out - echo starting $command, logging to $logfile - bookkeeper=$BK_HOME/bin/bookkeeper - nohup $bookkeeper $command "$@" > "$out" 2>&1 < /dev/null & - echo $! > $pid_file - sleep 1; head $out - sleep 2; - if ! kill -0 $! > /dev/null ; then +start() +{ + if [ -f $pid_file ]; then + PREVIOUS_PID=$(cat $pid_file) + if kill -0 $PREVIOUS_PID > /dev/null 2>&1; then + echo $command running as process $PREVIOUS_PID. Stop it first. exit 1 fi - ;; + fi - (stop) - if [ -f $pid_file ]; then - TARGET_PID=$(cat $pid_file) - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo stopping $command - kill $TARGET_PID - - count=0 - location=$BOOKIE_LOG_DIR - while kill -0 $TARGET_PID > /dev/null 2>&1; - do - echo "Shutdown is in progress... Please wait..." - sleep 1 - count=$(expr $count + 1) - - if [ "$count" = "$BOOKIE_STOP_TIMEOUT" ]; then - break - fi - done + rotate_out_log $out + echo starting $command, logging to $logfile + bookkeeper=$BK_HOME/bin/bookkeeper + nohup $bookkeeper $command "$@" > "$out" 2>&1 < /dev/null & + echo $! > $pid_file + sleep 1; head $out + sleep 2; + if ! kill -0 $! > /dev/null ; then + exit 1 + fi +} - if [ "$count" != "$BOOKIE_STOP_TIMEOUT" ]; then - echo "Shutdown completed." +stop() +{ + if [ -f $pid_file ]; then + TARGET_PID=$(cat $pid_file) + if kill -0 $TARGET_PID > /dev/null 2>&1; then + echo stopping $command + kill $TARGET_PID + + count=0 + location=$BOOKIE_LOG_DIR + while kill -0 $TARGET_PID > /dev/null 2>&1; + do + echo "Shutdown is in progress... Please wait..." + sleep 1 + count=$(expr $count + 1) + + if [ "$count" = "$BOOKIE_STOP_TIMEOUT" ]; then + break fi + done - if kill -0 $TARGET_PID > /dev/null 2>&1; then - fileName=$location/$command.out - $JAVA_HOME/bin/jstack $TARGET_PID > $fileName - echo Thread dumps are taken for analysis at $fileName - if [ "$1" == "-force" ] - then - echo forcefully stopping $command - kill -9 $TARGET_PID >/dev/null 2>&1 - echo Successfully stopped the process - else - echo "WARNNING : Bookie Server is not stopped completely." + if [ "$count" != "$BOOKIE_STOP_TIMEOUT" ]; then + echo "Shutdown completed." + fi + + if kill -0 $TARGET_PID > /dev/null 2>&1; then + fileName=$location/$command.out + # Check for the java to use + if [[ -z ${JAVA_HOME} ]]; then + JSTACK=$(which jstack) + if [ $? -ne 0 ]; then + echo "Error: JAVA_HOME not set, and no jstack executable found in $PATH." 1>&2 exit 1 fi + else + JSTACK=${JAVA_HOME}/bin/jstack + fi + $JSTACK $TARGET_PID > $fileName + echo Thread dumps are taken for analysis at $fileName + if [ "$1" == "-force" ] + then + echo forcefully stopping $command + kill -9 $TARGET_PID >/dev/null 2>&1 + echo Successfully stopped the process + else + echo "WARNNING : Bookie Server is not stopped completely." + exit 1 fi - else - echo no $command to stop fi - rm $pid_file else echo no $command to stop fi + rm $pid_file + else + echo no $command to stop + fi +} +case $startStop in + (start) + start "$*" ;; + (stop) + stop $1 + ;; + (restart) + forceStopFlag=$(echo "$*"|grep "\-force") + if [[ "$forceStopFlag" != "" ]] + then + stop "-force" + else + stop + fi + if [ "$?" == 0 ] + then + sleep 3 + paramaters="$*" + startParamaters=${paramaters//-force/} + start "$startParamaters" + else + echo "WARNNING : $command failed restart, for $command is not stopped completely." + fi + ;; (*) usage echo $supportedargs diff --git a/bin/common.sh b/bin/common.sh index ff5966143be..508a8921296 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -51,9 +51,7 @@ fi # Check for the java to use if [[ -z ${JAVA_HOME} ]]; then JAVA=$(which java) - if [ $? = 0 ]; then - echo "JAVA_HOME not set, using java from PATH. ($JAVA)" - else + if [ $? != 0 ]; then echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 exit 1 fi @@ -71,13 +69,12 @@ source ${BK_CONFDIR}/bkenv.sh source ${BK_CONFDIR}/bk_cli_env.sh detect_jdk8() { - - if [ -f "$JAVA_HOME/lib/modules" ]; then + local is_java_8=$($JAVA -version 2>&1 | grep version | grep '"1\.8') + if [ -z "$is_java_8" ]; then echo "0" else echo "1" fi - return } # default netty settings @@ -333,7 +330,7 @@ find_table_service() { TABLE_SERVICE_RELEASED="false" fi fi - + # check the configuration to see if table service is enabled or not. if [ -z "${ENABLE_TABLE_SERVICE}" ]; then # mask exit code if the configuration file doesn't contain `StreamStorageLifecycleComponent` @@ -346,7 +343,7 @@ find_table_service() { ENABLE_TABLE_SERVICE="true" fi fi - + # standalone only run if [ \( "x${SERVICE_COMMAND}" == "xstandalone" \) -a \( "x${TABLE_SERVICE_RELEASED}" == "xfalse" \) ]; then echo "The release binary is built without table service. Use \`localbookie \` instead of \`standalone\` for local development." diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml index d28a93425cb..2d0bc645e08 100644 --- a/bookkeeper-common/pom.xml +++ b/bookkeeper-common/pom.xml @@ -115,12 +115,13 @@ org.apache.maven.plugins maven-javadoc-plugin - ${maven-javadoc-plugin.version} ${src.dir} none + Bookkeeper Client diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 85d8a507577..c83516a4de2 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -251,7 +251,7 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] -- lib/net.java.dev.jna-jna-3.2.7.jar [18] +- lib/net.java.dev.jna-jna-5.13.0.jar [18] - lib/org.apache.commons-commons-collections4-4.1.jar [19] - lib/org.apache.commons-commons-lang3-3.6.jar [20] - lib/org.apache.zookeeper-zookeeper-3.8.0.jar [21] @@ -292,7 +292,6 @@ Apache Software License, Version 2. - lib/org.apache.curator-curator-client-5.1.0.jar [34] - lib/org.apache.curator-curator-framework-5.1.0.jar [34] - lib/org.apache.curator-curator-recipes-5.1.0.jar [34] -- lib/org.inferred-freebuilder-2.7.0.jar [35] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [36] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [37] - lib/org.jctools-jctools-core-2.1.2.jar [38] @@ -332,7 +331,7 @@ Apache Software License, Version 2. [15] Source available at https://github.com/eclipse/vert.x/tree/3.9.8 [16] Source available at https://github.com/vert-x3/vertx-web/tree/3.9.8 [17] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 -[18] Source available at https://github.com/java-native-access/jna/tree/3.2.7 +[18] Source available at https://github.com/java-native-access/jna/tree/5.13.0 [19] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 @@ -346,7 +345,6 @@ Apache Software License, Version 2. [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 [34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.1.0 -[35] Source available at https://github.com/inferred/FreeBuilder/tree/v2.7.0 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [38] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index d4b1c8fe14d..e830056e73e 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -240,7 +240,7 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16] -- lib/net.java.dev.jna-jna-3.2.7.jar [17] +- lib/net.java.dev.jna-jna-5.13.0.jar [17] - lib/org.apache.commons-commons-collections4-4.1.jar [18] - lib/org.apache.commons-commons-lang3-3.6.jar [19] - lib/org.apache.zookeeper-zookeeper-3.8.0.jar [20] @@ -271,7 +271,6 @@ Apache Software License, Version 2. - lib/org.apache.curator-curator-client-5.1.0.jar [33] - lib/org.apache.curator-curator-framework-5.1.0.jar [33] - lib/org.apache.curator-curator-recipes-5.1.0.jar [33] -- lib/org.inferred-freebuilder-2.7.0.jar [34] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [35] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [36] - lib/org.jctools-jctools-core-2.1.2.jar [37] @@ -302,7 +301,7 @@ Apache Software License, Version 2. [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 [11] Source available at https://github.com/netty/netty/tree/netty-4.1.75.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 -[17] Source available at https://github.com/java-native-access/jna/tree/3.2.7 +[17] Source available at https://github.com/java-native-access/jna/tree/5.13.0 [18] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [20] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 @@ -313,7 +312,6 @@ Apache Software License, Version 2. [29] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [32] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 [33] Source available at https://github.com/apache/curator/tree/apache-curator-5.1.0 -[34] Source available at https://github.com/inferred/FreeBuilder/tree/v2.7.0 [35] Source available at https://github.com/google/error-prone/tree/v2.9.0 [36] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [37] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 85ce30d4217..61e4771e773 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -251,7 +251,7 @@ Apache Software License, Version 2. - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17] - lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17] -- lib/net.java.dev.jna-jna-3.2.7.jar [18] +- lib/net.java.dev.jna-jna-5.13.0.jar [18] - lib/org.apache.commons-commons-collections4-4.1.jar [19] - lib/org.apache.commons-commons-lang3-3.6.jar [20] - lib/org.apache.zookeeper-zookeeper-3.8.0.jar [21] @@ -292,7 +292,6 @@ Apache Software License, Version 2. - lib/org.apache.curator-curator-client-5.1.0.jar [34] - lib/org.apache.curator-curator-framework-5.1.0.jar [34] - lib/org.apache.curator-curator-recipes-5.1.0.jar [34] -- lib/org.inferred-freebuilder-2.7.0.jar [35] - lib/com.google.errorprone-error_prone_annotations-2.9.0.jar [36] - lib/org.apache.yetus-audience-annotations-0.12.0.jar [37] - lib/org.jctools-jctools-core-2.1.2.jar [38] @@ -328,7 +327,7 @@ Apache Software License, Version 2. [15] Source available at https://github.com/eclipse/vert.x/tree/3.9.8 [16] Source available at https://github.com/vert-x3/vertx-web/tree/3.9.8 [17] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.18.0 -[18] Source available at https://github.com/java-native-access/jna/tree/3.2.7 +[18] Source available at https://github.com/java-native-access/jna/tree/5.13.0 [19] Source available at https://github.com/apache/commons-collections/tree/collections-4.1 [20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6 [21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0 @@ -342,7 +341,6 @@ Apache Software License, Version 2. [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.28.0 [33] Source available at https://github.com/grpc/grpc-java/tree/v1.56.0 [34] Source available at https://github.com/apache/curator/releases/tag/apache.curator-5.1.0 -[35] Source available at https://github.com/inferred/FreeBuilder/tree/v2.7.0 [36] Source available at https://github.com/google/error-prone/tree/v2.9.0 [37] Source available at https://github.com/apache/yetus/tree/rel/0.12.0 [38] Source available at https://github.com/JCTools/JCTools/tree/v2.1.2 diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml index 58728088e08..8aba8e14d8b 100644 --- a/bookkeeper-server/pom.xml +++ b/bookkeeper-server/pom.xml @@ -251,7 +251,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} false @@ -265,12 +264,13 @@ org.apache.maven.plugins maven-javadoc-plugin - ${maven-javadoc-plugin.version} ${src.dir} none + Bookkeeper diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBase.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBase.java index 58f26bf36d3..3bf7bdd767d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBase.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBase.java @@ -65,6 +65,7 @@ public class ZKMetadataDriverBase implements AutoCloseable { protected static final String SCHEME = "zk"; + private static final int ZK_CLIENT_WAIT_FOR_SHUTDOWN_TIMEOUT_MS = 5000; public static String getZKServersFromServiceUri(URI uri) { String authority = uri.getAuthority(); @@ -338,7 +339,7 @@ public void close() { } if (ownZKHandle && null != zk) { try { - zk.close(); + zk.close(ZK_CLIENT_WAIT_FOR_SHUTDOWN_TIMEOUT_MS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); log.warn("Interrupted on closing zookeeper client", e); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBaseTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBaseTest.java index 2386a7fe062..f2d9a7c99c8 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBaseTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBaseTest.java @@ -88,7 +88,7 @@ public void testInitialize() throws Exception { driver.close(); - verify(mockZkc, times(1)).close(); + verify(mockZkc, times(1)).close(5000); assertNull(driver.zk); } diff --git a/bookkeeper-stats/pom.xml b/bookkeeper-stats/pom.xml index 8d0263890a4..aad44eeb233 100644 --- a/bookkeeper-stats/pom.xml +++ b/bookkeeper-stats/pom.xml @@ -33,12 +33,13 @@ org.apache.maven.plugins maven-javadoc-plugin - ${maven-javadoc-plugin.version} ${src.dir} none + Bookkeeper Stats API diff --git a/buildtools/src/main/resources/bookkeeper/checkstyle.xml b/buildtools/src/main/resources/bookkeeper/checkstyle.xml index fda4fb619d6..0807c63061c 100644 --- a/buildtools/src/main/resources/bookkeeper/checkstyle.xml +++ b/buildtools/src/main/resources/bookkeeper/checkstyle.xml @@ -29,6 +29,21 @@ page at http://checkstyle.sourceforge.net/config.html --> + + + + + + + + + + @@ -53,12 +68,6 @@ page at http://checkstyle.sourceforge.net/config.html --> - - - - - - @@ -69,6 +78,13 @@ page at http://checkstyle.sourceforge.net/config.html --> + + + + + + + @@ -88,9 +104,10 @@ page at http://checkstyle.sourceforge.net/config.html --> - + + See: http://checkstyle.sourceforge.net/config_modifier.html#RedundantModifier --> + - + @@ -140,23 +158,15 @@ page at http://checkstyle.sourceforge.net/config.html --> JAVADOC CHECKS --> - - + - - - - - - - @@ -275,25 +285,9 @@ page at http://checkstyle.sourceforge.net/config.html --> - - - - - - - - - - - @@ -419,6 +413,7 @@ page at http://checkstyle.sourceforge.net/config.html --> + @@ -434,9 +429,6 @@ page at http://checkstyle.sourceforge.net/config.html --> - - - diff --git a/circe-checksum/pom.xml b/circe-checksum/pom.xml index 8ff9f3d8e74..a6f300b0f1f 100644 --- a/circe-checksum/pom.xml +++ b/circe-checksum/pom.xml @@ -63,25 +63,32 @@ org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - com.github.maven-nar nar-maven-plugin - ${nar-maven-plugin.version} true + + ${nar.runtime} + circe-checksum + + + jni + com.scurrilous.circe.checksum + + + + ${nar.cpp.optionSet} + false + false + full + + **/*.cpp + + + + true + org.apache.maven.plugins @@ -104,17 +111,6 @@ - - org.jacoco - jacoco-maven-plugin - ${jacoco-maven-plugin.version} - - - - com/scurrilous/circe/checksum/NarSystem* - - - @@ -132,7 +128,6 @@ com.github.maven-nar nar-maven-plugin - ${nar-maven-plugin.version} true @@ -146,17 +141,8 @@ org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} - 1.8 - 1.8 - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always -h ${project.build.directory}/nar/javah-include @@ -178,23 +164,9 @@ com.github.maven-nar nar-maven-plugin - ${nar-maven-plugin.version} true - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - + false @@ -212,23 +184,9 @@ com.github.maven-nar nar-maven-plugin - ${nar-maven-plugin.version} true - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - + false rt diff --git a/cpu-affinity/pom.xml b/cpu-affinity/pom.xml index 6ab40e3314e..ae1dd61f458 100644 --- a/cpu-affinity/pom.xml +++ b/cpu-affinity/pom.xml @@ -48,18 +48,6 @@ org.apache.maven.plugins maven-compiler-plugin - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - com.github.maven-nar @@ -129,17 +117,8 @@ org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} - 1.8 - 1.8 - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always -h ${project.build.directory}/nar/javah-include diff --git a/dependencies.gradle b/dependencies.gradle index 916fe31e25e..269069c82dc 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -60,7 +60,7 @@ depVersions = [ jetty: "9.4.46.v20220331", jmh: "1.19", jmock: "2.8.2", - jna: "3.2.7", + jna: "5.13.0", jsr305: "3.0.2", junit: "4.12", junitFoundation: "11.0.0", diff --git a/dev/ci-tool b/dev/ci-tool index 08bc2e1386c..0f26955f643 100755 --- a/dev/ci-tool +++ b/dev/ci-tool @@ -80,6 +80,45 @@ function ci_move_test_reports() { ) } +# Finds fastest up-to-date ubuntu mirror based on download speed +function ci_find_fast_ubuntu_mirror() { + local ubuntu_release=${1:-"$(lsb_release -c 2>/dev/null | cut -f2 || echo "jammy")"} + local ubuntu_arch=${2:-"$(dpkg --print-architecture 2>/dev/null || echo "amd64")"} + { + # choose mirrors that are up-to-date by checking the Last-Modified header for + { + # randomly choose up to 10 mirrors using http:// protocol + # (https isn't supported in docker containers that don't have ca-certificates installed) + curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10 + # also consider Azure's Ubuntu mirror + echo http://azure.archive.ubuntu.com/ubuntu/ + } | xargs -I {} sh -c "ubuntu_release=$ubuntu_release ubuntu_arch=$ubuntu_arch;"'echo "$(curl -m 5 -sI {}dists/${ubuntu_release}/Contents-${ubuntu_arch}.gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }' + } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \ + |sort -g -r |head -1| awk '{ print $2 }' +} + +function ci_pick_ubuntu_mirror() { + echo "Choosing fastest up-to-date ubuntu mirror based on download speed..." + UBUNTU_MIRROR=$(ci_find_fast_ubuntu_mirror) + if [ -z "$UBUNTU_MIRROR" ]; then + # fallback to no mirror + UBUNTU_MIRROR="http://archive.ubuntu.com/ubuntu/" + UBUNTU_SECURITY_MIRROR="http://security.ubuntu.com/ubuntu/" + else + UBUNTU_SECURITY_MIRROR="${UBUNTU_MIRROR}" + fi + echo "Picked '$UBUNTU_MIRROR'." + # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables + # that can be used by docker builds + export UBUNTU_MIRROR + export UBUNTU_SECURITY_MIRROR + # make environment variables available for later GitHub Actions steps + if [ -n "$GITHUB_ENV" ]; then + echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV + echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV + fi +} + if [ -z "$1" ]; then echo "usage: $0 [ci_tool_function_name]" echo "Available ci tool functions:" diff --git a/docker/Dockerfile b/docker/Dockerfile index d72e0f37df4..87b8899b39c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,10 +17,10 @@ # under the License. # -FROM centos:7 +FROM ubuntu:22.04 MAINTAINER Apache BookKeeper -ARG BK_VERSION=4.12.1 +ARG BK_VERSION=4.15.1 ARG DISTRO_NAME=bookkeeper-server-${BK_VERSION}-bin ARG DISTRO_URL=https://archive.apache.org/dist/bookkeeper/bookkeeper-${BK_VERSION}/${DISTRO_NAME}.tar.gz @@ -29,12 +29,26 @@ ENV BOOKIE_HTTP_PORT=8080 EXPOSE $BOOKIE_PORT $BOOKIE_HTTP_PORT ENV BK_USER=bookkeeper ENV BK_HOME=/opt/bookkeeper -ENV JAVA_HOME=/usr/lib/jvm/java-11 +ENV DEBIAN_FRONTEND=noninteractive +ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/ +ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ # Download Apache Bookkeeper, untar and clean up RUN set -x \ + && sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \ + -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ + && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ && adduser "${BK_USER}" \ - && yum install -y java-11-openjdk-devel wget bash python3 sudo\ + && apt-get update \ + && apt-get install -y ca-certificates apt-transport-https \ + && apt-get install -y --no-install-recommends openjdk-17-jdk \ + && apt-get install -y --no-install-recommends python3 pip \ + && ln -s /usr/bin/python3 /usr/bin/python \ + && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ && mkdir -pv /opt \ && cd /opt \ && wget -q "${DISTRO_URL}" \ @@ -47,15 +61,10 @@ RUN set -x \ && tar -xzf "$DISTRO_NAME.tar.gz" \ && mv bookkeeper-server-${BK_VERSION}/ /opt/bookkeeper/ \ && rm -rf "$DISTRO_NAME.tar.gz" "$DISTRO_NAME.tar.gz.asc" "$DISTRO_NAME.tar.gz.sha512" \ - # install zookeeper shell - && wget -q https://bootstrap.pypa.io/pip/2.7/get-pip.py \ - && python --version \ - && python get-pip.py \ && pip install zk-shell \ - && rm -rf get-pip.py \ - && yum remove -y wget \ - && yum clean all \ - && ls /usr/lib/jvm + && JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) \ + && echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security \ + && echo networkaddress.cache.negative.ttl=1 >> $JAVA_HOME/conf/security/java.security WORKDIR /opt/bookkeeper diff --git a/docker/README.md b/docker/README.md index 4bb3c782d86..cbb874d6085 100644 --- a/docker/README.md +++ b/docker/README.md @@ -13,7 +13,6 @@ Bookkeeper needs [Zookeeper](https://zookeeper.apache.org/) in order to preserve Just like running a BookKeeper cluster in one machine(http://bookkeeper.apache.org/docs/latest/getting-started/run-locally/), you can run a standalone BookKeeper in one docker container, the command is: ``` docker run -it \ - --env JAVA_HOME=/usr/lib/jvm/java-11 \ --entrypoint "/bin/bash" \ apache/bookkeeper \ -c "/opt/bookkeeper/bin/bookkeeper localbookie 3" diff --git a/docker/scripts/apply-config-from-env.py b/docker/scripts/apply-config-from-env.py index 3fccab98164..7b74b503411 100755 --- a/docker/scripts/apply-config-from-env.py +++ b/docker/scripts/apply-config-from-env.py @@ -23,23 +23,30 @@ ## based on the ENV variables ## export my-key=new-value ## -## ./apply-config-from-env config_dir +## ./apply-config-from-env file ... ## import os, sys -if len(sys.argv) != 2: - print 'Usage: %s ' + 'config_dir' % (sys.argv[0]) +if len(sys.argv) < 2: + print('Usage: %s file ...' % (sys.argv[0])) sys.exit(1) -def mylistdir(dir): - return [os.path.join(dir, filename) for filename in os.listdir(dir)] +def prepare_conf_files(files): + conf_files = [] + for f in files: + if os.path.isfile(f): + if not os.path.isabs(f): + f = os.path.join(os.getcwd(), f) + conf_files.append(f) + else: + print('%s is not a readable file' % f) + sys.exit(1) + return conf_files -# Always apply env config to all the files under conf -conf_dir = sys.argv[1] -conf_files = mylistdir(conf_dir) -print 'conf files: ' -print conf_files +conf_files = prepare_conf_files(sys.argv[1:]) +print('conf files: ') +print(conf_files) bk_env_prefix = 'BK_' zk_env_prefix = 'ZK_' @@ -75,13 +82,13 @@ def mylistdir(dir): if k.startswith(bk_env_prefix): search_key = k[len(bk_env_prefix):] if search_key in keys: - print '[%s] Applying config %s = %s' % (conf_filename, search_key, v) + print('[%s] Applying config %s = %s' % (conf_filename, search_key, v)) idx = keys[search_key] lines[idx] = '%s=%s\n' % (search_key, v) if k.startswith(zk_env_prefix): search_key = k[len(zk_env_prefix):] if search_key in keys: - print '[%s] Applying config %s = %s' % (conf_filename, search_key, v) + print('[%s] Applying config %s = %s' % (conf_filename, search_key, v)) idx = keys[search_key] lines[idx] = '%s=%s\n' % (search_key, v) diff --git a/docker/scripts/common.sh b/docker/scripts/common.sh index 5bbcd208072..0f745db4b45 100755 --- a/docker/scripts/common.sh +++ b/docker/scripts/common.sh @@ -71,7 +71,7 @@ echo " BK_STREAM_STORAGE_ROOT_PATH is ${BK_STREAM_STORAGE_ROOT_PATH}" echo " BK_NUM_STORAGE_CONTAINERS is ${BK_NUM_STORAGE_CONTAINERS}" echo " BOOKIE_GRPC_PORT is ${BOOKIE_GRPC_PORT}" -python scripts/apply-config-from-env.py ${BK_HOME}/conf +python scripts/apply-config-from-env.py ${BK_HOME}/conf/*.conf export BOOKIE_CONF=${BK_HOME}/conf/bk_server.conf export SERVICE_PORT=${PORT0} diff --git a/docker/scripts/init_zookeeper.sh b/docker/scripts/init_zookeeper.sh index 803ef91d786..cff981211c8 100755 --- a/docker/scripts/init_zookeeper.sh +++ b/docker/scripts/init_zookeeper.sh @@ -63,7 +63,7 @@ function create_zk_dynamic_conf() { function init_zookeeper() { # apply zookeeper envs - python scripts/apply-config-from-env.py ${BK_HOME}/conf + python scripts/apply-config-from-env.py ${BK_HOME}/conf/zookeeper.conf # create dirs if they don't exist create_zk_dirs diff --git a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java index bbc50ae62c9..e1391a68e02 100644 --- a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java +++ b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java @@ -42,8 +42,8 @@ import org.apache.bookkeeper.versioning.Version.Occurred; import org.apache.bookkeeper.versioning.Versioned; import org.apache.commons.compress.utils.Sets; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.Test; -import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils; /** * Integration test {@link KeySetReader}. diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml index 29e81eb7d77..5bcd1f71046 100644 --- a/microbenchmarks/pom.xml +++ b/microbenchmarks/pom.xml @@ -16,7 +16,7 @@ limitations under the License. --> - 4.0.0 + 4.0.0 org.apache.bookkeeper bookkeeper @@ -58,14 +58,24 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} - ${javac.target} - ${javac.target} - ${javac.target} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + @@ -82,7 +92,10 @@ ${uberjar.name} - org.openjdk.jmh.Main + + org.openjdk.jmh.Main + true + @@ -108,6 +121,13 @@ true + + com.github.spotbugs + spotbugs-maven-plugin + + true + + diff --git a/native-io/pom.xml b/native-io/pom.xml index e36aa1b7c0c..4146c29e6cf 100644 --- a/native-io/pom.xml +++ b/native-io/pom.xml @@ -30,6 +30,7 @@ dynamic + -msse4.2 -mpclmul @@ -48,18 +49,6 @@ org.apache.maven.plugins maven-compiler-plugin - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - com.github.maven-nar @@ -123,17 +112,8 @@ org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} - 1.8 - 1.8 - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always -h ${project.build.directory}/nar/javah-include diff --git a/pom.xml b/pom.xml index 946e7b0904f..fa455bd8728 100644 --- a/pom.xml +++ b/pom.xml @@ -108,9 +108,10 @@ + 8 + 8 UTF-8 UTF-8 - 1.8 true 2 src/main/java @@ -118,7 +119,7 @@ 1.18.2 - 1.6.0.Final + 1.8.0.Final 3.0.1 1.2 4.1 @@ -132,27 +133,28 @@ 5.1.0 4.1.12.1 0.5.11 - 2.7.0 + 2.8.0 3.0.2 2.9.0 1.54.1 32.0.1-jre 1.1.1 3.2.4 - 1.3 2.1.10 2.13.4.20221013 1.82 9.4.53.v20231009 - 1.19 + 1.37 2.8.2 1.14.3 - 3.2.7 - 4.12 - - 5.8.2 + 5.13.0 + 4.13.2 + 1.3 + 5.10.2 + 3.25.3 + 4.2.0 0.14.2 - 1.18.22 + 1.18.30 2.18.0 1.3.0 3.12.4 @@ -169,50 +171,50 @@ ${grpc.version} 0.9.11 6.29.4.1 - 3.0.1 + 3.3.0 1.7.32 1.32 4.6.0 1.3.2 - 1.15.1 + 1.19.4 3.9.8 3.8.0 1.1.10.1 2.1.2 - 0.12 + 0.16.1 2.7 4.3.0 1.4.13 1.6.8 - 1.6.0 - 1.13.1 + 3.0.0 1.6 0.8.0 1.18.20.0 1.8 - 3.1.0 - 3.0.0 - 2.5 - 3.8.1 - 3.0.2 - 2.7 - 2.5.1 - 2.4 - 3.1.1 - 3.2.0 - 2.2.1 - 3.0.0-M5 - 7.1.2 - 3.5.2 + 3.6.0 + 3.3.1 + 3.3.2 + 3.12.1 + 3.6.1 + 3.1.1 + 3.1.1 + 3.3.0 + 3.6.3 + 3.5.1 + 3.3.0 + 3.2.5 + 8.0.2 + 3.10.1 1.4.1.Final 0.6.1 - 6.19 - 4.6.0.0 + 9.3 + 4.7.3.2 1 4.0.0 3.0.1 - 4.0.3 + http://archive.ubuntu.com/ubuntu/ + http://security.ubuntu.com/ubuntu/ @@ -248,7 +250,6 @@ org.inferred freebuilder ${freebuilder.version} - true @@ -527,13 +528,6 @@ - - - org.junit.jupiter - junit-jupiter-api - ${junit5.version} - - org.xerial.snappy @@ -680,16 +674,44 @@ junit ${junit.version} + + org.junit + junit-bom + ${junit5.version} + pom + import + + + org.assertj + assertj-core + ${assertj-core.version} + org.hamcrest hamcrest-all ${hamcrest.version} + + org.hamcrest + hamcrest-core + ${hamcrest.version} + + + org.hamcrest + hamcrest-library + ${hamcrest.version} + org.jmock jmock ${jmock.version} + + org.awaitility + awaitility + ${awaitility.version} + test + org.mockito mockito-core @@ -726,6 +748,11 @@ + + javax.ws.rs + javax.ws.rs-api + 2.1.1 + org.jboss.arquillian.junit arquillian-junit-standalone @@ -737,11 +764,6 @@ - - org.jboss.arquillian.junit - arquillian-junit-container - ${arquillian-junit.version} - org.codehaus.groovy groovy-all @@ -760,20 +782,16 @@ org.testcontainers - testcontainers + testcontainers-bom ${testcontainers.version} + pom + import org.jsoup jsoup ${jsoup.version} - - org.awaitility - awaitility - ${awaitility.version} - test - @@ -825,6 +843,31 @@ junit test + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.assertj + assertj-core + test + org.hamcrest hamcrest-all @@ -872,7 +915,7 @@ buildtools/src/main/resources/bookkeeper/checkstyle.xml buildtools/src/main/resources/bookkeeper/suppressions.xml - UTF-8 + UTF-8 true true false @@ -888,6 +931,31 @@ + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + org.apache.maven.plugins + maven-failsafe-plugin + ${maven-surefire-plugin.version} + + + com.github.maven-nar + nar-maven-plugin + ${nar-maven-plugin.version} + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + true + none + + @@ -900,24 +968,28 @@ + org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} - ${javac.target} - ${javac.target} + UTF-8 + true + true + + + org.projectlombok + lombok + ${lombok.version} + + - -Werror - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - + -parameters + org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} ${redirectTestOutputToFile} @@ -928,15 +1000,19 @@ ${testRetryCount} + + org.apache.maven.plugins + maven-failsafe-plugin + org.apache.maven.plugins maven-javadoc-plugin - ${maven-javadoc-plugin.version} - -notimestamp - - none + true + Bookkeeper Client @@ -962,6 +1038,10 @@ site/_site/overview/index.html package false + + false @@ -992,7 +1072,10 @@ ${apache-rat-plugin.version} - + + dependency-reduced-pom.xml + + **/.idea/** @@ -1013,7 +1096,7 @@ **/src/main/resources/deps/** **/META-INF/** - + **/.classpath **/.project **/.checkstyle @@ -1021,6 +1104,8 @@ **/*.iml **/*.iws **/*.ipr + + **/.sdkmanrc .repository/** @@ -1036,6 +1121,9 @@ **/*.log + + **/*.json + data/** @@ -1055,7 +1143,6 @@ **/.pytest_cache/** **/__pycache__/** **/bookkeeper.egg-info/** - **/pip-selfcheck.json **/test_conf_2.conf @@ -1199,12 +1286,20 @@ ${project.basedir}/src/main/java ${project.build.directory}/generated-sources/delombok - + + + org.apache.maven.plugins + maven-javadoc-plugin + + + false + + @@ -1216,7 +1311,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} -Xmx2G -Djava.net.preferIPv4Stack=true false @@ -1237,9 +1331,8 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} - -Xmx2G -Djava.net.preferIPv4Stack=true -Dbookkeeper.log.root.level=INFO -Dbookkeeper.log.root.appender=CONSOLE + -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid -Dbookkeeper.log.root.level=INFO -Dbookkeeper.log.root.appender=CONSOLE false ${forkCount.variable} false @@ -1280,14 +1373,24 @@ [11,) + + + ${maven.compiler.target} + --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.nio.channels.spi=ALL-UNNAMED + --add-opens java.base/java.nio.file=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED --add-opens java.base/java.util.stream=ALL-UNNAMED + --add-opens java.base/java.util.zip=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens java.base/sun.net.dns=ALL-UNNAMED @@ -1296,6 +1399,20 @@ --add-opens java.xml/jdk.xml.internal=ALL-UNNAMED + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + ${maven.compiler.release} + + + + + apache-release @@ -1316,5 +1433,29 @@ + + ubuntu-mirror-set + + + env.UBUNTU_MIRROR + + + + + ${env.UBUNTU_MIRROR} + + + + ubuntu-security-mirror-set + + + env.UBUNTU_SECURITY_MIRROR + + + + + ${env.UBUNTU_SECURITY_MIRROR} + + diff --git a/shaded/bookkeeper-server-shaded/pom.xml b/shaded/bookkeeper-server-shaded/pom.xml index c2aa6dd23a6..bfc1fed478a 100644 --- a/shaded/bookkeeper-server-shaded/pom.xml +++ b/shaded/bookkeeper-server-shaded/pom.xml @@ -38,10 +38,18 @@ org.slf4j slf4j-log4j12 + + org.slf4j + slf4j-reload4j + log4j log4j + + ch.qos.reload4j + reload4j + diff --git a/shaded/bookkeeper-server-tests-shaded/pom.xml b/shaded/bookkeeper-server-tests-shaded/pom.xml index 340c8185d72..cef6d0772f5 100644 --- a/shaded/bookkeeper-server-tests-shaded/pom.xml +++ b/shaded/bookkeeper-server-tests-shaded/pom.xml @@ -39,10 +39,18 @@ org.slf4j slf4j-log4j12 + + org.slf4j + slf4j-reload4j + log4j log4j + + ch.qos.reload4j + reload4j + org.apache.bookkeeper bookkeeper-common @@ -103,7 +111,7 @@ - + org.codehaus.mojo license-maven-plugin diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml index a58c994b12f..8a4b2118c07 100644 --- a/shaded/distributedlog-core-shaded/pom.xml +++ b/shaded/distributedlog-core-shaded/pom.xml @@ -39,10 +39,18 @@ org.slf4j slf4j-log4j12 + + org.slf4j + slf4j-reload4j + log4j log4j + + ch.qos.reload4j + reload4j + io.netty netty-buffer diff --git a/stream/api/pom.xml b/stream/api/pom.xml index 0e0b5b7750d..3f637537e2b 100644 --- a/stream/api/pom.xml +++ b/stream/api/pom.xml @@ -41,9 +41,5 @@ io.netty netty-buffer - - org.inferred - freebuilder - diff --git a/stream/clients/java/base/pom.xml b/stream/clients/java/base/pom.xml index 02eedf30413..8182970146f 100644 --- a/stream/clients/java/base/pom.xml +++ b/stream/clients/java/base/pom.xml @@ -36,6 +36,11 @@ stream-storage-proto ${project.parent.version} + + org.inferred + freebuilder + true + org.apache.bookkeeper testtools @@ -45,6 +50,19 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.inferred + freebuilder + ${freebuilder.version} + + + + org.apache.maven.plugins maven-jar-plugin diff --git a/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/internal/TestStorageServerClientManagerImpl.java b/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/internal/TestStorageServerClientManagerImpl.java index 5f68d11c5cb..d72a69e431e 100644 --- a/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/internal/TestStorageServerClientManagerImpl.java +++ b/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/internal/TestStorageServerClientManagerImpl.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import com.google.common.collect.Lists; import io.grpc.stub.StreamObserver; import java.util.List; import org.apache.bookkeeper.clients.grpc.GrpcClientTestBase; @@ -36,7 +37,6 @@ import org.apache.bookkeeper.stream.proto.storage.OneStorageContainerEndpointResponse; import org.apache.bookkeeper.stream.proto.storage.RootRangeServiceGrpc.RootRangeServiceImplBase; import org.apache.bookkeeper.stream.proto.storage.StatusCode; -import org.inferred.freebuilder.shaded.com.google.common.collect.Lists; import org.junit.Test; /** diff --git a/stream/clients/python/setup.py b/stream/clients/python/setup.py index e746a24faa6..d95c99a27a8 100644 --- a/stream/clients/python/setup.py +++ b/stream/clients/python/setup.py @@ -31,7 +31,7 @@ 'six>=1.10.0', 'pytz', 'futures>=3.2.0;python_version<"3.2"', - 'grpcio<1.28,>=1.8.2', + 'grpcio>=1.8.2', 'pymmh3>=0.0.5' ] extras = { diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml index a47047e2117..c5888633b9b 100644 --- a/stream/distributedlog/common/pom.xml +++ b/stream/distributedlog/common/pom.xml @@ -34,6 +34,18 @@ org.slf4j slf4j-log4j12 + + org.slf4j + slf4j-reload4j + + + log4j + log4j + + + ch.qos.reload4j + reload4j + @@ -78,8 +90,8 @@ spotbugs-maven-plugin + org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} org.apache.maven.plugins @@ -96,11 +108,11 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} ${redirectTestOutputToFile} -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} - always + 1 + false 1800 diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml index 4428c2554ca..09089148a53 100644 --- a/stream/distributedlog/core/pom.xml +++ b/stream/distributedlog/core/pom.xml @@ -90,8 +90,8 @@ + org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} org.apache.maven.plugins @@ -108,12 +108,12 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} false ${redirectTestOutputToFile} -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G ${test.additional.args} - always + 1 + false 1800 diff --git a/stream/distributedlog/io/dlfs/pom.xml b/stream/distributedlog/io/dlfs/pom.xml index 6badaf75417..c3a3dc2c039 100644 --- a/stream/distributedlog/io/dlfs/pom.xml +++ b/stream/distributedlog/io/dlfs/pom.xml @@ -54,10 +54,18 @@ org.slf4j slf4j-log4j12 + + org.slf4j + slf4j-reload4j + log4j log4j + + ch.qos.reload4j + reload4j + org.apache.avro avro diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml index bf1be992121..64e985b8521 100644 --- a/stream/distributedlog/pom.xml +++ b/stream/distributedlog/pom.xml @@ -45,7 +45,6 @@ org.apache.maven.plugins maven-javadoc-plugin - ${maven-javadoc-plugin.version} ${src.dir} true @@ -74,11 +73,11 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} ${redirectTestOutputToFile} -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} - always + 1 + false 1800 diff --git a/stream/pom.xml b/stream/pom.xml index 42c57491a1e..425c1c2e8cf 100644 --- a/stream/pom.xml +++ b/stream/pom.xml @@ -53,13 +53,13 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} true ${redirectTestOutputToFile} -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID ${test.additional.args} - always + 1 + false 1800 diff --git a/stream/proto/pom.xml b/stream/proto/pom.xml index bc96faf3ba6..025593b006d 100644 --- a/stream/proto/pom.xml +++ b/stream/proto/pom.xml @@ -71,16 +71,9 @@ + org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - false false diff --git a/stream/tests-common/pom.xml b/stream/tests-common/pom.xml index 145b2348cca..53b9c393228 100644 --- a/stream/tests-common/pom.xml +++ b/stream/tests-common/pom.xml @@ -76,16 +76,9 @@ + org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - false false diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000000..4d79d4b6d3c --- /dev/null +++ b/tests/README.md @@ -0,0 +1,81 @@ +# Integration tests for Bookkeeper + +## Prerequisites + +### Workaround for running the tests with Mac Apple Silicon + +The current version of the outdated maven plugin requires a workaround. + +Install socat +```bash +brew install socat +``` + +Run a TCP -> Unix socket proxy for docker in a separate terminal +```bash +socat TCP-LISTEN:2375,bind=127.0.0.1,reuseaddr,fork UNIX-CLIENT:/var/run/docker.sock & +``` + +Set the `DOCKER_HOST` environment variable in the terminal where you run the tests +```bash +export DOCKER_HOST=tcp://localhost:2375 +``` + +Here's a shell script function to automate starting the proxy and setting `DOCKER_HOST`: +```bash +function docker_socket_proxy() { + local port=2375 + socat /dev/null TCP4:127.0.0.1:$port,connect-timeout=2 &> /dev/null + if [ $? -ne 0 ]; then + echo "Starting socat tcp proxy on port $port for docker socket /var/run/docker.sock" + socat TCP-LISTEN:$port,bind=127.0.0.1,reuseaddr,fork UNIX-CLIENT:/var/run/docker.sock &> /dev/null & + echo "Stop the proxy with 'kill $!'" + fi + export DOCKER_HOST=tcp://127.0.0.1:$port + echo "Added DOCKER_HOST=$DOCKER_HOST to environment" +} +``` +You can add the function to your shell profile (`~/.bashrc`, `~/.zshrc`, etc.). + +### Support for connecting to docker network from Mac host + +You will also need to install [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). It allows the tests to connect to the docker network from the Mac host. + +```bash +brew install chipmk/tap/docker-mac-net-connect +sudo brew services start chipmk/tap/docker-mac-net-connect +``` + +## Running the tests + +Remember to start the unix socket proxy for docker as described in the previous section and set the `DOCKER_HOST` environment variable in the terminal where you run the tests. + +### Building the docker images together with the project source code + +```bash +mvn -B -nsu clean install -Pdocker -DskipTests +docker images | grep apachebookkeeper +``` + +### Running integration tests + +```bash +# Run metadata driver tests +mvn -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO -DredirectTestOutputToFile=false -DtestRetryCount=0 + +# Run all integration tests +mvn -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO -DredirectTestOutputToFile=false -DtestRetryCount=0 +``` + +### Running backward compatibility tests + +```bash +# Test current server with old clients +mvn -DbackwardCompatTests -pl :backward-compat-current-server-old-clients test -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO -DredirectTestOutputToFile=false -DtestRetryCount=0 + +# Test progressive upgrade +mvn -DbackwardCompatTests -pl :upgrade test -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO -DredirectTestOutputToFile=false -DtestRetryCount=0 + +# Other backward compat tests +mvn -DbackwardCompatTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct,:yahoo-custom-version test -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO -DredirectTestOutputToFile=false -DtestRetryCount=0 +``` \ No newline at end of file diff --git a/tests/backward-compat/pom.xml b/tests/backward-compat/pom.xml index d85338b7965..398a15f3b16 100644 --- a/tests/backward-compat/pom.xml +++ b/tests/backward-compat/pom.xml @@ -38,6 +38,33 @@ yahoo-custom-version bc-non-fips + + + + com.github.docker-java + docker-java-bom + ${arquillian-cube.docker-java.version} + pom + import + + + org.arquillian.cube + arquillian-cube-docker + ${arquillian-cube.version} + + + org.yaml + snakeyaml + ${arquillian-cube.snakeyaml.version} + + + + + + org.arquillian.cube + arquillian-cube-docker + + @@ -47,6 +74,34 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + backwardCompatTests + + + backwardCompatTests + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + + + + + diff --git a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy index 2ee1c7d8aa5..448c0f83b83 100644 --- a/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy +++ b/tests/backward-compat/upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeDirect.groovy @@ -36,6 +36,12 @@ class TestCompatUpgradeDirect { private static final Logger LOG = LoggerFactory.getLogger(TestCompatUpgradeDirect.class) private static byte[] PASSWD = "foobar".getBytes() + static { + Thread.setDefaultUncaughtExceptionHandler { Thread t, Throwable e -> + LOG.error("Uncaught exception in thread {}", t, e) + } + } + @ArquillianResource DockerClient docker diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index aee3227ecd1..a8c761c7e88 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -17,16 +17,27 @@ # under the License. # -FROM openjdk:8-jdk +FROM eclipse-temurin:8-jdk MAINTAINER Apache BookKeeper ENV BK_JOURNALDIR=/opt/bookkeeper/data/journal ENV BK_LEDGERDIR=/opt/bookkeeper/data/ledgers ENV BK_ZKCONNECTSTRING=zookeeper1,zookeeper2,zookeeper3 +ENV DEBIAN_FRONTEND=noninteractive +ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/ +ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ -RUN apt-get update && apt-get install -qy wget supervisor bash \ +RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \ + -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ + && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ + && apt-get update && apt-get install -qy wget curl supervisor bash ca-certificates apt-transport-https \ + && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get install -y --no-install-recommends gpg gpg-agent sudo \ && echo "dash dash/sh boolean false" | debconf-set-selections \ - && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash \ + && JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) \ + && echo networkaddress.cache.ttl=1 >> $JAVA_HOME/jre/lib/security/java.security \ + && echo networkaddress.cache.negative.ttl=1 >> $JAVA_HOME/jre/lib/security/java.security RUN mkdir /tarballs WORKDIR /tarballs diff --git a/tests/docker-images/all-released-versions-image/image_builder.sh b/tests/docker-images/all-released-versions-image/image_builder.sh index a06c29e49b9..b89afad3e6d 100755 --- a/tests/docker-images/all-released-versions-image/image_builder.sh +++ b/tests/docker-images/all-released-versions-image/image_builder.sh @@ -29,4 +29,5 @@ SCRIPT_DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) ## BASE_DIR will be ./bookkeeper/ BASE_DIR=${SCRIPT_DIR}/../../../ -docker build -t ${IMAGE_NAME} "${BASE_DIR}"/tests/docker-images/all-released-versions-image \ No newline at end of file +docker build --build-arg UBUNTU_MIRROR="${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}" --build-arg UBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}" \ + -t ${IMAGE_NAME} "${BASE_DIR}"/tests/docker-images/all-released-versions-image \ No newline at end of file diff --git a/tests/docker-images/all-released-versions-image/pom.xml b/tests/docker-images/all-released-versions-image/pom.xml index 99a4c1133e8..f3e268984f9 100644 --- a/tests/docker-images/all-released-versions-image/pom.xml +++ b/tests/docker-images/all-released-versions-image/pom.xml @@ -62,6 +62,10 @@ apachebookkeeper/bookkeeper-all-released-versions ${project.version} false + + ${UBUNTU_MIRROR} + ${UBUNTU_SECURITY_MIRROR} + diff --git a/tests/docker-images/all-versions-image/image_builder.sh b/tests/docker-images/all-versions-image/image_builder.sh index 7b553362303..f021c5f034f 100755 --- a/tests/docker-images/all-versions-image/image_builder.sh +++ b/tests/docker-images/all-versions-image/image_builder.sh @@ -34,4 +34,5 @@ mkdir -p "${BASE_DIR}"/tests/docker-images/all-versions-image/build ls -la ${BASE_DIR}bookkeeper-dist/server/build/distributions cp ${BASE_DIR}bookkeeper-dist/server/build/distributions/bookkeeper-server-${VERSION}-bin.tar.gz "${BASE_DIR}"/tests/docker-images/all-versions-image/build/ TARBALL=build/bookkeeper-server-${VERSION}-bin.tar.gz -docker build -t ${IMAGE_NAME} "${BASE_DIR}"tests/docker-images/all-versions-image --build-arg BK_TARBALL="${TARBALL}" \ No newline at end of file +docker build --build-arg UBUNTU_MIRROR="${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}" --build-arg UBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}" \ + -t ${IMAGE_NAME} "${BASE_DIR}"tests/docker-images/all-versions-image --build-arg BK_TARBALL="${TARBALL}" \ No newline at end of file diff --git a/tests/docker-images/all-versions-image/pom.xml b/tests/docker-images/all-versions-image/pom.xml index f3516858f56..c9eaed5e72d 100644 --- a/tests/docker-images/all-versions-image/pom.xml +++ b/tests/docker-images/all-versions-image/pom.xml @@ -82,6 +82,8 @@ true target/bookkeeper-dist-server-${project.version}-bin.tar.gz + ${UBUNTU_MIRROR} + ${UBUNTU_SECURITY_MIRROR} diff --git a/tests/docker-images/current-version-image/Dockerfile b/tests/docker-images/current-version-image/Dockerfile new file mode 100644 index 00000000000..0e489eb5ea6 --- /dev/null +++ b/tests/docker-images/current-version-image/Dockerfile @@ -0,0 +1,75 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM ubuntu:22.04 +MAINTAINER Apache BookKeeper + +ARG BK_VERSION=DOESNOTEXISTS +ARG DISTRO_NAME=bookkeeper-dist-server-${BK_VERSION}-bin +ARG PKG_NAME=bookkeeper-server-${BK_VERSION} + +ENV BOOKIE_PORT=3181 +ENV BOOKIE_HTTP_PORT=8080 +ENV BOOKIE_GRPC_PORT=4181 +EXPOSE ${BOOKIE_PORT} ${BOOKIE_HTTP_PORT} ${BOOKIE_GRPC_PORT} +ENV BK_USER=bookkeeper +ENV BK_HOME=/opt/bookkeeper +ENV DEBIAN_FRONTEND=noninteractive +ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/ +ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ + +RUN set -x \ + && sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \ + -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ + && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ + && adduser "${BK_USER}" \ + && apt-get update \ + && apt-get install -y ca-certificates apt-transport-https \ + && apt-get install -y --no-install-recommends openjdk-17-jdk \ + && apt-get install -y --no-install-recommends python3 pip \ + && ln -s /usr/bin/python3 /usr/bin/python \ + && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \ + && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && pip install zk-shell \ + && JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) \ + && echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security \ + && echo networkaddress.cache.negative.ttl=1 >> $JAVA_HOME/conf/security/java.security + +# untar tarballs +ADD target/${DISTRO_NAME}.tar.gz /opt +RUN mv /opt/${PKG_NAME} /opt/bookkeeper + +WORKDIR /opt/bookkeeper + +COPY target/scripts /opt/bookkeeper/scripts +COPY scripts/install-python-client.sh /opt/bookkeeper/scripts +RUN chmod +x -R /opt/bookkeeper/scripts/ + +# copy the python client +ADD target/bookkeeper-client/ /opt/bookkeeper/bookkeeper-client +RUN /opt/bookkeeper/scripts/install-python-client.sh + +ENTRYPOINT [ "/bin/bash", "/opt/bookkeeper/scripts/entrypoint.sh" ] +CMD ["bookie"] + +#HEALTHCHECK --interval=10s --timeout=60s CMD /bin/bash /opt/bookkeeper/scripts/healthcheck.sh diff --git a/tests/docker-images/current-version-image/pom.xml b/tests/docker-images/current-version-image/pom.xml new file mode 100644 index 00000000000..ebc6610a93a --- /dev/null +++ b/tests/docker-images/current-version-image/pom.xml @@ -0,0 +1,149 @@ + + + + + org.apache.bookkeeper.tests + docker-images + 4.15.6-SNAPSHOT + + 4.0.0 + org.apache.bookkeeper.tests + current-version-image + Apache BookKeeper :: Tests :: Docker Images :: Current Version + pom + + + org.apache.bookkeeper + bookkeeper-dist-server + ${project.parent.version} + bin + tar.gz + provided + + + + + docker + + + integrationTests + + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + build-python-client + generate-resources + + exec + + + ${project.basedir}/target + ${project.basedir}/../../../stream/clients/python/scripts/docker_build.sh + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven-antrun-plugin.version} + + + generate-resources + + run + + + + copy python wheel file + + + copying docker scripts + + + + + + + + + com.spotify + dockerfile-maven-plugin + ${dockerfile-maven-plugin.version} + + + default + + build + + + + add-latest-tag + + tag + + + apachebookkeeper/bookkeeper-current + latest + + + + + apachebookkeeper/bookkeeper-current + ${project.version} + false + true + + ${project.version} + ${UBUNTU_MIRROR} + ${UBUNTU_SECURITY_MIRROR} + + + + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + copy-docker-dependencies + + copy-dependencies + + generate-resources + + ${project.build.directory}/ + bookkeeper-dist-server + true + + + + + + + + + diff --git a/.github/workflows/bot.yml b/tests/docker-images/current-version-image/scripts/install-python-client.sh similarity index 55% rename from .github/workflows/bot.yml rename to tests/docker-images/current-version-image/scripts/install-python-client.sh index d1eb093c56c..2719b335ac1 100644 --- a/.github/workflows/bot.yml +++ b/tests/docker-images/current-version-image/scripts/install-python-client.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -17,28 +18,7 @@ # under the License. # -name: Bot tests -on: - issue_comment: - types: [created] +set -x -jobs: - bot: - runs-on: ubuntu-latest - - steps: - - name: clone repository - uses: actions/checkout@v2 - - - name: bot actions - uses: actions/github-script@v1 - env: - PROVIDER: 'apache' - REPOSITORY: 'bookkeeper' - RERUN_CMD: 'rerun failure checks' - with: - github-token: ${{secrets.BKBOT_TOKEN}} - script: | - const path = require('path') - const scriptPath = path.resolve('.github/actions/bot/src/run.js') - require(scriptPath)({core}, {context}, {github}) \ No newline at end of file +WHEEL_FILE=`ls /opt/bookkeeper/bookkeeper-client/*.whl` +pip install ${WHEEL_FILE} diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index af85928fff1..ed5aaa51a76 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -29,5 +29,6 @@ all-released-versions-image all-versions-image + current-version-image diff --git a/tests/docker-images/statestore-image/Dockerfile b/tests/docker-images/statestore-image/Dockerfile index 961af1e2390..5a7a1688855 100644 --- a/tests/docker-images/statestore-image/Dockerfile +++ b/tests/docker-images/statestore-image/Dockerfile @@ -17,27 +17,40 @@ # under the License. # -FROM centos:7 -#ENV OPTS="$OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005" +FROM ubuntu:22.04 +MAINTAINER Apache BookKeeper + ENV BOOKIE_PORT=3181 ENV BOOKIE_HTTP_PORT=8080 ENV BOOKIE_GRPC_PORT=4181 EXPOSE ${BOOKIE_PORT} ${BOOKIE_HTTP_PORT} ${BOOKIE_GRPC_PORT} ENV BK_USER=bookkeeper ENV BK_HOME=/opt/bookkeeper -ENV JAVA_HOME=/usr/lib/jvm/java-11 - +ENV DEBIAN_FRONTEND=noninteractive +ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/ +ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ RUN set -x \ + && sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \ + -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ + && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ && adduser "${BK_USER}" \ - && yum install -y java-11-openjdk-devel wget bash python-pip python-devel sudo netcat gcc gcc-c++ \ - && wget -q https://bootstrap.pypa.io/pip/2.7/get-pip.py \ - && python get-pip.py \ + && apt-get update \ + && apt-get install -y ca-certificates apt-transport-https \ + && apt-get install -y --no-install-recommends openjdk-17-jdk \ + && apt-get install -y --no-install-recommends python3 pip \ + && ln -s /usr/bin/python3 /usr/bin/python \ + && apt-get -y install netcat dnsutils less procps iputils-ping \ + && apt-get install -y --no-install-recommends gpg gpg-agent wget sudo \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ && pip install zk-shell \ - && rm -rf get-pip.py \ && mkdir -pv /opt \ - && cd /opt \ - && yum clean all + && JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java)))) \ + && echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security \ + && echo networkaddress.cache.negative.ttl=1 >> $JAVA_HOME/conf/security/java.security WORKDIR /opt/bookkeeper RUN mkdir /opt/bookkeeper/lib diff --git a/tests/docker-images/statestore-image/image_builder.sh b/tests/docker-images/statestore-image/image_builder.sh index e17ee608b75..8f305d0ec1d 100755 --- a/tests/docker-images/statestore-image/image_builder.sh +++ b/tests/docker-images/statestore-image/image_builder.sh @@ -42,7 +42,8 @@ cp "${BASE_DIR}"/stream/server/build/distributions/server-bin.tar.gz "${BASE_DIR cp "${BASE_DIR}"/docker/scripts/* "${BASE_DIR}"/tests/docker-images/statestore-image/scripts cp "${BASE_DIR}"/conf/* "${BASE_DIR}"/tests/docker-images/statestore-image/temp_conf cp "${BASE_DIR}"/bin/* "${BASE_DIR}"/tests/docker-images/statestore-image/temp_bin -docker build -t ${IMAGE_NAME} "${BASE_DIR}"/tests/docker-images/statestore-image +docker build --build-arg UBUNTU_MIRROR="${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}" --build-arg UBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}" \ + -t ${IMAGE_NAME} "${BASE_DIR}"/tests/docker-images/statestore-image rm -rf "${BASE_DIR}"/tests/docker-images/statestore-image/dist rm -rf "${BASE_DIR}"/tests/docker-images/statestore-image/scripts diff --git a/tests/integration-tests-base-groovy/pom.xml b/tests/integration-tests-base-groovy/pom.xml index 2288f360e00..81f68f7a6f6 100644 --- a/tests/integration-tests-base-groovy/pom.xml +++ b/tests/integration-tests-base-groovy/pom.xml @@ -38,12 +38,12 @@ + org.apache.maven.plugins maven-compiler-plugin - ${javac.target} - ${javac.target} groovy-eclipse-compiler + true @@ -64,18 +64,6 @@ ${groovy-eclipse-compiler.version} true - - org.codehaus.gmavenplus - gmavenplus-plugin - ${gmavenplus-plugin.version} - - - - compileTests - - - - org.apache.maven.plugins maven-surefire-plugin @@ -93,6 +81,27 @@ + + + + com.github.docker-java + docker-java-bom + ${arquillian-cube.docker-java.version} + pom + import + + + org.arquillian.cube + arquillian-cube-docker + ${arquillian-cube.version} + + + org.yaml + snakeyaml + ${arquillian-cube.snakeyaml.version} + + + org.codehaus.groovy @@ -104,9 +113,8 @@ arquillian-cube-docker - org.jboss.arquillian.junit - arquillian-junit-container - test + javax.ws.rs + javax.ws.rs-api diff --git a/tests/integration-tests-utils/pom.xml b/tests/integration-tests-utils/pom.xml index 5986998fc89..ee723aedf7d 100644 --- a/tests/integration-tests-utils/pom.xml +++ b/tests/integration-tests-utils/pom.xml @@ -47,7 +47,7 @@ org.apache.zookeeper zookeeper - + org.arquillian.cube @@ -71,6 +71,11 @@ pom + + javax.ws.rs + javax.ws.rs-api + + @@ -78,12 +83,9 @@ org.apache.maven.plugins maven-surefire-plugin - - 2.8.1 1 + false false diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index 55f625c8bcb..27d633978c1 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -94,8 +94,12 @@ public static boolean zookeeperRunning(DockerClient docker, String containerId) public static void legacyMetadataFormat(DockerClient docker) throws Exception { @Cleanup ZooKeeper zk = BookKeeperClusterUtils.zookeeperClient(docker); - zk.create("/ledgers", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - zk.create("/ledgers/available", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + if (zk.exists("/ledgers", false) == null) { + zk.create("/ledgers", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + if (zk.exists("/ledgers/available", false) == null) { + zk.create("/ledgers/available", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } } public static boolean metadataFormatIfNeeded(DockerClient docker, String version) throws Exception { @@ -262,9 +266,6 @@ public static boolean waitAllBookieUp(DockerClient docker) { } private static String computeBinFilenameByVersion(String version) { - if (OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME.contains(version)) { - return "bookkeeper"; - } - return "bookkeeper_gradle"; + return "bookkeeper"; } } diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java index 15773091f86..d2a4d731a6e 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java @@ -54,9 +54,9 @@ public class DockerUtils { private static final Logger LOG = LoggerFactory.getLogger(DockerUtils.class); private static File getTargetDirectory(String containerId) { - String base = System.getProperty("gradle.buildDirectory"); + String base = System.getProperty("maven.buildDirectory"); if (base == null) { - base = "build"; + base = "target"; } File directory = new File(base + "/container-logs/" + containerId); if (!directory.exists() && !directory.mkdirs()) { diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java index 7c4137a6ffa..32d00370b1d 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java @@ -17,6 +17,8 @@ */ package org.apache.bookkeeper.tests.integration.utils; +import static org.jboss.shrinkwrap.resolver.api.maven.coordinate.MavenDependencies.createExclusion; + import com.google.common.collect.Lists; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import groovy.lang.Closure; @@ -41,22 +43,43 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; - +import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.archivers.ArchiveStreamFactory; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.jboss.shrinkwrap.resolver.api.maven.ConfigurableMavenResolverSystem; import org.jboss.shrinkwrap.resolver.api.maven.Maven; +import org.jboss.shrinkwrap.resolver.api.maven.MavenResolvedArtifact; import org.jboss.shrinkwrap.resolver.api.maven.ScopeType; +import org.jboss.shrinkwrap.resolver.api.maven.coordinate.MavenCoordinate; import org.jboss.shrinkwrap.resolver.api.maven.coordinate.MavenDependencies; import org.jboss.shrinkwrap.resolver.api.maven.coordinate.MavenDependency; /** * A maven class loader for resolving and loading maven artifacts. */ +@Slf4j public class MavenClassLoader implements AutoCloseable { + private static ScheduledExecutorService delayedCloseExecutor = createExecutorThatShutsDownIdleThreads(); + + private static ScheduledExecutorService createExecutorThatShutsDownIdleThreads() { + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + // Cast to ThreadPoolExecutor to access additional configuration methods + ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) executor; + // Set the timeout for idle threads + poolExecutor.setKeepAliveTime(10, TimeUnit.SECONDS); + // Allow core threads to time out + poolExecutor.allowCoreThreadTimeOut(true); + return executor; + } private static List currentVersionLibs; @@ -79,22 +102,45 @@ private static MavenClassLoader createClassLoader(ConfigurableMavenResolverSyste String mainArtifact) throws Exception { Optional slf4jVersion = Arrays.stream(resolver.resolve(mainArtifact) .withTransitivity().asResolvedArtifact()) - .filter((a) -> a.getCoordinate().getGroupId().equals("org.slf4j") - && a.getCoordinate().getArtifactId().equals("slf4j-1.2-api")) + .filter((a) -> a.getCoordinate().getGroupId().equals("org.slf4j")) .map((a) -> a.getCoordinate().getVersion()) .findFirst(); + MavenDependency dependency = MavenDependencies.createDependency(mainArtifact, ScopeType.COMPILE, false, + createExclusion("log4j", "log4j"), + createExclusion("org.slf4j", "slf4j-log4j12"), + createExclusion("ch.qos.reload4j", "log4j"), + createExclusion("org.slf4j", "slf4j-reload4j"), + createExclusion("org.apache.logging.log4j", "*") + ); List deps = Lists.newArrayList( - MavenDependencies.createDependency( - mainArtifact, ScopeType.COMPILE, false)); + dependency); if (slf4jVersion.isPresent()) { deps.add(MavenDependencies.createDependency("org.slf4j:slf4j-simple:" + slf4jVersion.get(), ScopeType.COMPILE, false)); + deps.add(MavenDependencies.createDependency("org.slf4j:jcl-over-slf4j:" + slf4jVersion.get(), + ScopeType.COMPILE, false)); } - File[] files = resolver.addDependencies(deps.toArray(new MavenDependency[0])) - .resolve().withTransitivity().asFile(); - return createClassLoader(files); + MavenResolvedArtifact[] resolvedArtifact = resolver.addDependencies(deps.toArray(new MavenDependency[0])) + .resolve().withTransitivity().asResolvedArtifact(); + File[] files = Arrays.stream(resolvedArtifact) + .filter((a) -> { + MavenCoordinate c = a.getCoordinate(); + // exclude log4j + if (c.getGroupId().equals("org.apache.logging.log4j") || c.getGroupId().equals("log4j") + || c.getGroupId().equals("ch.qos.reload4j") + || c.getGroupId().equals("commons-logging")) { + return false; + } + if (c.getArtifactId().contains("log4j") || c.getArtifactId().contains("commons-logging")) { + return false; + } + return true; + }).map(MavenResolvedArtifact::asFile) + .collect(Collectors.toList()) + .toArray(new File[0]); + return createClassLoader(files); } private static MavenClassLoader createClassLoader(File[] jars) { @@ -117,6 +163,12 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE try { loadedClass = findClass(name); } catch (ClassNotFoundException ignored) { + // never load these classes from the parent classloader + if (name.startsWith("org.apache") + || name.startsWith("org.slf4j") + || name.startsWith("log4j")) { + throw ignored; + } } if (loadedClass == null) { try { @@ -141,13 +193,22 @@ public static MavenClassLoader forBookKeeperVersion(String version) throws Excep return forArtifact("org.apache.bookkeeper:bookkeeper-server:" + version); } - private static MavenClassLoader forBookkeeperCurrentVersion() throws Exception { + private static synchronized MavenClassLoader forBookkeeperCurrentVersion() throws Exception { if (currentVersionLibs == null) { final String version = BookKeeperClusterUtils.CURRENT_VERSION; + String rootDirectory = System.getenv("GITHUB_WORKSPACE"); + if (rootDirectory == null) { + File gitDirectory = findGitRoot(new File(".")); + if (gitDirectory != null) { + rootDirectory = gitDirectory.getAbsolutePath(); + } else { + rootDirectory = System.getProperty("maven.buildDirectory", ".") + "/../../../.."; + } + } final String artifactName = "bookkeeper-server-" + version + "-bin"; - final Path tarFile = Paths.get("..", "..", "..", - "bookkeeper-dist", "server", "build", "distributions", artifactName + ".tar.gz"); - final File tempDir = new File("build"); + final Path tarFile = Paths.get(rootDirectory, + "bookkeeper-dist", "server", "target", artifactName + ".tar.gz").toAbsolutePath(); + final File tempDir = new File(System.getProperty("maven.buildDirectory", "target")); extractTarGz(tarFile.toFile(), tempDir); List jars = new ArrayList<>(); Files.list(Paths.get(tempDir.getAbsolutePath(), "bookkeeper-server-" + version, "lib")) @@ -159,6 +220,16 @@ private static MavenClassLoader forBookkeeperCurrentVersion() throws Exception { return createClassLoader(currentVersionLibs.toArray(new File[]{})); } + private static File findGitRoot(File currentDir) { + while (currentDir != null) { + if (new File(currentDir, ".git").exists()) { + return currentDir; + } + currentDir = currentDir.getParentFile(); + } + return null; + } + public Object callStaticMethod(String className, String methodName, ArrayList args) throws Exception { Class klass = Class.forName(className, true, classloader); @@ -218,26 +289,31 @@ public Object newInstance(String className, Object... args) throws Exception { } public Object newBookKeeper(String zookeeper) throws Exception { - Class clientConfigurationClass = Class - .forName("org.apache.bookkeeper.conf.ClientConfiguration", true, classloader); - Object clientConfiguration = newInstance("org.apache.bookkeeper.conf.ClientConfiguration"); - clientConfigurationClass - .getMethod("setZkServers", String.class) - .invoke(clientConfiguration, zookeeper); - - // relax timeouts in order to get tests passing in limited environments - clientConfigurationClass - .getMethod("setReadTimeout", int.class) - .invoke(clientConfiguration, 15); - - clientConfigurationClass - .getMethod("setZkTimeout", int.class) - .invoke(clientConfiguration, 30_000); - Class klass = Class.forName("org.apache.bookkeeper.client.BookKeeper", true, classloader); - return klass - .getConstructor(clientConfigurationClass) - .newInstance(clientConfiguration); - + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(classloader); + Class clientConfigurationClass = Class + .forName("org.apache.bookkeeper.conf.ClientConfiguration", true, classloader); + Object clientConfiguration = newInstance("org.apache.bookkeeper.conf.ClientConfiguration"); + clientConfigurationClass + .getMethod("setZkServers", String.class) + .invoke(clientConfiguration, zookeeper); + + // relax timeouts in order to get tests passing in limited environments + clientConfigurationClass + .getMethod("setReadTimeout", int.class) + .invoke(clientConfiguration, 15); + + clientConfigurationClass + .getMethod("setZkTimeout", int.class) + .invoke(clientConfiguration, 30_000); + Class klass = Class.forName("org.apache.bookkeeper.client.BookKeeper", true, classloader); + return klass + .getConstructor(clientConfigurationClass) + .newInstance(clientConfiguration); + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); + } } public Object digestType(String type) throws Exception { @@ -253,7 +329,14 @@ public Object digestType(String type) throws Exception { @Override public void close() throws Exception { if (classloader instanceof Closeable) { - ((Closeable) classloader).close(); + // delay closing the classloader so that currently executing asynchronous tasks can complete + delayedCloseExecutor.schedule(() -> { + try { + ((Closeable) classloader).close(); + } catch (Exception e) { + log.error("Failed to close classloader", e); + } + }, 5, TimeUnit.SECONDS); } } @@ -285,14 +368,16 @@ private static void unTar(final File inputFile, final File outputDir) throws Exc TarArchiveEntry entry; while ((entry = (TarArchiveEntry) debInputStream.getNextEntry()) != null) { final File outputFile = new File(outputDir, entry.getName()); + if (!outputFile.getParentFile().exists()) { + outputFile.getParentFile().mkdirs(); + } if (entry.isDirectory()) { - if (!outputFile.exists()) { - if (!outputFile.mkdirs()) { - throw new IllegalStateException( - String.format("Couldn't create directory %s.", outputFile.getAbsolutePath())); - } - } else { - outputFile.delete(); + if (outputFile.exists()) { + FileUtils.deleteDirectory(outputFile); + } + if (!outputFile.mkdirs()) { + throw new IllegalStateException( + String.format("Couldn't create directory %s.", outputFile.getAbsolutePath())); } } else { try (final OutputStream outputFileStream = new FileOutputStream(outputFile)) { diff --git a/tests/integration/cluster/pom.xml b/tests/integration/cluster/pom.xml index 1c4aec2ce77..a811fe2034c 100644 --- a/tests/integration/cluster/pom.xml +++ b/tests/integration/cluster/pom.xml @@ -74,7 +74,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} 0 diff --git a/tests/integration/smoke/pom.xml b/tests/integration/smoke/pom.xml index 682485045b7..2e194920dd3 100644 --- a/tests/integration/smoke/pom.xml +++ b/tests/integration/smoke/pom.xml @@ -29,6 +29,28 @@ jar Apache BookKeeper :: Tests :: Integration :: Smoke test + + + + com.github.docker-java + docker-java-bom + ${arquillian-cube.docker-java.version} + pom + import + + + org.arquillian.cube + arquillian-cube-docker + ${arquillian-cube.version} + + + org.yaml + snakeyaml + ${arquillian-cube.snakeyaml.version} + + + + org.apache.bookkeeper @@ -56,6 +78,11 @@ arquillian-junit-standalone test + + + org.arquillian.cube + arquillian-cube-docker + @@ -63,7 +90,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} 0 diff --git a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/BookieShellTestBase.java b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/BookieShellTestBase.java index 05ff030470a..59626881da0 100644 --- a/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/BookieShellTestBase.java +++ b/tests/integration/smoke/src/test/java/org/apache/bookkeeper/tests/integration/BookieShellTestBase.java @@ -22,13 +22,17 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; /** * Test Base for testing bookie shell scripts. */ @Slf4j public abstract class BookieShellTestBase { + @Rule + public Timeout testTimeout = Timeout.seconds(300); String currentVersion = System.getProperty("currentVersion"); String bkScript; diff --git a/tests/integration/standalone/pom.xml b/tests/integration/standalone/pom.xml index 065354e5ecd..5746f2468ff 100644 --- a/tests/integration/standalone/pom.xml +++ b/tests/integration/standalone/pom.xml @@ -59,7 +59,6 @@ org.apache.maven.plugins maven-surefire-plugin - ${maven-surefire-plugin.version} 0 diff --git a/tests/pom.xml b/tests/pom.xml index fb596e0b385..412e419178d 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -28,7 +28,10 @@ Apache BookKeeper :: Tests - 3.0.11 + 3.0.20 + + 3.2.14 + 1.19 diff --git a/tests/shaded/bookkeeper-server-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-shaded-test/pom.xml index 5951a716460..ede06a8ee1c 100644 --- a/tests/shaded/bookkeeper-server-shaded-test/pom.xml +++ b/tests/shaded/bookkeeper-server-shaded-test/pom.xml @@ -54,8 +54,8 @@ spotbugs-maven-plugin + org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version}