Merge pull request #1143 from HathorNetwork/master

Release-candidate v0.63.0-rc.1

.envrc

@@ -0,0 +1,7 @@
+if [[ $(type -t use_flake) != function ]]; then
+  echo "ERROR: use_flake function missing."
+  echo "Please update direnv to v2.30.0 or later."
+  exit 1
+fi
+
+use flake

.github/actions/setup-hathor-env/action.yml

@@ -0,0 +1,38 @@
+name: setup-hathor-env
+description: Setup Hathor node environment
+inputs:
+  python:
+    description: The python version
+  os:
+    description: The OS name
+runs:
+  using: composite
+  steps:
+    - name: Install Poetry
+      shell: bash
+      run: pipx install poetry
+
+    - name: Set up Python ${{ inputs.python }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python }}
+        cache: 'poetry'
+
+    - name: Install Ubuntu dependencies
+      if: startsWith(inputs.os, 'ubuntu')
+      run: |
+        sudo apt-get -qy update
+        sudo apt-get -qy install graphviz librocksdb-dev libsnappy-dev liblz4-dev
+      shell: bash
+
+    - name: Install macOS dependencies
+      if: startsWith(inputs.os, 'macos')
+      run: |
+        brew cleanup -q
+        # brew update -q
+        brew install -q graphviz rocksdb pkg-config
+      shell: bash
+
+    - name: Install Poetry dependencies
+      run: poetry install -n --no-root
+      shell: bash

.github/workflows/base_benchmarks.yml

@@ -0,0 +1,61 @@
+# yamllint disable rule:line-length
+name: benchmarking
+on: # yamllint disable-line rule:truthy
+  push:
+    branches:
+      - master
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark_base_branch:
+    name: Continuous Benchmarking base branch
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: bencherdev/bencher@main
+      - name: Install hyperfine
+        run: |
+          wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
+          sudo dpkg -i hyperfine_1.12.0_amd64.deb
+      - uses: ./.github/actions/setup-hathor-env
+        name: Setup Hathor node environment
+        with:
+          python: 3.11
+          os: ubuntu-22.04
+      - name: Set env vars
+        run: cat ./extras/benchmarking/sync_v2/.env >> $GITHUB_ENV
+      - name: Download benchmark data
+        run: |
+          wget $BENCH_DATA_URL
+          tar -xzf $BENCH_DATA_FILE_NAME.tar.gz
+          mkdir $SERVER_DATA_DIR
+          poetry run hathor-cli load-from-logs \
+            --testnet \
+            --data $SERVER_DATA_DIR \
+            --cache \
+            --cache-size $CACHE_SIZE \
+            --x-localhost-only \
+            --log-dump $BENCH_DATA_FILE_NAME
+      - name: Run server node
+        run: |
+          poetry run hathor-cli run_node \
+            --testnet \
+            --data $SERVER_DATA_DIR \
+            --cache \
+            --cache-size $CACHE_SIZE \
+            --x-localhost-only \
+            --listen tcp:$TCP_PORT \
+            &
+      - name: Track base branch benchmarks with Bencher
+        run: |
+          bencher run \
+            --project hathor-core \
+            --token '${{ secrets.BENCHER_API_TOKEN }}' \
+            --branch master \
+            --testbed ubuntu-22.04 \
+            --adapter shell_hyperfine \
+            --err \
+            --file $BENCH_FILE \
+            './extras/benchmarking/sync_v2/benchmark_sync_v2.sh'

.github/workflows/docker.yml

@@ -28,7 +28,7 @@ jobs:
           - '3.12'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Prepare base version
         id: prep
         run: |
@@ -49,28 +49,28 @@ jobs:
           VERSION: ${{ steps.prep.outputs.check-version }}
         run: make check-custom
       - name: Set up QEMU  # arm64 is not available natively
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           version: latest
           install: true
           driver-opts: network=host
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: steps.prep.outputs.login-dockerhub == 'true'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: steps.prep.outputs.login-ghcr == 'true'
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         if: steps.prep_base_version.outputs.is-nightly == 'false'
         with:
           path: /tmp/.buildx-cache
@@ -79,7 +79,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-buildx-${{ matrix.python-impl }}${{ matrix.python-version }}-refs/heads/master-
       - name: Build and export to Docker
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ steps.prep.outputs.dockerfile }}
@@ -92,7 +92,7 @@ jobs:
       - name: Test image
         run: docker run --rm ${{ env.TEST_TAG }} quick_test --data / --testnet
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         if: ${{ !env.ACT }}  # Skip this step when testing locally with https://github.com/nektos/act
         with:
           context: .

.github/workflows/main.yml

@@ -25,7 +25,7 @@ jobs:
           full_matrix = {
             'python': ['3.10', '3.11', '3.12'],
             # available OS's: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on
-            'os': ['ubuntu-22.04', 'macos-12', 'windows-2022'],
+            'os': ['ubuntu-22.04', 'macos-13'],
           }
           # this is the fastest one:
           reduced_matrix = {
@@ -62,29 +62,14 @@ jobs:
       matrix: ${{fromJson(needs.matrix.outputs.matrix)}}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install Poetry
-        run: pipx install poetry
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-hathor-env
+        name: Setup Hathor node environment
         with:
-          python-version: ${{ matrix.python }}
-          cache: 'poetry'
-      - name: Install Ubuntu dependencies
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          sudo apt-get -qy update
-          sudo apt-get -qy install graphviz librocksdb-dev libsnappy-dev liblz4-dev
-      - name: Install macOS dependencies
-        if: startsWith(matrix.os, 'macos')
-        run: |
-          brew cleanup -q
-          # brew update -q
-          brew install -q graphviz rocksdb pkg-config
-      - name: Install Poetry dependencies
-        run: poetry install -n --no-root
+          python: ${{ matrix.python }}
+          os: ${{ matrix.os }}
       - name: Cache mypy
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: .mypy_cache
           # this key is setup such that every branch has its cache and new branches can reuse dev's cache, but not the other way around

.github/workflows/pr_benchmarks.yml

@@ -0,0 +1,69 @@
+# yamllint disable rule:line-length
+name: benchmarking
+on: # yamllint disable-line rule:truthy
+  pull_request:
+    branches:
+      - master
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark_pr_branch:
+    name: Continuous Benchmarking PRs
+    # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: bencherdev/bencher@main
+      - name: Install hyperfine
+        run: |
+          wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
+          sudo dpkg -i hyperfine_1.12.0_amd64.deb
+      - uses: ./.github/actions/setup-hathor-env
+        name: Setup Hathor node environment
+        with:
+          python: 3.11
+          os: ubuntu-22.04
+      - name: Set env vars
+        run: cat ./extras/benchmarking/sync_v2/.env >> $GITHUB_ENV
+      - name: Download benchmark data
+        run: |
+          wget $BENCH_DATA_URL
+          tar -xzf $BENCH_DATA_FILE_NAME.tar.gz
+          mkdir $SERVER_DATA_DIR
+          poetry run hathor-cli load-from-logs \
+            --testnet \
+            --data $SERVER_DATA_DIR \
+            --cache \
+            --cache-size $CACHE_SIZE \
+            --x-localhost-only \
+            --log-dump $BENCH_DATA_FILE_NAME
+      - name: Run server node
+        run: |
+          poetry run hathor-cli run_node \
+            --testnet \
+            --data $SERVER_DATA_DIR \
+            --cache \
+            --cache-size $CACHE_SIZE \
+            --x-localhost-only \
+            --listen tcp:$TCP_PORT \
+            &
+      - name: Track PR Benchmarks with Bencher
+        run: |
+          bencher run \
+            --project hathor-core \
+            --token '${{ secrets.BENCHER_API_TOKEN }}' \
+            --branch '${{ github.head_ref }}' \
+            --start-point '${{ github.base_ref }}' \
+            --start-point-hash '${{ github.event.pull_request.base.sha }}' \
+            --start-point-clone-thresholds \
+            --testbed ubuntu-22.04 \
+            --adapter shell_hyperfine \
+            --err \
+            --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+            --file $BENCH_FILE \
+            './extras/benchmarking/sync_v2/benchmark_sync_v2.sh'

.gitignore

@@ -22,3 +22,6 @@ extras/docker/envvars
 /dist/
 /requirements.txt
 *.egg-info
+
+# Nix
+.direnv/

Dockerfile

@@ -1,6 +1,6 @@
 # before changing these variables, make sure the tag $PYTHON-alpine$ALPINE exists first
 # list of valid tags hese: https://hub.docker.com/_/python
-ARG PYTHON=3.10
+ARG PYTHON=3.11
 ARG DEBIAN=bullseye
 
 # stage-0: copy pyproject.toml/poetry.lock and install the production set of dependencies

docs/legacy/ref/p2p.rst

@@ -27,7 +27,7 @@ The :py:mod:`hathor.p2p.states` has all states and messages of the p2p network.
 to send new messages and handle the new incoming ones.
 
 
-The :py:class:`hathor.p2p.peer_id.PeerId` stores the peer's identity, entrypoint, reputation and history.
+The :py:class:`hathor.p2p.peer.Peer` stores the peer's identity, entrypoint, reputation and history.
 
 
 

extras/benchmarking/sync_v2/.env

@@ -0,0 +1,9 @@
+BENCH_DATA_FILE_NAME=ci_testnet_upto_20k
+BENCH_DATA_URL=https://hathor-public-files.s3.amazonaws.com/hathor-core-ci/ci_testnet_upto_20k.tar.gz
+N_BLOCKS=20000
+CACHE_SIZE=100000
+SERVER_DATA_DIR=server-data
+TCP_PORT=40403
+N_RUNS=2
+BENCH_FILE=testnet_upto_20k_results.json
+BENCH_DATA_DIR=bench-data

extras/benchmarking/sync_v2/benchmark_sync_v2.sh

@@ -0,0 +1,16 @@
+hyperfine \
+  --warmup 1 \
+  --runs $N_RUNS \
+  --export-json $BENCH_FILE \
+  --command-name "sync-v2 (up to $N_BLOCKS blocks)" \
+  --prepare "rm -rf $BENCH_DATA_DIR && mkdir $BENCH_DATA_DIR" \
+  "
+    poetry run hathor-cli quick_test \
+      --testnet \
+      --data $BENCH_DATA_DIR \
+      --cache \
+      --cache-size $CACHE_SIZE \
+      --x-localhost-only \
+      --bootstrap tcp://localhost:$TCP_PORT \
+      --quit-after-n-blocks $N_BLOCKS
+  "