Binary release to GHA (#10154)

First try to implement the release pipeline in GHA.

.github/workflows/master_fuzzer_binaries.yml

@@ -1,4 +1,4 @@
-name: Fuzz Master
+name: Build fuzz targets from master
 on:
   push:
     branches:

.github/workflows/near_crates_publish.yml

@@ -0,0 +1,32 @@
+name: near crates publish
+
+on:
+  push:
+    branches: master
+  workflow_dispatch:
+
+jobs:
+  publish-cargo-crates:
+    name: "Publish near-workspaces on crates.io https://crates.io/crates/near-workspaces"
+    runs-on: "ubuntu-latest"
+    environment: deploy
+    permissions:
+      contents: write # required for crates push
+    timeout-minutes: 30 
+
+    steps:
+      - name: Publish near-workspaces on crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          set -x
+          cargo install --git https://github.com/miraclx/cargo-workspaces --tag v0.3.0 cargo-workspaces
+          cargo ws publish --yes --allow-dirty --force '*' \
+              --no-git-commit --no-git-push --no-individual-tags --tag-prefix 'crates-' \
+              --tag-msg $$'crates.io snapshot\n---%{\n- %n - https://crates.io/crates/%n/%v}'
+
+      - name: Create tag on https://github.com/near/nearcore
+        run: |
+          git push --no-follow-tags https://github.com/near/nearcore.git tag 'crates-*'
+
+

.github/workflows/nearcore_deploy.yml

@@ -0,0 +1,58 @@
+name: Nearcore release
+
+on:
+  push:
+    branches: master
+
+  workflow_dispatch:
+    inputs:
+      branch:
+        default: 'master'
+        description: "Nearcore branch to build and publish"
+        type: string
+        required: true
+
+jobs:
+  binary-release:
+    name: "Build and publish neard binary"
+    runs-on: "ubuntu-22.04-8core"
+    environment: deploy
+    permissions:
+      id-token: write # required to use OIDC authentication
+
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::754641474505:role/GitHubActionsRunner
+          aws-region: us-west-1
+
+      - name: Checkout ${{ github.event.inputs.branch }} branch
+        if: ${{ github.event_name == 'workflow_dispatch'}}
+        uses: actions/checkout@v4
+        with:
+          repository: near/nearcore
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: Checkout nearcore master repository
+        if: ${{ github.event_name != 'workflow_dispatch'}}
+        uses: actions/checkout@v4
+        with:
+          repository: near/nearcore
+          ref: master
+
+      - name: Log rust and cargo versions 
+        run: |
+          rustup --version
+          cargo --version
+
+      - name: Neard binary build and upload to S3
+        run: |
+          chmod +x nearcore/scripts/binary-release.sh
+          nearcore/scripts/binary-release.sh
+
+      - name: Update latest version metadata in S3
+        run: |
+          echo $(git rev-parse HEAD) > latest
+          branch=$(git rev-parse --abbrev-ref HEAD)
+          # aws s3 cp --acl public-read latest s3://build.nearprotocol.com/nearcore/$(uname)/${branch}/latest

.github/workflows/ondemand_fuzzer_binaries.yml

@@ -22,7 +22,8 @@ on:
 
 jobs:
   build_fuzzers:
-    runs-on: "ubuntu-20.04"
+    name: Build Fuzzers
+    runs-on: "ubuntu-22.04-8core"
 
     permissions:
       contents: "read"
@@ -40,17 +41,6 @@ jobs:
           rustup install nightly
           rustup default nightly
 
-      - name: Set swap to 10G
-        shell: bash
-        run: |
-          export SWAP_FILE=$(swapon --show=NAME | tail -n 1)
-          sudo swapoff $SWAP_FILE
-          sudo rm $SWAP_FILE
-          sudo fallocate -l 10G $SWAP_FILE
-          sudo chmod 600 $SWAP_FILE
-          sudo mkswap $SWAP_FILE
-          sudo swapon $SWAP_FILE
-
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-bolero
@@ -86,7 +76,7 @@ jobs:
         run: echo "branch_type=${{ github.event.inputs.branch_type }}" >> "$GITHUB_ENV"
 
       - name: "Compile fuzzers and upload to GCS"
-      - run: |
+        run: |
           NAME="nearcore-$branch_type-$(env TZ=Etc/UTC  date +"%Y%m%d%H%M%S")"
           cargo +nightly bolero build-clusterfuzz --all-features --profile fuzz
           gsutil cp -Z target/fuzz/clusterfuzz.tar "gs://fuzzer_targets/$branch_type/$NAME.tar.gz"

scripts/binary_release.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+set -eo pipefail
+
+release="${1:-release}"
+
+case "$release" in
+  release|nightly-release|perf-release|assertions-release)
+    ;;
+  *)  
+    echo "Unsupported release type '$release'. Please provide no argument for normal release or provide nightly-release for nightly."
+    exit 1
+    ;;
+esac
+
+branch=${BUILDKITE_BRANCH:-${GITHUB_REF##*/}}
+commit=${BUILDKITE_COMMIT:-${GITHUB_SHA}}
+if [[ ${commit} = "HEAD" ]]; then
+    commit=$(git rev-parse HEAD)
+fi
+
+os=$(uname)
+arch=$(uname -m)
+os_and_arch=${os}-${arch}
+
+function tar_binary {
+  mkdir -p $1/${os_and_arch}
+  cp target/release/$1 $1/${os_and_arch}/
+  tar -C $1 -czvf $1.tar.gz ${os_and_arch}
+}
+
+make $release
+
+function upload_binary {
+  if [ "$release" = "release" ]
+  then
+    tar_binary $1
+    tar_file=$1.tar.gz
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os}/${branch}/$1
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os}/${branch}/${commit}/$1
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os}/${branch}/${commit}/stable/$1
+
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os_and_arch}/${branch}/$1
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os_and_arch}/${branch}/${commit}/$1
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os_and_arch}/${branch}/${commit}/stable/$1
+
+    aws s3 cp --acl public-read ${tar_file} s3://build.nearprotocol.com/nearcore/${os_and_arch}/${branch}/${tar_file}
+    aws s3 cp --acl public-read ${tar_file} s3://build.nearprotocol.com/nearcore/${os_and_arch}/${branch}/${commit}/${tar_file}
+    aws s3 cp --acl public-read ${tar_file} s3://build.nearprotocol.com/nearcore/${os_and_arch}/${branch}/${commit}/stable/${tar_file}
+
+  else
+    folder="${release%-release}"
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os}/${branch}/${commit}/${folder}/$1
+    aws s3 cp --acl public-read target/release/$1 s3://build.nearprotocol.com/nearcore/${os_and_arch}/${branch}/${commit}/${folder}/$1
+  fi
+}
+
+upload_binary neard
+
+# disabled until we clarify why we need this binary in S3
+# if [ "$release" != "assertions-release" ]
+# then
+#   upload_binary store-validator
+# fi
+
+if [ "$release" = "release" ]
+then
+  upload_binary near-sandbox
+fi