A0-2699: Weekly sync from snapshot tests (#1597)

# Description

Add two weekly tests for syncing the DB from a snapshot. One downloads
the pruned ParityDB snapshot and synces to testnet on Tuesday, and
another does the same with a non-pruned one on Wednesday.

## Type of change

Automated tests.

# Checklist:

- I have added tests
- I have made neccessary updates to the Infrastructure

.github/scripts/get_top_block.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from sys import argv
+from substrateinterface import SubstrateInterface
+chain = SubstrateInterface(url=argv[1])
+number = chain.get_block()['header']['number']
+print(number)

.github/scripts/test_testnet_db_sync.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+set -euo pipefail
+echo "Starting DB sync test."
+
+BOOT_NODES=/dns4/bootnode-eu-central-1-0.test.azero.dev/tcp/30333/p2p/12D3KooWRkGLz4YbVmrsWK75VjFTs8NvaBu42xhAmQaP4KeJpw1L/dns4/bootnode-eu-west-1-0.test.azero.dev/tcp/30333/p2p/12D3KooWFVXnvJdPuGnGYMPn5qLQAQYwmRBgo6SmEQsKZSrDoo2k/dns4/bootnode-eu-west-2-0.test.azero.dev/tcp/30333/p2p/12D3KooWAkqYFFKMEJn6fnPjYnbuBBsBZq6fRFJZYR6rxnuCZWCC/dns4/bootnode-us-east-1-0.test.azero.dev/tcp/30333/p2p/12D3KooWQFkkFr5aM5anGEiUCQiGUdRyWgrdpvSjBgWAUS9srLE4/dns4/bootnode-us-east-2-0.test.azero.dev/tcp/30333/p2p/12D3KooWD5s2dkifJua69RbLwEREDdJjsNHvavNRGxdCvzhoeaLc
+BASE_PATH="running/"
+CHAINSPEC="${BASE_PATH}/chainspec.json"
+DB_SNAPSHOT_URL="http://db.test.azero.dev.s3-website.eu-central-1.amazonaws.com/latest-parity.html"
+DB_PATH="chains/testnet/"
+DB_ARG="--database paritydb"
+TOP_BLOCK_SCRIPT="./.github/scripts/get_top_block.py"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -p | --pruned)
+            echo "Using pruned DB."
+            DB_SNAPSHOT_URL="http://db.test.azero.dev.s3-website.eu-central-1.amazonaws.com/latest-parity-pruned.html"
+            DB_ARG="--enable-pruning"
+            shift;;
+        *)
+            echo "Unrecognized argument: $1"
+            exit 1;;
+    esac
+done
+
+initialize() {
+    pip install substrate-interface
+    mkdir -p "${BASE_PATH}"
+}
+
+get_snapshot () {
+    echo "Downloading the snapshot...  "
+    DB_SNAPSHOT_PATH=${BASE_PATH}/${DB_PATH}
+    mkdir -p "${DB_SNAPSHOT_PATH}"
+    pushd "${DB_SNAPSHOT_PATH}" > /dev/null
+
+    set +e
+    wget -q -O - ${DB_SNAPSHOT_URL} | tar xzf -
+    if [[ 0 -ne $? ]]
+    then
+        error "Failed to download and unpack the snapshot."
+    fi
+    set -e
+    popd > /dev/null
+}
+
+copy_chainspec () {
+    echo "Copying the chainspec...   "
+    cp ./bin/node/src/resources/testnet_chainspec.json "${CHAINSPEC}"
+}
+
+get_target_block() {
+    echo "Determining target block...   "
+    TARGET_BLOCK=`${TOP_BLOCK_SCRIPT} "wss://ws.test.azero.dev"`
+}
+
+get_current_block() {
+    echo "Determining current block...   "
+    CURRENT_BLOCK=`${TOP_BLOCK_SCRIPT} "ws://localhost:9944"`
+}
+
+initialize
+copy_chainspec
+get_snapshot
+
+get_target_block
+
+chmod +x aleph-node
+./aleph-node \
+    --chain "${CHAINSPEC}" \
+    --base-path "${BASE_PATH}" \
+    --rpc-port 9944 \
+    --name sync-from-snapshot-tester \
+    --bootnodes "${BOOT_NODES}" \
+    --node-key-file "${BASE_PATH}/p2p_secret" \
+    ${DB_ARG} \
+    --no-mdns 1>/dev/null 2>/dev/null &
+
+echo "Waiting a moment for the node to start up..."
+sleep 1m
+
+get_current_block
+echo "Syncing to ${TARGET_BLOCK} starting at ${CURRENT_BLOCK}."
+
+while [ $CURRENT_BLOCK -le $TARGET_BLOCK ]; do
+    sleep 1m
+    get_current_block
+    echo "Sync status: ${CURRENT_BLOCK}/${TARGET_BLOCK}".
+done

.github/workflows/weekly-sync-from-snapshot-testnet-pruned.yml

@@ -4,9 +4,8 @@
 name: Weekly sync from snapshot test, pruned
 on:
   # At 03:00 on Tuesday
-  # TODO: fix the time to actually be correct
   schedule:
-    - cron: '0 3 2 4 2'
+    - cron: '0 3 * * 2'
   workflow_dispatch:
 
 concurrency:
@@ -18,3 +17,48 @@ jobs:
     name: Check vars and secrets
     uses: ./.github/workflows/_check-vars-and-secrets.yml
     secrets: inherit
+
+  build-production-node-and-runtime:
+    needs: [check-vars-and-secrets]
+    name: Build production node and runtime
+    uses: ./.github/workflows/_build-production-node-and-runtime.yml
+    with:
+      ref: ${{ github.ref }}
+
+  get-snapshot-and-sync:
+    needs: [build-production-node-and-runtime]
+    name: Download snapshot and run
+    runs-on: [self-hosted, Linux, X64, medium-1000GB]
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Download release artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: aleph-production-node
+
+      - name: Download snapshot
+        timeout-minutes: 180
+        run: |
+          ./.github/scripts/test_testnet_db_sync.sh \
+            --pruned
+
+
+  slack-notification:
+    name: Slack notification
+    runs-on: ubuntu-20.04
+    needs: [get-snapshot-and-sync]
+    if: >
+      !cancelled() &&
+      github.event_name != 'workflow_dispatch'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Send Slack message
+        uses: ./.github/actions/slack-notification
+        with:
+          notify-on: "failure"
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }}

.github/workflows/weekly-sync-from-snapshot-testnet.yml

@@ -4,9 +4,8 @@
 name: Weekly sync from snapshot test, non-pruned
 on:
   # At 03:00 on Wednesday
-  # TODO: fix the time to actually be correct
   schedule:
-    - cron: '0 3 2 4 3'
+    - cron: '0 3 * * 3'
   workflow_dispatch:
 
 concurrency:
@@ -18,3 +17,47 @@ jobs:
     name: Check vars and secrets
     uses: ./.github/workflows/_check-vars-and-secrets.yml
     secrets: inherit
+
+  build-production-node-and-runtime:
+    needs: [check-vars-and-secrets]
+    name: Build production node and runtime
+    uses: ./.github/workflows/_build-production-node-and-runtime.yml
+    with:
+      ref: ${{ github.ref }}
+
+  get-snapshot-and-sync:
+    needs: [build-production-node-and-runtime]
+    name: Download snapshot and run
+    runs-on: [self-hosted, Linux, X64, medium-1000GB]
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Download release artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: aleph-production-node
+
+      - name: Download snapshot
+        timeout-minutes: 360
+        run: |
+          ./.github/scripts/test_testnet_db_sync.sh
+
+
+  slack-notification:
+    name: Slack notification
+    runs-on: ubuntu-20.04
+    needs: [get-snapshot-and-sync]
+    if: >
+      !cancelled() &&
+      github.event_name != 'workflow_dispatch'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Send Slack message
+        uses: ./.github/actions/slack-notification
+        with:
+          notify-on: "failure"
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_DEV_ONDUTY }}