Prepare consensus scripts to add tests to CI

.gitignore

@@ -18,6 +18,10 @@ build/**/*
 # Ignore development SQL dump
 data/**/*
 
+# Ignore test artifacts
+*/**/precomputed_*
+*/**/mina
+
 # Ignore coverage
 coverage/**/*
 

package.json

@@ -19,7 +19,8 @@
     "coverage": "vitest run --coverage",
     "lint": "eslint . --ext .ts",
     "format": "prettier --write .",
-    "generate": "graphql-codegen"
+    "generate": "graphql-codegen",
+    "consensus:test": "./tests/consensus/scripts/run-consensus-apps.sh"
   },
   "keywords": [],
   "author": "",

tests/consensus/run-compare.ts

@@ -1,15 +1,9 @@
-/**
- * Usage: npx ts-node ./tests/consensus/run-compare.ts
- */
-
 import pwd from 'process';
 import path from 'path';
 import fs from 'fs/promises';
 
-const TYPESCRIPT_OUTPUT_DIR_NAME =
-  process.env.TYPESCRIPT_OUTPUT_DIR_NAME || 'precomputed_ts';
-const OCAML_OUTPUT_DIR_NAME =
-  process.env.OCAML_OUTPUT_DIR_NAME || 'precomputed_ocaml';
+const TYPESCRIPT_OUTPUT_DIR_NAME = process.argv[2];
+const OCAML_OUTPUT_DIR_NAME = process.argv[3];
 
 const TYPESCRIPT_DIR = path.join(pwd.cwd(), TYPESCRIPT_OUTPUT_DIR_NAME);
 const OCAML_DIR = path.join(pwd.cwd(), OCAML_OUTPUT_DIR_NAME);
@@ -23,7 +17,7 @@ async function compareFiles() {
     console.error('Directories do not contain the same number of files!');
     console.error(`TypeScript: ${typescriptFiles.length}`);
     console.error(`OCaml: ${ocamlFiles.length}`);
-    return;
+    process.exit(1);
   }
 
   // Iterate through each file
@@ -58,10 +52,11 @@ async function compareFiles() {
       console.error(`File contents do not match! (${tsFile}), (${ocamlFile}})`);
       console.error(`TypeScript: ${JSON.stringify(tsContentJson)}`);
       console.error(`OCaml: ${JSON.stringify(ocamlContentJson)}`);
-      return;
+      process.exit(1);
     }
   }
   console.log('🎉 All files match!');
+  process.exit(0);
 }
 
 compareFiles();

tests/consensus/scripts/download-precomputed.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# This script is designed to download the latest precomputed blocks from a Google Cloud Storage bucket.
+# It fetches the latest file based on a naming convention, downloads it to the specified directory (or a default location),
+# unzips it, and then removes the zip file.
+# Usage:
+#   ./tests/consensus/scripts/download-precomputed.sh [download_location]
+#   Example: ./tests/consensus/scripts/download-precomputed.sh ./tests/consensus
+
+set -e
+set -o pipefail
+
+# Google Cloud Storage bucket where the zip files are uploaded
+bucket="gs://mina-consensus-precomputed-blocks-json/"
+
+# Get the latest file name from the bucket
+latest_file=$(gsutil ls $bucket | grep 'precomputed_blocks_' | sort | tail -1)
+
+if [ -z "$latest_file" ]; then
+  echo "No files found in the bucket."
+  exit 1
+fi
+
+# Use provided directory or default to the parent directory of the script's location
+download_location="${1:-$(dirname "$0")/..}"
+
+# Get the base name of the latest file to use when downloading
+base_name=$(basename "$latest_file")
+
+# Download the latest file to the provided location
+gsutil cp "$latest_file" "$download_location/$base_name"
+
+echo "Downloaded: $latest_file to $download_location"
+
+# Unzip the file in the provided location
+unzip -o "$download_location/$base_name" -d "$download_location"
+
+# Remove the zip file
+rm "$download_location/$base_name"
+
+echo "Unzipped and removed: $download_location/$base_name"

tests/consensus/scripts/run-consensus-apps.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# This script facilitates the process of setting up and testing the Mina consensus protocol.
+# It handles the following tasks:
+#   1. Checks prerequisites like npm and necessary directories.
+#   2. Clones or updates the Mina repository from GitHub.
+#   3. Builds the OCaml app and runs it.
+#   4. Runs the TypeScript app and a comparison script.
+
+# Usage:
+#   Simply execute this script. No arguments required.
+#   ./tests/consensus/scripts/run-consensus-apps.sh
+
+set -eo pipefail
+
+REPO_URL="[email protected]:MinaProtocol/mina.git"
+BRANCH="feat/berkeley-pos-test-client" # TODO replace this with "berkeley"
+DIR="./tests/consensus/mina"
+APP_PATH="./src/app/consensus_test"
+EXE_PATH="./_build/default/src/app/consensus_test/consensus_test.exe"
+PRECOMPUTED_DIR="../precomputed_blocks"
+OUTPUT_DIR="../precomputed_ocaml"
+CONFIG_FILE="./genesis_ledgers/berkeley.json"
+
+if ! command -v npm &> /dev/null; then
+    echo "Error: npm is not installed. Please install npm first as npx is required."
+    exit 1
+fi
+
+# Check if node_modules directory exists
+if [ ! -d "node_modules" ]; then
+    echo "node_modules directory not found. Running npm install..."
+    npm install
+fi
+
+# Check if $PRECOMPUTED_DIR exists
+if [ ! -d "$PRECOMPUTED_DIR" ]; then
+    echo "$PRECOMPUTED_DIR does not exist. Downloading precomputed data..."
+    ./tests/consensus/scripts/download-precomputed.sh ./tests/consensus/
+fi
+
+# Clone the repository if it doesn't exist, otherwise pull
+if [ ! -d "$DIR" ]; then
+    git clone -b $BRANCH $REPO_URL $DIR
+else
+    pushd $DIR
+    echo "Pulling $BRANCH from $REPO_URL"
+    git pull origin $BRANCH
+    popd
+fi
+
+# Navigate to the repository directory
+pushd $DIR
+
+# Update submodules
+echo "Updating submodules"
+git submodule update --init --recursive
+
+# Build the app
+echo "Building OCaml app"
+time dune b $APP_PATH
+
+# Run the app
+echo "Running OCaml app"
+echo "precomputed-dir: $PRECOMPUTED_DIR"
+$EXE_PATH --precomputed-dir $PRECOMPUTED_DIR --output-dir $OUTPUT_DIR --config-file $CONFIG_FILE
+
+# Return to the original directory
+popd
+
+# Run the TypeScript app and the compare script
+echo "Running TypeScript app"
+npx ts-node tests/consensus/select-consensus-precomputed.ts ./tests/consensus/precomputed_blocks ./tests/consensus/precomputed_ts
+
+echo "Running compare script"
+npx ts-node tests/consensus/run-compare.ts ./tests/consensus/precomputed_ts ./tests/consensus/precomputed_ocaml

tests/consensus/scripts/upload-precomputed-zip.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# This script automates the process of downloading block data with a given prefix from a specified Google Cloud Storage bucket.
+# It then processes the downloaded files, checking for a certain time threshold and removing files that don't meet the criteria.
+# Afterwards, the script archives the remaining files into a zip and uploads them to another GCS bucket.
+# Usage:
+#   ./tests/consensus/scripts/upload-precomputed-zip.sh [directory_for_precomputed_blocks]
+#   Example: ./tests/consensus/scripts/upload-precomputed-zip.sh ./tests/consensus/precomputed_blocks
+
+set -e
+set -o pipefail
+
+BUCKET="gs://mina_network_block_data/"
+UPLOAD_BUCKET="gs://mina-consensus-precomputed-blocks-json/"
+PREFIX="berkeley-"
+START_DATE=$(date -d "2023-08-13" "+%s")000 # Date the current Berkeley network was deployed
+
+# Use provided directory or default to the original location
+DIR="${1:-$(dirname "$0")/../precomputed_blocks}"
+
+# Initialize
+mkdir -p "$DIR"
+
+# Download files from bucket with given prefix and range
+download_files() {
+  for i in $(seq 2 5); do
+    gsutil -m cp "${BUCKET}${PREFIX}${i}-*" "$DIR/"
+  done
+}
+
+# Process downloaded files
+process_files() {
+  for file in $(ls "$DIR"); do
+    local file_path="$DIR/$file"
+    echo "Processing file $file_path"
+
+    local scheduled_time
+    scheduled_time=$(jq -r '.data.scheduled_time' "$file_path")
+
+    # Remove the file if scheduled_time is not later than START_DATE
+    [[ "$scheduled_time" -le "$START_DATE" ]] && rm "$file_path"
+  done
+}
+
+# Archive and upload
+archive_and_upload() {
+  echo "Archiving and uploading files in $DIR"
+  local current_date
+  current_date=$(date "+%Y-%m-%d")
+  local zip_name="precomputed_blocks_$current_date.zip"
+  local parent_dir="$(dirname "$DIR")"
+
+  # Change directory to the parent of $DIR
+  pushd "$parent_dir"
+  echo "Archiving $(basename "$DIR") to $zip_name"
+  zip -r $zip_name "$(basename "$DIR")/"
+  popd
+
+  echo "Uploading $parent_dir/$zip_name to $UPLOAD_BUCKET"
+  gsutil cp "$parent_dir/$zip_name" $UPLOAD_BUCKET
+}
+
+
+download_files
+process_files
+archive_and_upload

tests/consensus/select-consensus-precomputed.ts

@@ -9,15 +9,14 @@ import { select } from '../../src/consensus/mina-consensus';
 import type { BlockInfo } from '../../src/blockchain/types';
 import { type BlockFileOutput, type PrecomputedBlock, GetSlot } from './types';
 
-const outputDir = process.env.OUTPUT_DIR || 'precomputed_ts';
+const inputDir = process.argv[2];
+const outputDir = process.argv[3];
 
 function readPrecomputedBlocks() {
-  const directoryPath = path.join(process.cwd(), 'precomputed_berkeley');
-  const files = fs.readdirSync(directoryPath);
-
+  const files = fs.readdirSync(inputDir);
   const blocks: BlockInfo[] = [];
   for (const file of files) {
-    const data = fs.readFileSync(path.join(directoryPath, file));
+    const data = fs.readFileSync(path.join(inputDir, file));
     const block = mapPrecomputedToBlockInfo(data.toString());
     blocks.push(block);
   }