Merge pull request #8 from cartesi/feature/migration-db-v2-beta

Feature/migration db v2 beta

.github/workflows/loadtest.yaml

@@ -46,7 +46,8 @@ jobs:
         run: |
           export POSTGRES_GRAPHQL_DB_URL="postgres://postgres:password@localhost:5432/rlgraphql?sslmode=disable"
           export POSTGRES_NODE_DB_URL="postgres://postgres:password@localhost:5432/rollupsdb?sslmode=disable"
-          nohup ./cartesi-rollups-graphql -d --raw-enabled --db-implementation=postgres &
+          nohup ./cartesi-rollups-graphql -d --db-implementation=postgres &
+          sleep 10
 
       - name: Run k6 tests
         id: k6

.gitignore

@@ -7,3 +7,6 @@ TODO
 sqlite3
 .idea/
 .direnv/
+/rollups-graphql
+/rollups-node
+/rollups-espresso-reader

.vscode/settings.json

@@ -9,8 +9,10 @@
         "celestiaorg",
         "CHAINID",
         "codegen",
+        "configtest",
         "ctsi",
         "CTSI",
+        "CTYPE",
         "czvf",
         "darwin",
         "dataavailability",
@@ -38,16 +40,20 @@
         "golangci",
         "gqlgen",
         "gqlparser",
+        "grafana",
         "graphile",
         "Graphile",
         "gsrpc",
         "hexutil",
+        "hlgraphql",
+        "iapplication",
         "idempotency",
         "initdb",
         "inputbox",
         "inputter",
         "Inputter",
         "isatty",
+        "isready",
         "jmoiron",
         "joho",
         "keyrings",
@@ -63,6 +69,7 @@
         "mypassword",
         "myuser",
         "ncruces",
+        "nohup",
         "nolint",
         "nonodomodel",
         "oapi",
@@ -73,12 +80,15 @@
         "posix",
         "postgraphile",
         "postgres",
+        "postgresql",
         "Preparex",
+        "psql",
         "pubkey",
         "Queryx",
         "randao",
         "Randao",
         "readerclient",
+        "rlgraphql",
         "rollup",
         "rollups",
         "rollupsdb",

DEV2DEV.md

@@ -4,6 +4,12 @@
 watchexec --exts go --watch . 'make test && make lint'
 ```
 
+To run just one test:
+
+```shell
+watchexec --exts go --watch . 'go test -p 1 ./... -testify.m ^TestNoDuplicateInputs$'
+```
+
 uint64 type is based on [rollups_outputs.rs](https://github.com/cartesi/rollups-node/blob/392c75972037352ecf94fb482619781b1b09083f/offchain/rollups-events/src/rollups_outputs.rs#L41)
 
 ```go
@@ -47,9 +53,9 @@ go run . --enable-debug
 ```
 
 ```shell
-export POSTGRES_GRAPHQL_DB_URL="postgres://myuser:mypassword@127.0.0.1:5432/mydatabase?sslmode=disable"
+export POSTGRES_GRAPHQL_DB_URL="postgres://postgres:password@127.0.0.1:5432/hlgraphql?sslmode=disable"
 export POSTGRES_NODE_DB_URL="postgres://postgres:password@localhost:5432/rollupsdb?sslmode=disable"
-go run . --http-address=0.0.0.0 --enable-debug --db-implementation postgres
+go run . --http-address=0.0.0.0 --enable-debug
 ```
 
 ## Environment Variables

README.md

@@ -31,8 +31,32 @@ Start a Postgres instance locally using docker compose.
 make up-db-raw
 ```
 
-```sh
-export POSTGRES_GRAPHQL_DB_URL="postgres://postgres:password@localhost:5432/rlgraphql?sslmode=disable"
+```shell
+export POSTGRES_GRAPHQL_DB_URL="postgres://postgres:[email protected]:5432/hlgraphql?sslmode=disable"
+export POSTGRES_NODE_DB_URL="postgres://postgres:password@localhost:5432/rollupsdb?sslmode=disable"
+go run . --http-address=0.0.0.0
+```
+
+## Running with Node V2
+
+Build Node V2 and then start it.
+
+Create the rollups graphql database:
+
+```shell
+docker exec -i postgres psql -U postgres -d hlgraphql < ./postgres/raw/hlgraphql.sql
+```
+
+Compile:
+
+```shell
+go build -o cartesi-rollups-graphql
+```
+
+Run the rollups graphql:
+
+```shell
+export POSTGRES_GRAPHQL_DB_URL="postgres://postgres:password@localhost:5432/hlgraphql?sslmode=disable"
 export POSTGRES_NODE_DB_URL="postgres://postgres:password@localhost:5432/rollupsdb?sslmode=disable"
 ./cartesi-rollups-graphql
 ```

ci/Dockerfile

@@ -0,0 +1,147 @@
+# (c) Cartesi and individual authors (see AUTHORS)
+# SPDX-License-Identifier: Apache-2.0 (see LICENSE)
+
+# syntax=docker.io/docker/dockerfile:1
+
+ARG EMULATOR_VERSION=0.18.1
+
+# Build directories.
+ARG GO_BUILD_PATH=/build/cartesi/go
+
+FROM cartesi/machine-emulator:${EMULATOR_VERSION} AS common-env
+
+USER root
+
+# Re-declare ARGs so they can be used in the RUN block
+ARG GO_BUILD_PATH
+
+# Install ca-certificates and curl (setup).
+RUN <<EOF
+    set -e
+    apt-get update
+    apt-get install -y --no-install-recommends ca-certificates curl wget build-essential pkg-config libssl-dev
+    mkdir -p /opt/go ${GO_BUILD_PATH}/rollups-node
+    chown -R cartesi:cartesi /opt/go ${GO_BUILD_PATH}
+EOF
+
+USER cartesi
+
+# =============================================================================
+# STAGE: go-installer
+#
+# This stage installs Go in the /opt directory.
+# =============================================================================
+
+FROM common-env AS go-installer
+# Download and verify Go based on the target architecture
+RUN <<EOF
+    set -e
+    ARCH=$(dpkg --print-architecture)
+    wget -O /tmp/go.tar.gz "https://go.dev/dl/go1.22.7.linux-${ARCH}.tar.gz"
+    sha256sum /tmp/go.tar.gz
+    case "$ARCH" in
+        amd64) echo "fc5d49b7a5035f1f1b265c17aa86e9819e6dc9af8260ad61430ee7fbe27881bb  /tmp/go.tar.gz" | sha256sum --check ;;
+        arm64) echo "ed695684438facbd7e0f286c30b7bc2411cfc605516d8127dc25c62fe5b03885  /tmp/go.tar.gz" | sha256sum --check ;;
+        *) echo "unsupported architecture: $ARCH"; exit 1 ;;
+    esac
+    tar -C /opt -xzf /tmp/go.tar.gz
+    rm /tmp/go.tar.gz
+EOF
+
+# Set up Go environment variables
+ENV PATH="/opt/go/bin:$PATH"
+
+# =============================================================================
+# STAGE: go-prepare
+#
+# This stage prepares the Go build environment. It downloads the external
+# =============================================================================
+
+FROM go-installer AS go-prepare
+
+ARG GO_BUILD_PATH
+WORKDIR ${GO_BUILD_PATH}
+
+ENV GOCACHE=${GO_BUILD_PATH}/.cache
+ENV GOENV=${GO_BUILD_PATH}/.config/go/env
+ENV GOPATH=${GO_BUILD_PATH}/.go
+
+# Download external dependencies.
+COPY --chown=cartesi:cartesi ./rollups-node/go.mod ${GO_BUILD_PATH}/rollups-node/
+COPY --chown=cartesi:cartesi ./rollups-node/go.sum ${GO_BUILD_PATH}/rollups-node/
+RUN cd ${GO_BUILD_PATH}/rollups-node && go mod download
+
+# =============================================================================
+# STAGE: go-builder
+#
+# This stage builds the node Go binaries. First it downloads the external
+# dependencies and then it builds the binaries.
+# =============================================================================
+
+FROM go-prepare AS go-builder
+
+ARG GO_BUILD_PATH
+
+COPY --chown=cartesi:cartesi . ${GO_BUILD_PATH}/
+
+# Build espresso reader.
+RUN go build -o cartesi-rollups-espresso-reader
+
+# Build rollups node.
+RUN cd ${GO_BUILD_PATH}/rollups-node && make build-go
+
+
+# =============================================================================
+# STAGE: rollups-node
+#
+# This stage prepares the final Docker image that will be used in the production
+# environment. It installs in /usr/bin all the binaries necessary to run the
+# node.
+#
+# (This stage copies the binaries from previous stages.)
+# =============================================================================
+
+FROM cartesi/machine-emulator:${EMULATOR_VERSION} AS rollups-node
+
+ARG NODE_RUNTIME_DIR=/var/lib/cartesi-rollups-node
+
+USER root
+
+# Download system dependencies required at runtime.
+ARG DEBIAN_FRONTEND=noninteractive
+RUN <<EOF
+    set -e
+    apt-get update
+    apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        procps \
+        tini
+    rm -rf /var/lib/apt/lists/*
+    mkdir -p ${NODE_RUNTIME_DIR}/snapshots ${NODE_RUNTIME_DIR}/data
+    chown -R cartesi:cartesi ${NODE_RUNTIME_DIR}
+EOF
+
+# Copy Go binary.
+ARG GO_BUILD_PATH
+COPY --from=go-builder ${GO_BUILD_PATH}/rollups-node/cartesi-rollups-* /usr/bin
+COPY --from=go-builder ${GO_BUILD_PATH}/cartesi-rollups-* /usr/bin
+
+RUN curl -L https://github.com/cartesi/image-kernel/releases/download/v0.20.0/linux-6.5.13-ctsi-1-v0.20.0.bin -o /usr/share/cartesi-machine/images/linux.bin
+
+RUN curl -L https://github.com/cartesi/machine-emulator-tools/releases/download/v0.16.1/rootfs-tools-v0.16.1.ext2 -o /usr/share/cartesi-machine/images/rootfs.ext2
+
+# Set user to low-privilege.
+USER cartesi
+
+WORKDIR ${NODE_RUNTIME_DIR}
+
+HEALTHCHECK --interval=1s --timeout=1s --retries=5 \
+    CMD curl -G -f -H 'Content-Type: application/json' http://127.0.0.1:10000/healthz
+
+RUN mkdir applications
+RUN cartesi-machine --ram-length=128Mi --store=applications/echo-dapp --final-hash -- ioctl-echo-loop --vouchers=1 --notices=1 --reports=1 --verbose=1
+
+COPY --chown=cartesi:cartesi ./ci/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+CMD ["tini", "--", "/entrypoint.sh"]

ci/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+cartesi-rollups-node &
+
+# TODO: remove in the future when we stop test the node v2 without espresso in this repo.
+if [ "${CARTESI_FEATURE_ESPRESSO_READER_ENABLED:-false}" != "false" ]; then
+    cartesi-rollups-espresso-reader
+fi
+
+wait

ci/env.nodev2-local

@@ -0,0 +1,11 @@
+CARTESI_BLOCKCHAIN_HTTP_ENDPOINT=http://localhost:8545
+CARTESI_BLOCKCHAIN_ID=31337
+CARTESI_LOG_LEVEL=debug
+CARTESI_CONTRACTS_INPUT_BOX_ADDRESS=0x593E5BCf894D6829Dd26D0810DA7F064406aebB6
+CARTESI_CONTRACTS_INPUT_BOX_DEPLOYMENT_BLOCK_NUMBER=10
+CARTESI_AUTH_MNEMONIC=test test test test test test test test test test test junk
+CARTESI_POSTGRES_ENDPOINT=postgres://postgres:password@localhost:5432/rollupsdb?sslmode=disable
+CARTESI_FEATURE_CLAIM_SUBMISSION_ENABLED=true
+CARTESI_FEATURE_INPUT_READER_ENABLED=true
+CARTESI_FEATURE_ESPRESSO_READER_ENABLED=false
+CARTESI_BLOCKCHAIN_WS_ENDPOINT=ws://localhost:8545

ci/env.nodev2-sepolia

@@ -0,0 +1,16 @@
+CARTESI_AUTH_MNEMONIC=never gonna give you up never gonna let you down never gonna run around and desert you never gonna make you cry never gonna say goodbye never gonna tell a lie and hurt you
+CARTESI_BLOCKCHAIN_HTTP_ENDPOINT=https://eth-sepolia.g.alchemy.com/v2/YourApiKey
+CARTESI_BLOCKCHAIN_ID=11155111
+CARTESI_BLOCKCHAIN_WS_ENDPOINT=wss://eth-sepolia.g.alchemy.com/v2/YourApiKey
+CARTESI_FEATURE_CLAIM_SUBMISSION_ENABLED=false
+CARTESI_LOG_LEVEL=debug
+CARTESI_LOG_PRETTY_ENABLED=true
+CARTESI_FEATURE_INPUT_READER_ENABLED=false
+CARTESI_FEATURE_ESPRESSO_READER_ENABLED=true
+CARTESI_CONTRACTS_INPUT_BOX_ADDRESS=0x593E5BCf894D6829Dd26D0810DA7F064406aebB6
+ESPRESSO_STARTING_BLOCK=1485399
+ESPRESSO_SERVICE_ENDPOINT=0.0.0.0:8080
+ESPRESSO_BASE_URL=https://query.decaf.testnet.espresso.network
+ESPRESSO_NAMESPACE=55555
+CARTESI_POSTGRES_ENDPOINT=postgres://postgres:password@localhost:5432/rollupsdb?sslmode=disable
+CARTESI_CONTRACTS_INPUT_BOX_DEPLOYMENT_BLOCK_NUMBER=6850934

ci/env.nodev2-sepolia-no-reader

@@ -0,0 +1,16 @@
+CARTESI_AUTH_MNEMONIC=never gonna give you up never gonna let you down never gonna run around and desert you never gonna make you cry never gonna say goodbye never gonna tell a lie and hurt you
+CARTESI_BLOCKCHAIN_HTTP_ENDPOINT=https://eth-sepolia.g.alchemy.com/v2/YourApiKey
+CARTESI_BLOCKCHAIN_ID=11155111
+CARTESI_BLOCKCHAIN_WS_ENDPOINT=wss://eth-sepolia.g.alchemy.com/v2/YourApiKey
+CARTESI_FEATURE_CLAIM_SUBMISSION_ENABLED=false
+CARTESI_LOG_LEVEL=debug
+CARTESI_LOG_PRETTY_ENABLED=true
+CARTESI_FEATURE_INPUT_READER_ENABLED=false
+CARTESI_FEATURE_ESPRESSO_READER_ENABLED=false
+ESPRESSO_STARTING_BLOCK=1482301
+ESPRESSO_SERVICE_ENDPOINT=0.0.0.0:8080
+ESPRESSO_BASE_URL=https://query.decaf.testnet.espresso.network
+CARTESI_CONTRACTS_INPUT_BOX_ADDRESS=0x593E5BCf894D6829Dd26D0810DA7F064406aebB6
+CARTESI_CONTRACTS_INPUT_BOX_DEPLOYMENT_BLOCK_NUMBER=6850934
+CARTESI_POSTGRES_ENDPOINT=postgres://postgres:password@localhost:5432/rollupsdb?sslmode=disable
+ESPRESSO_NAMESPACE=55555

ci/scripts/check-db-output.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+MAX_RETRIES=30
+for i in $(seq 1 $MAX_RETRIES); do
+  RESULT=$(docker exec -i postgres psql -U postgres -d rollupsdb -t -c "SELECT * FROM public.output;")
+  if [[ "$RESULT" =~ "deadbeef" ]]; then
+    echo "Result found: $RESULT"
+    exit 0
+  fi
+  echo "Result: $RESULT"
+  echo "Waiting for result... attempt $i"
+  sleep 5
+done
+echo "Timeout reached: result not found"
+exit 1