Merge pull request #9059 from MinaProtocol/compatible

Merge back to develop

.gitmodules

@@ -3,16 +3,16 @@
 	url = https://github.com/o1-labs/snarky
 [submodule "src/external/ppx_optcomp"]
 	path = src/external/ppx_optcomp
-	url = https://github.com/CodaProtocol/ppx_optcomp
+	url = https://github.com/MinaProtocol/ppx_optcomp
 [submodule "src/external/async_kernel"]
 	path = src/external/async_kernel
-	url = https://github.com/CodaProtocol/async_kernel
+	url = https://github.com/MinaProtocol/async_kernel
 [submodule "src/external/ocaml-extlib"]
 	path = src/external/ocaml-extlib
-	url = https://github.com/CodaProtocol/ocaml-extlib.git
+	url = https://github.com/MinaProtocol/ocaml-extlib.git
 [submodule "src/external/rpc_parallel"]
 	path = src/external/rpc_parallel
-	url = https://github.com/CodaProtocol/rpc_parallel.git
+	url = https://github.com/MinaProtocol/rpc_parallel.git
 [submodule "frontend/wallet/tablecloth"]
 	path = frontend/wallet/tablecloth
 	url = https://github.com/bkase/tablecloth
@@ -25,3 +25,7 @@
 [submodule "src/external/ppx_version"]
 	path = src/external/ppx_version
 	url = https://github.com/o1-labs/ppx_version.git
+[submodule "src/external/ppx_deriving_yojson"]
+	path = src/external/ppx_deriving_yojson
+	url = https://github.com/MinaProtocol/ppx_deriving_yojson.git
+	branch = master

buildkite/scripts/export-git-env-vars.sh

@@ -26,7 +26,7 @@ export BUILD_URL=${BUILDKITE_BUILD_URL}
 
 [[ -n "$BUILDKITE_BRANCH" ]] && export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g')
 
-if [[ -n ${THIS_COMMIT_TAG} ]]; then # If the commit is tagged
+if [[ -n "${THIS_COMMIT_TAG}" ]]; then # If the commit is tagged
     export VERSION="${GITTAG}-${GITHASH}"
     export GENERATE_KEYPAIR_VERSION=${VERSION}
     export DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!g; s!_!-!g')"
@@ -44,7 +44,7 @@ export MINA_DEB_CODENAME=stretch
 case $GITBRANCH in
     master)
         RELEASE=stable ;;
-    compatible|master|release/*) # whitelist of branches that can be tagged
+    compatible|master|release*) # whitelist of branches that can be tagged
         case "${THIS_COMMIT_TAG}" in
           *alpha*) # any tag including the string `alpha`
             RELEASE=alpha ;;

buildkite/src/Constants/ContainerImages.dhall

@@ -2,7 +2,7 @@
 
 {
   toolchainBase = "codaprotocol/ci-toolchain-base:v3",
-  minaToolchain = "codaprotocol/mina-toolchain@sha256:03958744ae145185132fe6d5f681a3dc992811ab2c036e1825daeda0d71ce0f9",
+  minaToolchain = "codaprotocol/mina-toolchain@sha256:61701a8c0382384f862888b7a0947f1209b5561af46dcca9d3ccd2aec04dea70",
   elixirToolchain = "elixir:1.10-alpine",
   rustToolchain = "codaprotocol/coda:toolchain-rust-e855336d087a679f76f2dd2bbdc3fdfea9303be3",
   nodeToolchain = "node:14.13.1-stretch-slim",

dockerfiles/Dockerfile

@@ -1,7 +1,7 @@
-FROM codaprotocol/coda:toolchain-14cb996cc7d8910a2ff6ae3ce132cea571bbb69c
+FROM codaprotocol/coda:toolchain-77e1b03e2785a003827e9bb61af4f4c51f37fdb0
 
 # same as in Dockerfile-toolchain
-ARG OCAML_VERSION=4.07.1
+ARG OCAML_VERSION=4.11.2
 
 ENV OPAM_DIR             "/home/opam/.opam/$OCAML_VERSION"
 ENV PATH                 "${OPAM_DIR}/bin:$PATH"

dockerfiles/Dockerfile-rosetta

@@ -1,14 +1,14 @@
 #################################################################################################
 # The "build-deps" stage
-# - Installs all compilers/interpreters, tools, and OS packages on debian buster-slim
+# - Installs all compilers/interpreters, tools, and OS packages on debian stretch-slim
 #################################################################################################
 FROM debian:stretch-slim AS build-deps
 
 # Ocaml Version
-ARG OCAML_VERSION=4.07
-ARG OCAML_REVISION=.1
-ARG OCAML_VARIANT=+logoom
-ARG OCAML_PACKAGE=ocaml-variants
+ARG OCAML_VERSION=4.11
+ARG OCAML_REVISION=.2
+ARG OCAML_VARIANT=
+ARG OCAML_PACKAGE=
 ARG OPAM_VERSION=2.0.7
 
 # Golang version number used to detemine tarball name
@@ -129,7 +129,7 @@ RUN git clone \
   /home/opam/opam-repository \
   && opam init --disable-sandboxing -k git -a ~/opam-repository --bare \
   && opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git \
-  && opam switch create "${OCAML_VERSION}" "${OCAML_PACKAGE}.${OCAML_VERSION}${OCAML_REVISION}${OCAML_VARIANT}" \
+  && opam switch create "${OCAML_VERSION}" "${OCAML_PACKAGE}${OCAML_VERSION}${OCAML_REVISION}${OCAML_VARIANT}" \
   && opam switch "${OCAML_VERSION}"
 
 # Alternate variants for 4.07 that are included in the official opam image
@@ -150,11 +150,11 @@ FROM build-deps AS opam-deps
 
 # location of repo used for pins and external package commits
 ARG MINA_DIR=mina
-# branch to checkout on first clone (this will be the only availible branch in the container)
+# branch to checkout on first clone (this will be the only available branch in the container)
 # can also be a tagged release
 # TODO: change this to two distinct variables, one for opam/dependency clone
 #       and a distinct one for the mina codebase in the next stage
-ARG OPAM_BRANCH=develop
+ARG OPAM_BRANCH=compatible
 
 # location of external packages
 ARG EXTERNAL_PKG_DIR=$MINA_DIR/src/external
@@ -205,6 +205,7 @@ RUN eval $(opam config env) \
   && opam pin add src/external/async_kernel \
   && opam pin add src/external/coda_base58 \
   && opam pin add src/external/graphql_ppx \
+  && opam pin add src/external/ppx_deriving_yojson \
   && opam clean --logs -cs
 
 
@@ -222,7 +223,7 @@ ARG DUNE_PROFILE=testnet_postake_medium_curves
 # can also be a tagged release
 # TODO: change this to two distinct variables, one for opam/dependency clone
 #       and a distinct one for the mina codebase in the next stage
-ARG MINA_BRANCH=develop
+ARG MINA_BRANCH=compatible
 
 # repo to checkout the branch from
 ARG MINA_REPO=https://github.com/MinaProtocol/mina
@@ -293,7 +294,6 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN mkdir /usr/share/man/man7 /usr/share/man/man1
 
 # Dependencies
-# buster-slim configures apt to not store any cache, so no need to rm it
 # TODO: make sure this is the minimum runtime deps
 RUN apt-get -y update \
   && apt -y install \

dockerfiles/Dockerfile-toolchain

@@ -1,8 +1,7 @@
-#FROM ocaml/opam2:debian-10-ocaml-4.08
-FROM gcr.io/o1labs-192920/mina-rosetta-build-deps@sha256:3cc21ddf546350407f5a5f63bb59c4aec9bbaf680b2ac89bb834e23c5a0ae07d
+FROM gcr.io/o1labs-192920/mina-rosetta-build-deps@sha256:eb4d3ad2c15127e00c68b7b2c07dd8f887ed20c499d1302e963cbf02740b9eaa
 
 # if updated, also change in Dockerfile
-ARG OCAML_VERSION=4.07.1
+ARG OCAML_VERSION=4.11.2
 ARG DOCKER_VERSION=19.03.4
 ARG TERRAFORM_VERSION=0.12.29
 

dockerfiles/scripts/cron_job_dump_ledger.sh

@@ -7,6 +7,20 @@ sleep 480
 
 echo "done sleeping"
 
+# wait until daemon is sync'ed
+while true; do
+  STATUS=$(mina client status | grep "Sync status")
+  echo "$STATUS" | grep "Synced"
+  RESULT=$?
+  if [ $RESULT -eq 0 ] ; then
+      echo "daemon is synced"
+      break
+  else
+      echo "waiting for daemon to sync"
+      sleep 60
+  fi
+done
+
 # retry getting the staking ledger until the node is fully up and the command returns exit code 0
 while true; do
   mina ledger export staking-epoch-ledger > staking_epoch_ledger.json
@@ -39,6 +53,12 @@ NEXT_STAKING_MD5="$(md5sum next_epoch_ledger.json | cut -d " " -f 1 )"
 NEXT_FILENAME=next-staking-"$EPOCHNUM"-"$NEXT_STAKING_HASH"-"$NEXT_STAKING_MD5".json
 mv ./next_epoch_ledger.json ./$NEXT_FILENAME
 
+EXPORTED_LOGS="local-logs"
+LOGS_FILENAME="daemon-logs-epoch-$EPOCHNUM.tgz"
+mina client export-local-logs --tarfile $EXPORTED_LOGS
+mv /root/.mina-config/exported_logs/$EXPORTED_LOGS.tar.gz $LOGS_FILENAME
+
 echo "upload to a GCP cloud storage bucket"
 gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $LEDGER_FILENAME gs://mina-staking-ledgers
 gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $NEXT_FILENAME gs://mina-staking-ledgers
+gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $LOGS_FILENAME gs://mina-staking-ledgers

helm/staking-ledger-cron/mainnet-dump-staking-ledger-cronjob.yaml

@@ -5,28 +5,80 @@ kind: CronJob
 metadata:
   name: mainnet-dump-staking-ledger-cronjob
 spec:
-  schedule: "0 11 * * *"
+  concurrencyPolicy: Replace
+  failedJobsHistoryLimit: 1
   jobTemplate:
     spec:
       template:
         spec:
+          containers:
+          - command:
+            - /bin/bash
+            - -c
+            - 'mina daemon --generate-genesis-proof true --peer-list-url https://storage.googleapis.com/mina-seed-lists/mainnet_seeds.txt --background;
+               sleep 480;echo "done sleeping";
+               while true; do
+                 STATUS=$(mina client status | grep "Sync status");
+                 echo "$STATUS" | grep "Synced";
+                 RESULT=$?;
+                 if [ $RESULT -eq 0 ]; then
+                   echo "daemon is synced";
+                   break;
+                 else
+                   echo "waiting for daemon to sync";
+                   sleep 60;
+                 fi;
+               done;
+               while true;
+                do mina ledger export staking-epoch-ledger>staking_epoch_ledger.json;
+                 if [ "$?" -eq 0 ]&&[ "$(cat staking_epoch_ledger.json)" != "Ledger not found: current staking ledger not available" ];
+                  then echo "staking epoch ledger dumped!";break;
+                  else echo "waiting for staking ledger to become available, sleeping for 30s";sleep 30;
+                 fi;
+               done;
+               mina ledger export next-epoch-ledger>next_epoch_ledger.json;
+               echo "next epoch ledger dumped!";
+               EPOCHNUM="$(mina client status|grep "Best tip consensus time"|grep -o "epoch=[0-9]*"|sed "s/[^0-9]*//g")";
+               STAKING_HASH="$(mina ledger hash --ledger-file staking_epoch_ledger.json)";
+               STAKING_MD5="$(md5sum staking_epoch_ledger.json|cut -d " " -f 1)";
+               LEDGER_FILENAME=staking-"$EPOCHNUM"-"$STAKING_HASH"-"$STAKING_MD5".json;
+               mv ./staking_epoch_ledger.json ./$LEDGER_FILENAME;NEXT_STAKING_HASH="$(mina ledger hash --ledger-file next_epoch_ledger.json)";
+               NEXT_STAKING_MD5="$(md5sum next_epoch_ledger.json|cut -d " " -f 1)";
+               NEXT_FILENAME=next-staking-"$EPOCHNUM"-"$NEXT_STAKING_HASH"-"$NEXT_STAKING_MD5".json;
+               mv ./next_epoch_ledger.json ./$NEXT_FILENAME;
+               EXPORTED_LOGS="local-logs";
+               LOGS_FILENAME="daemon-logs-epoch-$EPOCHNUM.tgz";
+               mina client export-local-logs --tarfile $EXPORTED_LOGS;
+               mv /root/.mina-config/exported_logs/$EXPORTED_LOGS.tar.gz $LOGS_FILENAME;
+               echo "upload to a GCP cloud storage bucket";
+               gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $LEDGER_FILENAME gs://mina-staking-ledgers;
+               gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $NEXT_FILENAME gs://mina-staking-ledgers;
+               gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $LOGS_FILENAME gs://mina-staking-ledgers'
+            env:
+            - name: GCLOUD_KEYFILE
+              value: /gcloud/keyfile.json
+            image: minaprotocol/mina-daemon-baked:1.1.5-a42bdee
+            imagePullPolicy: IfNotPresent
+            name: mainnet-dump-staking-ledger-container
+            resources: {}
+            terminationMessagePath: /dev/termination-log
+            terminationMessagePolicy: File
+            volumeMounts:
+            - mountPath: /gcloud/
+              name: gcloud-keyfile
+          dnsPolicy: ClusterFirst
+          restartPolicy: Never
+          schedulerName: default-scheduler
+          securityContext: {}
+          terminationGracePeriodSeconds: 30
           volumes:
           - name: gcloud-keyfile
             secret:
-              secretName: gcloud-keyfile
               defaultMode: 256
               items:
               - key: keyfile
                 path: keyfile.json
-          containers:
-          - name: mainnet-dump-staking-ledger-container
-            image: minaprotocol/mina-daemon-baked:1.1.5-a42bdee
-            env:
-            - name: GCLOUD_KEYFILE
-              value: "/gcloud/keyfile.json"
-            volumeMounts:
-            - name: gcloud-keyfile
-              mountPath: "/gcloud/"
-            # command: ["/usr/bin/dumb-init", '/cron_job_dump_ledger.sh']
-            command: ["/bin/bash", "-c", 'mina daemon --generate-genesis-proof true --peer-list-url https://storage.googleapis.com/mina-seed-lists/mainnet_seeds.txt --background;sleep 480;echo "done sleeping";while true;do mina ledger export staking-epoch-ledger>staking_epoch_ledger.json;if [ "$?" -eq 0 ]&&[ "$(cat staking_epoch_ledger.json)" != "Ledger not found: current staking ledger not available" ];then echo "staking epoch ledger dumped!";break;else echo "waiting for staking ledger to become available, sleeping for 30s";sleep 30;fi;done;mina ledger export next-epoch-ledger>next_epoch_ledger.json;echo "next epoch ledger dumped!";EPOCHNUM="$(mina client status|grep "Best tip consensus time"|grep -o "epoch=[0-9]*"|sed "s/[^0-9]*//g")";STAKING_HASH="$(mina ledger hash --ledger-file staking_epoch_ledger.json)";STAKING_MD5="$(md5sum staking_epoch_ledger.json|cut -d " " -f 1)";LEDGER_FILENAME=staking-"$EPOCHNUM"-"$STAKING_HASH"-"$STAKING_MD5".json;mv ./staking_epoch_ledger.json ./$LEDGER_FILENAME;NEXT_STAKING_HASH="$(mina ledger hash --ledger-file next_epoch_ledger.json)";NEXT_STAKING_MD5="$(md5sum next_epoch_ledger.json|cut -d " " -f 1)";NEXT_FILENAME=next-staking-"$EPOCHNUM"-"$NEXT_STAKING_HASH"-"$NEXT_STAKING_MD5".json;mv ./next_epoch_ledger.json ./$NEXT_FILENAME;echo "upload to a GCP cloud storage bucket";gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $LEDGER_FILENAME gs://mina-staking-ledgers;gsutil -o Credentials:gs_service_key_file=/gcloud/keyfile.json cp $NEXT_FILENAME gs://mina-staking-ledgers' ]
-          restartPolicy: Never
+              secretName: gcloud-keyfile
+  schedule: 0 11 * * *
+  successfulJobsHistoryLimit: 3
+  suspend: false

scripts/pin-external-packages.sh

@@ -2,7 +2,7 @@
 
 # update and pin packages, used by CI
 
-PACKAGES="ocaml-sodium rpc_parallel ocaml-extlib ocaml-extlib async_kernel coda_base58 graphql_ppx"
+PACKAGES="ocaml-sodium rpc_parallel ocaml-extlib ocaml-extlib async_kernel coda_base58 graphql_ppx ppx_deriving_yojson"
 
 git submodule sync && git submodule update --init --recursive
 

scripts/setup-opam.sh

@@ -10,7 +10,7 @@ export OPAMYES=1
 # Set term to xterm if not set
 export TERM=${TERM:-xterm}
 
-SWITCH='ocaml-variants.4.07.1+logoom'
+SWITCH='4.11.2'
 
 if [[ -d ~/.opam ]]; then
   # ocaml environment
@@ -72,6 +72,7 @@ sudo chmod -R u+rw ~/.opam
 opam pin add src/external/async_kernel
 opam pin add src/external/coda_base58
 opam pin add src/external/graphql_ppx
+opam pin add src/external/ppx_deriving_yojson
 eval $(opam config env)
 
 # show switch list at end

src/app/archive/archive_lib/processor.ml

@@ -395,7 +395,7 @@ module User_command = struct
                {sql| INSERT INTO user_commands (type, fee_payer_id, source_id,
                       receiver_id, fee_token, token, nonce, amount, fee,
                       valid_until, memo, hash)
-                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    VALUES (?::user_command_type, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                     RETURNING id |sql})
             { typ=
                 ( match via with
@@ -489,7 +489,7 @@ module User_command = struct
          {sql| INSERT INTO user_commands (type, fee_payer_id, source_id,
                       receiver_id, fee_token, token, nonce, amount, fee,
                       valid_until, memo, hash)
-                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    VALUES (?::user_command_type, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                     RETURNING id
          |sql})
       { typ= user_cmd.typ
@@ -540,7 +540,7 @@ module Internal_command = struct
       (Caqti_request.find_opt
          Caqti_type.(tup2 string string)
          Caqti_type.int
-         "SELECT id FROM internal_commands WHERE hash = $1 AND type = $2")
+         "SELECT id FROM internal_commands WHERE hash = $1 AND type = $2::internal_command_type")
       (Transaction_hash.to_base58_check transaction_hash, typ)
 
   let load (module Conn : CONNECTION) ~(id : int) =
@@ -570,7 +570,7 @@ module Internal_command = struct
           (Caqti_request.find typ Caqti_type.int
              {sql| INSERT INTO internal_commands
                     (type, receiver_id, fee, token,hash)
-                   VALUES (?, ?, ?, ?, ?)
+                   VALUES (?::internal_command_type, ?, ?, ?, ?)
                    RETURNING id
              |sql})
           { typ= internal_cmd.typ
@@ -640,7 +640,7 @@ module Fee_transfer = struct
           (Caqti_request.find typ Caqti_type.int
              {sql| INSERT INTO internal_commands
                     (type, receiver_id, fee, token, hash)
-                   VALUES (?, ?, ?, ?, ?)
+                   VALUES (?::internal_command_type, ?, ?, ?, ?)
                    RETURNING id
              |sql})
           { kind
@@ -688,7 +688,7 @@ module Coinbase = struct
           (Caqti_request.find typ Caqti_type.int
              {sql| INSERT INTO internal_commands
                     (type, receiver_id, fee, token, hash)
-                   VALUES (?, ?, ?, ?, ?)
+                   VALUES (?::internal_command_type, ?, ?, ?, ?)
                    RETURNING id
              |sql})
           { receiver_id
@@ -881,7 +881,7 @@ module Block_and_signed_command = struct
                  fee_payer_balance,
                  source_balance,
                  receiver_balance)
-               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+               VALUES (?, ?, ?, ?::user_command_status, ?, ?, ?, ?, ?, ?, ?)
          |sql})
       { block_id
       ; user_command_id

src/app/archive/cli/archive_cli.ml

@@ -88,7 +88,7 @@ let command_prune =
            Caqti_async.connect postgres.value
          in
          let%bind () = Conn.start () in
-         match%bind.Async
+         match%bind.Async.Deferred
            let%bind () =
              Archive_lib.Processor.Block.delete_if_older_than ?height
                ?num_blocks ?timestamp conn
@@ -98,7 +98,7 @@ let command_prune =
          | Ok () ->
              return ()
          | Error err ->
-             let%bind.Async _ = Conn.rollback () in
+             let%bind.Async.Deferred _ = Conn.rollback () in
              Deferred.Result.fail err
        in
        let logger = Logger.create () in
@@ -109,7 +109,7 @@ let command_prune =
            ; Option.map timestamp ~f:(fun v ->
                  ("timestamp", `String (Int64.to_string v)) ) ]
        in
-       match%map.Async go () with
+       match%map.Async.Deferred go () with
        | Ok () ->
            [%log info] "Successfully purged blocks." ~metadata:cmd_metadata
        | Error err ->