fix(state): Verify the state inclusion proof without splitting keys (#…

…1483)

## Overview

There has to be better way of fixing this, but I'm unsure of how we want
to handle it, so submitting this for now.

The bug is that, when verifying state inclusion proofs, tendermint will
perform [some
processing](https://github.com/celestiaorg/celestia-core/blob/a0ccbd068927b638705f63acdf6303d3091da985/crypto/merkle/proof_key_path.go#L85-L110)
on the keypath that you give it that the sdk simply doesn't have to do.
The iavl doesn't do this, as we just pass it the key it wants, not the
path. It just sticks the key that you give in the [request right
back](https://github.com/cosmos/cosmos-sdk/blob/b46d83850270b82496e4cd61421d9ebc8bac87f2/store/iavl/store.go#L324-L326),
and because the processing in tendermint sometimes results in a
different key, the proof verification
[fails](https://github.com/celestiaorg/celestia-core/blob/a0ccbd068927b638705f63acdf6303d3091da985/crypto/merkle/proof_op.go#L52).

Specifically, the string encoding of the bytes we pass it sometimes have
the string `/` and `%` in them from the address, the processing
tendermint does thinks its supposed to do something when it actually
isn't, and then the correct key returned by the sdk no longer matches.
This is why we're only seeing some addresses break!

[bad processing point
1](https://github.com/celestiaorg/celestia-core/blob/a0ccbd068927b638705f63acdf6303d3091da985/crypto/merkle/proof_key_path.go#L91)
[bad processing point
2](https://github.com/celestiaorg/celestia-core/blob/a0ccbd068927b638705f63acdf6303d3091da985/crypto/merkle/proof_key_path.go#L102)
(idek why this is there)

My quick and dirty solution was to copy paste fork ~~the file from
tendermint~~ a few methods that verify proofs, then change the api so we
don't have to split the string 😆 . We should fix it properly
soon. This might be a bug with tendermint, but I was admittedly too
tired to look into it or write up an issue. We might just need to pass a
different formatted string(?)

as a bonus, I added the foundation for an integration test for the core
accessor, that should make it waaaaaaaayyyy easier to round trip test
things! 🙂 🎄

pls feel to edit this PR and add whatever else is needed to get this
across the finish line

closes #1461 

## Checklist

- [x] New and updated code has appropriate documentation
- [x] New and updated code has new and/or updated testing
- [x] Required CI checks are passing
- [x] Visual proof for any user facing features like CLI or
documentation updates
- [x] Linked issues closed with keywords

Co-authored-by: rene <[email protected]>

go.mod

@@ -327,5 +327,5 @@ replace (
 	github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20221014072825-395797efb659
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	github.com/libp2p/go-libp2p-pubsub v0.7.0 => github.com/celestiaorg/go-libp2p-pubsub v0.6.2-0.20220812132010-46b2a019f2f2
-	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.5.0-tm-v0.34.20
+	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.5.0-tm-v0.34.20-verify-key-patch
 )

go.sum

@@ -218,8 +218,8 @@ github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOC
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/celestiaorg/celestia-app v0.11.0 h1:F76mBdLJZ1LLjWEPVEjoFPJA4skaY7Wrg+5o1LVlHGY=
 github.com/celestiaorg/celestia-app v0.11.0/go.mod h1:6k/zcNDEgOyJRGnAgWw1VsrwTKcVjOyYG5LPTHcZR+w=
-github.com/celestiaorg/celestia-core v1.5.0-tm-v0.34.20 h1:BqlcOQqL2UqdDTcdCtrOLXDlmwxIA8DiKiY79oahxkQ=
-github.com/celestiaorg/celestia-core v1.5.0-tm-v0.34.20/go.mod h1:f4R8qNJrP1CDH0SNwj4jA3NymBLQM4lNdx6Ijmfllbw=
+github.com/celestiaorg/celestia-core v1.5.0-tm-v0.34.20-verify-key-patch h1:b0swFbc0JqivwVz1UwnzFPlXYsVFkoHAmJ3LIfP4/2o=
+github.com/celestiaorg/celestia-core v1.5.0-tm-v0.34.20-verify-key-patch/go.mod h1:f4R8qNJrP1CDH0SNwj4jA3NymBLQM4lNdx6Ijmfllbw=
 github.com/celestiaorg/cosmos-sdk v1.4.0-sdk-v0.46.0 h1:65gnQ92mfz+9XNVTHeVwMp+SZuBqmToEnz8+WdDRmQ8=
 github.com/celestiaorg/cosmos-sdk v1.4.0-sdk-v0.46.0/go.mod h1:ByQ2rOrZs7s2OnPfeaiTMC8IOlcrT195xIRPgevydCI=
 github.com/celestiaorg/dagstore v0.0.0-20221014072825-395797efb659 h1:f3205vw3GYBtMiNoS+qB6IuHSs50Iwqsm9lNIikLTCk=

state/core_access.go

@@ -7,12 +7,13 @@ import (
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/api/tendermint/abci"
-	"github.com/cosmos/cosmos-sdk/store/rootmulti"
+	storetypes "github.com/cosmos/cosmos-sdk/store/types"
 	sdktypes "github.com/cosmos/cosmos-sdk/types"
 	sdktx "github.com/cosmos/cosmos-sdk/types/tx"
 	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
 	logging "github.com/ipfs/go-log/v2"
+	"github.com/tendermint/tendermint/crypto/merkle"
 	rpcclient "github.com/tendermint/tendermint/rpc/client"
 	"github.com/tendermint/tendermint/rpc/client/http"
 	"google.golang.org/grpc"
@@ -44,6 +45,8 @@ type CoreAccessor struct {
 	stakingCli stakingtypes.QueryClient
 	rpcCli     rpcclient.ABCIClient
 
+	prt *merkle.ProofRuntime
+
 	coreConn *grpc.ClientConn
 	coreIP   string
 	rpcPort  string
@@ -63,12 +66,17 @@ func NewCoreAccessor(
 	rpcPort string,
 	grpcPort string,
 ) *CoreAccessor {
+	// create verifier
+	prt := merkle.DefaultProofRuntime()
+	prt.RegisterOpDecoder(storetypes.ProofOpIAVLCommitment, storetypes.CommitmentOpDecoder)
+	prt.RegisterOpDecoder(storetypes.ProofOpSimpleMerkleCommitment, storetypes.CommitmentOpDecoder)
 	return &CoreAccessor{
 		signer:   signer,
 		getter:   getter,
 		coreIP:   coreIP,
 		rpcPort:  rpcPort,
 		grpcPort: grpcPort,
+		prt:      prt,
 	}
 }
 
@@ -97,6 +105,7 @@ func (ca *CoreAccessor) Start(ctx context.Context) error {
 		return err
 	}
 	ca.rpcCli = cli
+
 	return nil
 }
 
@@ -221,9 +230,12 @@ func (ca *CoreAccessor) BalanceForAddress(ctx context.Context, addr Address) (*B
 		return nil, fmt.Errorf("cannot convert %s into sdktypes.Int", string(value))
 	}
 	// verify balance
-	path := fmt.Sprintf("/%s/%s", banktypes.StoreKey, string(prefixedAccountKey))
-	prt := rootmulti.DefaultProofRuntime()
-	err = prt.VerifyValue(result.Response.GetProofOps(), head.AppHash, path, value)
+	err = ca.prt.VerifyValueKeys(
+		result.Response.GetProofOps(),
+		head.AppHash,
+		[][]byte{[]byte(banktypes.StoreKey),
+			prefixedAccountKey,
+		}, value)
 	if err != nil {
 		return nil, err
 	}

state/integration_test.go

@@ -0,0 +1,132 @@
+package state
+
+import (
+	"context"
+	"testing"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
+	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	"google.golang.org/grpc"
+
+	"github.com/celestiaorg/celestia-app/app"
+	"github.com/celestiaorg/celestia-app/testutil/testnode"
+	blobtypes "github.com/celestiaorg/celestia-app/x/payment/types"
+
+	"github.com/celestiaorg/celestia-node/header"
+)
+
+func TestIntegrationTestSuite(t *testing.T) {
+	suite.Run(t, new(IntegrationTestSuite))
+}
+
+type IntegrationTestSuite struct {
+	suite.Suite
+
+	cleanups []func() error
+	accounts []string
+	cctx     testnode.Context
+
+	accessor *CoreAccessor
+}
+
+func (s *IntegrationTestSuite) SetupSuite() {
+	if testing.Short() {
+		s.T().Skip("skipping test in unit-tests or race-detector mode.")
+	}
+
+	s.T().Log("setting up integration test suite")
+	require := s.Require()
+
+	// we create an arbitrary number of funded accounts
+	for i := 0; i < 25; i++ {
+		s.accounts = append(s.accounts, tmrand.Str(9))
+	}
+
+	tmNode, app, cctx, err := testnode.New(
+		s.T(),
+		testnode.DefaultParams(),
+		testnode.DefaultTendermintConfig(),
+		false,
+		s.accounts...,
+	)
+	require.NoError(err)
+
+	cctx, stopNode, err := testnode.StartNode(tmNode, cctx)
+	require.NoError(err)
+	s.cleanups = append(s.cleanups, stopNode)
+
+	cctx, cleanupGRPC, err := testnode.StartGRPCServer(app, testnode.DefaultAppConfig(), cctx)
+	require.NoError(err)
+	s.cleanups = append(s.cleanups, cleanupGRPC)
+
+	s.cctx = cctx
+	require.NoError(cctx.WaitForNextBlock())
+
+	signer := blobtypes.NewKeyringSigner(s.cctx.Keyring, s.accounts[0], cctx.ChainID)
+
+	accessor := NewCoreAccessor(signer, localHeader{s.cctx.Client}, "", "", "")
+	setClients(accessor, s.cctx.GRPCClient, s.cctx.Client)
+	s.accessor = accessor
+}
+
+func setClients(ca *CoreAccessor, conn *grpc.ClientConn, abciCli rpcclient.ABCIClient) {
+	ca.coreConn = conn
+	// create the query client
+	queryCli := banktypes.NewQueryClient(ca.coreConn)
+	ca.queryCli = queryCli
+	// create the staking query client
+	stakingCli := stakingtypes.NewQueryClient(ca.coreConn)
+	ca.stakingCli = stakingCli
+
+	ca.rpcCli = abciCli
+}
+
+func (s *IntegrationTestSuite) TearDownSuite() {
+	s.T().Log("tearing down integration test suite")
+	require := s.Require()
+	require.NoError(s.accessor.Stop(s.cctx.GoContext()))
+	for _, c := range s.cleanups {
+		err := c()
+		require.NoError(err)
+	}
+}
+
+func (s *IntegrationTestSuite) getAddress(acc string) sdk.Address {
+	rec, err := s.cctx.Keyring.Key(acc)
+	require.NoError(s.T(), err)
+
+	addr, err := rec.GetAddress()
+	require.NoError(s.T(), err)
+
+	return addr
+}
+
+type localHeader struct {
+	client rpcclient.Client
+}
+
+func (l localHeader) Head(ctx context.Context) (*header.ExtendedHeader, error) {
+	latest, err := l.client.Block(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	h := &header.ExtendedHeader{
+		RawHeader: latest.Block.Header,
+	}
+	return h, nil
+}
+
+func (s *IntegrationTestSuite) TestGetBalance() {
+	require := s.Require()
+	expectedBal := sdk.NewCoin(app.BondDenom, sdk.NewInt(int64(99999999999999999)))
+	for _, acc := range s.accounts {
+		bal, err := s.accessor.BalanceForAddress(context.Background(), s.getAddress(acc))
+		require.NoError(err)
+		require.Equal(&expectedBal, bal)
+	}
+}