Consistently use Merkle Tree root as ECChain key (#834)

* Consistently use Merkle Tree root as ECChain key

Remove the custom ECChain key generation inside `gpbft` in favor of
consistently using Merkle Tree root of chain as key. This key is already
used for signature payload generation, and in the new chain exchange as
a key to identify a chain. Doing so enables signature validation of
messages without having to know the chain.

Consistent use of Merkle Tree root hash as key enables a number of
simplifications across the repo:
* unblocks the unification of validation logic between partial and full
  validators, since caching can use a consistent way to identify
  messages whether they're partial or not.
* reduce various type gymnastics across root and chain exchange packages
  by repurposing gpbft ECChain Key.

The work here also introduces lazy-loading for the ECChain, which
requires ECChain type to be converted to `struct`, and be passed by
pointer. As part of this work, TipSet is also turned into a pointer
consistently across various interfaces.

The ECChain receiver functions are adopted to accommodate potential nil
values for a chain.

Fixes #825

* Bump timeout for `TestF3LateBootstrap` to unblock progress

* Bump timeout for `TestF3PauseResumeCatchup` to unblock progress

* Bump timeout for `TestF3LateBootstrap` to unblock progress

Bumped timeout to 2 m, just to see if CI passes.

certchain/certchain.go

@@ -21,7 +21,7 @@ var (
 type FinalityCertificateProvider func(context.Context, uint64) (*certs.FinalityCertificate, error)
 
 type tipSetWithPowerTable struct {
-	gpbft.TipSet
+	*gpbft.TipSet
 	Beacon     []byte
 	PowerTable *gpbft.PowerTable
 }
@@ -31,7 +31,7 @@ type CertChain struct {
 
 	rng              *rand.Rand
 	certificates     []*certs.FinalityCertificate
-	generateProposal func(context.Context, uint64) (gpbft.ECChain, error)
+	generateProposal func(context.Context, uint64) (*gpbft.ECChain, error)
 }
 
 func New(o ...Option) (*CertChain, error) {
@@ -63,7 +63,7 @@ func (cc *CertChain) GetCommittee(instance uint64) (*gpbft.Committee, error) {
 	return cc.getCommittee(tspt)
 }
 
-func (cc *CertChain) GetProposal(instance uint64) (*gpbft.SupplementalData, gpbft.ECChain, error) {
+func (cc *CertChain) GetProposal(instance uint64) (*gpbft.SupplementalData, *gpbft.ECChain, error) {
 	//TODO refactor ProposalProvider in gpbft to take context.
 	ctx := context.TODO()
 	proposal, err := cc.generateProposal(ctx, instance)
@@ -120,7 +120,7 @@ func (cc *CertChain) getTipSetWithPowerTableByEpoch(ctx context.Context, epoch i
 		return nil, err
 	}
 	return &tipSetWithPowerTable{
-		TipSet: gpbft.TipSet{
+		TipSet: &gpbft.TipSet{
 			Epoch:      epoch,
 			Key:        ts.Key(),
 			PowerTable: ptCid,
@@ -130,12 +130,12 @@ func (cc *CertChain) getTipSetWithPowerTableByEpoch(ctx context.Context, epoch i
 	}, nil
 }
 
-func (cc *CertChain) generateRandomProposal(ctx context.Context, base gpbft.TipSet, len int) (gpbft.ECChain, error) {
+func (cc *CertChain) generateRandomProposal(ctx context.Context, base *gpbft.TipSet, len int) (*gpbft.ECChain, error) {
 	if len == 0 {
 		return gpbft.NewChain(base)
 	}
 
-	suffix := make([]gpbft.TipSet, len-1)
+	suffix := make([]*gpbft.TipSet, len-1)
 	for i := range suffix {
 		epoch := base.Epoch + 1 + int64(i)
 		gTS, err := cc.getTipSetWithPowerTableByEpoch(ctx, epoch)
@@ -250,7 +250,7 @@ func (cc *CertChain) sign(ctx context.Context, committee *gpbft.Committee, paylo
 func (cc *CertChain) Generate(ctx context.Context, length uint64) ([]*certs.FinalityCertificate, error) {
 	cc.certificates = make([]*certs.FinalityCertificate, 0, length)
 
-	cc.generateProposal = func(ctx context.Context, instance uint64) (gpbft.ECChain, error) {
+	cc.generateProposal = func(ctx context.Context, instance uint64) (*gpbft.ECChain, error) {
 		var baseEpoch int64
 		if instance == cc.m.InitialInstance {
 			baseEpoch = cc.m.BootstrapEpoch - cc.m.EC.Finality

certexchange/polling/common_test.go

@@ -17,7 +17,7 @@ const TestNetworkName gpbft.NetworkName = "testnet"
 
 func MakeCertificate(t *testing.T, rng *rand.Rand, tsg *sim.TipSetGenerator, backend signing.Backend, base *gpbft.TipSet, instance uint64, powerTable, nextPowerTable gpbft.PowerEntries) *certs.FinalityCertificate {
 	chainLen := rng.Intn(23) + 1
-	chain, err := gpbft.NewChain(*base)
+	chain, err := gpbft.NewChain(base)
 	require.NoError(t, err)
 
 	for i := 0; i < chainLen; i++ {

certexchange/protocol_test.go

@@ -55,11 +55,23 @@ func TestClientServer(t *testing.T) {
 	cs, err := certstore.CreateStore(ctx, ds, 0, pt)
 	require.NoError(t, err)
 
-	cert := &certs.FinalityCertificate{GPBFTInstance: 0, SupplementalData: supp, ECChain: gpbft.ECChain{{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: pcid}}}
+	cert := &certs.FinalityCertificate{GPBFTInstance: 0, SupplementalData: supp,
+		ECChain: &gpbft.ECChain{
+			TipSets: []*gpbft.TipSet{
+				{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: pcid},
+			},
+		},
+	}
 	err = cs.Put(ctx, cert)
 	require.NoError(t, err)
 
-	cert = &certs.FinalityCertificate{GPBFTInstance: 1, SupplementalData: supp, ECChain: gpbft.ECChain{{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: pcid}}}
+	cert = &certs.FinalityCertificate{GPBFTInstance: 1, SupplementalData: supp,
+		ECChain: &gpbft.ECChain{
+			TipSets: []*gpbft.TipSet{
+				{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: pcid},
+			},
+		},
+	}
 	err = cs.Put(ctx, cert)
 	require.NoError(t, err)
 
@@ -149,7 +161,13 @@ func TestClientServer(t *testing.T) {
 	}
 
 	// Until we've added a new certificate.
-	cert = &certs.FinalityCertificate{GPBFTInstance: 2, SupplementalData: supp, ECChain: gpbft.ECChain{{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: pcid}}}
+	cert = &certs.FinalityCertificate{GPBFTInstance: 2, SupplementalData: supp,
+		ECChain: &gpbft.ECChain{
+			TipSets: []*gpbft.TipSet{
+				{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: pcid},
+			},
+		},
+	}
 	require.NoError(t, cs.Put(ctx, cert))
 
 	{

certs/certs.go

@@ -36,7 +36,7 @@ type FinalityCertificate struct {
 	GPBFTInstance uint64
 	// The ECChain finalized during this instance, starting with the last tipset finalized in
 	// the previous instance.
-	ECChain gpbft.ECChain
+	ECChain *gpbft.ECChain
 	// Additional data signed by the participants in this instance. Currently used to certify
 	// the power table used in the next instance.
 	SupplementalData gpbft.SupplementalData
@@ -89,7 +89,7 @@ func NewFinalityCertificate(powerDelta PowerTableDiff, justification *gpbft.Just
 // finalized, the instance of the first invalid finality certificate, and the power table that
 // should be used to validate that finality certificate, along with the error encountered.
 func ValidateFinalityCertificates(verifier gpbft.Verifier, network gpbft.NetworkName, prevPowerTable gpbft.PowerEntries, nextInstance uint64, base *gpbft.TipSet,
-	certs ...*FinalityCertificate) (_nextInstance uint64, chain gpbft.ECChain, newPowerTable gpbft.PowerEntries, err error) {
+	certs ...*FinalityCertificate) (_nextInstance uint64, chain *gpbft.ECChain, newPowerTable gpbft.PowerEntries, err error) {
 	for _, cert := range certs {
 		if cert.GPBFTInstance != nextInstance {
 			return nextInstance, chain, prevPowerTable, fmt.Errorf("expected instance %d, found instance %d", nextInstance, cert.GPBFTInstance)
@@ -133,7 +133,7 @@ func ValidateFinalityCertificates(verifier gpbft.Verifier, network gpbft.Network
 				cert.GPBFTInstance, cert.SupplementalData.PowerTable, powerTableCid)
 		}
 		nextInstance++
-		chain = append(chain, cert.ECChain.Suffix()...)
+		chain = chain.Append(cert.ECChain.Suffix()...)
 		prevPowerTable = newPowerTable
 		base = cert.ECChain.Head()
 	}

certs/certs_test.go

@@ -212,7 +212,7 @@ func TestFinalityCertificates(t *testing.T) {
 
 	rng := rand.New(rand.NewSource(1234))
 	tsg := sim.NewTipSetGenerator(rng.Uint64())
-	base := gpbft.TipSet{Epoch: 0, Key: tsg.Sample(), PowerTable: tableCid}
+	base := &gpbft.TipSet{Epoch: 0, Key: tsg.Sample(), PowerTable: tableCid}
 
 	certificates := make([]*certs.FinalityCertificate, 10)
 	powerTables := make([]gpbft.PowerEntries, 10)
@@ -224,14 +224,14 @@ func TestFinalityCertificates(t *testing.T) {
 		cert, err := certs.NewFinalityCertificate(certs.MakePowerTableDiff(powerTables[i], powerTable), justification)
 		require.NoError(t, err)
 		certificates[i] = cert
-		base = *justification.Vote.Value.Head()
+		base = justification.Vote.Value.Head()
 	}
 
 	// Validate one.
 	nextInstance, chain, newPowerTable, err := certs.ValidateFinalityCertificates(backend, networkName, powerTables[0], 0, certificates[0].ECChain.Base(), certificates[0])
 	require.NoError(t, err)
 	require.EqualValues(t, 1, nextInstance)
-	require.True(t, chain.Eq(certificates[0].ECChain.Suffix()))
+	require.Equal(t, chain.TipSets, certificates[0].ECChain.Suffix())
 	require.Equal(t, powerTables[1], newPowerTable)
 
 	// Validate multiple
@@ -240,14 +240,14 @@ func TestFinalityCertificates(t *testing.T) {
 	require.EqualValues(t, 4, nextInstance)
 	require.Equal(t, powerTables[4], newPowerTable)
 	require.True(t, certificates[3].ECChain.Head().Equal(chain.Head()))
-	require.True(t, certificates[0].ECChain[1].Equal(chain.Base()))
+	require.True(t, certificates[0].ECChain.TipSets[1].Equal(chain.Base()))
 
 	nextInstance, chain, newPowerTable, err = certs.ValidateFinalityCertificates(backend, networkName, powerTables[nextInstance], nextInstance, nil, certificates[nextInstance:]...)
 	require.NoError(t, err)
 	require.EqualValues(t, len(certificates), nextInstance)
 	require.Equal(t, powerTable, newPowerTable)
 	require.True(t, certificates[len(certificates)-1].ECChain.Head().Equal(chain.Head()))
-	require.True(t, certificates[4].ECChain[1].Equal(chain.Base()))
+	require.True(t, certificates[4].ECChain.TipSets[1].Equal(chain.Base()))
 }
 
 func TestBadFinalityCertificates(t *testing.T) {
@@ -257,7 +257,7 @@ func TestBadFinalityCertificates(t *testing.T) {
 	tsg := sim.NewTipSetGenerator(rng.Uint64())
 	tableCid, err := certs.MakePowerTableCID(powerTable)
 	require.NoError(t, err)
-	base := gpbft.TipSet{Epoch: 0, Key: tsg.Sample(), PowerTable: tableCid}
+	base := &gpbft.TipSet{Epoch: 0, Key: tsg.Sample(), PowerTable: tableCid}
 
 	nextPowerTable, _ := randomizePowerTable(rng, backend, 200, powerTable, nil)
 
@@ -393,8 +393,10 @@ func TestBadFinalityCertificates(t *testing.T) {
 	// Chain is invalid.
 	{
 		certCpy := *certificate
-		certCpy.ECChain = slices.Clone(certCpy.ECChain)
-		slices.Reverse(certCpy.ECChain)
+		certCpy.ECChain = &gpbft.ECChain{
+			TipSets: slices.Clone(certificate.ECChain.TipSets),
+		}
+		slices.Reverse(certCpy.ECChain.TipSets)
 		nextInstance, chain, newPowerTable, err := certs.ValidateFinalityCertificates(backend, networkName, powerTable, 1, nil, &certCpy)
 		require.ErrorContains(t, err, "chain must have increasing epochs")
 		require.EqualValues(t, 1, nextInstance)
@@ -476,7 +478,7 @@ func randomPowerTable(backend signing.Backend, entries int64) gpbft.PowerEntries
 	return powerTable
 }
 
-func makeJustification(t *testing.T, rng *rand.Rand, tsg *sim.TipSetGenerator, backend signing.Backend, base gpbft.TipSet, instance uint64, powerTable, nextPowerTable gpbft.PowerEntries) *gpbft.Justification {
+func makeJustification(t *testing.T, rng *rand.Rand, tsg *sim.TipSetGenerator, backend signing.Backend, base *gpbft.TipSet, instance uint64, powerTable, nextPowerTable gpbft.PowerEntries) *gpbft.Justification {
 	chainLen := rng.Intn(23) + 1
 	chain, err := gpbft.NewChain(base)
 	require.NoError(t, err)

certstore/certstore_test.go

@@ -19,7 +19,11 @@ func makeCert(instance uint64, supp gpbft.SupplementalData) *certs.FinalityCerti
 	return &certs.FinalityCertificate{
 		GPBFTInstance:    instance,
 		SupplementalData: supp,
-		ECChain:          gpbft.ECChain{{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: supp.PowerTable}},
+		ECChain: &gpbft.ECChain{
+			TipSets: []*gpbft.TipSet{
+				{Epoch: 0, Key: gpbft.TipSetKey("tsk0"), PowerTable: supp.PowerTable},
+			},
+		},
 	}
 }