goethereum dependency to v1.14~ (#14351)

* updating the goethereum dependency

* fixing dependencies

* reverting workspace

* more fixes, work in progress

* trying with upgraded geth version

* fixing deprecated functions except for the time related ones on eth1 distance due to time issues

* fixing time issues

* gaz

* fixing test and upgrading some dependencies and reverting others

* Disable cgo in hid, delete old vendored usb library

* changelog

* rolling back dependencies

* fixing go mod tidy

* Geth v1.13.6

* fix tests

* Add ping interval, set to 500ms for tests. This didnt work

* Update to v1.14.8

* Spread it out to different bootnodes

* Fix it

* Remove Memsize

* Update all out of date dependencies

* Fix geth body change

* Fix Test

* Fix Build

* Fix Tests

* Fix Tests Again

* Fix Tests Again

* Fix Tests

* Fix Test

* Copy USB Package for HID

* Push it up

* Finally fix all tests with felix's changes

* updating geth dependency

* Update go-ethereum to v1.14.11

* fixing import

* reverting blob change

* fixing Implicit memory aliasing in for loop.

* WIP changes

* wip getting a little further on e2e runs

* getting a little further

* getting a little further

* setting everything to capella

* more partial fixes

* more fixes but still WIP

* fixing access list transactions"

* some cleanup

* making configs dynamic

* reverting time

* skip lower bound in builder

* updating to geth v1.14.12

* fixing verify blob to pointer

* go mod tidy

* fixing linting

* missed removing another terminal difficulty item

* another missed update

* updating more dependencies to fix cicd

* fixing holiman dependency update

* downgrading geth to 1.14.11 due to p2p loop issue

* reverting builder middleware caused by downgrade

* fixing more rollback issues

* upgrading back to 1.14.12 after discussing with preston

* mod tidy

* gofmt

* partial review feedback

* trying to start e2e from bellatrix instead

* reverting some changes

---------

Co-authored-by: Preston Van Loon <[email protected]>
Co-authored-by: nisdas <[email protected]>

CHANGELOG.md

@@ -44,6 +44,8 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - Fixed Metadata errors for peers connected via QUIC.
 - Process light client finality updates only for new finalized epochs instead of doing it for every block.
 - Update blobs by rpc topics from V2 to V1.
+- Updated geth to 1.14 ~
+- E2e tests start from bellatrix
 
 ### Deprecated
 
@@ -52,6 +54,7 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 
 - Cleanup ProcessSlashings method to remove unnecessary argument.
 - Remove `/proto/eth/v2` directory. [PR](https://github.com/prysmaticlabs/prysm/pull/14765)
+- Remove `/memsize/` pprof endpoint as it will no longer be supported in go 1.23, geth also removed in https://github.com/ethereum/go-ethereum/commit/e4675771eda550e7eeb63a8884816982c1980644
 
 ### Fixed
 

beacon-chain/blockchain/kzg/validation.go

@@ -20,16 +20,17 @@ func Verify(sidecars ...blocks.ROBlob) error {
 	cmts := make([]GoKZG.KZGCommitment, len(sidecars))
 	proofs := make([]GoKZG.KZGProof, len(sidecars))
 	for i, sidecar := range sidecars {
-		blobs[i] = bytesToBlob(sidecar.Blob)
+		blobs[i] = *bytesToBlob(sidecar.Blob)
 		cmts[i] = bytesToCommitment(sidecar.KzgCommitment)
 		proofs[i] = bytesToKZGProof(sidecar.KzgProof)
 	}
 	return kzgContext.VerifyBlobKZGProofBatch(blobs, cmts, proofs)
 }
 
-func bytesToBlob(blob []byte) (ret GoKZG.Blob) {
+func bytesToBlob(blob []byte) *GoKZG.Blob {
+	var ret GoKZG.Blob
 	copy(ret[:], blob)
-	return
+	return &ret
 }
 
 func bytesToCommitment(commitment []byte) (ret GoKZG.KZGCommitment) {

beacon-chain/blockchain/kzg/validation_test.go

@@ -10,11 +10,11 @@ import (
 )
 
 func GenerateCommitmentAndProof(blob GoKZG.Blob) (GoKZG.KZGCommitment, GoKZG.KZGProof, error) {
-	commitment, err := kzgContext.BlobToKZGCommitment(blob, 0)
+	commitment, err := kzgContext.BlobToKZGCommitment(&blob, 0)
 	if err != nil {
 		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
 	}
-	proof, err := kzgContext.ComputeBlobKZGProof(blob, commitment, 0)
+	proof, err := kzgContext.ComputeBlobKZGProof(&blob, commitment, 0)
 	if err != nil {
 		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
 	}
@@ -31,7 +31,7 @@ func TestBytesToAny(t *testing.T) {
 	blob := GoKZG.Blob{0x01, 0x02}
 	commitment := GoKZG.KZGCommitment{0x01, 0x02}
 	proof := GoKZG.KZGProof{0x01, 0x02}
-	require.DeepEqual(t, blob, bytesToBlob(bytes))
+	require.DeepEqual(t, blob, *bytesToBlob(bytes))
 	require.DeepEqual(t, commitment, bytesToCommitment(bytes))
 	require.DeepEqual(t, proof, bytesToKZGProof(bytes))
 }

beacon-chain/execution/BUILD.bazel

@@ -131,11 +131,11 @@ go_test(
         "//testing/util:go_default_library",
         "//time/slots:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
-        "@com_github_ethereum_go_ethereum//accounts/abi/bind/backends:go_default_library",
         "@com_github_ethereum_go_ethereum//beacon/engine:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//ethclient/simulated:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",

beacon-chain/execution/block_reader_test.go

@@ -12,6 +12,7 @@ import (
 	dbutil "github.com/prysmaticlabs/prysm/v5/beacon-chain/db/testing"
 	mockExecution "github.com/prysmaticlabs/prysm/v5/beacon-chain/execution/testing"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/execution/types"
+	"github.com/prysmaticlabs/prysm/v5/config/params"
 	contracts "github.com/prysmaticlabs/prysm/v5/contracts/deposit"
 	"github.com/prysmaticlabs/prysm/v5/contracts/deposit/mock"
 	"github.com/prysmaticlabs/prysm/v5/testing/assert"
@@ -44,7 +45,7 @@ func TestLatestMainchainInfo_OK(t *testing.T) {
 	web3Service = setDefaultMocks(web3Service)
 	web3Service.rpcClient = &mockExecution.RPCClient{Backend: testAcc.Backend}
 
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	require.NoError(t, err)
 	testAcc.Backend.Commit()
 
@@ -141,7 +142,7 @@ func TestBlockExists_ValidHash(t *testing.T) {
 	web3Service = setDefaultMocks(web3Service)
 	web3Service.rpcClient = &mockExecution.RPCClient{Backend: testAcc.Backend}
 	testAcc.Backend.Commit()
-	block, err := testAcc.Backend.BlockByNumber(context.Background(), big.NewInt(0))
+	block, err := testAcc.Backend.Client().BlockByNumber(context.Background(), big.NewInt(0))
 	assert.NoError(t, err)
 
 	exists, height, err := web3Service.BlockExists(context.Background(), block.Hash())
@@ -201,8 +202,10 @@ func TestBlockExists_UsesCachedBlockInfo(t *testing.T) {
 }
 
 func TestService_BlockNumberByTimestamp(t *testing.T) {
+	ctx := context.Background()
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
+
 	require.NoError(t, err, "Unable to set up simulated backend")
 	server, endpoint, err := mockExecution.SetupRPCServer()
 	require.NoError(t, err)
@@ -216,16 +219,22 @@ func TestService_BlockNumberByTimestamp(t *testing.T) {
 	require.NoError(t, err)
 	web3Service = setDefaultMocks(web3Service)
 	web3Service.rpcClient = &mockExecution.RPCClient{Backend: testAcc.Backend}
-
+	// simulated backend sets eth1 block
+	params.SetupTestConfigCleanup(t)
+	conf := params.BeaconConfig().Copy()
+	conf.SecondsPerETH1Block = 1
+	params.OverrideBeaconConfig(conf)
+	initialHead, err := testAcc.Backend.Client().HeaderByNumber(ctx, nil)
+	require.NoError(t, err)
 	for i := 0; i < 200; i++ {
 		testAcc.Backend.Commit()
 	}
-	ctx := context.Background()
-	hd, err := testAcc.Backend.HeaderByNumber(ctx, nil)
+
+	hd, err := testAcc.Backend.Client().HeaderByNumber(ctx, nil)
 	require.NoError(t, err)
 	web3Service.latestEth1Data.BlockTime = hd.Time
 	web3Service.latestEth1Data.BlockHeight = hd.Number.Uint64()
-	blk, err := web3Service.BlockByTimestamp(ctx, 1000 /* time */)
+	blk, err := web3Service.BlockByTimestamp(ctx, initialHead.Time+100 /* time */)
 	require.NoError(t, err)
 	if blk.Number.Cmp(big.NewInt(0)) == 0 {
 		t.Error("Returned a block with zero number, expected to be non zero")
@@ -253,7 +262,7 @@ func TestService_BlockNumberByTimestampLessTargetTime(t *testing.T) {
 		testAcc.Backend.Commit()
 	}
 	ctx := context.Background()
-	hd, err := testAcc.Backend.HeaderByNumber(ctx, nil)
+	hd, err := testAcc.Backend.Client().HeaderByNumber(ctx, nil)
 	require.NoError(t, err)
 	web3Service.latestEth1Data.BlockTime = hd.Time
 	// Use extremely small deadline to illustrate that context deadlines are respected.
@@ -291,7 +300,7 @@ func TestService_BlockNumberByTimestampMoreTargetTime(t *testing.T) {
 		testAcc.Backend.Commit()
 	}
 	ctx := context.Background()
-	hd, err := testAcc.Backend.HeaderByNumber(ctx, nil)
+	hd, err := testAcc.Backend.Client().HeaderByNumber(ctx, nil)
 	require.NoError(t, err)
 	web3Service.latestEth1Data.BlockTime = hd.Time
 	// Use extremely small deadline to illustrate that context deadlines are respected.

beacon-chain/execution/log_processing_test.go

@@ -47,7 +47,7 @@ func TestProcessDepositLog_OK(t *testing.T) {
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	require.NoError(t, err)
 
 	testAcc.Backend.Commit()
@@ -72,7 +72,7 @@ func TestProcessDepositLog_OK(t *testing.T) {
 		},
 	}
 
-	logs, err := testAcc.Backend.FilterLogs(web3Service.ctx, query)
+	logs, err := testAcc.Backend.Client().FilterLogs(web3Service.ctx, query)
 	require.NoError(t, err, "Unable to retrieve logs")
 
 	if len(logs) == 0 {
@@ -116,7 +116,7 @@ func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) {
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	require.NoError(t, err)
 
 	testAcc.Backend.Commit()
@@ -144,7 +144,7 @@ func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) {
 		},
 	}
 
-	logs, err := testAcc.Backend.FilterLogs(web3Service.ctx, query)
+	logs, err := testAcc.Backend.Client().FilterLogs(web3Service.ctx, query)
 	require.NoError(t, err, "Unable to retrieve logs")
 
 	web3Service.chainStartData.Chainstarted = true
@@ -176,7 +176,7 @@ func TestUnpackDepositLogData_OK(t *testing.T) {
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	require.NoError(t, err)
 
 	testAcc.Backend.Commit()
@@ -199,7 +199,7 @@ func TestUnpackDepositLogData_OK(t *testing.T) {
 		},
 	}
 
-	logz, err := testAcc.Backend.FilterLogs(web3Service.ctx, query)
+	logz, err := testAcc.Backend.Client().FilterLogs(web3Service.ctx, query)
 	require.NoError(t, err, "Unable to retrieve logs")
 
 	loggedPubkey, withCreds, _, loggedSig, index, err := contracts.UnpackDepositLogData(logz[0].Data)
@@ -232,7 +232,7 @@ func TestProcessETH2GenesisLog_8DuplicatePubkeys(t *testing.T) {
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	require.NoError(t, err)
 
 	params.SetupTestConfigCleanup(t)
@@ -269,7 +269,7 @@ func TestProcessETH2GenesisLog_8DuplicatePubkeys(t *testing.T) {
 		},
 	}
 
-	logs, err := testAcc.Backend.FilterLogs(web3Service.ctx, query)
+	logs, err := testAcc.Backend.Client().FilterLogs(web3Service.ctx, query)
 	require.NoError(t, err, "Unable to retrieve logs")
 
 	for i := range logs {
@@ -307,7 +307,7 @@ func TestProcessETH2GenesisLog(t *testing.T) {
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	web3Service.rpcClient = &mockExecution.RPCClient{Backend: testAcc.Backend}
 	require.NoError(t, err)
 	params.SetupTestConfigCleanup(t)
@@ -342,7 +342,7 @@ func TestProcessETH2GenesisLog(t *testing.T) {
 		},
 	}
 
-	logs, err := testAcc.Backend.FilterLogs(web3Service.ctx, query)
+	logs, err := testAcc.Backend.Client().FilterLogs(web3Service.ctx, query)
 	require.NoError(t, err, "Unable to retrieve logs")
 	require.Equal(t, depositsReqForChainStart, len(logs))
 
@@ -400,16 +400,18 @@ func TestProcessETH2GenesisLog_CorrectNumOfDeposits(t *testing.T) {
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	require.NoError(t, err)
 	web3Service.rpcClient = &mockExecution.RPCClient{Backend: testAcc.Backend}
-	web3Service.httpLogger = testAcc.Backend
+	web3Service.httpLogger = testAcc.Backend.Client()
 	web3Service.latestEth1Data.LastRequestedBlock = 0
-	web3Service.latestEth1Data.BlockHeight = testAcc.Backend.Blockchain().CurrentBlock().Number.Uint64()
-	web3Service.latestEth1Data.BlockTime = testAcc.Backend.Blockchain().CurrentBlock().Time
+	block, err := testAcc.Backend.Client().BlockByNumber(context.Background(), nil)
+	require.NoError(t, err)
+	web3Service.latestEth1Data.BlockHeight = block.NumberU64()
+	web3Service.latestEth1Data.BlockTime = block.Time()
 	bConfig := params.MinimalSpecConfig().Copy()
 	bConfig.MinGenesisTime = 0
-	bConfig.SecondsPerETH1Block = 10
+	bConfig.SecondsPerETH1Block = 1
 	params.OverrideBeaconConfig(bConfig)
 	nConfig := params.BeaconNetworkConfig()
 	nConfig.ContractDeploymentBlock = 0
@@ -444,8 +446,10 @@ func TestProcessETH2GenesisLog_CorrectNumOfDeposits(t *testing.T) {
 	for i := uint64(0); i < params.BeaconConfig().Eth1FollowDistance; i++ {
 		testAcc.Backend.Commit()
 	}
-	web3Service.latestEth1Data.BlockHeight = testAcc.Backend.Blockchain().CurrentBlock().Number.Uint64()
-	web3Service.latestEth1Data.BlockTime = testAcc.Backend.Blockchain().CurrentBlock().Time
+	b, err := testAcc.Backend.Client().BlockByNumber(context.Background(), nil)
+	require.NoError(t, err)
+	web3Service.latestEth1Data.BlockHeight = b.NumberU64()
+	web3Service.latestEth1Data.BlockTime = b.Time()
 
 	// Set up our subscriber now to listen for the chain started event.
 	stateChannel := make(chan *feed.Event, 1)
@@ -497,13 +501,15 @@ func TestProcessETH2GenesisLog_LargePeriodOfNoLogs(t *testing.T) {
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(testAcc.ContractAddr, testAcc.Backend.Client())
 	require.NoError(t, err)
 	web3Service.rpcClient = &mockExecution.RPCClient{Backend: testAcc.Backend}
-	web3Service.httpLogger = testAcc.Backend
+	web3Service.httpLogger = testAcc.Backend.Client()
 	web3Service.latestEth1Data.LastRequestedBlock = 0
-	web3Service.latestEth1Data.BlockHeight = testAcc.Backend.Blockchain().CurrentBlock().Number.Uint64()
-	web3Service.latestEth1Data.BlockTime = testAcc.Backend.Blockchain().CurrentBlock().Time
+	b, err := testAcc.Backend.Client().BlockByNumber(context.Background(), nil)
+	require.NoError(t, err)
+	web3Service.latestEth1Data.BlockHeight = b.NumberU64()
+	web3Service.latestEth1Data.BlockTime = b.Time()
 	bConfig := params.MinimalSpecConfig().Copy()
 	bConfig.SecondsPerETH1Block = 10
 	params.OverrideBeaconConfig(bConfig)
@@ -540,14 +546,19 @@ func TestProcessETH2GenesisLog_LargePeriodOfNoLogs(t *testing.T) {
 	for i := uint64(0); i < 1500; i++ {
 		testAcc.Backend.Commit()
 	}
-	wantedGenesisTime := testAcc.Backend.Blockchain().CurrentBlock().Time
+	genesisBlock, err := testAcc.Backend.Client().BlockByNumber(context.Background(), nil)
+	require.NoError(t, err)
+
+	wantedGenesisTime := genesisBlock.Time()
 
 	// Forward the chain to account for the follow distance
 	for i := uint64(0); i < params.BeaconConfig().Eth1FollowDistance; i++ {
 		testAcc.Backend.Commit()
 	}
-	web3Service.latestEth1Data.BlockHeight = testAcc.Backend.Blockchain().CurrentBlock().Number.Uint64()
-	web3Service.latestEth1Data.BlockTime = testAcc.Backend.Blockchain().CurrentBlock().Time
+	currBlock, err := testAcc.Backend.Client().BlockByNumber(context.Background(), nil)
+	require.NoError(t, err)
+	web3Service.latestEth1Data.BlockHeight = currBlock.NumberU64()
+	web3Service.latestEth1Data.BlockTime = currBlock.Time()
 
 	// Set the genesis time 500 blocks ahead of the last
 	// deposit log.
@@ -608,7 +619,7 @@ func newPowchainService(t *testing.T, eth1Backend *mock.TestAccount, beaconDB db
 	)
 	require.NoError(t, err, "unable to setup web3 ETH1.0 chain service")
 	web3Service = setDefaultMocks(web3Service)
-	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(eth1Backend.ContractAddr, eth1Backend.Backend)
+	web3Service.depositContractCaller, err = contracts.NewDepositContractCaller(eth1Backend.ContractAddr, eth1Backend.Backend.Client())
 	require.NoError(t, err)
 
 	web3Service.rpcClient = &mockExecution.RPCClient{Backend: eth1Backend.Backend}