From 273dc1982d2ee6ecc935880ceaa1050c4471c8e4 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Thu, 23 Jan 2025 16:20:32 +0100 Subject: [PATCH 1/2] clean verkle code and update besu-verkle-trie library Signed-off-by: Karim Taam --- .../trie/diffbased/verkle/LeafBuilder.java} | 38 ++++---- .../verkle/cache/preloader/StemPreloader.java | 90 +++++++------------ .../flat/VerkleStemFlatDbStrategy.java | 4 +- .../verkle/worldview/VerkleWorldState.java | 57 +++++++----- .../flat/VerkleStemFlatDbStrategyTest.java | 3 +- .../stateless/Eip4762AccessWitness.java | 8 +- platform/build.gradle | 2 +- 7 files changed, 94 insertions(+), 108 deletions(-) rename ethereum/{verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleEntryFactory.java => core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LeafBuilder.java} (79%) diff --git a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleEntryFactory.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LeafBuilder.java similarity index 79% rename from ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleEntryFactory.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LeafBuilder.java index d09a3f1d8d5..afbd6fab512 100644 --- a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleEntryFactory.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LeafBuilder.java @@ -12,14 +12,14 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.verkletrie; +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyBatchAdapter; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyFactory; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils; import org.hyperledger.besu.ethereum.trie.verkle.util.SuffixTreeEncoder; import java.util.HashMap; @@ -33,38 +33,38 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -public class VerkleEntryFactory { +public class LeafBuilder { - private final TrieKeyBatchAdapter trieKeyAdapter; + private final TrieKeyFactory trieKeyFactory; private final HashSet keysForRemoval = new HashSet<>(); private final HashMap nonStorageKeyValuesForUpdate = new HashMap<>(); private final HashMap> storageKeyValuesForUpdate = new HashMap<>(); - public VerkleEntryFactory(final Hasher hasher) { - trieKeyAdapter = new TrieKeyBatchAdapter(hasher); + public LeafBuilder(final TrieKeyFactory trieKeyFactory) { + this.trieKeyFactory = trieKeyFactory; } public void generateAccountKeyForRemoval(final Address address) { - keysForRemoval.add(trieKeyAdapter.basicDataKey(address)); + keysForRemoval.add(trieKeyFactory.basicDataKey(address)); } public void generateCodeKeysForRemoval(final Address address, final Bytes code) { - keysForRemoval.add(trieKeyAdapter.basicDataKey(address)); - keysForRemoval.add(trieKeyAdapter.codeHashKey(address)); - List codeChunks = trieKeyAdapter.chunkifyCode(code); + keysForRemoval.add(trieKeyFactory.basicDataKey(address)); + keysForRemoval.add(trieKeyFactory.codeHashKey(address)); + List codeChunks = TrieKeyUtils.chunkifyCode(code); for (int i = 0; i < codeChunks.size(); i++) { - keysForRemoval.add(trieKeyAdapter.codeChunkKey(address, UInt256.valueOf(i))); + keysForRemoval.add(trieKeyFactory.codeChunkKey(address, UInt256.valueOf(i))); } } public void generateStorageKeyForRemoval(final Address address, final StorageSlotKey storageKey) { - keysForRemoval.add(trieKeyAdapter.storageKey(address, storageKey.getSlotKey().orElseThrow())); + keysForRemoval.add(trieKeyFactory.storageKey(address, storageKey.getSlotKey().orElseThrow())); } public void generateAccountKeyValueForUpdate( final Address address, final long nonce, final Wei balance) { - Bytes32 basicDataKey = trieKeyAdapter.basicDataKey(address); + Bytes32 basicDataKey = trieKeyFactory.basicDataKey(address); Bytes32 basicDataValue; if ((basicDataValue = nonStorageKeyValuesForUpdate.get(basicDataKey)) == null) { basicDataValue = Bytes32.ZERO; @@ -81,7 +81,7 @@ public void generateAccountKeyValueForUpdate( } public void generateCodeSizeKeyValueForUpdate(final Address address, final int size) { - Bytes32 basicDataKey = trieKeyAdapter.basicDataKey(address); + Bytes32 basicDataKey = trieKeyFactory.basicDataKey(address); Bytes32 basicDataValue; if ((basicDataValue = nonStorageKeyValuesForUpdate.get(basicDataKey)) == null) { basicDataValue = Bytes32.ZERO; @@ -96,7 +96,7 @@ public void generateCodeSizeKeyValueForUpdate(final Address address, final int s } public void generateCodeHashKeyValueForUpdate(final Address address, final Hash codeHash) { - nonStorageKeyValuesForUpdate.put(trieKeyAdapter.codeHashKey(address), codeHash); + nonStorageKeyValuesForUpdate.put(trieKeyFactory.codeHashKey(address), codeHash); } public void generateCodeKeyValuesForUpdate( @@ -104,10 +104,10 @@ public void generateCodeKeyValuesForUpdate( generateCodeSizeKeyValueForUpdate(address, code.size()); generateCodeHashKeyValueForUpdate(address, codeHash); - List codeChunks = trieKeyAdapter.chunkifyCode(code); + List codeChunks = TrieKeyUtils.chunkifyCode(code); for (int i = 0; i < codeChunks.size(); i++) { nonStorageKeyValuesForUpdate.put( - trieKeyAdapter.codeChunkKey(address, UInt256.valueOf(i)), codeChunks.get(i)); + trieKeyFactory.codeChunkKey(address, UInt256.valueOf(i)), codeChunks.get(i)); } } @@ -116,7 +116,7 @@ public void generateStorageKeyValueForUpdate( storageKeyValuesForUpdate.put( storageSlotKey, new Pair<>( - trieKeyAdapter.storageKey(address, storageSlotKey.getSlotKey().orElseThrow()), value)); + trieKeyFactory.storageKey(address, storageSlotKey.getSlotKey().orElseThrow()), value)); } public Set getKeysForRemoval() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/preloader/StemPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/preloader/StemPreloader.java index e97ff2cb9db..9d5d9da45f9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/preloader/StemPreloader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/preloader/StemPreloader.java @@ -16,19 +16,14 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils; import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.StemHasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.builder.StemHasherBuilder; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.cache.InMemoryCacheStrategy; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; @@ -52,23 +47,18 @@ * through the {@code STEM_CACHE_SIZE} constant. The {@link Hasher} used for stem generation is * configurable, allowing for different hashing strategies (e.g., Pedersen hashing) to be employed. * - * @see org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter + * @see org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyFactory * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher - * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher */ public class StemPreloader { + private static final int HASHER_CACHE_SIZE = 10_000; private static final int STEM_CACHE_SIZE = 10_000; + private static final int ADDRESS_COMMITMENT_CACHE_SIZE = 10_000; - // Cache of stem by address. The sub-map will contain the trie index as the key and the stem as - // the value. - private final Cache pedersenHasherCache = - CacheBuilder.newBuilder().maximumSize(STEM_CACHE_SIZE).build(); + private final Cache stemHasherByAddress = + CacheBuilder.newBuilder().maximumSize(HASHER_CACHE_SIZE).build(); - private final TrieKeyAdapter trieKeyAdapter; - - public StemPreloader() { - this.trieKeyAdapter = new TrieKeyAdapter(new PedersenHasher()); - } + public StemPreloader() {} /** * Creates a preloaded hasher context for a given updated address, storage, and code. This method @@ -83,44 +73,43 @@ public StemPreloader() { * @param keys a list of keys to use for stem generation * @return the preloaded stems */ - public Map preloadStems(final Address address, final Set keys) { - return getHasherByAddress(address).manyStems(address, new ArrayList<>(keys)); + public Map preloadStems(final Address address, final List keys) { + return getStemHasherByAddress(address).manyStems(address, keys); } public Bytes preloadAccountStem(final Address address) { - return getHasherByAddress(address).computeStem(address, UInt256.ZERO); + return getStemHasherByAddress(address).computeStem(address, UInt256.ZERO); } public Bytes preloadSlotStems(final Address address, final StorageSlotKey storageSlotKey) { - return getHasherByAddress(address) - .computeStem( - address, - trieKeyAdapter.getStorageKeyTrieIndex(storageSlotKey.getSlotKey().orElseThrow())); + final Bytes32 storageKeyTrieIndex = + TrieKeyUtils.getStorageKeyTrieIndex(storageSlotKey.getSlotKey().orElseThrow()); + return getStemHasherByAddress(address).computeStem(address, storageKeyTrieIndex); } public Map preloadCodeChunckStems(final Address address, final Bytes codeUpdate) { - return getHasherByAddress(address).manyStems(address, generateCodeChunkKeyIds(codeUpdate)); + return getStemHasherByAddress(address) + .manyStems(address, TrieKeyUtils.getCodeChunkKeyTrieIndexes(codeUpdate)); } /** - * Retrieves the cache that maps account addresses to their corresponding cached stems. + * Retrieves the cache that maps account addresses to their corresponding stem hasher. * - * @return the cache mapping account addresses to trie stems + * @return the stem hasher linked to the address */ @VisibleForTesting - public CachedPedersenHasher getHasherByAddress(final Address address) { - CachedPedersenHasher ifPresent = pedersenHasherCache.getIfPresent(address); + public StemHasher getStemHasherByAddress(final Address address) { + StemHasher ifPresent = stemHasherByAddress.getIfPresent(address); if (ifPresent != null) { return ifPresent; } - final CachedPedersenHasher defaultHasher = - new CachedPedersenHasher(STEM_CACHE_SIZE, new ConcurrentHashMap<>()); - pedersenHasherCache.put(address, defaultHasher); - return defaultHasher; - } - - public Bytes getStorageKeySuffix(final Bytes32 storageKey) { - return trieKeyAdapter.getStorageKeySuffix(storageKey); + final StemHasher stemHasher = + StemHasherBuilder.builder() + .withStemCache(new InMemoryCacheStrategy<>(STEM_CACHE_SIZE)) + .withAddressCommitmentCache(new InMemoryCacheStrategy<>(ADDRESS_COMMITMENT_CACHE_SIZE)) + .build(); + stemHasherByAddress.put(address, stemHasher); + return stemHasher; } /** @@ -129,25 +118,6 @@ public Bytes getStorageKeySuffix(final Bytes32 storageKey) { * with performance monitoring or testing. */ public void reset() { - pedersenHasherCache.invalidateAll(); - } - - public Bytes32 generateAccountKeyId() { - return Parameters.BASIC_DATA_LEAF_KEY; - } - - public List generateCodeChunkKeyIds(final Bytes code) { - return IntStream.range(0, trieKeyAdapter.getNbChunk(code)) - .mapToObj(UInt256::valueOf) - .collect(Collectors.toUnmodifiableList()); - } - - public List generateStorageKeyIds(final Set storageSlotKeys) { - return storageSlotKeys.stream() - .map( - storageSlotKey -> - trieKeyAdapter.getStorageKeyTrieIndex(storageSlotKey.getSlotKey().orElseThrow())) - .map(Bytes32::wrap) - .toList(); + stemHasherByAddress.invalidateAll(); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategy.java index b746c4a7342..015bc0f08ac 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategy.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.preloader.StemPreloader; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils; import org.hyperledger.besu.ethereum.trie.verkle.util.SuffixTreeDecoder; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -120,8 +121,7 @@ public Optional getFlatStorageValueByStorageSlotKey( stem.flatMap( values -> values.get( - stemPreloader - .getStorageKeySuffix(storageSlotKey.getSlotKey().orElseThrow()) + TrieKeyUtils.getStorageKeySuffix(storageSlotKey.getSlotKey().orElseThrow()) .toInt())); if (storageFound.isPresent()) { getStorageValueFlatDatabaseCounter.inc(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index ae2bb27b98e..d92e13594d1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldStateConfig; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.LeafBuilder; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.preloader.StemPreloader; @@ -36,21 +37,23 @@ import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.flat.VerkleStemFlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyFactory; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.StemHasher; import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; -import org.hyperledger.besu.ethereum.verkletrie.VerkleEntryFactory; import org.hyperledger.besu.ethereum.verkletrie.VerkleTrie; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; import org.apache.tuweni.bytes.Bytes; @@ -121,21 +124,25 @@ protected Hash internalCalculateRootHash( final VerkleTrie stateTrie = createTrie((location, hash) -> worldStateKeyValueStorage.getStateTrieNode(location)); - final Map preloadedHashers = new ConcurrentHashMap<>(); + final StemPreloader stemPreloader = verklePreloader.stemPreloader(); + // For each address that needs to be updated in the state, generate all the necessary leaf keys + // (basic data leaf, code, code hash, storage slots, etc.). Then, preload the stems in an + // optimized + // and parallelized manner. These stems will later be used to update the trie. Some stems may + // have + // already been generated during block processing, so they will not be regenerated here. final Set
addressesToPersist = getAddressesToPersist(worldStateUpdater); addressesToPersist.parallelStream() .forEach( accountKey -> { - final StemPreloader stemPreloader = verklePreloader.stemPreloader(); - final DiffBasedValue accountUpdate = worldStateUpdater.getAccountsToUpdate().get(accountKey); // generate account triekeys - final Set leafKeys = new HashSet<>(); + final List leafKeys = new ArrayList<>(); if (accountUpdate != null && !accountUpdate.isUnchanged()) { - leafKeys.add(stemPreloader.generateAccountKeyId()); + leafKeys.add(TrieKeyUtils.getAccountKeyTrieIndex()); if (accountUpdate.getPrior() == null) { leafKeys.add(Parameters.CODE_HASH_LEAF_KEY); } @@ -146,10 +153,22 @@ protected Hash internalCalculateRootHash( storageAccountUpdate = worldStateUpdater.getStorageToUpdate().get(accountKey); boolean isStorageUpdateNeeded; if (storageAccountUpdate != null) { - final Set storageSlotKeys = storageAccountUpdate.keySet(); + final List storageSlotKeys = + storageAccountUpdate.keySet().stream() + .map( + storageSlotKey -> { + // Throw an exception if the slot key is missing + UInt256 slotKey = + storageSlotKey + .getSlotKey() + .orElseThrow( + () -> new IllegalStateException("Slot key is missing")); + return Bytes32.wrap(slotKey); + }) + .toList(); isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); if (isStorageUpdateNeeded) { - leafKeys.addAll(stemPreloader.generateStorageKeyIds(storageSlotKeys)); + leafKeys.addAll(TrieKeyUtils.getStorageKeyTrieIndexes(storageSlotKeys)); } } @@ -166,13 +185,11 @@ protected Hash internalCalculateRootHash( if (isCodeUpdateNeeded) { leafKeys.add(Parameters.CODE_HASH_LEAF_KEY); leafKeys.addAll( - stemPreloader.generateCodeChunkKeyIds( + TrieKeyUtils.getCodeChunkKeyTrieIndexes( updatedCode == null ? previousCode : updatedCode)); } } stemPreloader.preloadStems(accountKey, leafKeys); - - preloadedHashers.put(accountKey, stemPreloader.getHasherByAddress(accountKey)); }); for (final Address accountKey : addressesToPersist) { @@ -180,7 +197,7 @@ protected Hash internalCalculateRootHash( accountKey, stateTrie, maybeStateUpdater, - preloadedHashers.get(accountKey), + stemPreloader.getStemHasherByAddress(accountKey), worldStateUpdater); } @@ -210,7 +227,7 @@ private static boolean codeIsEmpty(final Bytes value) { private void generateAccountValues( final Address accountKey, - final VerkleEntryFactory verkleEntryFactory, + final LeafBuilder verkleEntryFactory, final Optional maybeStateUpdater, final VerkleWorldStateUpdateAccumulator worldStateUpdater) { var accountUpdate = worldStateUpdater.getAccountsToUpdate().get(accountKey); @@ -239,7 +256,7 @@ private void generateAccountValues( private void handleCoupledCodeAccountUpdates( final Address accountKey, - final VerkleEntryFactory verkleEntryFactory, + final LeafBuilder verkleEntryFactory, final DiffBasedValue accountUpdate, final VerkleWorldStateUpdateAccumulator worldStateUpdater) { final VerkleAccount priorAccount = accountUpdate.getPrior(); @@ -257,7 +274,7 @@ private void handleCoupledCodeAccountUpdates( private void generateCodeValues( final Address accountKey, - final VerkleEntryFactory verkleEntryFactory, + final LeafBuilder verkleEntryFactory, final Optional maybeStateUpdater, final DiffBasedValue codeUpdate) { if (codeUpdate == null @@ -287,7 +304,7 @@ private void generateCodeValues( private void generateStorageValues( final Address accountKey, - final VerkleEntryFactory verkleEntryFactory, + final LeafBuilder verkleEntryFactory, final Optional maybeStateUpdater, final StorageConsumingMap> storageAccountUpdate) { if (storageAccountUpdate == null || storageAccountUpdate.keySet().isEmpty()) { @@ -321,10 +338,10 @@ private void updateState( final Address accountKey, final VerkleTrie stateTrie, final Optional maybeStateUpdater, - final Hasher hasher, + final StemHasher stemHasher, final VerkleWorldStateUpdateAccumulator worldStateUpdater) { - final VerkleEntryFactory verkleEntryFactory = new VerkleEntryFactory(hasher); + final LeafBuilder verkleEntryFactory = new LeafBuilder(new TrieKeyFactory(stemHasher)); generateAccountValues(accountKey, verkleEntryFactory, maybeStateUpdater, worldStateUpdater); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java index 259fdddfe13..8cc6b08d82e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.preloader.StemPreloader; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils; import org.hyperledger.besu.ethereum.trie.verkle.node.LeafNode; import org.hyperledger.besu.ethereum.trie.verkle.node.StemNode; import org.hyperledger.besu.ethereum.trie.verkle.util.SuffixTreeEncoder; @@ -194,7 +195,7 @@ private Bytes prepareStemForSlot( final Address address, final StorageSlotKey storageSlotKey, final Bytes keySuffix) { final Bytes stem = Bytes32.random(); when(stemPreloader.preloadSlotStems(eq(address), eq(storageSlotKey))).thenReturn(stem); - when(stemPreloader.getStorageKeySuffix(eq(storageSlotKey.getSlotKey().get()))) + when(TrieKeyUtils.getStorageKeySuffix(eq(storageSlotKey.getSlotKey().get()))) .thenReturn(keySuffix); return stem; } diff --git a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java index 5af08a480d7..b5259fe2921 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java @@ -23,8 +23,7 @@ import org.hyperledger.besu.datatypes.AccessWitness; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils; import java.util.HashMap; import java.util.List; @@ -40,7 +39,6 @@ public class Eip4762AccessWitness implements AccessWitness { private static final Logger LOG = LoggerFactory.getLogger(Eip4762AccessWitness.class); - private static final TrieKeyAdapter TRIE_KEY_ADAPTER = new TrieKeyAdapter(new PedersenHasher()); private static final UInt256 zeroTreeIndex = UInt256.ZERO; private final Map leaves; @@ -264,8 +262,8 @@ private long touchAddressOnReadAndComputeGas( private List getStorageSlotTreeIndexes(final UInt256 storageKey) { return List.of( - TRIE_KEY_ADAPTER.getStorageKeyTrieIndex(storageKey), - UInt256.fromBytes(TRIE_KEY_ADAPTER.getStorageKeySuffix(storageKey))); + UInt256.fromBytes(TrieKeyUtils.getStorageKeyTrieIndex(storageKey)), + UInt256.fromBytes(TrieKeyUtils.getStorageKeySuffix(storageKey))); } @Override diff --git a/platform/build.gradle b/platform/build.gradle index a1d18d6f7e4..c62c1911512 100644 --- a/platform/build.gradle +++ b/platform/build.gradle @@ -182,7 +182,7 @@ dependencies { api 'org.hyperledger.besu:ipa-multipoint:0.8.4-SNAPSHOT' - api 'org.hyperledger.besu:besu-verkle-trie:0.0.3-20241107.124625-9' + api 'org.hyperledger.besu:besu-verkle-trie:0.0.3-20250123.150439-12' //a5cac6cb8e5e927ac070ff8d3607c73c0301d3e6 } } From e3c45d4b39c35d0d4854831c4fd60f162ceb965a Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Mon, 27 Jan 2025 10:32:45 +0100 Subject: [PATCH 2/2] Addressed review feedback Signed-off-by: Karim Taam --- .../verkle/worldview/VerkleWorldState.java | 43 +++++++++---------- .../flat/VerkleStemFlatDbStrategyTest.java | 28 ++++++------ 2 files changed, 32 insertions(+), 39 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index d92e13594d1..f80f4550143 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -227,7 +227,7 @@ private static boolean codeIsEmpty(final Bytes value) { private void generateAccountValues( final Address accountKey, - final LeafBuilder verkleEntryFactory, + final LeafBuilder leafBuilder, final Optional maybeStateUpdater, final VerkleWorldStateUpdateAccumulator worldStateUpdater) { var accountUpdate = worldStateUpdater.getAccountsToUpdate().get(accountKey); @@ -235,18 +235,17 @@ private void generateAccountValues( return; } if (accountUpdate.getUpdated() == null) { - verkleEntryFactory.generateAccountKeyForRemoval(accountKey); + leafBuilder.generateAccountKeyForRemoval(accountKey); final Hash addressHash = hashAndSavePreImage(accountKey); maybeStateUpdater.ifPresent( verkleUpdater -> verkleUpdater.removeAccountInfoState(addressHash)); return; } - handleCoupledCodeAccountUpdates( - accountKey, verkleEntryFactory, accountUpdate, worldStateUpdater); + handleCoupledCodeAccountUpdates(accountKey, leafBuilder, accountUpdate, worldStateUpdater); final VerkleAccount updatedAcount = accountUpdate.getUpdated(); - verkleEntryFactory.generateAccountKeyValueForUpdate( + leafBuilder.generateAccountKeyValueForUpdate( accountKey, updatedAcount.getNonce(), updatedAcount.getBalance()); maybeStateUpdater.ifPresent( verkleUpdater -> @@ -256,25 +255,24 @@ private void generateAccountValues( private void handleCoupledCodeAccountUpdates( final Address accountKey, - final LeafBuilder verkleEntryFactory, + final LeafBuilder leafBuilder, final DiffBasedValue accountUpdate, final VerkleWorldStateUpdateAccumulator worldStateUpdater) { final VerkleAccount priorAccount = accountUpdate.getPrior(); final VerkleAccount updatedAccount = accountUpdate.getUpdated(); if (priorAccount == null) { - verkleEntryFactory.generateCodeHashKeyValueForUpdate( - accountKey, updatedAccount.getCodeHash()); + leafBuilder.generateCodeHashKeyValueForUpdate(accountKey, updatedAccount.getCodeHash()); return; } Optional currentCode = worldStateUpdater.getCode(accountKey, updatedAccount.getCodeHash()); currentCode.ifPresent( - code -> verkleEntryFactory.generateCodeSizeKeyValueForUpdate(accountKey, code.size())); + code -> leafBuilder.generateCodeSizeKeyValueForUpdate(accountKey, code.size())); } private void generateCodeValues( final Address accountKey, - final LeafBuilder verkleEntryFactory, + final LeafBuilder leafBuilder, final Optional maybeStateUpdater, final DiffBasedValue codeUpdate) { if (codeUpdate == null @@ -284,7 +282,7 @@ private void generateCodeValues( } if (codeUpdate.getUpdated() == null) { final Hash priorCodeHash = Hash.hash(codeUpdate.getPrior()); - verkleEntryFactory.generateCodeKeysForRemoval(accountKey, codeUpdate.getPrior()); + leafBuilder.generateCodeKeysForRemoval(accountKey, codeUpdate.getPrior()); final Hash accountHash = accountKey.addressHash(); maybeStateUpdater.ifPresent( verkleUpdater -> verkleUpdater.removeCode(accountHash, priorCodeHash)); @@ -292,8 +290,7 @@ private void generateCodeValues( } final Hash accountHash = accountKey.addressHash(); final Hash codeHash = Hash.hash(codeUpdate.getUpdated()); - verkleEntryFactory.generateCodeKeyValuesForUpdate( - accountKey, codeUpdate.getUpdated(), codeHash); + leafBuilder.generateCodeKeyValuesForUpdate(accountKey, codeUpdate.getUpdated(), codeHash); if (codeUpdate.getUpdated().isEmpty()) { maybeStateUpdater.ifPresent(verkleUpdater -> verkleUpdater.removeCode(accountHash, codeHash)); } else { @@ -304,7 +301,7 @@ private void generateCodeValues( private void generateStorageValues( final Address accountKey, - final LeafBuilder verkleEntryFactory, + final LeafBuilder leafBuilder, final Optional maybeStateUpdater, final StorageConsumingMap> storageAccountUpdate) { if (storageAccountUpdate == null || storageAccountUpdate.keySet().isEmpty()) { @@ -318,12 +315,12 @@ private void generateStorageValues( if (!storageUpdate.getValue().isUnchanged()) { final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); if (updatedStorage == null) { - verkleEntryFactory.generateStorageKeyForRemoval(accountKey, storageUpdate.getKey()); + leafBuilder.generateStorageKeyForRemoval(accountKey, storageUpdate.getKey()); maybeStateUpdater.ifPresent( verkleUpdater -> verkleUpdater.removeStorageValueBySlotHash(updatedAddressHash, slotHash)); } else { - verkleEntryFactory.generateStorageKeyValueForUpdate( + leafBuilder.generateStorageKeyValueForUpdate( accountKey, storageUpdate.getKey(), updatedStorage); maybeStateUpdater.ifPresent( verkleUpdater -> @@ -341,37 +338,37 @@ private void updateState( final StemHasher stemHasher, final VerkleWorldStateUpdateAccumulator worldStateUpdater) { - final LeafBuilder verkleEntryFactory = new LeafBuilder(new TrieKeyFactory(stemHasher)); + final LeafBuilder leafBuilder = new LeafBuilder(new TrieKeyFactory(stemHasher)); - generateAccountValues(accountKey, verkleEntryFactory, maybeStateUpdater, worldStateUpdater); + generateAccountValues(accountKey, leafBuilder, maybeStateUpdater, worldStateUpdater); generateCodeValues( accountKey, - verkleEntryFactory, + leafBuilder, maybeStateUpdater, worldStateUpdater.getCodeToUpdate().get(accountKey)); generateStorageValues( accountKey, - verkleEntryFactory, + leafBuilder, maybeStateUpdater, worldStateUpdater.getStorageToUpdate().get(accountKey)); - verkleEntryFactory + leafBuilder .getKeysForRemoval() .forEach( key -> { System.out.println("remove key " + key); stateTrie.remove(key); }); - verkleEntryFactory + leafBuilder .getNonStorageKeyValuesForUpdate() .forEach( (key, value) -> { System.out.println("add key " + key + " leaf value " + value); stateTrie.put(key, value); }); - verkleEntryFactory + leafBuilder .getStorageKeyValuesForUpdate() .forEach( (storageSlotKey, pair) -> { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java index 8cc6b08d82e..2d35baa002f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategyTest.java @@ -31,9 +31,9 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.preloader.StemPreloader; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils; import org.hyperledger.besu.ethereum.trie.verkle.node.LeafNode; import org.hyperledger.besu.ethereum.trie.verkle.node.StemNode; +import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; import org.hyperledger.besu.ethereum.trie.verkle.util.SuffixTreeEncoder; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -90,7 +90,7 @@ void testGetFlatAccountFound() { final Bytes codeSize = Bytes.repeat((byte) 0x1, 3); final Bytes32 suffix = encodeAccountProperties(nonce, balance, codeSize); - mockStorageWithStemNode(stem, suffix); + mockStorageWithStemNode(stem, (byte) Parameters.BASIC_DATA_LEAF_KEY.toInt(), suffix); final Optional result = strategy.getFlatAccount(address, context, stemPreloader, storage); @@ -118,10 +118,10 @@ void testGetFlatAccountNotFound() { void testGetFlatStorageValueByStorageSlotKeyFound() { final Address address = Address.fromHexString("0x1"); final StorageSlotKey slotKey = createSlotKey(); - final Bytes stem = prepareStemForSlot(address, slotKey, Bytes.of(0)); + final Bytes stem = prepareStemForSlot(address, slotKey); final Bytes expectedValue = Bytes32.fromHexString("0x1234"); - mockStorageWithStemNode(stem, Bytes.EMPTY, expectedValue); + mockStorageWithStemNode(stem, (byte) Parameters.HEADER_STORAGE_OFFSET.toInt(), expectedValue); final Optional result = strategy.getFlatStorageValueByStorageSlotKey(address, slotKey, stemPreloader, storage); @@ -135,7 +135,7 @@ void testGetFlatStorageValueByStorageSlotKeyFound() { void testGetFlatStorageValueByStorageSlotKeyNotFound() { final Address address = Address.fromHexString("0x1"); final StorageSlotKey slotKey = createSlotKey(); - final Bytes stem = prepareStemForSlot(address, slotKey, Bytes.of(0)); + final Bytes stem = prepareStemForSlot(address, slotKey); when(storage.get(eq(TRIE_BRANCH_STORAGE), eq(stem.toArrayUnsafe()))) .thenReturn(Optional.empty()); @@ -169,17 +169,16 @@ private Bytes32 encodeAccountProperties( return SuffixTreeEncoder.setCodeSizeInValue(suffix, codeSize); } - private void mockStorageWithStemNode(final Bytes stem, final Bytes... children) { - final StemNode stemNode = createStemNode(stem, children); + private void mockStorageWithStemNode(final Bytes stem, final byte index, final Bytes children) { + final StemNode stemNode = createStemNode(stem, index, children); when(storage.get(eq(TRIE_BRANCH_STORAGE), eq(stem.toArrayUnsafe()))) .thenReturn(Optional.of(stemNode.getEncodedValue().toArrayUnsafe())); } - private StemNode createStemNode(final Bytes stem, final Bytes... children) { + private StemNode createStemNode( + final Bytes stem, final byte index, final Bytes children) { final StemNode stemNode = new StemNode<>(Bytes.EMPTY, stem); - for (Bytes child : children) { - stemNode.replaceChild((byte) 0, new LeafNode<>(Bytes.EMPTY, child)); - } + stemNode.replaceChild(index, new LeafNode<>(Bytes.EMPTY, children)); stemNode.replaceHash( Bytes32.ZERO, Bytes.EMPTY, Bytes32.ZERO, Bytes.EMPTY, Bytes32.ZERO, Bytes.EMPTY); return stemNode; @@ -191,17 +190,14 @@ private Bytes prepareStemForAccount(final Address address) { return stem; } - private Bytes prepareStemForSlot( - final Address address, final StorageSlotKey storageSlotKey, final Bytes keySuffix) { + private Bytes prepareStemForSlot(final Address address, final StorageSlotKey storageSlotKey) { final Bytes stem = Bytes32.random(); when(stemPreloader.preloadSlotStems(eq(address), eq(storageSlotKey))).thenReturn(stem); - when(TrieKeyUtils.getStorageKeySuffix(eq(storageSlotKey.getSlotKey().get()))) - .thenReturn(keySuffix); return stem; } private StorageSlotKey createSlotKey() { - return new StorageSlotKey(UInt256.fromBytes(Bytes32.random())); + return new StorageSlotKey(UInt256.valueOf(0x00)); } private void assertAccountProperties(