Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

clean verkle code and update besu-verkle-trie library #8162

Merged
merged 3 commits into from
Jan 31, 2025
Merged

clean verkle code and update besu-verkle-trie library #8162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
273dc19
clean verkle code and update besu-verkle-trie library
matkt Jan 23, 2025
c7ef16c
Merge branch 'verkle' into feature/update-besu-verkle-trie-library
matkt Jan 23, 2025
e3c45d4
Addressed review feedback
matkt Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions ...hereum/verkletrie/VerkleEntryFactory.java → ...um/trie/diffbased/verkle/LeafBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.ethereum.verkletrie;
package org.hyperledger.besu.ethereum.trie.diffbased.verkle;

import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyBatchAdapter;
import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher;
import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyFactory;
import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils;
import org.hyperledger.besu.ethereum.trie.verkle.util.SuffixTreeEncoder;

import java.util.HashMap;
Expand All @@ -33,38 +33,38 @@
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt256;

public class VerkleEntryFactory {
public class LeafBuilder {

private final TrieKeyBatchAdapter trieKeyAdapter;
private final TrieKeyFactory trieKeyFactory;
private final HashSet<Bytes32> keysForRemoval = new HashSet<>();
private final HashMap<Bytes32, Bytes32> nonStorageKeyValuesForUpdate = new HashMap<>();
private final HashMap<StorageSlotKey, Pair<Bytes32, Bytes32>> storageKeyValuesForUpdate =
new HashMap<>();

public VerkleEntryFactory(final Hasher hasher) {
trieKeyAdapter = new TrieKeyBatchAdapter(hasher);
public LeafBuilder(final TrieKeyFactory trieKeyFactory) {
this.trieKeyFactory = trieKeyFactory;
}

public void generateAccountKeyForRemoval(final Address address) {
keysForRemoval.add(trieKeyAdapter.basicDataKey(address));
keysForRemoval.add(trieKeyFactory.basicDataKey(address));
}

public void generateCodeKeysForRemoval(final Address address, final Bytes code) {
keysForRemoval.add(trieKeyAdapter.basicDataKey(address));
keysForRemoval.add(trieKeyAdapter.codeHashKey(address));
List<UInt256> codeChunks = trieKeyAdapter.chunkifyCode(code);
keysForRemoval.add(trieKeyFactory.basicDataKey(address));
keysForRemoval.add(trieKeyFactory.codeHashKey(address));
List<UInt256> codeChunks = TrieKeyUtils.chunkifyCode(code);
for (int i = 0; i < codeChunks.size(); i++) {
keysForRemoval.add(trieKeyAdapter.codeChunkKey(address, UInt256.valueOf(i)));
keysForRemoval.add(trieKeyFactory.codeChunkKey(address, UInt256.valueOf(i)));
}
}

public void generateStorageKeyForRemoval(final Address address, final StorageSlotKey storageKey) {
keysForRemoval.add(trieKeyAdapter.storageKey(address, storageKey.getSlotKey().orElseThrow()));
keysForRemoval.add(trieKeyFactory.storageKey(address, storageKey.getSlotKey().orElseThrow()));
}

public void generateAccountKeyValueForUpdate(
final Address address, final long nonce, final Wei balance) {
Bytes32 basicDataKey = trieKeyAdapter.basicDataKey(address);
Bytes32 basicDataKey = trieKeyFactory.basicDataKey(address);
Bytes32 basicDataValue;
if ((basicDataValue = nonStorageKeyValuesForUpdate.get(basicDataKey)) == null) {
basicDataValue = Bytes32.ZERO;
Expand All @@ -81,7 +81,7 @@ public void generateAccountKeyValueForUpdate(
}

public void generateCodeSizeKeyValueForUpdate(final Address address, final int size) {
Bytes32 basicDataKey = trieKeyAdapter.basicDataKey(address);
Bytes32 basicDataKey = trieKeyFactory.basicDataKey(address);
Bytes32 basicDataValue;
if ((basicDataValue = nonStorageKeyValuesForUpdate.get(basicDataKey)) == null) {
basicDataValue = Bytes32.ZERO;
Expand All @@ -96,18 +96,18 @@ public void generateCodeSizeKeyValueForUpdate(final Address address, final int s
}

public void generateCodeHashKeyValueForUpdate(final Address address, final Hash codeHash) {
nonStorageKeyValuesForUpdate.put(trieKeyAdapter.codeHashKey(address), codeHash);
nonStorageKeyValuesForUpdate.put(trieKeyFactory.codeHashKey(address), codeHash);
}

public void generateCodeKeyValuesForUpdate(
final Address address, final Bytes code, final Hash codeHash) {
generateCodeSizeKeyValueForUpdate(address, code.size());
generateCodeHashKeyValueForUpdate(address, codeHash);

List<UInt256> codeChunks = trieKeyAdapter.chunkifyCode(code);
List<UInt256> codeChunks = TrieKeyUtils.chunkifyCode(code);
for (int i = 0; i < codeChunks.size(); i++) {
nonStorageKeyValuesForUpdate.put(
trieKeyAdapter.codeChunkKey(address, UInt256.valueOf(i)), codeChunks.get(i));
trieKeyFactory.codeChunkKey(address, UInt256.valueOf(i)), codeChunks.get(i));
}
}

Expand All @@ -116,7 +116,7 @@ public void generateStorageKeyValueForUpdate(
storageKeyValuesForUpdate.put(
storageSlotKey,
new Pair<>(
trieKeyAdapter.storageKey(address, storageSlotKey.getSlotKey().orElseThrow()), value));
trieKeyFactory.storageKey(address, storageSlotKey.getSlotKey().orElseThrow()), value));
}

public Set<Bytes32> getKeysForRemoval() {
Expand Down
90 changes: 30 additions & 60 deletions ...va/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/preloader/StemPreloader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,14 @@

import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter;
import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher;
import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils;
import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher;
import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher;
import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters;
import org.hyperledger.besu.ethereum.trie.verkle.hasher.StemHasher;
import org.hyperledger.besu.ethereum.trie.verkle.hasher.builder.StemHasherBuilder;
import org.hyperledger.besu.ethereum.trie.verkle.hasher.cache.InMemoryCacheStrategy;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
Expand All @@ -52,23 +47,18 @@
* through the {@code STEM_CACHE_SIZE} constant. The {@link Hasher} used for stem generation is
* configurable, allowing for different hashing strategies (e.g., Pedersen hashing) to be employed.
*
* @see org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter
* @see org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyFactory
* @see org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher
* @see org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher
*/
public class StemPreloader {
private static final int HASHER_CACHE_SIZE = 10_000;
private static final int STEM_CACHE_SIZE = 10_000;
private static final int ADDRESS_COMMITMENT_CACHE_SIZE = 10_000;

// Cache of stem by address. The sub-map will contain the trie index as the key and the stem as
// the value.
private final Cache<Address, CachedPedersenHasher> pedersenHasherCache =
CacheBuilder.newBuilder().maximumSize(STEM_CACHE_SIZE).build();
private final Cache<Address, StemHasher> stemHasherByAddress =
CacheBuilder.newBuilder().maximumSize(HASHER_CACHE_SIZE).build();

private final TrieKeyAdapter trieKeyAdapter;

public StemPreloader() {
this.trieKeyAdapter = new TrieKeyAdapter(new PedersenHasher());
}
public StemPreloader() {}

/**
* Creates a preloaded hasher context for a given updated address, storage, and code. This method
Expand All @@ -83,44 +73,43 @@ public StemPreloader() {
* @param keys a list of keys to use for stem generation
* @return the preloaded stems
*/
public Map<Bytes32, Bytes> preloadStems(final Address address, final Set<Bytes32> keys) {
return getHasherByAddress(address).manyStems(address, new ArrayList<>(keys));
public Map<Bytes32, Bytes> preloadStems(final Address address, final List<Bytes32> keys) {
return getStemHasherByAddress(address).manyStems(address, keys);
}

public Bytes preloadAccountStem(final Address address) {
return getHasherByAddress(address).computeStem(address, UInt256.ZERO);
return getStemHasherByAddress(address).computeStem(address, UInt256.ZERO);
}

public Bytes preloadSlotStems(final Address address, final StorageSlotKey storageSlotKey) {
return getHasherByAddress(address)
.computeStem(
address,
trieKeyAdapter.getStorageKeyTrieIndex(storageSlotKey.getSlotKey().orElseThrow()));
final Bytes32 storageKeyTrieIndex =
TrieKeyUtils.getStorageKeyTrieIndex(storageSlotKey.getSlotKey().orElseThrow());
return getStemHasherByAddress(address).computeStem(address, storageKeyTrieIndex);
}

public Map<Bytes32, Bytes> preloadCodeChunckStems(final Address address, final Bytes codeUpdate) {
return getHasherByAddress(address).manyStems(address, generateCodeChunkKeyIds(codeUpdate));
return getStemHasherByAddress(address)
.manyStems(address, TrieKeyUtils.getCodeChunkKeyTrieIndexes(codeUpdate));
}

/**
* Retrieves the cache that maps account addresses to their corresponding cached stems.
* Retrieves the cache that maps account addresses to their corresponding stem hasher.
*
* @return the cache mapping account addresses to trie stems
* @return the stem hasher linked to the address
*/
@VisibleForTesting
public CachedPedersenHasher getHasherByAddress(final Address address) {
CachedPedersenHasher ifPresent = pedersenHasherCache.getIfPresent(address);
public StemHasher getStemHasherByAddress(final Address address) {
StemHasher ifPresent = stemHasherByAddress.getIfPresent(address);
if (ifPresent != null) {
return ifPresent;
}
final CachedPedersenHasher defaultHasher =
new CachedPedersenHasher(STEM_CACHE_SIZE, new ConcurrentHashMap<>());
pedersenHasherCache.put(address, defaultHasher);
return defaultHasher;
}

public Bytes getStorageKeySuffix(final Bytes32 storageKey) {
return trieKeyAdapter.getStorageKeySuffix(storageKey);
final StemHasher stemHasher =
StemHasherBuilder.builder()
.withStemCache(new InMemoryCacheStrategy<>(STEM_CACHE_SIZE))
.withAddressCommitmentCache(new InMemoryCacheStrategy<>(ADDRESS_COMMITMENT_CACHE_SIZE))
.build();
stemHasherByAddress.put(address, stemHasher);
return stemHasher;
}

/**
Expand All @@ -129,25 +118,6 @@ public Bytes getStorageKeySuffix(final Bytes32 storageKey) {
* with performance monitoring or testing.
*/
public void reset() {
pedersenHasherCache.invalidateAll();
}

public Bytes32 generateAccountKeyId() {
return Parameters.BASIC_DATA_LEAF_KEY;
}

public List<Bytes32> generateCodeChunkKeyIds(final Bytes code) {
return IntStream.range(0, trieKeyAdapter.getNbChunk(code))
.mapToObj(UInt256::valueOf)
.collect(Collectors.toUnmodifiableList());
}

public List<Bytes32> generateStorageKeyIds(final Set<StorageSlotKey> storageSlotKeys) {
return storageSlotKeys.stream()
.map(
storageSlotKey ->
trieKeyAdapter.getStorageKeyTrieIndex(storageSlotKey.getSlotKey().orElseThrow()))
.map(Bytes32::wrap)
.toList();
stemHasherByAddress.invalidateAll();
}
}
4 changes: 2 additions & 2 deletions ...yperledger/besu/ethereum/trie/diffbased/verkle/storage/flat/VerkleStemFlatDbStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView;
import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount;
import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.preloader.StemPreloader;
import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyUtils;
import org.hyperledger.besu.ethereum.trie.verkle.util.SuffixTreeDecoder;
import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.plugin.services.MetricsSystem;
Expand Down Expand Up @@ -120,8 +121,7 @@ public Optional<Bytes> getFlatStorageValueByStorageSlotKey(
stem.flatMap(
values ->
values.get(
stemPreloader
.getStorageKeySuffix(storageSlotKey.getSlotKey().orElseThrow())
TrieKeyUtils.getStorageKeySuffix(storageSlotKey.getSlotKey().orElseThrow())
.toInt()));
if (storageFound.isPresent()) {
getStorageValueFlatDatabaseCounter.inc();
Expand Down
Loading
Loading