From eac84716edcb78b223ec921175903afe86422d1f Mon Sep 17 00:00:00 2001 From: cxkoda Date: Tue, 15 Nov 2022 12:22:50 +0100 Subject: [PATCH] Add appending as base64 to DynamicBuffer (#60) * Add base64 encoding to dynamic buffer * Refactor appendSafeBase64 and add tests * Fix linting * Make capacity and overflow checks available for users * Revert back to text-based errors since the we cannot parse them in go yet * Remove name shadowing * Improvements after Arran's review * Move a comment closer to the relevant code * Merge branch 'main' into cx/appendSafeBase64 * Push minor version --- contracts/utils/DynamicBuffer.sol | 132 +++++++++++++++++--- package.json | 2 +- tests/utils/TestableDynamicBuffer.sol | 20 +++ tests/utils/dynamicbuffer_test.go | 169 ++++++++++++++++++++++++++ 4 files changed, 308 insertions(+), 15 deletions(-) diff --git a/contracts/utils/DynamicBuffer.sol b/contracts/utils/DynamicBuffer.sol index e38d8cb..2ad772c 100644 --- a/contracts/utils/DynamicBuffer.sol +++ b/contracts/utils/DynamicBuffer.sol @@ -14,12 +14,12 @@ pragma solidity >=0.8.0; /// bounds checking is required. library DynamicBuffer { /// @notice Allocates container space for the DynamicBuffer - /// @param capacity The intended max amount of bytes in the buffer + /// @param capacity_ The intended max amount of bytes in the buffer /// @return buffer The memory location of the buffer - /// @dev Allocates `capacity + 0x60` bytes of space + /// @dev Allocates `capacity_ + 0x60` bytes of space /// The buffer array starts at the first container data position, /// (i.e. `buffer = container + 0x20`) - function allocate(uint256 capacity) + function allocate(uint256 capacity_) internal pure returns (bytes memory buffer) @@ -32,14 +32,14 @@ library DynamicBuffer { { // Add 2 x 32 bytes in size for the two length fields // Add 32 bytes safety space for 32B chunked copy - let size := add(capacity, 0x60) + let size := add(capacity_, 0x60) let newNextFree := add(container, size) mstore(0x40, newNextFree) } // Set the correct container length { - let length := add(capacity, 0x40) + let length := add(capacity_, 0x40) mstore(container, length) } @@ -90,17 +90,121 @@ library DynamicBuffer { /// @param data the data to append /// @dev Performs out-of-bound checks and calls `appendUnchecked`. function appendSafe(bytes memory buffer, bytes memory data) internal pure { - uint256 capacity; - uint256 length; + checkOverflow(buffer, data.length); + appendUnchecked(buffer, data); + } + + /// @notice Appends data encoded as Base64 to buffer. + /// @param fileSafe Whether to replace '+' with '-' and '/' with '_'. + /// @param noPadding Whether to strip away the padding. + /// @dev Encodes `data` using the base64 encoding described in RFC 4648. + /// See: https://datatracker.ietf.org/doc/html/rfc4648 + /// Author: Modified from Solady (https://github.com/vectorized/solady/blob/main/src/utils/Base64.sol) + /// Author: Modified from Solmate (https://github.com/transmissions11/solmate/blob/main/src/utils/Base64.sol) + /// Author: Modified from (https://github.com/Brechtpd/base64/blob/main/base64.sol) by Brecht Devos. + function appendSafeBase64( + bytes memory buffer, + bytes memory data, + bool fileSafe, + bool noPadding + ) internal pure { + uint256 dataLength = data.length; + + if (data.length == 0) { + return; + } + + uint256 encodedLength; + uint256 r; assembly { - capacity := sub(mload(sub(buffer, 0x20)), 0x40) - length := mload(buffer) + // For each 3 bytes block, we will have 4 bytes in the base64 + // encoding: `encodedLength = 4 * divCeil(dataLength, 3)`. + // The `shl(2, ...)` is equivalent to multiplying by 4. + encodedLength := shl(2, div(add(dataLength, 2), 3)) + + r := mod(dataLength, 3) + if noPadding { + // if r == 0 => no modification + // if r == 1 => encodedLength -= 2 + // if r == 2 => encodedLength -= 1 + encodedLength := sub( + encodedLength, + add(iszero(iszero(r)), eq(r, 1)) + ) + } } - require( - length + data.length <= capacity, - "DynamicBuffer: Appending out of bounds." - ); - appendUnchecked(buffer, data); + checkOverflow(buffer, encodedLength); + + assembly { + let nextFree := mload(0x40) + + // Store the table into the scratch space. + // Offsetted by -1 byte so that the `mload` will load the character. + // We will rewrite the free memory pointer at `0x40` later with + // the allocated size. + mstore(0x1f, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef") + mstore( + 0x3f, + sub( + "ghijklmnopqrstuvwxyz0123456789-_", + // The magic constant 0x0230 will translate "-_" + "+/". + mul(iszero(fileSafe), 0x0230) + ) + ) + + // Skip the first slot, which stores the length. + let ptr := add(add(buffer, 0x20), mload(buffer)) + let end := add(data, dataLength) + + // Run over the input, 3 bytes at a time. + // prettier-ignore + // solhint-disable-next-line no-empty-blocks + for {} 1 {} { + data := add(data, 3) // Advance 3 bytes. + let input := mload(data) + + // Write 4 bytes. Optimized for fewer stack operations. + mstore8( ptr , mload(and(shr(18, input), 0x3F))) + mstore8(add(ptr, 1), mload(and(shr(12, input), 0x3F))) + mstore8(add(ptr, 2), mload(and(shr( 6, input), 0x3F))) + mstore8(add(ptr, 3), mload(and( input , 0x3F))) + + ptr := add(ptr, 4) // Advance 4 bytes. + // prettier-ignore + if iszero(lt(data, end)) { break } + } + + if iszero(noPadding) { + // Offset `ptr` and pad with '='. We can simply write over the end. + mstore8(sub(ptr, iszero(iszero(r))), 0x3d) // Pad at `ptr - 1` if `r > 0`. + mstore8(sub(ptr, shl(1, eq(r, 1))), 0x3d) // Pad at `ptr - 2` if `r == 1`. + } + + mstore(buffer, add(mload(buffer), encodedLength)) + mstore(0x40, nextFree) + } + } + + /// @notice Returns the capacity of a given buffer. + function capacity(bytes memory buffer) internal pure returns (uint256) { + uint256 cap; + assembly { + cap := sub(mload(sub(buffer, 0x20)), 0x40) + } + return cap; + } + + /// @notice Reverts if the buffer will overflow after appending a given + /// number of bytes. + function checkOverflow(bytes memory buffer, uint256 addedLength) + internal + pure + { + uint256 cap = capacity(buffer); + uint256 newLength = buffer.length + addedLength; + if (cap < newLength) { + revert("DynamicBuffer: Appending out of bounds."); + } } } diff --git a/package.json b/package.json index 82f39a7..905cb51 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@divergencetech/ethier", - "version": "0.39.0", + "version": "0.40.0", "description": "Golang and Solidity SDK to make Ethereum development ethier", "main": "\"\"", "scripts": { diff --git a/tests/utils/TestableDynamicBuffer.sol b/tests/utils/TestableDynamicBuffer.sol index 111f22b..7f48dd6 100644 --- a/tests/utils/TestableDynamicBuffer.sol +++ b/tests/utils/TestableDynamicBuffer.sol @@ -27,4 +27,24 @@ contract TestableDynamicBuffer { return string(buffer); } + + /** + @notice Allocates a buffer with a given capacity and safely appends data for + a given number of times encoded as base64. + */ + function allocateAndAppendRepeatedBase64( + uint256 capacity, + bytes memory data, + uint256 repetitions, + bool fileSafe, + bool noPadding + ) public pure returns (string memory) { + bytes memory buffer = DynamicBuffer.allocate(capacity); + + for (uint256 idx = 0; idx < repetitions; ++idx) { + buffer.appendSafeBase64(data, fileSafe, noPadding); + } + + return string(buffer); + } } diff --git a/tests/utils/dynamicbuffer_test.go b/tests/utils/dynamicbuffer_test.go index 3910b75..d4f9d85 100644 --- a/tests/utils/dynamicbuffer_test.go +++ b/tests/utils/dynamicbuffer_test.go @@ -2,8 +2,11 @@ package utils import ( "math/big" + "strings" "testing" + "encoding/base64" + "github.com/divergencetech/ethier/ethtest" "github.com/h-fam/errdiff" ) @@ -131,3 +134,169 @@ func TestDynamicBuffer(t *testing.T) { }) } } + +func base64Len(data string) int { + return base64.StdEncoding.EncodedLen(len(data)) +} + +func base64LenUnpadded(data string) int { + return base64Len(data) - len(data)%3 +} + +func TestDynamicBufferBase64(t *testing.T) { + sim := ethtest.NewSimulatedBackendTB(t, 3) + _, _, dynBuf, err := DeployTestableDynamicBuffer(sim.Acc(0), sim) + if err != nil { + t.Fatalf("DeployTestableDynamicBuffer() error %v", err) + } + const ( + testStrWithoutPadding = "This is a really long string without padding." + testStrWithPadding = "This is a short string" + outOfBoundsMsg = "DynamicBuffer: Appending out of bounds." + ) + + if base64Len(testStrWithoutPadding) != base64LenUnpadded(testStrWithoutPadding) { + t.Fatalf("The length of %q must be a multiple of 3", testStrWithoutPadding) + } + + if base64Len(testStrWithPadding) == base64LenUnpadded(testStrWithPadding) { + t.Fatalf("The length of %q must not be a multiple of 3", testStrWithPadding) + } + + tests := []struct { + name string + capacity int + appendString string + repetitions int + fileSafe bool + noPadding bool + errDiffAgainst interface{} + }{ + { + name: "Single append", + capacity: base64Len(testStrWithoutPadding), + appendString: testStrWithoutPadding, + repetitions: 1, + }, + { + name: "Double append out-of-bound", + capacity: base64Len(testStrWithoutPadding), + appendString: testStrWithoutPadding, + repetitions: 2, + errDiffAgainst: outOfBoundsMsg, + }, + { + name: "Mutliple append", + capacity: 420 * base64Len(testStrWithoutPadding), + appendString: testStrWithoutPadding, + repetitions: 420, + }, + { + name: "Mutliple append out-of-bound", + capacity: 420 * base64Len(testStrWithoutPadding), + appendString: testStrWithoutPadding, + repetitions: 421, + errDiffAgainst: outOfBoundsMsg, + }, + { + name: "Single append short", + capacity: base64Len(testStrWithPadding), + appendString: testStrWithPadding, + repetitions: 1, + }, + { + name: "Double append short out-of-bound", + capacity: base64Len(testStrWithPadding), + repetitions: 2, + appendString: testStrWithPadding, + errDiffAgainst: outOfBoundsMsg, + }, + { + name: "Mutliple append short", + capacity: 420 * base64Len(testStrWithPadding), + appendString: testStrWithPadding, + repetitions: 420, + }, + { + name: "Mutliple append short out-of-bound", + capacity: 420 * base64Len(testStrWithPadding), + appendString: testStrWithPadding, + repetitions: 421, + errDiffAgainst: outOfBoundsMsg, + }, + { + name: "Mutliple append short out-of-bound", + capacity: 420 * base64Len(testStrWithPadding), + appendString: testStrWithPadding, + repetitions: 421, + errDiffAgainst: outOfBoundsMsg, + }, + { + name: "Single (testStrWithoutPadding) append with noPadding ", + capacity: base64Len(testStrWithoutPadding), + appendString: testStrWithoutPadding, + repetitions: 1, + noPadding: true, + }, + { + name: "Double (testStrWithoutPadding) append with noPadding ", + capacity: 2 * base64Len(testStrWithoutPadding), + appendString: testStrWithoutPadding, + repetitions: 2, + noPadding: true, + }, + { + name: "Single (testStrWithPadding) append with noPadding ", + capacity: base64LenUnpadded(testStrWithPadding), + appendString: testStrWithPadding, + repetitions: 1, + noPadding: true, + }, + { + name: "Mutliple (testStrWithPadding) append with noPadding ", + capacity: 42 * base64LenUnpadded(testStrWithPadding), + appendString: testStrWithPadding, + repetitions: 42, + noPadding: true, + }, + { + name: "Mutliple append file safe", + capacity: 42 * base64Len(testStrWithoutPadding), + appendString: testStrWithoutPadding, + repetitions: 42, + fileSafe: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := dynBuf.AllocateAndAppendRepeatedBase64(nil, big.NewInt(int64(tt.capacity)), []byte(tt.appendString), big.NewInt(int64(tt.repetitions)), tt.fileSafe, tt.noPadding) + + if diff := errdiff.Check(err, tt.errDiffAgainst); diff != "" { + t.Fatalf("AllocateAndAppendRepeatedBase64(%d, %q, %d, %v, %v) %s", tt.capacity, tt.appendString, tt.repetitions, tt.fileSafe, tt.noPadding, diff) + } + + if tt.errDiffAgainst != nil { + return + } + + var want string + for i := 0; i < tt.repetitions; i++ { + want = want + base64.StdEncoding.EncodeToString([]byte(tt.appendString)) + } + + if tt.noPadding { + want = strings.ReplaceAll(want, "=", "") + } + + if tt.fileSafe { + want = strings.ReplaceAll(want, "+", "-") + want = strings.ReplaceAll(want, "/", "_") + } + + if got != want { + t.Errorf("AllocateAndAppendRepeatedBase64(%v, %q, %v, %v, %v) got %q; want %q", tt.capacity, tt.appendString, tt.repetitions, tt.fileSafe, tt.noPadding, got, want) + } + }) + } +}