Add appending as base64 to DynamicBuffer (#60)

* Add base64 encoding to dynamic buffer

* Refactor appendSafeBase64 and add tests

* Fix linting

* Make capacity and overflow checks available for users

* Revert back to text-based errors since the we cannot parse them in go yet

* Remove name shadowing

* Improvements after Arran's review

* Move a comment closer to the relevant code

* Merge branch 'main' into cx/appendSafeBase64

* Push minor version

contracts/utils/DynamicBuffer.sol

@@ -14,12 +14,12 @@ pragma solidity >=0.8.0;
 ///      bounds checking is required.
 library DynamicBuffer {
     /// @notice Allocates container space for the DynamicBuffer
-    /// @param capacity The intended max amount of bytes in the buffer
+    /// @param capacity_ The intended max amount of bytes in the buffer
     /// @return buffer The memory location of the buffer
-    /// @dev Allocates `capacity + 0x60` bytes of space
+    /// @dev Allocates `capacity_ + 0x60` bytes of space
     ///      The buffer array starts at the first container data position,
     ///      (i.e. `buffer = container + 0x20`)
-    function allocate(uint256 capacity)
+    function allocate(uint256 capacity_)
         internal
         pure
         returns (bytes memory buffer)
@@ -32,14 +32,14 @@ library DynamicBuffer {
             {
                 // Add 2 x 32 bytes in size for the two length fields
                 // Add 32 bytes safety space for 32B chunked copy
-                let size := add(capacity, 0x60)
+                let size := add(capacity_, 0x60)
                 let newNextFree := add(container, size)
                 mstore(0x40, newNextFree)
             }
 
             // Set the correct container length
             {
-                let length := add(capacity, 0x40)
+                let length := add(capacity_, 0x40)
                 mstore(container, length)
             }
 
@@ -90,17 +90,121 @@ library DynamicBuffer {
     /// @param data the data to append
     /// @dev Performs out-of-bound checks and calls `appendUnchecked`.
     function appendSafe(bytes memory buffer, bytes memory data) internal pure {
-        uint256 capacity;
-        uint256 length;
+        checkOverflow(buffer, data.length);
+        appendUnchecked(buffer, data);
+    }
+
+    /// @notice Appends data encoded as Base64 to buffer.
+    /// @param fileSafe  Whether to replace '+' with '-' and '/' with '_'.
+    /// @param noPadding Whether to strip away the padding.
+    /// @dev Encodes `data` using the base64 encoding described in RFC 4648.
+    /// See: https://datatracker.ietf.org/doc/html/rfc4648
+    /// Author: Modified from Solady (https://github.com/vectorized/solady/blob/main/src/utils/Base64.sol)
+    /// Author: Modified from Solmate (https://github.com/transmissions11/solmate/blob/main/src/utils/Base64.sol)
+    /// Author: Modified from (https://github.com/Brechtpd/base64/blob/main/base64.sol) by Brecht Devos.
+    function appendSafeBase64(
+        bytes memory buffer,
+        bytes memory data,
+        bool fileSafe,
+        bool noPadding
+    ) internal pure {
+        uint256 dataLength = data.length;
+
+        if (data.length == 0) {
+            return;
+        }
+
+        uint256 encodedLength;
+        uint256 r;
         assembly {
-            capacity := sub(mload(sub(buffer, 0x20)), 0x40)
-            length := mload(buffer)
+            // For each 3 bytes block, we will have 4 bytes in the base64
+            // encoding: `encodedLength = 4 * divCeil(dataLength, 3)`.
+            // The `shl(2, ...)` is equivalent to multiplying by 4.
+            encodedLength := shl(2, div(add(dataLength, 2), 3))
+
+            r := mod(dataLength, 3)
+            if noPadding {
+                // if r == 0 => no modification
+                // if r == 1 => encodedLength -= 2
+                // if r == 2 => encodedLength -= 1
+                encodedLength := sub(
+                    encodedLength,
+                    add(iszero(iszero(r)), eq(r, 1))
+                )
+            }
         }
 
-        require(
-            length + data.length <= capacity,
-            "DynamicBuffer: Appending out of bounds."
-        );
-        appendUnchecked(buffer, data);
+        checkOverflow(buffer, encodedLength);
+
+        assembly {
+            let nextFree := mload(0x40)
+
+            // Store the table into the scratch space.
+            // Offsetted by -1 byte so that the `mload` will load the character.
+            // We will rewrite the free memory pointer at `0x40` later with
+            // the allocated size.
+            mstore(0x1f, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef")
+            mstore(
+                0x3f,
+                sub(
+                    "ghijklmnopqrstuvwxyz0123456789-_",
+                    // The magic constant 0x0230 will translate "-_" + "+/".
+                    mul(iszero(fileSafe), 0x0230)
+                )
+            )
+
+            // Skip the first slot, which stores the length.
+            let ptr := add(add(buffer, 0x20), mload(buffer))
+            let end := add(data, dataLength)
+
+            // Run over the input, 3 bytes at a time.
+            // prettier-ignore
+            // solhint-disable-next-line no-empty-blocks
+            for {} 1 {} {
+                    data := add(data, 3) // Advance 3 bytes.
+                    let input := mload(data)
+
+                    // Write 4 bytes. Optimized for fewer stack operations.
+                    mstore8(    ptr    , mload(and(shr(18, input), 0x3F)))
+                    mstore8(add(ptr, 1), mload(and(shr(12, input), 0x3F)))
+                    mstore8(add(ptr, 2), mload(and(shr( 6, input), 0x3F)))
+                    mstore8(add(ptr, 3), mload(and(        input , 0x3F)))
+
+                    ptr := add(ptr, 4) // Advance 4 bytes.
+                    // prettier-ignore
+                    if iszero(lt(data, end)) { break }
+                }
+
+            if iszero(noPadding) {
+                // Offset `ptr` and pad with '='. We can simply write over the end.
+                mstore8(sub(ptr, iszero(iszero(r))), 0x3d) // Pad at `ptr - 1` if `r > 0`.
+                mstore8(sub(ptr, shl(1, eq(r, 1))), 0x3d) // Pad at `ptr - 2` if `r == 1`.
+            }
+
+            mstore(buffer, add(mload(buffer), encodedLength))
+            mstore(0x40, nextFree)
+        }
+    }
+
+    /// @notice Returns the capacity of a given buffer.
+    function capacity(bytes memory buffer) internal pure returns (uint256) {
+        uint256 cap;
+        assembly {
+            cap := sub(mload(sub(buffer, 0x20)), 0x40)
+        }
+        return cap;
+    }
+
+    /// @notice Reverts if the buffer will overflow after appending a given
+    /// number of bytes.
+    function checkOverflow(bytes memory buffer, uint256 addedLength)
+        internal
+        pure
+    {
+        uint256 cap = capacity(buffer);
+        uint256 newLength = buffer.length + addedLength;
+        if (cap < newLength) {
+            revert("DynamicBuffer: Appending out of bounds.");
+        }
     }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@divergencetech/ethier",
-  "version": "0.39.0",
+  "version": "0.40.0",
   "description": "Golang and Solidity SDK to make Ethereum development ethier",
   "main": "\"\"",
   "scripts": {

tests/utils/TestableDynamicBuffer.sol

@@ -27,4 +27,24 @@ contract TestableDynamicBuffer {
 
         return string(buffer);
     }
+
+    /**
+    @notice Allocates a buffer with a given capacity and safely appends data for
+    a given number of times encoded as base64.
+     */
+    function allocateAndAppendRepeatedBase64(
+        uint256 capacity,
+        bytes memory data,
+        uint256 repetitions,
+        bool fileSafe,
+        bool noPadding
+    ) public pure returns (string memory) {
+        bytes memory buffer = DynamicBuffer.allocate(capacity);
+
+        for (uint256 idx = 0; idx < repetitions; ++idx) {
+            buffer.appendSafeBase64(data, fileSafe, noPadding);
+        }
+
+        return string(buffer);
+    }
 }

tests/utils/dynamicbuffer_test.go

@@ -2,8 +2,11 @@ package utils
 
 import (
 	"math/big"
+	"strings"
 	"testing"
 
+	"encoding/base64"
+
 	"github.com/divergencetech/ethier/ethtest"
 	"github.com/h-fam/errdiff"
 )
@@ -131,3 +134,169 @@ func TestDynamicBuffer(t *testing.T) {
 		})
 	}
 }
+
+func base64Len(data string) int {
+	return base64.StdEncoding.EncodedLen(len(data))
+}
+
+func base64LenUnpadded(data string) int {
+	return base64Len(data) - len(data)%3
+}
+
+func TestDynamicBufferBase64(t *testing.T) {
+	sim := ethtest.NewSimulatedBackendTB(t, 3)
+	_, _, dynBuf, err := DeployTestableDynamicBuffer(sim.Acc(0), sim)
+	if err != nil {
+		t.Fatalf("DeployTestableDynamicBuffer() error %v", err)
+	}
+	const (
+		testStrWithoutPadding = "This is a really long string without padding."
+		testStrWithPadding    = "This is a short string"
+		outOfBoundsMsg        = "DynamicBuffer: Appending out of bounds."
+	)
+
+	if base64Len(testStrWithoutPadding) != base64LenUnpadded(testStrWithoutPadding) {
+		t.Fatalf("The length of %q must be a multiple of 3", testStrWithoutPadding)
+	}
+
+	if base64Len(testStrWithPadding) == base64LenUnpadded(testStrWithPadding) {
+		t.Fatalf("The length of %q must not be a multiple of 3", testStrWithPadding)
+	}
+
+	tests := []struct {
+		name           string
+		capacity       int
+		appendString   string
+		repetitions    int
+		fileSafe       bool
+		noPadding      bool
+		errDiffAgainst interface{}
+	}{
+		{
+			name:         "Single append",
+			capacity:     base64Len(testStrWithoutPadding),
+			appendString: testStrWithoutPadding,
+			repetitions:  1,
+		},
+		{
+			name:           "Double append out-of-bound",
+			capacity:       base64Len(testStrWithoutPadding),
+			appendString:   testStrWithoutPadding,
+			repetitions:    2,
+			errDiffAgainst: outOfBoundsMsg,
+		},
+		{
+			name:         "Mutliple append",
+			capacity:     420 * base64Len(testStrWithoutPadding),
+			appendString: testStrWithoutPadding,
+			repetitions:  420,
+		},
+		{
+			name:           "Mutliple append out-of-bound",
+			capacity:       420 * base64Len(testStrWithoutPadding),
+			appendString:   testStrWithoutPadding,
+			repetitions:    421,
+			errDiffAgainst: outOfBoundsMsg,
+		},
+		{
+			name:         "Single append short",
+			capacity:     base64Len(testStrWithPadding),
+			appendString: testStrWithPadding,
+			repetitions:  1,
+		},
+		{
+			name:           "Double append short out-of-bound",
+			capacity:       base64Len(testStrWithPadding),
+			repetitions:    2,
+			appendString:   testStrWithPadding,
+			errDiffAgainst: outOfBoundsMsg,
+		},
+		{
+			name:         "Mutliple append short",
+			capacity:     420 * base64Len(testStrWithPadding),
+			appendString: testStrWithPadding,
+			repetitions:  420,
+		},
+		{
+			name:           "Mutliple append short out-of-bound",
+			capacity:       420 * base64Len(testStrWithPadding),
+			appendString:   testStrWithPadding,
+			repetitions:    421,
+			errDiffAgainst: outOfBoundsMsg,
+		},
+		{
+			name:           "Mutliple append short out-of-bound",
+			capacity:       420 * base64Len(testStrWithPadding),
+			appendString:   testStrWithPadding,
+			repetitions:    421,
+			errDiffAgainst: outOfBoundsMsg,
+		},
+		{
+			name:         "Single (testStrWithoutPadding) append with noPadding ",
+			capacity:     base64Len(testStrWithoutPadding),
+			appendString: testStrWithoutPadding,
+			repetitions:  1,
+			noPadding:    true,
+		},
+		{
+			name:         "Double (testStrWithoutPadding) append with noPadding ",
+			capacity:     2 * base64Len(testStrWithoutPadding),
+			appendString: testStrWithoutPadding,
+			repetitions:  2,
+			noPadding:    true,
+		},
+		{
+			name:         "Single (testStrWithPadding) append with noPadding ",
+			capacity:     base64LenUnpadded(testStrWithPadding),
+			appendString: testStrWithPadding,
+			repetitions:  1,
+			noPadding:    true,
+		},
+		{
+			name:         "Mutliple (testStrWithPadding) append with noPadding ",
+			capacity:     42 * base64LenUnpadded(testStrWithPadding),
+			appendString: testStrWithPadding,
+			repetitions:  42,
+			noPadding:    true,
+		},
+		{
+			name:         "Mutliple append file safe",
+			capacity:     42 * base64Len(testStrWithoutPadding),
+			appendString: testStrWithoutPadding,
+			repetitions:  42,
+			fileSafe:     true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := dynBuf.AllocateAndAppendRepeatedBase64(nil, big.NewInt(int64(tt.capacity)), []byte(tt.appendString), big.NewInt(int64(tt.repetitions)), tt.fileSafe, tt.noPadding)
+
+			if diff := errdiff.Check(err, tt.errDiffAgainst); diff != "" {
+				t.Fatalf("AllocateAndAppendRepeatedBase64(%d, %q, %d, %v, %v) %s", tt.capacity, tt.appendString, tt.repetitions, tt.fileSafe, tt.noPadding, diff)
+			}
+
+			if tt.errDiffAgainst != nil {
+				return
+			}
+
+			var want string
+			for i := 0; i < tt.repetitions; i++ {
+				want = want + base64.StdEncoding.EncodeToString([]byte(tt.appendString))
+			}
+
+			if tt.noPadding {
+				want = strings.ReplaceAll(want, "=", "")
+			}
+
+			if tt.fileSafe {
+				want = strings.ReplaceAll(want, "+", "-")
+				want = strings.ReplaceAll(want, "/", "_")
+			}
+
+			if got != want {
+				t.Errorf("AllocateAndAppendRepeatedBase64(%v, %q, %v, %v, %v) got %q; want %q", tt.capacity, tt.appendString, tt.repetitions, tt.fileSafe, tt.noPadding, got, want)
+			}
+		})
+	}
+}