Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: finish msg inclusion api #681

Merged
merged 7 commits into from
Sep 9, 2022
Merged

feat: finish msg inclusion api #681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
01cb836
feat: add GetCommit func to complete message inclusion API
evan-forbes Sep 2, 2022
52788e9
import shares package
evan-forbes Sep 2, 2022
404ea37
linter
evan-forbes Sep 2, 2022
7ef087a
pull fix from feature branch
evan-forbes Sep 6, 2022
09e022e
reanme contiguous to compact
evan-forbes Sep 7, 2022
0949647
remove redundant info
evan-forbes Sep 7, 2022
537ec74
remove redundant info
evan-forbes Sep 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions pkg/inclusion/get_commit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package inclusion

import (
"errors"

"github.com/tendermint/tendermint/crypto/merkle"
"github.com/tendermint/tendermint/pkg/da"
)

func GetCommit(cacher *EDSSubTreeRootCacher, dah da.DataAvailabilityHeader, start, msgShareLen int) ([]byte, error) {
originalSquareSize := len(dah.RowsRoots) / 2
if start+msgShareLen > originalSquareSize*originalSquareSize {
return nil, errors.New("cannot get commit for message that doesn't fit in square")
}
paths := calculateCommitPaths(originalSquareSize, start, msgShareLen)
commits := make([][]byte, len(paths))
for i, path := range paths {
// here we prepend false (walk left down the tree) because we only need
// the commits to the original square
orignalSquarePath := append(append(make([]WalkInstruction, 0, len(path.instructions)+1), WalkLeft), path.instructions...)
commit, err := cacher.getSubTreeRoot(dah, path.row, orignalSquarePath)
if err != nil {
return nil, err
}
commits[i] = commit

}
return merkle.HashFromByteSlices(commits), nil
}
4 changes: 2 additions & 2 deletions pkg/inclusion/nmt_caching.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,9 @@ func (stc *EDSSubTreeRootCacher) Constructor() rsmt2d.Tree {
return &newTree
}

// GetSubTreeRoot traverses the nmt of the selected row and returns the
// getSubTreeRoot traverses the nmt of the selected row and returns the
// subtree root. An error is thrown if the subtree cannot be found.
func (stc *EDSSubTreeRootCacher) GetSubTreeRoot(dah da.DataAvailabilityHeader, row int, path []WalkInstruction) ([]byte, error) {
func (stc *EDSSubTreeRootCacher) getSubTreeRoot(dah da.DataAvailabilityHeader, row int, path []WalkInstruction) ([]byte, error) {
if len(stc.caches) != len(dah.RowsRoots) {
return nil, fmt.Errorf("data availability header has unexpected number of row roots: expected %d got %d", len(stc.caches), len(dah.RowsRoots))
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/inclusion/nmt_caching_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func TestEDSSubRootCacher(t *testing.T) {
require.NotNil(t, expectedSubTreeRoots)
// note: the depth is one greater than expected because we're dividing
// the row in half when we calculate the expected roots.
result, err := stc.GetSubTreeRoot(dah, i, []WalkInstruction{false, false, false})
result, err := stc.getSubTreeRoot(dah, i, []WalkInstruction{false, false, false})
require.NoError(t, err)
assert.Equal(t, expectedSubTreeRoots[0], result)
}
Expand Down
66 changes: 55 additions & 11 deletions pkg/inclusion/paths.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,61 @@
package inclusion

import (
"math"

"github.com/celestiaorg/celestia-app/pkg/shares"
)

type path struct {
instructions []WalkInstruction
row int
}

// calculateCommitPaths calculates all of the paths to subtree roots needed to
// create the commitment for a given message.
func calculateCommitPaths(origSquareSize, start, msgShareLen int) []path {
// todo: make the non-interactive defaults optional. by calculating the
// NextAlignedPowerOfTwo, we are forcing use of the non-interactive
// defaults. If we want to make this optional in the future, we have to move
// this next line out of this function.
start, _ = shares.NextAlignedPowerOfTwo(start, msgShareLen, origSquareSize)
startRow, endRow := start/origSquareSize, (start+msgShareLen-1)/origSquareSize
normalizedStartIndex := start % origSquareSize
normalizedEndIndex := (start + msgShareLen) - endRow*origSquareSize
paths := []path{}
maxDepth := int(math.Log2(float64(origSquareSize)))
for i := startRow; i <= endRow; i++ {
start, end := 0, origSquareSize
if i == startRow {
start = normalizedStartIndex
}
if i == endRow {
end = normalizedEndIndex
}
coord := calculateSubTreeRootCoordinates(maxDepth, start, end)
for _, c := range coord {
paths = append(paths, path{
instructions: genSubTreeRootPath(c.depth, uint(c.position)),
row: i,
})
}
}

return paths
}

// genSubTreeRootPath calculates the path to a given subtree root of a node, given the
// depth and position of the node. note: the root of the tree is depth 0.
// The following nolint can be removed after this function is used.
//nolint:unused,deadcode
Copy link
Member

@rach-id rach-id Sep 8, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Non Blocking][Suggestion]

Probably can be removed, as this function is now used.

Suggested change
//nolint:unused,deadcode

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice! good find, I'll if there is other feedback, I'll include this in this PR, if not, I'll be sure to include this in the next ProcessProposal PR as to not dismiss reviews.

func genSubTreeRootPath(depth int, pos uint) []bool {
path := make([]bool, depth)
func genSubTreeRootPath(depth int, pos uint) []WalkInstruction {
path := make([]WalkInstruction, depth)
counter := 0
for i := depth - 1; i >= 0; i-- {
if (pos & (1 << i)) == 0 {
path[counter] = false
path[counter] = WalkLeft
} else {
path[counter] = true
path[counter] = WalkRight
}
counter++
}
Expand All @@ -30,10 +74,10 @@ func genSubTreeRootPath(depth int, pos uint) []bool {
// 3 0 1 2 3 4 5 6 7
type coord struct {
// depth is the typical depth of a tree, 0 being the root
depth uint64
depth int
// position is the index of a node of a given depth, 0 being the left most
// node
position uint64
position int
}

// climb is a state transition function to simulate climbing a balanced binary
Expand All @@ -57,7 +101,7 @@ func (c coord) canClimbRight() bool {
// end does not exceed the range of a tree of the provided depth, and that end
// >= start. This function works by starting at the first index of the msg and
// working our way right.
func calculateSubTreeRootCoordinates(maxDepth, start, end uint64) []coord {
func calculateSubTreeRootCoordinates(maxDepth, start, end int) []coord {
cds := []coord{}
// leafCursor keeps track of the current leaf that we are starting with when
// finding the subtree root for some set. When leafCursor == end, we are
Expand All @@ -76,7 +120,7 @@ func calculateSubTreeRootCoordinates(maxDepth, start, end uint64) []coord {
// nodeRangeCursor keeps track of the number of leaves that are under the
// current tree node. We could calculate this each time, but this acts as a
// cache
nodeRangeCursor := uint64(1)
nodeRangeCursor := 1
// reset is used to reset the above state after finding a subtree root. We
// reset by setting the node cursors to the values equal to the next leaf
// node.
Expand All @@ -87,7 +131,7 @@ func calculateSubTreeRootCoordinates(maxDepth, start, end uint64) []coord {
depth: maxDepth,
position: leafCursor,
}
nodeRangeCursor = uint64(1)
nodeRangeCursor = 1
}
// recursively climb the tree starting at the left most leaf node (the
// starting leaf), and save each subtree root as we find it. After finding a
Expand All @@ -96,12 +140,12 @@ func calculateSubTreeRootCoordinates(maxDepth, start, end uint64) []coord {
for {
switch {
// check if we're finished, if so add the last coord and return
case leafCursor == end:
case leafCursor+1 == end:
cds = append(cds, nodeCursor)
return cds
// check if we've climbed too high in the tree. if so, add the last
// highest node and proceed.
case leafCursor > end:
case leafCursor+1 > end:
cds = append(cds, lastNodeCursor)
leafCursor = lastLeafCursor + 1
reset()
Expand Down
102 changes: 81 additions & 21 deletions pkg/inclusion/paths_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package inclusion

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
Expand All @@ -9,14 +10,14 @@ import (
func Test_calculateSubTreeRootCoordinates(t *testing.T) {
type test struct {
name string
start, end, maxDepth uint64
start, end, maxDepth int
expected []coord
}
tests := []test{
{
name: "first four shares of an 8 leaf tree",
start: 0,
end: 3,
end: 4,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -28,7 +29,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "second set of four shares of an 8 leaf tree",
start: 4,
end: 7,
end: 8,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -40,7 +41,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "middle 2 shares of an 8 leaf tree",
start: 3,
end: 4,
end: 5,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -56,7 +57,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "third lone share of an 8 leaf tree",
start: 3,
end: 3,
end: 4,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -68,7 +69,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "middle 3 shares of an 8 leaf tree",
start: 3,
end: 5,
end: 6,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -84,7 +85,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "middle 6 shares of an 8 leaf tree",
start: 1,
end: 6,
end: 7,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -108,7 +109,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "first 5 shares of an 8 leaf tree",
start: 0,
end: 4,
end: 5,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -124,7 +125,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "first 7 shares of an 8 leaf tree",
start: 0,
end: 6,
end: 7,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -144,7 +145,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "all shares of an 8 leaf tree",
start: 0,
end: 7,
end: 8,
maxDepth: 3,
expected: []coord{
{
Expand All @@ -156,7 +157,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "first 32 shares of a 128 leaf tree",
start: 0,
end: 31,
end: 32,
maxDepth: 7,
expected: []coord{
{
Expand All @@ -168,7 +169,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "first 33 shares of a 128 leaf tree",
start: 0,
end: 32,
end: 33,
maxDepth: 7,
expected: []coord{
{
Expand All @@ -184,7 +185,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "first 31 shares of a 128 leaf tree",
start: 0,
end: 30,
end: 31,
maxDepth: 7,
expected: []coord{
{
Expand Down Expand Up @@ -212,7 +213,7 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
{
name: "first 64 shares of a 128 leaf tree",
start: 0,
end: 63,
end: 64,
maxDepth: 7,
expected: []coord{
{
Expand All @@ -221,6 +222,18 @@ func Test_calculateSubTreeRootCoordinates(t *testing.T) {
},
},
},
{
name: "single leaf square size 4",
start: 0,
end: 1,
maxDepth: 2,
expected: []coord{
{
depth: 2,
position: 0,
},
},
},
}
for _, tt := range tests {
res := calculateSubTreeRootCoordinates(tt.maxDepth, tt.start, tt.end)
Expand All @@ -232,18 +245,65 @@ func Test_genSubTreeRootPath(t *testing.T) {
type test struct {
depth int
pos uint
expected []bool
expected []WalkInstruction
}
tests := []test{
{2, 0, []bool{false, false}},
{0, 0, []bool{}},
{3, 0, []bool{false, false, false}},
{3, 1, []bool{false, false, true}},
{3, 2, []bool{false, true, false}},
{5, 16, []bool{true, false, false, false, false}},
{2, 0, []WalkInstruction{WalkLeft, WalkLeft}},
{0, 0, []WalkInstruction{}},
{3, 0, []WalkInstruction{WalkLeft, WalkLeft, WalkLeft}},
{3, 1, []WalkInstruction{WalkLeft, WalkLeft, WalkRight}},
{3, 2, []WalkInstruction{WalkLeft, WalkRight, WalkLeft}},
{5, 16, []WalkInstruction{WalkRight, WalkLeft, WalkLeft, WalkLeft, WalkLeft}},
}
for _, tt := range tests {
path := genSubTreeRootPath(tt.depth, tt.pos)
assert.Equal(t, tt.expected, path)
}
}

func Test_calculateCommitPaths(t *testing.T) {
type test struct {
size, start, msgLen int
expected []path
}
tests := []test{
{2, 0, 1, []path{{instructions: []WalkInstruction{WalkLeft}, row: 0}}},
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[no change needed] visual

row 0 -> |1| |   
row 1 -> | | |

merkle tree of row 0

   x
 /   \
1     x

where I assume x doesn't matter

{2, 2, 2, []path{{instructions: []WalkInstruction{}, row: 1}}},
{2, 1, 2, []path{{instructions: []WalkInstruction{}, row: 1}}},
{4, 2, 2, []path{{instructions: []WalkInstruction{WalkRight}, row: 0}}},
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[no change needed] visual

row 0 -> | | |2|2|
row 2 -> | | | | |
row 3 -> | | | | |
row 4 -> | | | | |

merkle tree of row 0

        x
       / \
      x   y
         / \
        2   2

where y is what we want

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exactly!

{4, 2, 4, []path{{instructions: []WalkInstruction{}, row: 1}}},
{4, 3, 4, []path{{instructions: []WalkInstruction{}, row: 1}}},
{4, 2, 9, []path{
{instructions: []WalkInstruction{}, row: 1},
{instructions: []WalkInstruction{}, row: 2},
{instructions: []WalkInstruction{WalkLeft, WalkLeft}, row: 3},
}},
{8, 3, 16, []path{
{instructions: []WalkInstruction{}, row: 1},
{instructions: []WalkInstruction{}, row: 2},
}},
{64, 144, 32, []path{
{instructions: []WalkInstruction{WalkRight}, row: 2},
}},
{64, 4032, 33, []path{
{instructions: []WalkInstruction{WalkLeft}, row: 63},
{instructions: []WalkInstruction{WalkRight, WalkLeft, WalkLeft, WalkLeft, WalkLeft, WalkLeft}, row: 63},
}},
{64, 4032, 63, []path{
{instructions: []WalkInstruction{WalkLeft}, row: 63},
{instructions: []WalkInstruction{WalkRight, WalkLeft}, row: 63},
{instructions: []WalkInstruction{WalkRight, WalkRight, WalkLeft}, row: 63},
{instructions: []WalkInstruction{WalkRight, WalkRight, WalkRight, WalkLeft}, row: 63},
{instructions: []WalkInstruction{WalkRight, WalkRight, WalkRight, WalkRight, WalkLeft}, row: 63},
{instructions: []WalkInstruction{WalkRight, WalkRight, WalkRight, WalkRight, WalkRight, WalkLeft}, row: 63},
}},
}
for i, tt := range tests {
t.Run(
fmt.Sprintf("test %d: square size %d start %d msgLen %d", i, tt.size, tt.start, tt.msgLen),
func(t *testing.T) {
assert.Equal(t, tt.expected, calculateCommitPaths(tt.size, tt.start, tt.msgLen))
},
)
}
}
Loading