From 4e1000769b5590caa8f8c5fcb94b060d2e848a5f Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 28 Mar 2023 13:48:09 -0300 Subject: [PATCH 01/15] add batch method for creating leaf nodes Signed-off-by: Ignacio Hagopian --- tree.go | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tree.go b/tree.go index d505d65b..f6009873 100644 --- a/tree.go +++ b/tree.go @@ -1391,3 +1391,89 @@ func (n *LeafNode) serializeWithCompressedCommitments(c1Bytes [32]byte, c2Bytes return result } + +type BatchNewLeafNodeData struct { + Stem []byte + Values [][]byte +} + +func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { + cfg := GetConfig() + + ret := make([]LeafNode, len(nodesValues)) + c1c2points := make([]*Point, 2*len(nodesValues)) + c1c2frs := make([]*Fr, 2*len(nodesValues)) + for i, nv := range nodesValues { + ret[i] = LeafNode{ + values: nv.Values, + stem: nv.Stem, + c1: Generator(), + c2: Generator(), + } + + var c1poly, c2poly [256]Fr + + fillSuffixTreePoly(c1poly[:], nv.Values[:128]) + ret[i].c1 = cfg.CommitToPoly(c1poly[:], 0) + fillSuffixTreePoly(c2poly[:], nv.Values[128:]) + ret[i].c2 = cfg.CommitToPoly(c2poly[:], 0) + + c1c2points[2*i], c1c2points[2*i+1] = ret[i].c1, ret[i].c2 + c1c2frs[2*i], c1c2frs[2*i+1] = new(Fr), new(Fr) + } + + toFrMultiple(c1c2frs, c1c2points) + + var poly [256]Fr + poly[0].SetUint64(1) + for i, nv := range nodesValues { + StemFromBytes(&poly[1], nv.Stem) + poly[2] = *c1c2frs[i] + poly[3] = *c1c2frs[i+1] + + ret[i].commitment = cfg.CommitToPoly(poly[:], 252) + } + + return ret +} + +func BatchSerialize(nodes []*LeafNode) ([]SerializedNode, error) { + pointsToCompress := make([]*Point, 0, 3*len(nodes)) + compressedPointsIdxs := make(map[*LeafNode]int, 3*len(nodes)) + for i := range nodes { + pointsToCompress = append(pointsToCompress, n.commitment, n.c1, n.c2) + compressedPointsIdxs[n] = len(pointsToCompress) - 3 + } + } + + compressedPoints := banderwagon.ElementsToBytes(pointsToCompress) + + // Now we that we did the heavy CPU work, we have to do the rest of `nodes` serialization + // taking the compressed points from this single list. + ret := make([]SerializedNode, 0, len(nodes)) + idx := 0 + for i := range nodes { + switch n := nodes[i].(type) { + case *InternalNode: + sn := SerializedNode{ + Node: n, + CommitmentBytes: compressedPoints[idx], + SerializedBytes: n.serializeWithCompressedChildren(compressedPointsIdxs, compressedPoints), + } + ret = append(ret, sn) + idx++ + case *LeafNode: + c1Bytes := compressedPoints[idx+1] + c2Bytes := compressedPoints[idx+2] + sn := SerializedNode{ + Node: n, + CommitmentBytes: compressedPoints[idx], + SerializedBytes: n.serializeWithCompressedCommitments(c1Bytes, c2Bytes), + } + ret = append(ret, sn) + idx += 3 + } + } + + return ret, nil +} \ No newline at end of file From 430f21f4b333ab2cecbc8b39209b978b629024aa Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 28 Mar 2023 19:43:53 -0300 Subject: [PATCH 02/15] new optimized batch insert ordered leaves api Signed-off-by: Ignacio Hagopian --- tree.go | 100 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/tree.go b/tree.go index f6009873..9e9ff75b 100644 --- a/tree.go +++ b/tree.go @@ -29,6 +29,7 @@ import ( "bytes" "errors" "fmt" + "sort" "github.com/crate-crypto/go-ipa/banderwagon" ) @@ -1428,52 +1429,81 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { poly[0].SetUint64(1) for i, nv := range nodesValues { StemFromBytes(&poly[1], nv.Stem) - poly[2] = *c1c2frs[i] - poly[3] = *c1c2frs[i+1] + poly[2] = *c1c2frs[2*i] + poly[3] = *c1c2frs[2*i+1] ret[i].commitment = cfg.CommitToPoly(poly[:], 252) } + sort.Slice(ret, func(i, j int) bool { + return bytes.Compare(ret[i].stem[:], ret[j].stem[:]) < 0 + }) + return ret } -func BatchSerialize(nodes []*LeafNode) ([]SerializedNode, error) { - pointsToCompress := make([]*Point, 0, 3*len(nodes)) - compressedPointsIdxs := make(map[*LeafNode]int, 3*len(nodes)) - for i := range nodes { - pointsToCompress = append(pointsToCompress, n.commitment, n.c1, n.c2) - compressedPointsIdxs[n] = len(pointsToCompress) - 3 - } - } +func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { + var currentBranch [31]*InternalNode - compressedPoints := banderwagon.ElementsToBytes(pointsToCompress) + // Initial state. + currentBranch[0] = New().(*InternalNode) + currentBranch[0].cowChild(leaves[0].stem[0]) + currentBranch[0].children[leaves[0].stem[0]] = &leaves[0] - // Now we that we did the heavy CPU work, we have to do the rest of `nodes` serialization - // taking the compressed points from this single list. - ret := make([]SerializedNode, 0, len(nodes)) - idx := 0 - for i := range nodes { - switch n := nodes[i].(type) { - case *InternalNode: - sn := SerializedNode{ - Node: n, - CommitmentBytes: compressedPoints[idx], - SerializedBytes: n.serializeWithCompressedChildren(compressedPointsIdxs, compressedPoints), + prevLeaf := &leaves[0] + leaves = leaves[1:] + for i := range leaves { + // currentBranch[0].Commit() + // fmt.Printf("Commitment: %v\n", currentBranch[0].Hash()) + + leaf := &leaves[i] + idx := firstDiffByteIdx(prevLeaf.stem, leaf.stem) + + if currentBranch[idx] != nil { + currentBranch[idx].cowChild(leaf.stem[idx]) + currentBranch[idx].children[leaf.stem[idx]] = leaf + leaf.setDepth(currentBranch[idx].depth + 1) + for i := idx + 1; i < len(currentBranch); i++ { + currentBranch[i] = nil } - ret = append(ret, sn) - idx++ - case *LeafNode: - c1Bytes := compressedPoints[idx+1] - c2Bytes := compressedPoints[idx+2] - sn := SerializedNode{ - Node: n, - CommitmentBytes: compressedPoints[idx], - SerializedBytes: n.serializeWithCompressedCommitments(c1Bytes, c2Bytes), + } else { + // Create the new internal node. + // Make the immediate previous internal node point to this new node. + prevNonNilIdx := 0 + for i := idx - 1; i >= 0; i-- { + if currentBranch[i] != nil { + prevNonNilIdx = i + break + } + } + for k := prevNonNilIdx + 1; k <= idx; k++ { + currentBranch[k] = newInternalNode(currentBranch[k-1].depth + 1).(*InternalNode) + currentBranch[k-1].cowChild(leaf.stem[k-1]) + currentBranch[k-1].children[leaf.stem[k-1]] = currentBranch[k] + } + + currentBranch[idx].cowChild(prevLeaf.stem[idx]) + currentBranch[idx].children[prevLeaf.stem[idx]] = prevLeaf + prevLeaf.setDepth(currentBranch[idx].depth + 1) + currentBranch[idx].cowChild(leaf.stem[idx]) + currentBranch[idx].children[leaf.stem[idx]] = leaf + + for i := idx + 1; i < len(currentBranch); i++ { + currentBranch[i] = nil } - ret = append(ret, sn) - idx += 3 } + + prevLeaf = leaf } - return ret, nil -} \ No newline at end of file + return currentBranch[0] +} + +func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { + for i := range stem1 { + if stem1[i] != stem2[i] { + return i + } + } + panic("stems are equal") +} From 9020773f9838526d6f906666e2719b8221330a23 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Wed, 29 Mar 2023 17:37:23 -0300 Subject: [PATCH 03/15] tree: use map for leaf values instead of a slice Signed-off-by: Ignacio Hagopian --- tree.go | 59 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/tree.go b/tree.go index 9e9ff75b..446014a3 100644 --- a/tree.go +++ b/tree.go @@ -172,7 +172,7 @@ type ( LeafNode struct { stem []byte - values [][]byte + values map[int][]byte commitment *Point c1, c2 *Point @@ -232,10 +232,14 @@ func NewLeafNode(stem []byte, values [][]byte) *LeafNode { StemFromBytes(&poly[1], stem) toFrMultiple([]*Fr{&poly[2], &poly[3]}, []*Point{c1, c2}) + vals := make(map[int][]byte, len(values)) + for i, v := range values { + vals[i] = v + } return &LeafNode{ // depth will be 0, but the commitment calculation // does not need it, and so it won't be free. - values: values, + values: vals, stem: stem, commitment: cfg.CommitToPoly(poly[:], NodeWidth-4), c1: c1, @@ -247,10 +251,14 @@ func NewLeafNode(stem []byte, values [][]byte) *LeafNode { // commitments. The created node's commitments are intended to be // initialized with `SetTrustedBytes` in a deserialization context. func NewLeafNodeWithNoComms(stem []byte, values [][]byte) *LeafNode { + vals := make(map[int][]byte, len(values)) + for i, v := range values { + vals[i] = v + } return &LeafNode{ // depth will be 0, but the commitment calculation // does not need it, and so it won't be free. - values: values, + values: vals, stem: stem, } } @@ -376,7 +384,11 @@ func (n *InternalNode) GetStem(stem []byte, resolver NodeResolverFn) ([][]byte, return n.GetStem(stem, resolver) case *LeafNode: if equalPaths(child.stem, stem) { - return child.values, nil + values := make([][]byte, NodeWidth) + for i, v := range child.values { + values[i] = v + } + return values, nil } return nil, nil case *InternalNode: @@ -847,7 +859,7 @@ func (n *LeafNode) updateCn(index byte, value []byte, c *Point) { // do not include it. The result should be the same, // but the computation time should be faster as one doesn't need to // compute 1 - 1 mod N. - leafToComms(old[:], n.values[index]) + leafToComms(old[:], n.values[int(index)]) leafToComms(newH[:], value) newH[0].Sub(&newH[0], &old[0]) @@ -883,7 +895,7 @@ func (n *LeafNode) updateLeaf(index byte, value []byte) { cxIndex := 2 + int(index)/(NodeWidth/2) // [1, stem, -> C1, C2 <-] n.updateC(cxIndex, frs[0], frs[1]) - n.values[index] = value + n.values[int(index)] = value } func (n *LeafNode) updateMultipleLeaves(values [][]byte) { @@ -957,7 +969,7 @@ func (n *LeafNode) Get(k []byte, _ NodeResolverFn) ([]byte, error) { return nil, nil } // value can be nil, as expected by geth - return n.values[k[31]], nil + return n.values[int(k[31])], nil } func (n *LeafNode) Hash() *Fr { @@ -1096,11 +1108,14 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte suffPoly [256]Fr // suffix-level polynomial count int ) - + valsslice := make([][]byte, 256) + for idx := range n.values { + valsslice[idx] = n.values[idx] + } if suffix >= 128 { - count = fillSuffixTreePoly(suffPoly[:], n.values[128:]) + count = fillSuffixTreePoly(suffPoly[:], valsslice[128:]) } else { - count = fillSuffixTreePoly(suffPoly[:], n.values[:128]) + count = fillSuffixTreePoly(suffPoly[:], valsslice[:128]) } // Proof of absence: case of a missing suffix tree. @@ -1134,7 +1149,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte // only happen when the leaf has never been written to // since after deletion the value would be set to zero // but still contain the leaf marker 2^128. - if n.values[suffix] == nil { + if n.values[int(suffix)] == nil { pe.Cis = append(pe.Cis, scomm, scomm) pe.Zis = append(pe.Zis, 2*suffix, 2*suffix+1) pe.Yis = append(pe.Yis, &FrZero, &FrZero) @@ -1148,7 +1163,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte // suffix tree is present and contains the key var leaves [2]Fr - leafToComms(leaves[:], n.values[suffix]) + leafToComms(leaves[:], n.values[int(suffix)]) pe.Cis = append(pe.Cis, scomm, scomm) pe.Zis = append(pe.Zis, 2*suffix, 2*suffix+1) pe.Yis = append(pe.Yis, &leaves[0], &leaves[1]) @@ -1172,7 +1187,8 @@ func (n *LeafNode) Serialize() ([]byte, error) { func (n *LeafNode) Copy() VerkleNode { l := &LeafNode{} l.stem = make([]byte, len(n.stem)) - l.values = make([][]byte, len(n.values)) + l.values = make(map[int][]byte, len(n.values)) + l.depth = n.depth copy(l.stem, n.stem) for i, v := range n.values { @@ -1223,7 +1239,11 @@ func (n *LeafNode) setDepth(d byte) { } func (n *LeafNode) Values() [][]byte { - return n.values + valsslice := make([][]byte, 256) + for idx := range n.values { + valsslice[idx] = n.values[idx] + } + return valsslice } func setBit(bitlist []byte, index int) { @@ -1395,7 +1415,7 @@ func (n *LeafNode) serializeWithCompressedCommitments(c1Bytes [32]byte, c2Bytes type BatchNewLeafNodeData struct { Stem []byte - Values [][]byte + Values map[int][]byte } func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { @@ -1414,9 +1434,14 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { var c1poly, c2poly [256]Fr - fillSuffixTreePoly(c1poly[:], nv.Values[:128]) + valsslice := make([][]byte, 256) + for idx := range nv.Values { + valsslice[idx] = nv.Values[idx] + } + + fillSuffixTreePoly(c1poly[:], valsslice[:128]) ret[i].c1 = cfg.CommitToPoly(c1poly[:], 0) - fillSuffixTreePoly(c2poly[:], nv.Values[128:]) + fillSuffixTreePoly(c2poly[:], valsslice[128:]) ret[i].c2 = cfg.CommitToPoly(c2poly[:], 0) c1c2points[2*i], c1c2points[2*i+1] = ret[i].c1, ret[i].c2 From 1590957026e4d28ef556e09e9d62d240dc5a0db7 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Thu, 30 Mar 2023 15:54:50 -0300 Subject: [PATCH 04/15] add api to merge second level partial trees Signed-off-by: Ignacio Hagopian --- tree.go | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tree.go b/tree.go index 446014a3..a3c12f65 100644 --- a/tree.go +++ b/tree.go @@ -1478,9 +1478,6 @@ func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { prevLeaf := &leaves[0] leaves = leaves[1:] for i := range leaves { - // currentBranch[0].Commit() - // fmt.Printf("Commitment: %v\n", currentBranch[0].Hash()) - leaf := &leaves[i] idx := firstDiffByteIdx(prevLeaf.stem, leaf.stem) @@ -1532,3 +1529,33 @@ func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { } panic("stems are equal") } + +func MergeLevelTwoPartitions(roots []*InternalNode) *InternalNode { + firstLevelIdx := 0 + for i := 0; i < NodeWidth; i++ { + if _, ok := roots[0].children[i].(*InternalNode); !ok { + continue + } + firstLevelIdx = i + break + } + secondLevelRoot := newInternalNode(1).(*InternalNode) + for i := 0; i < NodeWidth; i++ { + for j := range roots { + proot := roots[j].children[firstLevelIdx].(*InternalNode) + in, ok := proot.children[i].(*InternalNode) + if !ok { + continue + } + secondLevelRoot.cowChild(byte(i)) + secondLevelRoot.children[i] = in + break + } + } + + root := newInternalNode(0).(*InternalNode) + root.cowChild(byte(firstLevelIdx)) + root.children[firstLevelIdx] = secondLevelRoot + + return root +} From 1507f84d0d16ce5322fe80d9f4d1c66c0e8198e7 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Mon, 3 Apr 2023 15:19:01 -0300 Subject: [PATCH 05/15] fix setChild bug Signed-off-by: Ignacio Hagopian --- tree.go | 150 +------------------------------------------------------- 1 file changed, 1 insertion(+), 149 deletions(-) diff --git a/tree.go b/tree.go index a3c12f65..b17f7dc8 100644 --- a/tree.go +++ b/tree.go @@ -29,7 +29,6 @@ import ( "bytes" "errors" "fmt" - "sort" "github.com/crate-crypto/go-ipa/banderwagon" ) @@ -268,7 +267,7 @@ func (n *InternalNode) Children() []VerkleNode { } func (n *InternalNode) SetChild(i int, c VerkleNode) error { - if i >= NodeWidth-1 { + if i >= NodeWidth { return errors.New("child index higher than node width") } n.children[i] = c @@ -1412,150 +1411,3 @@ func (n *LeafNode) serializeWithCompressedCommitments(c1Bytes [32]byte, c2Bytes return result } - -type BatchNewLeafNodeData struct { - Stem []byte - Values map[int][]byte -} - -func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { - cfg := GetConfig() - - ret := make([]LeafNode, len(nodesValues)) - c1c2points := make([]*Point, 2*len(nodesValues)) - c1c2frs := make([]*Fr, 2*len(nodesValues)) - for i, nv := range nodesValues { - ret[i] = LeafNode{ - values: nv.Values, - stem: nv.Stem, - c1: Generator(), - c2: Generator(), - } - - var c1poly, c2poly [256]Fr - - valsslice := make([][]byte, 256) - for idx := range nv.Values { - valsslice[idx] = nv.Values[idx] - } - - fillSuffixTreePoly(c1poly[:], valsslice[:128]) - ret[i].c1 = cfg.CommitToPoly(c1poly[:], 0) - fillSuffixTreePoly(c2poly[:], valsslice[128:]) - ret[i].c2 = cfg.CommitToPoly(c2poly[:], 0) - - c1c2points[2*i], c1c2points[2*i+1] = ret[i].c1, ret[i].c2 - c1c2frs[2*i], c1c2frs[2*i+1] = new(Fr), new(Fr) - } - - toFrMultiple(c1c2frs, c1c2points) - - var poly [256]Fr - poly[0].SetUint64(1) - for i, nv := range nodesValues { - StemFromBytes(&poly[1], nv.Stem) - poly[2] = *c1c2frs[2*i] - poly[3] = *c1c2frs[2*i+1] - - ret[i].commitment = cfg.CommitToPoly(poly[:], 252) - } - - sort.Slice(ret, func(i, j int) bool { - return bytes.Compare(ret[i].stem[:], ret[j].stem[:]) < 0 - }) - - return ret -} - -func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { - var currentBranch [31]*InternalNode - - // Initial state. - currentBranch[0] = New().(*InternalNode) - currentBranch[0].cowChild(leaves[0].stem[0]) - currentBranch[0].children[leaves[0].stem[0]] = &leaves[0] - - prevLeaf := &leaves[0] - leaves = leaves[1:] - for i := range leaves { - leaf := &leaves[i] - idx := firstDiffByteIdx(prevLeaf.stem, leaf.stem) - - if currentBranch[idx] != nil { - currentBranch[idx].cowChild(leaf.stem[idx]) - currentBranch[idx].children[leaf.stem[idx]] = leaf - leaf.setDepth(currentBranch[idx].depth + 1) - for i := idx + 1; i < len(currentBranch); i++ { - currentBranch[i] = nil - } - } else { - // Create the new internal node. - // Make the immediate previous internal node point to this new node. - prevNonNilIdx := 0 - for i := idx - 1; i >= 0; i-- { - if currentBranch[i] != nil { - prevNonNilIdx = i - break - } - } - for k := prevNonNilIdx + 1; k <= idx; k++ { - currentBranch[k] = newInternalNode(currentBranch[k-1].depth + 1).(*InternalNode) - currentBranch[k-1].cowChild(leaf.stem[k-1]) - currentBranch[k-1].children[leaf.stem[k-1]] = currentBranch[k] - } - - currentBranch[idx].cowChild(prevLeaf.stem[idx]) - currentBranch[idx].children[prevLeaf.stem[idx]] = prevLeaf - prevLeaf.setDepth(currentBranch[idx].depth + 1) - currentBranch[idx].cowChild(leaf.stem[idx]) - currentBranch[idx].children[leaf.stem[idx]] = leaf - - for i := idx + 1; i < len(currentBranch); i++ { - currentBranch[i] = nil - } - } - - prevLeaf = leaf - } - - return currentBranch[0] -} - -func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { - for i := range stem1 { - if stem1[i] != stem2[i] { - return i - } - } - panic("stems are equal") -} - -func MergeLevelTwoPartitions(roots []*InternalNode) *InternalNode { - firstLevelIdx := 0 - for i := 0; i < NodeWidth; i++ { - if _, ok := roots[0].children[i].(*InternalNode); !ok { - continue - } - firstLevelIdx = i - break - } - secondLevelRoot := newInternalNode(1).(*InternalNode) - for i := 0; i < NodeWidth; i++ { - for j := range roots { - proot := roots[j].children[firstLevelIdx].(*InternalNode) - in, ok := proot.children[i].(*InternalNode) - if !ok { - continue - } - secondLevelRoot.cowChild(byte(i)) - secondLevelRoot.children[i] = in - break - } - } - - root := newInternalNode(0).(*InternalNode) - root.cowChild(byte(firstLevelIdx)) - root.children[firstLevelIdx] = secondLevelRoot - - return root -} From a853c3b676efd301e68be91f8c97737d14394853 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Mon, 3 Apr 2023 15:19:12 -0300 Subject: [PATCH 06/15] move migration helpers to separate file Signed-off-by: Ignacio Hagopian --- migration.go | 193 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 migration.go diff --git a/migration.go b/migration.go new file mode 100644 index 00000000..02a1541c --- /dev/null +++ b/migration.go @@ -0,0 +1,193 @@ +package verkle + +import ( + "bytes" + "fmt" + "sort" +) + +type BatchNewLeafNodeData struct { + Stem []byte + Values map[int][]byte +} + +func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { + cfg := GetConfig() + + ret := make([]LeafNode, len(nodesValues)) + c1c2points := make([]*Point, 2*len(nodesValues)) + c1c2frs := make([]*Fr, 2*len(nodesValues)) + for i, nv := range nodesValues { + ret[i] = LeafNode{ + values: nv.Values, + stem: nv.Stem, + c1: Generator(), + c2: Generator(), + } + + var c1poly, c2poly [256]Fr + + valsslice := make([][]byte, 256) + for idx := range nv.Values { + valsslice[idx] = nv.Values[idx] + } + + fillSuffixTreePoly(c1poly[:], valsslice[:128]) + ret[i].c1 = cfg.CommitToPoly(c1poly[:], 0) + fillSuffixTreePoly(c2poly[:], valsslice[128:]) + ret[i].c2 = cfg.CommitToPoly(c2poly[:], 0) + + c1c2points[2*i], c1c2points[2*i+1] = ret[i].c1, ret[i].c2 + c1c2frs[2*i], c1c2frs[2*i+1] = new(Fr), new(Fr) + } + + toFrMultiple(c1c2frs, c1c2points) + + var poly [256]Fr + poly[0].SetUint64(1) + for i, nv := range nodesValues { + StemFromBytes(&poly[1], nv.Stem) + poly[2] = *c1c2frs[2*i] + poly[3] = *c1c2frs[2*i+1] + + ret[i].commitment = cfg.CommitToPoly(poly[:], 252) + } + + sort.Slice(ret, func(i, j int) bool { + return bytes.Compare(ret[i].stem[:], ret[j].stem[:]) < 0 + }) + + return ret +} + +func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { + var currentBranch [31]*InternalNode + + // Initial state. + currentBranch[0] = New().(*InternalNode) + currentBranch[0].cowChild(leaves[0].stem[0]) + currentBranch[0].children[leaves[0].stem[0]] = &leaves[0] + + prevLeaf := &leaves[0] + leaves = leaves[1:] + for i := range leaves { + leaf := &leaves[i] + idx := firstDiffByteIdx(prevLeaf.stem, leaf.stem) + + if currentBranch[idx] != nil { + currentBranch[idx].cowChild(leaf.stem[idx]) + currentBranch[idx].children[leaf.stem[idx]] = leaf + leaf.setDepth(currentBranch[idx].depth + 1) + for i := idx + 1; i < len(currentBranch); i++ { + currentBranch[i] = nil + } + } else { + // Create the new internal node. + // Make the immediate previous internal node point to this new node. + prevNonNilIdx := 0 + for i := idx - 1; i >= 0; i-- { + if currentBranch[i] != nil { + prevNonNilIdx = i + break + } + } + for k := prevNonNilIdx + 1; k <= idx; k++ { + currentBranch[k] = newInternalNode(currentBranch[k-1].depth + 1).(*InternalNode) + currentBranch[k-1].cowChild(leaf.stem[k-1]) + currentBranch[k-1].children[leaf.stem[k-1]] = currentBranch[k] + } + + currentBranch[idx].cowChild(prevLeaf.stem[idx]) + currentBranch[idx].children[prevLeaf.stem[idx]] = prevLeaf + prevLeaf.setDepth(currentBranch[idx].depth + 1) + currentBranch[idx].cowChild(leaf.stem[idx]) + currentBranch[idx].children[leaf.stem[idx]] = leaf + + for i := idx + 1; i < len(currentBranch); i++ { + currentBranch[i] = nil + } + } + + prevLeaf = leaf + } + + return currentBranch[0] +} + +func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { + for i := range stem1 { + if stem1[i] != stem2[i] { + return i + } + } + panic("stems are equal") +} + +func MergeLevelTwoPartitions(roots []*InternalNode) *InternalNode { + firstLevelIdx := 0 + for i := 0; i < NodeWidth; i++ { + if _, ok := roots[0].children[i].(*InternalNode); !ok { + continue + } + firstLevelIdx = i + break + } + secondLevelRoot := newInternalNode(1).(*InternalNode) + for i := 0; i < NodeWidth; i++ { + for j := range roots { + proot := roots[j].children[firstLevelIdx].(*InternalNode) + in, ok := proot.children[i].(*InternalNode) + if !ok { + continue + } + secondLevelRoot.cowChild(byte(i)) + secondLevelRoot.children[i] = in + break + } + } + + root := newInternalNode(0).(*InternalNode) + root.cowChild(byte(firstLevelIdx)) + root.children[firstLevelIdx] = secondLevelRoot + + return root +} + +func GetInternalNodeCommitment(node *InternalNode, partialStem []byte) (*Point, error) { + for i := range partialStem { + var ok bool + node, ok = node.children[partialStem[i]].(*InternalNode) + if !ok { + return nil, fmt.Errorf("partial stem is not a prefix of the tree") + } + } + + return node.commitment, nil +} + +func BuildFirstTwoLayers(commitments [256][256][32]byte) *InternalNode { + var secondLevelInternalNodes [256]*InternalNode + for stemFirstByte := range commitments { + for stemSecondByte := range commitments[stemFirstByte] { + if commitments[stemFirstByte][stemSecondByte] == [32]byte{} { + continue + } + if secondLevelInternalNodes[stemFirstByte] == nil { + secondLevelInternalNodes[stemFirstByte] = newInternalNode(1).(*InternalNode) + } + hashedNode := HashedNode{commitment: commitments[stemFirstByte][stemSecondByte][:]} + secondLevelInternalNodes[stemFirstByte].cowChild(byte(stemSecondByte)) + secondLevelInternalNodes[stemFirstByte].SetChild(stemSecondByte, &hashedNode) + } + } + root := newInternalNode(0).(*InternalNode) + for i, node := range secondLevelInternalNodes { + if node == nil { + continue + } + root.cowChild(byte(i)) + root.SetChild(i, node) + } + + return root +} From 5b0b63ce42d9c65544aff4fcd5822bb3a3f92ed9 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 4 Apr 2023 11:18:24 -0300 Subject: [PATCH 07/15] add conversion comments Signed-off-by: Ignacio Hagopian --- migration.go | 84 +++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/migration.go b/migration.go index 02a1541c..ce16dd0a 100644 --- a/migration.go +++ b/migration.go @@ -6,11 +6,14 @@ import ( "sort" ) +// BatchNewLeafNodeData is a struct that contains the data needed to create a new leaf node. type BatchNewLeafNodeData struct { Stem []byte Values map[int][]byte } +// BatchNewLeafNode creates a new leaf node from the given data. It optimizes LeafNode creation +// by batching expensive cryptography operations. It returns the LeafNodes sorted by stem. func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { cfg := GetConfig() @@ -60,30 +63,43 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { return ret } +// BatchInsertOrderedLeaves creates a tree under from an ordered and deduplicated list of leaves. func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { - var currentBranch [31]*InternalNode + // currentBranch is a representaion of the current branch we're in. + // The length of the branch is at most StemSize, and it might only + // have non-nil values in the first N levels. + var currentBranch [StemSize]*InternalNode - // Initial state. + // Initial state is a branch with only a root node at the top, pointing to + // the first leaf. currentBranch[0] = New().(*InternalNode) currentBranch[0].cowChild(leaves[0].stem[0]) currentBranch[0].children[leaves[0].stem[0]] = &leaves[0] prevLeaf := &leaves[0] leaves = leaves[1:] + // The idea is that we compare the newLeaf with the previousLeaf, and + // depending on how their stems differ, we adjust our currentBranch structure. for i := range leaves { - leaf := &leaves[i] - idx := firstDiffByteIdx(prevLeaf.stem, leaf.stem) + newLeaf := &leaves[i] + // We get the first index in their stems that is different. + idx := firstDiffByteIdx(prevLeaf.stem, newLeaf.stem) + + // If the currentBranch has a node at that index, we simply set the children + // to the newLeaf. if currentBranch[idx] != nil { - currentBranch[idx].cowChild(leaf.stem[idx]) - currentBranch[idx].children[leaf.stem[idx]] = leaf - leaf.setDepth(currentBranch[idx].depth + 1) + currentBranch[idx].cowChild(newLeaf.stem[idx]) + currentBranch[idx].children[newLeaf.stem[idx]] = newLeaf + newLeaf.setDepth(currentBranch[idx].depth + 1) for i := idx + 1; i < len(currentBranch); i++ { currentBranch[i] = nil } } else { - // Create the new internal node. - // Make the immediate previous internal node point to this new node. + // In this case there's no InternalNode in the current branch at the index. + // We need to "fill the gap" between the previous non-nil internal node up to + // the idx with new internal nodes. Then we set the last created internal node + // to the previous and new leaf. prevNonNilIdx := 0 for i := idx - 1; i >= 0; i-- { if currentBranch[i] != nil { @@ -93,27 +109,29 @@ func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { } for k := prevNonNilIdx + 1; k <= idx; k++ { currentBranch[k] = newInternalNode(currentBranch[k-1].depth + 1).(*InternalNode) - currentBranch[k-1].cowChild(leaf.stem[k-1]) - currentBranch[k-1].children[leaf.stem[k-1]] = currentBranch[k] + currentBranch[k-1].cowChild(newLeaf.stem[k-1]) + currentBranch[k-1].children[newLeaf.stem[k-1]] = currentBranch[k] } currentBranch[idx].cowChild(prevLeaf.stem[idx]) currentBranch[idx].children[prevLeaf.stem[idx]] = prevLeaf prevLeaf.setDepth(currentBranch[idx].depth + 1) - currentBranch[idx].cowChild(leaf.stem[idx]) - currentBranch[idx].children[leaf.stem[idx]] = leaf + currentBranch[idx].cowChild(newLeaf.stem[idx]) + currentBranch[idx].children[newLeaf.stem[idx]] = newLeaf for i := idx + 1; i < len(currentBranch); i++ { currentBranch[i] = nil } } - prevLeaf = leaf + prevLeaf = newLeaf } return currentBranch[0] } +// firstDiffByteIdx will return the first index in which the two stems differ. +// Both stems *must* be different. func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { for i := range stem1 { if stem1[i] != stem2[i] { @@ -123,36 +141,10 @@ func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { panic("stems are equal") } -func MergeLevelTwoPartitions(roots []*InternalNode) *InternalNode { - firstLevelIdx := 0 - for i := 0; i < NodeWidth; i++ { - if _, ok := roots[0].children[i].(*InternalNode); !ok { - continue - } - firstLevelIdx = i - break - } - secondLevelRoot := newInternalNode(1).(*InternalNode) - for i := 0; i < NodeWidth; i++ { - for j := range roots { - proot := roots[j].children[firstLevelIdx].(*InternalNode) - in, ok := proot.children[i].(*InternalNode) - if !ok { - continue - } - secondLevelRoot.cowChild(byte(i)) - secondLevelRoot.children[i] = in - break - } - } - - root := newInternalNode(0).(*InternalNode) - root.cowChild(byte(firstLevelIdx)) - root.children[firstLevelIdx] = secondLevelRoot - - return root -} - +// GetInternalNodeCommitment returns the commitment of the internal node at +// the partialStem. e.g: if partialStem is [a, b] it will walk to the a-th +// children of the node, and then to the b-th children of that node, returning +// its commitment.. func GetInternalNodeCommitment(node *InternalNode, partialStem []byte) (*Point, error) { for i := range partialStem { var ok bool @@ -165,6 +157,10 @@ func GetInternalNodeCommitment(node *InternalNode, partialStem []byte) (*Point, return node.commitment, nil } +// BuildFirstTwoLayers builds the first two layers of the tree from all the precalculated +// commitments of the children of the second level. This method is generally used if tree +// construction was done in partitions, and you want to glue them together without having +// the whole tree in memory. func BuildFirstTwoLayers(commitments [256][256][32]byte) *InternalNode { var secondLevelInternalNodes [256]*InternalNode for stemFirstByte := range commitments { From b7b260f6275d8f988cab3c7ddf39800bd96b9771 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Tue, 4 Apr 2023 11:18:44 -0300 Subject: [PATCH 08/15] rename file Signed-off-by: Ignacio Hagopian --- migration.go => conversion.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename migration.go => conversion.go (100%) diff --git a/migration.go b/conversion.go similarity index 100% rename from migration.go rename to conversion.go From da4656dfeefb7136ce566b732d25b2f4686ddba0 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Wed, 5 Apr 2023 09:52:22 -0300 Subject: [PATCH 09/15] tree: use byte as leaf values key Signed-off-by: Ignacio Hagopian --- conversion.go | 2 +- tree.go | 33 ++++++++++++++++++--------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/conversion.go b/conversion.go index ce16dd0a..30bcb8b1 100644 --- a/conversion.go +++ b/conversion.go @@ -9,7 +9,7 @@ import ( // BatchNewLeafNodeData is a struct that contains the data needed to create a new leaf node. type BatchNewLeafNodeData struct { Stem []byte - Values map[int][]byte + Values map[byte][]byte } // BatchNewLeafNode creates a new leaf node from the given data. It optimizes LeafNode creation diff --git a/tree.go b/tree.go index b17f7dc8..565ac39e 100644 --- a/tree.go +++ b/tree.go @@ -171,7 +171,7 @@ type ( LeafNode struct { stem []byte - values map[int][]byte + values map[byte][]byte commitment *Point c1, c2 *Point @@ -231,9 +231,9 @@ func NewLeafNode(stem []byte, values [][]byte) *LeafNode { StemFromBytes(&poly[1], stem) toFrMultiple([]*Fr{&poly[2], &poly[3]}, []*Point{c1, c2}) - vals := make(map[int][]byte, len(values)) + vals := make(map[byte][]byte, len(values)) for i, v := range values { - vals[i] = v + vals[byte(i)] = v } return &LeafNode{ // depth will be 0, but the commitment calculation @@ -250,9 +250,9 @@ func NewLeafNode(stem []byte, values [][]byte) *LeafNode { // commitments. The created node's commitments are intended to be // initialized with `SetTrustedBytes` in a deserialization context. func NewLeafNodeWithNoComms(stem []byte, values [][]byte) *LeafNode { - vals := make(map[int][]byte, len(values)) + vals := make(map[byte][]byte, len(values)) for i, v := range values { - vals[i] = v + vals[byte(i)] = v } return &LeafNode{ // depth will be 0, but the commitment calculation @@ -858,7 +858,7 @@ func (n *LeafNode) updateCn(index byte, value []byte, c *Point) { // do not include it. The result should be the same, // but the computation time should be faster as one doesn't need to // compute 1 - 1 mod N. - leafToComms(old[:], n.values[int(index)]) + leafToComms(old[:], n.values[index]) leafToComms(newH[:], value) newH[0].Sub(&newH[0], &old[0]) @@ -894,7 +894,7 @@ func (n *LeafNode) updateLeaf(index byte, value []byte) { cxIndex := 2 + int(index)/(NodeWidth/2) // [1, stem, -> C1, C2 <-] n.updateC(cxIndex, frs[0], frs[1]) - n.values[int(index)] = value + n.values[index] = value } func (n *LeafNode) updateMultipleLeaves(values [][]byte) { @@ -905,7 +905,7 @@ func (n *LeafNode) updateMultipleLeaves(values [][]byte) { // commitment. We copy the original point in oldC1 and oldC2, so we can batch their Fr transformation // after this loop. for i, v := range values { - if len(v) != 0 && !bytes.Equal(v, n.values[i]) { + if len(v) != 0 && !bytes.Equal(v, n.values[byte(i)]) { if i < NodeWidth/2 { // First time we touch C1? Save the original point for later. if oldC1 == nil { @@ -923,7 +923,7 @@ func (n *LeafNode) updateMultipleLeaves(values [][]byte) { // We update C2 directly in `n`. We have our original copy in oldC2. n.updateCn(byte(i), v, n.c2) } - n.values[i] = v + n.values[byte(i)] = v } } @@ -968,7 +968,7 @@ func (n *LeafNode) Get(k []byte, _ NodeResolverFn) ([]byte, error) { return nil, nil } // value can be nil, as expected by geth - return n.values[int(k[31])], nil + return n.values[k[31]], nil } func (n *LeafNode) Hash() *Fr { @@ -1148,7 +1148,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte // only happen when the leaf has never been written to // since after deletion the value would be set to zero // but still contain the leaf marker 2^128. - if n.values[int(suffix)] == nil { + if n.values[suffix] == nil { pe.Cis = append(pe.Cis, scomm, scomm) pe.Zis = append(pe.Zis, 2*suffix, 2*suffix+1) pe.Yis = append(pe.Yis, &FrZero, &FrZero) @@ -1162,7 +1162,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte // suffix tree is present and contains the key var leaves [2]Fr - leafToComms(leaves[:], n.values[int(suffix)]) + leafToComms(leaves[:], n.values[suffix]) pe.Cis = append(pe.Cis, scomm, scomm) pe.Zis = append(pe.Zis, 2*suffix, 2*suffix+1) pe.Yis = append(pe.Yis, &leaves[0], &leaves[1]) @@ -1186,7 +1186,7 @@ func (n *LeafNode) Serialize() ([]byte, error) { func (n *LeafNode) Copy() VerkleNode { l := &LeafNode{} l.stem = make([]byte, len(n.stem)) - l.values = make(map[int][]byte, len(n.values)) + l.values = make(map[byte][]byte, len(n.values)) l.depth = n.depth copy(l.stem, n.stem) @@ -1218,7 +1218,10 @@ func (n *LeafNode) Key(i int) []byte { } func (n *LeafNode) Value(i int) []byte { - return n.values[i] + if i >= NodeWidth { + panic("leaf node index out of range") + } + return n.values[byte(i)] } func (n *LeafNode) toDot(parent, path string) string { @@ -1391,7 +1394,7 @@ func (n *LeafNode) serializeWithCompressedCommitments(c1Bytes [32]byte, c2Bytes var bitlist [bitlistSize]byte for i, v := range n.values { if v != nil { - setBit(bitlist[:], i) + setBit(bitlist[:], int(i)) children = append(children, v...) if padding := emptyValue[:LeafValueSize-len(v)]; len(padding) != 0 { children = append(children, padding...) From 87a1680b29dc8b2e27a9779dcdaa2cdcdbacdf1d Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Wed, 5 Apr 2023 12:12:24 -0300 Subject: [PATCH 10/15] tree: fix & use constant Signed-off-by: Ignacio Hagopian --- tree.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tree.go b/tree.go index 565ac39e..7b5d5d26 100644 --- a/tree.go +++ b/tree.go @@ -233,7 +233,9 @@ func NewLeafNode(stem []byte, values [][]byte) *LeafNode { vals := make(map[byte][]byte, len(values)) for i, v := range values { - vals[byte(i)] = v + if v != nil { + vals[byte(i)] = v + } } return &LeafNode{ // depth will be 0, but the commitment calculation @@ -252,7 +254,9 @@ func NewLeafNode(stem []byte, values [][]byte) *LeafNode { func NewLeafNodeWithNoComms(stem []byte, values [][]byte) *LeafNode { vals := make(map[byte][]byte, len(values)) for i, v := range values { - vals[byte(i)] = v + if v != nil { + vals[byte(i)] = v + } } return &LeafNode{ // depth will be 0, but the commitment calculation @@ -795,7 +799,7 @@ func (n *InternalNode) setDepth(d byte) { func MergeTrees(subroots []*InternalNode) VerkleNode { root := New().(*InternalNode) for _, subroot := range subroots { - for i := 0; i < 256; i++ { + for i := 0; i < NodeWidth; i++ { if _, ok := subroot.children[i].(Empty); ok { continue } @@ -849,7 +853,7 @@ func (n *LeafNode) updateCn(index byte, value []byte, c *Point) { var ( old, newH [2]Fr diff Point - poly [256]Fr + poly [NodeWidth]Fr ) // Optimization idea: @@ -1033,8 +1037,8 @@ func leafToComms(poly []Fr, val []byte) { func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte) { var ( - poly [256]Fr // top-level polynomial - pe = &ProofElements{ + poly [NodeWidth]Fr // top-level polynomial + pe = &ProofElements{ Cis: []*Point{n.commitment, n.commitment}, Zis: []byte{0, 1}, Yis: []*Fr{&poly[0], &poly[1]}, // Should be 0 @@ -1104,17 +1108,17 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte var ( suffix = key[31] - suffPoly [256]Fr // suffix-level polynomial + suffPoly [NodeWidth]Fr // suffix-level polynomial count int ) - valsslice := make([][]byte, 256) + vals := make([][]byte, NodeWidth) for idx := range n.values { - valsslice[idx] = n.values[idx] + vals[idx] = n.values[idx] } if suffix >= 128 { - count = fillSuffixTreePoly(suffPoly[:], valsslice[128:]) + count = fillSuffixTreePoly(suffPoly[:], vals[128:]) } else { - count = fillSuffixTreePoly(suffPoly[:], valsslice[:128]) + count = fillSuffixTreePoly(suffPoly[:], vals[:128]) } // Proof of absence: case of a missing suffix tree. @@ -1241,11 +1245,11 @@ func (n *LeafNode) setDepth(d byte) { } func (n *LeafNode) Values() [][]byte { - valsslice := make([][]byte, 256) + vals := make([][]byte, NodeWidth) for idx := range n.values { - valsslice[idx] = n.values[idx] + vals[idx] = n.values[idx] } - return valsslice + return vals } func setBit(bitlist []byte, index int) { From 28c42699a0b912b3c940a06efae65ec25eb0d59f Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Wed, 5 Apr 2023 12:48:11 -0300 Subject: [PATCH 11/15] tree: bugfix Signed-off-by: Ignacio Hagopian --- tree.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tree.go b/tree.go index 7b5d5d26..0a5ad15b 100644 --- a/tree.go +++ b/tree.go @@ -1396,7 +1396,11 @@ func (n *LeafNode) serializeWithCompressedCommitments(c1Bytes [32]byte, c2Bytes // Create bitlist and store in children LeafValueSize (padded) values. children := make([]byte, 0, NodeWidth*LeafValueSize) var bitlist [bitlistSize]byte - for i, v := range n.values { + vals := make([][]byte, NodeWidth) + for i := range n.values { + vals[i] = n.values[i] + } + for i, v := range vals { if v != nil { setBit(bitlist[:], int(i)) children = append(children, v...) From dd4cc11d1c063f99b066f7e3c63a63c564c2b91d Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Thu, 6 Apr 2023 13:11:23 -0300 Subject: [PATCH 12/15] lint fixes Signed-off-by: Ignacio Hagopian --- conversion.go | 2 +- tree.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conversion.go b/conversion.go index 30bcb8b1..9839619e 100644 --- a/conversion.go +++ b/conversion.go @@ -65,7 +65,7 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { // BatchInsertOrderedLeaves creates a tree under from an ordered and deduplicated list of leaves. func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { - // currentBranch is a representaion of the current branch we're in. + // currentBranch is a representation of the current branch we're in. // The length of the branch is at most StemSize, and it might only // have non-nil values in the first N levels. var currentBranch [StemSize]*InternalNode diff --git a/tree.go b/tree.go index 0a5ad15b..b3c2c845 100644 --- a/tree.go +++ b/tree.go @@ -1402,7 +1402,7 @@ func (n *LeafNode) serializeWithCompressedCommitments(c1Bytes [32]byte, c2Bytes } for i, v := range vals { if v != nil { - setBit(bitlist[:], int(i)) + setBit(bitlist[:], i) children = append(children, v...) if padding := emptyValue[:LeafValueSize-len(v)]; len(padding) != 0 { children = append(children, padding...) From 44e8d936bb33e0a6887bade874838cc6dd587ee2 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Mon, 10 Apr 2023 13:45:54 -0300 Subject: [PATCH 13/15] lints Signed-off-by: Ignacio Hagopian --- conversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion.go b/conversion.go index 9839619e..45c81108 100644 --- a/conversion.go +++ b/conversion.go @@ -57,7 +57,7 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { } sort.Slice(ret, func(i, j int) bool { - return bytes.Compare(ret[i].stem[:], ret[j].stem[:]) < 0 + return bytes.Compare(ret[i].stem, ret[j].stem) < 0 }) return ret From 4dcbb6a81c9e5a5bf4409c134b1ed64e81e6cb58 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Thu, 13 Apr 2023 12:01:57 -0300 Subject: [PATCH 14/15] conversion: make adjustements to assume at least two leves in partial tree building Signed-off-by: Ignacio Hagopian --- conversion.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/conversion.go b/conversion.go index 45c81108..4d4ed683 100644 --- a/conversion.go +++ b/conversion.go @@ -2,7 +2,6 @@ package verkle import ( "bytes" - "fmt" "sort" ) @@ -64,6 +63,8 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { } // BatchInsertOrderedLeaves creates a tree under from an ordered and deduplicated list of leaves. +// There's weak assumption that each subtree of the first stem-byte has more than 1 leaf node. +// If the whole tree has more than 2000 leaves the chance of that not being true is 0.033~=0. func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { // currentBranch is a representation of the current branch we're in. // The length of the branch is at most StemSize, and it might only @@ -72,9 +73,13 @@ func BatchInsertOrderedLeaves(leaves []LeafNode) *InternalNode { // Initial state is a branch with only a root node at the top, pointing to // the first leaf. + currentBranch[1] = newInternalNode(1).(*InternalNode) + currentBranch[1].cowChild(leaves[0].stem[1]) + currentBranch[1].children[leaves[0].stem[1]] = &leaves[0] + currentBranch[0] = New().(*InternalNode) currentBranch[0].cowChild(leaves[0].stem[0]) - currentBranch[0].children[leaves[0].stem[0]] = &leaves[0] + currentBranch[0].children[leaves[0].stem[0]] = currentBranch[1] prevLeaf := &leaves[0] leaves = leaves[1:] @@ -145,16 +150,16 @@ func firstDiffByteIdx(stem1 []byte, stem2 []byte) int { // the partialStem. e.g: if partialStem is [a, b] it will walk to the a-th // children of the node, and then to the b-th children of that node, returning // its commitment.. -func GetInternalNodeCommitment(node *InternalNode, partialStem []byte) (*Point, error) { +func GetInternalNodeCommitment(node *InternalNode, partialStem []byte) *Point { for i := range partialStem { - var ok bool - node, ok = node.children[partialStem[i]].(*InternalNode) + nextNode, ok := node.children[partialStem[i]].(*InternalNode) if !ok { - return nil, fmt.Errorf("partial stem is not a prefix of the tree") + return node.children[partialStem[i]].(*LeafNode).commitment } + node = nextNode } - return node.commitment, nil + return node.commitment } // BuildFirstTwoLayers builds the first two layers of the tree from all the precalculated @@ -163,6 +168,7 @@ func GetInternalNodeCommitment(node *InternalNode, partialStem []byte) (*Point, // the whole tree in memory. func BuildFirstTwoLayers(commitments [256][256][32]byte) *InternalNode { var secondLevelInternalNodes [256]*InternalNode + for stemFirstByte := range commitments { for stemSecondByte := range commitments[stemFirstByte] { if commitments[stemFirstByte][stemSecondByte] == [32]byte{} { @@ -176,6 +182,7 @@ func BuildFirstTwoLayers(commitments [256][256][32]byte) *InternalNode { secondLevelInternalNodes[stemFirstByte].SetChild(stemSecondByte, &hashedNode) } } + root := newInternalNode(0).(*InternalNode) for i, node := range secondLevelInternalNodes { if node == nil { From 16043577ef1d6ed237830b57cb18608675e75bd5 Mon Sep 17 00:00:00 2001 From: Ignacio Hagopian Date: Thu, 13 Apr 2023 13:50:55 -0300 Subject: [PATCH 15/15] conversion: use constants Signed-off-by: Ignacio Hagopian --- conversion.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conversion.go b/conversion.go index 4d4ed683..05d6f65c 100644 --- a/conversion.go +++ b/conversion.go @@ -27,16 +27,16 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { c2: Generator(), } - var c1poly, c2poly [256]Fr + var c1poly, c2poly [NodeWidth]Fr - valsslice := make([][]byte, 256) + valsslice := make([][]byte, NodeWidth) for idx := range nv.Values { valsslice[idx] = nv.Values[idx] } - fillSuffixTreePoly(c1poly[:], valsslice[:128]) + fillSuffixTreePoly(c1poly[:], valsslice[:NodeWidth/2]) ret[i].c1 = cfg.CommitToPoly(c1poly[:], 0) - fillSuffixTreePoly(c2poly[:], valsslice[128:]) + fillSuffixTreePoly(c2poly[:], valsslice[NodeWidth/2:]) ret[i].c2 = cfg.CommitToPoly(c2poly[:], 0) c1c2points[2*i], c1c2points[2*i+1] = ret[i].c1, ret[i].c2 @@ -45,7 +45,7 @@ func BatchNewLeafNode(nodesValues []BatchNewLeafNodeData) []LeafNode { toFrMultiple(c1c2frs, c1c2points) - var poly [256]Fr + var poly [NodeWidth]Fr poly[0].SetUint64(1) for i, nv := range nodesValues { StemFromBytes(&poly[1], nv.Stem) @@ -166,8 +166,8 @@ func GetInternalNodeCommitment(node *InternalNode, partialStem []byte) *Point { // commitments of the children of the second level. This method is generally used if tree // construction was done in partitions, and you want to glue them together without having // the whole tree in memory. -func BuildFirstTwoLayers(commitments [256][256][32]byte) *InternalNode { - var secondLevelInternalNodes [256]*InternalNode +func BuildFirstTwoLayers(commitments [NodeWidth][NodeWidth][32]byte) *InternalNode { + var secondLevelInternalNodes [NodeWidth]*InternalNode for stemFirstByte := range commitments { for stemSecondByte := range commitments[stemFirstByte] {