From 875602232bc9db6ebefacefa37db013db550c5d7 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 2 Dec 2021 16:21:04 +0100 Subject: [PATCH 01/14] add stateless node --- stateless.go | 459 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 stateless.go diff --git a/stateless.go b/stateless.go new file mode 100644 index 00000000..ea1e8b4d --- /dev/null +++ b/stateless.go @@ -0,0 +1,459 @@ +// This is free and unencumbered software released into the public domain. +// +// Anyone is free to copy, modify, publish, use, compile, sell, or +// distribute this software, either in source code form or as a compiled +// binary, for any purpose, commercial or non-commercial, and by any +// means. +// +// In jurisdictions that recognize copyright laws, the author or authors +// of this software dedicate any and all copyright interest in the +// software to the public domain. We make this dedication for the benefit +// of the public at large and to the detriment of our heirs and +// successors. We intend this dedication to be an overt act of +// relinquishment in perpetuity of all present and future rights to this +// software under copyright law. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// +// For more information, please refer to + +package verkle + +import ( + "errors" + "fmt" +) + +type ( + InternalNodeStateless struct { + // List of available child nodes of this internal node. + children map[byte]VerkleNode + + // node depth in the tree, in bits + depth int + + // child count, used for the special case in + // commitment calculations. + count uint + + // Cache the field representation of the hash + // of the current node. + hash *Fr + + // Cache the commitment value + commitment *Point + + committer Committer + } +) + +func (n *InternalNodeStateless) Children() []VerkleNode { + var children []VerkleNode + for _, child := range n.children { + children = append(children, child) + } + return children +} + +func (n *InternalNodeStateless) SetChild(i int, c VerkleNode) error { + if i >= NodeWidth-1 { + return errors.New("child index higher than node width") + } + n.children[byte(i)] = c + return nil +} + +func (n *InternalNodeStateless) Insert(key []byte, value []byte, resolver NodeResolverFn) error { + // Clear cached commitment on modification + if n.commitment != nil { + n.commitment = nil + n.hash = nil + } + + nChild := offset2key(key, n.depth) + + switch child := n.children[nChild].(type) { + case Empty: + lastNode := &LeafNode{ + stem: key[:31], + values: make([][]byte, NodeWidth), + committer: n.committer, + } + lastNode.values[key[31]] = value + n.children[nChild] = lastNode + n.count++ + case *HashedNode: + if resolver != nil { + hash := child.ComputeCommitment().Bytes() + serialized, err := resolver(hash[:]) + if err != nil { + return fmt.Errorf("verkle tree: error resolving node %x: %w", key, err) + } + resolved, err := ParseNode(serialized, n.depth+NodeBitWidth) + if err != nil { + return fmt.Errorf("verkle tree: error parsing resolved node %x: %w", key, err) + } + n.children[nChild] = resolved + return n.children[nChild].Insert(key, value, resolver) + } + return errInsertIntoHash + case *LeafNode: + // Need to add a new branch node to differentiate + // between two keys, if the keys are different. + // Otherwise, just update the key. + if equalPaths(child.stem, key) { + if err := child.Insert(key, value, resolver); err != nil { + return err + } + } else { + // A new branch node has to be inserted. Depending + // on the next word in both keys, a recursion into + // the moved leaf node can occur. + nextWordInExistingKey := offset2key(child.stem, n.depth+NodeBitWidth) + newBranch := newInternalNode(n.depth+NodeBitWidth, n.committer).(*InternalNodeStateless) + newBranch.count = 1 + n.children[nChild] = newBranch + newBranch.children[nextWordInExistingKey] = child + + nextWordInInsertedKey := offset2key(key, n.depth+NodeBitWidth) + if nextWordInInsertedKey != nextWordInExistingKey { + // Next word differs, so this was the last level. + // Insert it directly into its final slot. + lastNode := &LeafNode{ + stem: key[:31], + values: make([][]byte, NodeWidth), + committer: n.committer, + } + lastNode.values[key[31]] = value + newBranch.children[nextWordInInsertedKey] = lastNode + newBranch.count++ + } else if err := newBranch.Insert(key, value, resolver); err != nil { + return err + } + } + default: // InternalNode + return child.Insert(key, value, resolver) + } + return nil +} + +func (n *InternalNodeStateless) toHashedNode() *HashedNode { + return &HashedNode{n.hash, n.commitment} +} + +func (n *InternalNodeStateless) InsertOrdered(key []byte, value []byte, flush NodeFlushFn) error { + // Clear cached commitment on modification + //if n.commitment != nil { + //n.commitment = nil + //n.hash = nil + //} + + //nChild := offset2key(key, n.depth) + + //switch child := n.children[nChild].(type) { + //case Empty: + // Insert into a new subtrie, which means that the + // subtree directly preceding this new one, can + // safely be calculated. + //searchFirstNonEmptyChild: + //for i := int(nChild) - 1; i >= 0; i-- { + //switch child := n.children[i].(type) { + //case Empty: + //continue + //case *LeafNode: + //child.ComputeCommitment() + //if flush != nil { + //flush(child) + //} + //n.children[i] = child.toHashedNode() + //break searchFirstNonEmptyChild + //case *HashedNode: + //break searchFirstNonEmptyChild + //case *InternalNode: + //n.children[i].ComputeCommitment() + //if flush != nil { + //child.Flush(flush) + //} + //n.children[i] = child.toHashedNode() + //break searchFirstNonEmptyChild + //} + //} + + // NOTE: these allocations are inducing a noticeable slowdown + //lastNode := &LeafNode{ + //stem: key[:31], + //values: make([][]byte, NodeWidth), + //committer: n.committer, + //} + //lastNode.values[key[31]] = value + //n.children[nChild] = lastNode + //n.count++ + + // If the node was already created, then there was at least one + // child. As a result, inserting this new leaf means there are + // now more than one child in this node. + //case *HashedNode: + //return errInsertIntoHash + //case *LeafNode: + // Need to add a new branch node to differentiate + // between two keys, if the keys are different. + // Otherwise, just update the key. + //if equalPaths(child.stem, key) { + //child.values[key[31]] = value + //} else { + // A new branch node has to be inserted. Depending + // on the next word in both keys, a recursion into + // the moved leaf node can occur. + //nextWordInExistingKey := offset2key(child.stem, n.depth+NodeBitWidth) + //newBranch := newInternalNode(n.depth+NodeBitWidth, n.committer).(*InternalNodeStateless) + //newBranch.count = 1 + //n.children[nChild] = newBranch + + //nextWordInInsertedKey := offset2key(key, n.depth+NodeBitWidth) + //if nextWordInInsertedKey != nextWordInExistingKey { + // Directly hash the (left) node that was already + // inserted. + //child.ComputeCommitment() + //if flush != nil { + //flush(child) + //} + //newBranch.children[nextWordInExistingKey] = child.toHashedNode() + // Next word differs, so this was the last level. + // Insert it directly into its final slot. + //lastNode := &LeafNode{ + //stem: key[:31], + //values: make([][]byte, NodeWidth), + //committer: n.committer, + //} + //lastNode.values[key[31]] = value + //newBranch.children[nextWordInInsertedKey] = lastNode + //newBranch.count++ + //} else { + // Reinsert the leaf in order to recurse + //newBranch.children[nextWordInExistingKey] = child + //if err := newBranch.InsertOrdered(key, value, flush); err != nil { + //return err + //} + //} + //} + //default: // InternalNode + //return child.InsertOrdered(key, value, flush) + //} + //return nil + return errors.New("not implemented yet") +} + +func (n *InternalNodeStateless) Delete(key []byte) error { + // Clear cached commitment on modification + n.commitment = nil + n.hash = nil + + nChild := offset2key(key, n.depth) + switch child := n.children[nChild].(type) { + case Empty: + return errDeleteNonExistent + case *HashedNode: + return errDeleteHash + default: + return child.Delete(key) + } +} + +// Flush hashes the children of an internal node and replaces them +// with HashedNode. It also sends the current node on the flush channel. +func (n *InternalNodeStateless) Flush(flush NodeFlushFn) { + for i, child := range n.children { + if c, ok := child.(*InternalNodeStateless); ok { + if c.commitment == nil { + c.ComputeCommitment() + } + c.Flush(flush) + n.children[i] = c.toHashedNode() + } else if c, ok := child.(*LeafNode); ok { + if c.commitment == nil { + c.ComputeCommitment() + } + flush(n.children[i]) + n.children[i] = c.toHashedNode() + } + } + flush(n) +} + +func (n *InternalNodeStateless) Get(k []byte, getter NodeResolverFn) ([]byte, error) { + nChild := offset2key(k, n.depth) + + switch child := n.children[nChild].(type) { + case Empty, nil: + // Return nil as a signal that the value isn't + // present in the tree. This matches the behavior + // of SecureTrie in Geth. + return nil, nil + case *HashedNode: + // if a resolution function is set, resolve the + // current hash node. + if getter == nil { + return nil, errReadFromInvalid + } + + commitment := child.hash.Bytes() + payload, err := getter(commitment[:]) + if err != nil { + return nil, err + } + + // deserialize the payload and set it as the child + c, err := ParseNode(payload, n.depth+NodeWidth) + if err != nil { + return nil, err + } + c.ComputeCommitment() + n.children[nChild] = c + + return c.Get(k, getter) + default: // InternalNode + return child.Get(k, getter) + } +} + +func (n *InternalNodeStateless) ComputeCommitment() *Fr { + if n.hash != nil { + return n.hash + } + + if n.count == 0 { + if n.depth != 0 { + panic("internal node should be empty node") + } + + n.commitment = new(Point) + n.commitment.Identity() + n.hash = new(Fr) + toFr(n.hash, n.commitment) + return n.hash + } + + n.hash = new(Fr) + + emptyChildren := 0 + poly := make([]Fr, NodeWidth) + for idx, child := range n.children { + switch child.(type) { + case Empty: + emptyChildren++ + default: + CopyFr(&poly[idx], child.ComputeCommitment()) + } + } + + // All the coefficients have been computed, evaluate the polynomial, + // serialize and hash the resulting point - this is the commitment. + n.commitment = n.committer.CommitToPoly(poly, emptyChildren) + toFr(n.hash, n.commitment) + + return n.hash +} + +func (n *InternalNodeStateless) GetCommitmentsAlongPath(key []byte) *ProofElements { + childIdx := offset2key(key, n.depth) + + // Build the list of elements for this level + var yi Fr + fi := make([]Fr, NodeWidth) + for i, child := range n.children { + CopyFr(&fi[i], child.ComputeCommitment()) + + if i == childIdx { + CopyFr(&yi, &fi[i]) + } + } + + // The proof elements that are to be added at this level + pe := &ProofElements{ + Cis: []*Point{n.commitment}, + Zis: []uint8{childIdx}, + Yis: []*Fr{&yi}, // Should be 0 + Fis: [][]Fr{fi}, + } + + // Special case of a proof of absence: no children + // commitment, as the value is 0. + if _, ok := n.children[childIdx].(Empty); ok { + return pe + } + + pec := n.children[childIdx].GetCommitmentsAlongPath(key) + pe.Merge(pec) + return pe +} + +func (n *InternalNodeStateless) Serialize() ([]byte, error) { + var bitlist [32]uint8 + children := make([]byte, 0, NodeWidth*32) + for i, c := range n.children { + if _, ok := c.(Empty); !ok { + setBit(bitlist[:], int(i)) + digits := c.ComputeCommitment().Bytes() + children = append(children, digits[:]...) + } + } + return append(append([]byte{internalRLPType}, bitlist[:]...), children...), nil +} + +func (n *InternalNodeStateless) Copy() VerkleNode { + ret := &InternalNode{ + children: make([]VerkleNode, len(n.children)), + commitment: new(Point), + depth: n.depth, + committer: n.committer, + count: n.count, + } + + for i, child := range n.children { + ret.children[i] = child.Copy() + } + + if n.hash != nil { + ret.hash = new(Fr) + CopyFr(ret.hash, n.hash) + } + if n.commitment != nil { + CopyPoint(ret.commitment, n.commitment) + } + + return ret +} + +// clearCache sets the commitment field of node +// and all of its children (recursively) to nil. +func (n *InternalNodeStateless) clearCache() { + for _, c := range n.children { + in, ok := c.(*InternalNodeStateless) + if !ok { + continue + } + in.clearCache() + } + n.commitment = nil +} + +func (n *InternalNodeStateless) toDot(parent, path string) string { + n.ComputeCommitment() + me := fmt.Sprintf("internal%s", path) + ret := fmt.Sprintf("%s [label=\"I: %x\"]\n", me, n.hash.Bytes()) + if len(parent) > 0 { + ret = fmt.Sprintf("%s %s -> %s\n", ret, parent, me) + } + + for i, child := range n.children { + ret = fmt.Sprintf("%s%s", ret, child.toDot(me, fmt.Sprintf("%s%02x", path, i))) + } + + return ret +} From 639fc4e1ef85c9bfcbdb61f19c6106f5dfb23506 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 7 Dec 2021 23:58:37 +0100 Subject: [PATCH 02/14] Add leaf insertion test and fix issues --- ipa.go | 4 + stateless.go | 376 ++++++++++++++++++++++++---------------------- stateless_test.go | 53 +++++++ tree.go | 3 + 4 files changed, 257 insertions(+), 179 deletions(-) create mode 100644 stateless_test.go diff --git a/ipa.go b/ipa.go index 746e85fb..6c595171 100644 --- a/ipa.go +++ b/ipa.go @@ -64,3 +64,7 @@ func fromBytes(fr *Fr, data []byte) { func Equal(fr *Fr, other *Fr) bool { return fr.Equal(other) } + +func Generator() *Point { + return new(Point).Identity() +} diff --git a/stateless.go b/stateless.go index ea1e8b4d..de97b8f9 100644 --- a/stateless.go +++ b/stateless.go @@ -30,30 +30,46 @@ import ( "fmt" ) -type ( - InternalNodeStateless struct { - // List of available child nodes of this internal node. - children map[byte]VerkleNode +// StatelessNode represents a node for execution in a stateless context, +// i.e. that its children/values are not all known. It can represent both +// an InternalNode or a LeafNode. +type StatelessNode struct { + // List of available child nodes of this internal node, + // nil if this is an extension node. + children map[byte]*StatelessNode - // node depth in the tree, in bits - depth int + // List of values, nil if this is an internal node. + values map[byte][]byte - // child count, used for the special case in - // commitment calculations. - count uint + stem []byte - // Cache the field representation of the hash - // of the current node. - hash *Fr + // node depth in the tree, in bits + depth int - // Cache the commitment value - commitment *Point + // child count, used for the special case in + // commitment calculations. + count uint - committer Committer + // Cache the field representation of the hash + // of the current node. + hash *Fr + + // Cache the commitment value + commitment, c1, c2 *Point + + committer Committer +} + +func NewStateless() *StatelessNode { + return &StatelessNode{ + children: make(map[byte]*StatelessNode), + hash: &FrZero, + committer: GetConfig(), + commitment: Generator(), } -) +} -func (n *InternalNodeStateless) Children() []VerkleNode { +func (n *StatelessNode) Children() []VerkleNode { var children []VerkleNode for _, child := range n.children { children = append(children, child) @@ -61,93 +77,120 @@ func (n *InternalNodeStateless) Children() []VerkleNode { return children } -func (n *InternalNodeStateless) SetChild(i int, c VerkleNode) error { +func (n *StatelessNode) SetChild(i int, v VerkleNode) error { if i >= NodeWidth-1 { return errors.New("child index higher than node width") } + c, ok := v.(*StatelessNode) + if !ok { + return errors.New("inserting non-stateless node into a stateless node") + } n.children[byte(i)] = c return nil } -func (n *InternalNodeStateless) Insert(key []byte, value []byte, resolver NodeResolverFn) error { - // Clear cached commitment on modification - if n.commitment != nil { - n.commitment = nil - n.hash = nil - } - - nChild := offset2key(key, n.depth) +func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn) error { + // Save the value of the initial commitment + var pre Point + //CopyPoint(&pre, n.commitment) - switch child := n.children[nChild].(type) { - case Empty: - lastNode := &LeafNode{ - stem: key[:31], - values: make([][]byte, NodeWidth), - committer: n.committer, - } - lastNode.values[key[31]] = value - n.children[nChild] = lastNode - n.count++ - case *HashedNode: - if resolver != nil { - hash := child.ComputeCommitment().Bytes() - serialized, err := resolver(hash[:]) - if err != nil { - return fmt.Errorf("verkle tree: error resolving node %x: %w", key, err) - } - resolved, err := ParseNode(serialized, n.depth+NodeBitWidth) - if err != nil { - return fmt.Errorf("verkle tree: error parsing resolved node %x: %w", key, err) - } - n.children[nChild] = resolved - return n.children[nChild].Insert(key, value, resolver) - } - return errInsertIntoHash - case *LeafNode: + // if this is a leaf value and the stems are different, intermediate + // nodes need to be inserted. + if n.values != nil { // Need to add a new branch node to differentiate // between two keys, if the keys are different. // Otherwise, just update the key. - if equalPaths(child.stem, key) { - if err := child.Insert(key, value, resolver); err != nil { - return err + if equalPaths(n.stem, key) { + if n.values[key[31]] == nil { + // only increase the count if no value is + // overwritten. + n.count++ } + n.values[key[31]] = value + // TODO: instead of invalidating the commitment + // and recalulating it entirely, compute the diff. + n.hash = nil + n.ComputeCommitment() } else { // A new branch node has to be inserted. Depending // on the next word in both keys, a recursion into // the moved leaf node can occur. - nextWordInExistingKey := offset2key(child.stem, n.depth+NodeBitWidth) - newBranch := newInternalNode(n.depth+NodeBitWidth, n.committer).(*InternalNodeStateless) - newBranch.count = 1 - n.children[nChild] = newBranch - newBranch.children[nextWordInExistingKey] = child + nextWordInExistingKey := offset2key(n.stem, n.depth+NodeBitWidth) + oldExtNode := &StatelessNode{ + depth: n.depth + NodeBitWidth, + committer: n.committer, + count: n.count, + values: n.values, + stem: n.stem, + commitment: n.commitment, + hash: n.hash, + } + n.children = map[byte]*StatelessNode{ + nextWordInExistingKey: oldExtNode, + } + n.values = nil + n.stem = nil nextWordInInsertedKey := offset2key(key, n.depth+NodeBitWidth) if nextWordInInsertedKey != nextWordInExistingKey { - // Next word differs, so this was the last level. - // Insert it directly into its final slot. - lastNode := &LeafNode{ + // Next word differs, so the branching point + // has been reached. Create the "new" child. + n.children[nextWordInInsertedKey] = &StatelessNode{ stem: key[:31], - values: make([][]byte, NodeWidth), + values: make(map[byte][]byte, NodeWidth), committer: n.committer, + count: 1, } - lastNode.values[key[31]] = value - newBranch.children[nextWordInInsertedKey] = lastNode - newBranch.count++ - } else if err := newBranch.Insert(key, value, resolver); err != nil { + n.count++ + } + + // recurse into the newly created child + if err := n.children[nextWordInInsertedKey].Insert(key, value, resolver); err != nil { return err } + n.children[nextWordInInsertedKey].ComputeCommitment() + + n.commitment.Add(n.commitment, n.children[nextWordInInsertedKey].commitment.ScalarMul(&GetConfig().conf.SRS[nextWordInInsertedKey], n.children[nextWordInInsertedKey].hash)) + } + } else { + // internal node + nChild := offset2key(key, n.depth) + + // special case: missing child, insert a leaf + if n.children[nChild] == nil { + n.children[nChild] = &StatelessNode{ + depth: n.depth + NodeBitWidth, + count: 1, + values: map[byte][]byte{key[31]: value}, + committer: n.committer, + stem: key[:31], + commitment: Generator(), + } + n.children[nChild].ComputeCommitment() + var diff Point + diff.ScalarMul(&GetConfig().conf.SRS[nChild], n.children[nChild].hash) + n.commitment.Add(n.commitment, &diff) + toFr(n.hash, n.commitment) + return nil } - default: // InternalNode - return child.Insert(key, value, resolver) + child := n.children[nChild] + // node is an internal node, pick the child and insert + if err := child.Insert(key, value, resolver); err != nil { + return err + } + + // update the commitment + n.commitment.Add(n.commitment, pre.Sub(n.children[nChild].commitment, &pre)) } + + toFr(n.hash, n.commitment) return nil } -func (n *InternalNodeStateless) toHashedNode() *HashedNode { +func (n *StatelessNode) toHashedNode() *HashedNode { return &HashedNode{n.hash, n.commitment} } -func (n *InternalNodeStateless) InsertOrdered(key []byte, value []byte, flush NodeFlushFn) error { // Clear cached commitment on modification //if n.commitment != nil { //n.commitment = nil @@ -247,82 +290,47 @@ func (n *InternalNodeStateless) InsertOrdered(key []byte, value []byte, flush No //} //return nil return errors.New("not implemented yet") +func (n *StatelessNode) InsertOrdered(key []byte, value []byte, flush NodeFlushFn) error { + return errors.New("not implemented") } -func (n *InternalNodeStateless) Delete(key []byte) error { - // Clear cached commitment on modification - n.commitment = nil - n.hash = nil +func (n *StatelessNode) Delete(key []byte) error { + // Case of an ext node + if n.values != nil { + var zero [32]byte + // Set the value to 0, data can not be deleted + n.values[key[31]] = zero[:] + n.ComputeCommitment() + return nil + } nChild := offset2key(key, n.depth) - switch child := n.children[nChild].(type) { - case Empty: - return errDeleteNonExistent - case *HashedNode: - return errDeleteHash - default: - return child.Delete(key) + child := n.children[nChild] + var pre Point + CopyPoint(&pre, child.commitment) + if err := child.Delete(key); err != nil { + return err } + + n.commitment.Add(n.commitment, pre.Sub(&pre, child.commitment)) + return nil } -// Flush hashes the children of an internal node and replaces them -// with HashedNode. It also sends the current node on the flush channel. -func (n *InternalNodeStateless) Flush(flush NodeFlushFn) { - for i, child := range n.children { - if c, ok := child.(*InternalNodeStateless); ok { - if c.commitment == nil { - c.ComputeCommitment() - } - c.Flush(flush) - n.children[i] = c.toHashedNode() - } else if c, ok := child.(*LeafNode); ok { - if c.commitment == nil { - c.ComputeCommitment() - } - flush(n.children[i]) - n.children[i] = c.toHashedNode() - } +func (n *StatelessNode) Get(k []byte, getter NodeResolverFn) ([]byte, error) { + if n.values != nil { + return n.values[k[31]], nil } - flush(n) -} -func (n *InternalNodeStateless) Get(k []byte, getter NodeResolverFn) ([]byte, error) { nChild := offset2key(k, n.depth) - switch child := n.children[nChild].(type) { - case Empty, nil: - // Return nil as a signal that the value isn't - // present in the tree. This matches the behavior - // of SecureTrie in Geth. + child := n.children[nChild] + if child == nil { return nil, nil - case *HashedNode: - // if a resolution function is set, resolve the - // current hash node. - if getter == nil { - return nil, errReadFromInvalid - } - - commitment := child.hash.Bytes() - payload, err := getter(commitment[:]) - if err != nil { - return nil, err - } - - // deserialize the payload and set it as the child - c, err := ParseNode(payload, n.depth+NodeWidth) - if err != nil { - return nil, err - } - c.ComputeCommitment() - n.children[nChild] = c - - return c.Get(k, getter) - default: // InternalNode - return child.Get(k, getter) } + return child.Get(k, getter) } -func (n *InternalNodeStateless) ComputeCommitment() *Fr { +func (n *StatelessNode) ComputeCommitment() *Fr { if n.hash != nil { return n.hash } @@ -341,26 +349,48 @@ func (n *InternalNodeStateless) ComputeCommitment() *Fr { n.hash = new(Fr) - emptyChildren := 0 - poly := make([]Fr, NodeWidth) - for idx, child := range n.children { - switch child.(type) { - case Empty: - emptyChildren++ - default: + if n.values != nil { + // leaf node: go over each value, and set them in the + // polynomial for the corresponding suffix node. + count1, count2 := 0, 0 + var poly, c1poly, c2poly [256]Fr + poly[0].SetUint64(1) + fromBytes(&poly[1], n.stem) + + for idx, val := range n.values { + if idx < 128 { + leafToComms(c1poly[idx<<1:], val) + count1++ + } else { + leafToComms(c2poly[(idx<<1)&0xFF:], val) + count2++ + } + } + n.c1 = n.committer.CommitToPoly(c1poly[:], 256-count1) + toFr(&poly[2], n.c1) + n.c2 = n.committer.CommitToPoly(c2poly[:], 256-count2) + toFr(&poly[3], n.c2) + + n.commitment = n.committer.CommitToPoly(poly[:], 252) + toFr(n.hash, n.commitment) + } else { + // internal node + emptyChildren := 0 + poly := make([]Fr, NodeWidth) + for idx, child := range n.children { CopyFr(&poly[idx], child.ComputeCommitment()) } - } - // All the coefficients have been computed, evaluate the polynomial, - // serialize and hash the resulting point - this is the commitment. - n.commitment = n.committer.CommitToPoly(poly, emptyChildren) - toFr(n.hash, n.commitment) + // All the coefficients have been computed, evaluate the polynomial, + // serialize and hash the resulting point - this is the commitment. + n.commitment = n.committer.CommitToPoly(poly, emptyChildren) + toFr(n.hash, n.commitment) + } return n.hash } -func (n *InternalNodeStateless) GetCommitmentsAlongPath(key []byte) *ProofElements { +func (n *StatelessNode) GetCommitmentsAlongPath(key []byte) *ProofElements { childIdx := offset2key(key, n.depth) // Build the list of elements for this level @@ -384,7 +414,7 @@ func (n *InternalNodeStateless) GetCommitmentsAlongPath(key []byte) *ProofElemen // Special case of a proof of absence: no children // commitment, as the value is 0. - if _, ok := n.children[childIdx].(Empty); ok { + if n.children[childIdx] == nil { return pe } @@ -393,20 +423,11 @@ func (n *InternalNodeStateless) GetCommitmentsAlongPath(key []byte) *ProofElemen return pe } -func (n *InternalNodeStateless) Serialize() ([]byte, error) { - var bitlist [32]uint8 - children := make([]byte, 0, NodeWidth*32) - for i, c := range n.children { - if _, ok := c.(Empty); !ok { - setBit(bitlist[:], int(i)) - digits := c.ComputeCommitment().Bytes() - children = append(children, digits[:]...) - } - } - return append(append([]byte{internalRLPType}, bitlist[:]...), children...), nil +func (n *StatelessNode) Serialize() ([]byte, error) { + return nil, errors.New("not supported") } -func (n *InternalNodeStateless) Copy() VerkleNode { +func (n *StatelessNode) Copy() VerkleNode { ret := &InternalNode{ children: make([]VerkleNode, len(n.children)), commitment: new(Point), @@ -430,29 +451,26 @@ func (n *InternalNodeStateless) Copy() VerkleNode { return ret } -// clearCache sets the commitment field of node -// and all of its children (recursively) to nil. -func (n *InternalNodeStateless) clearCache() { - for _, c := range n.children { - in, ok := c.(*InternalNodeStateless) - if !ok { - continue - } - in.clearCache() - } - n.commitment = nil -} - -func (n *InternalNodeStateless) toDot(parent, path string) string { +func (n *StatelessNode) toDot(parent, path string) string { n.ComputeCommitment() me := fmt.Sprintf("internal%s", path) - ret := fmt.Sprintf("%s [label=\"I: %x\"]\n", me, n.hash.Bytes()) - if len(parent) > 0 { - ret = fmt.Sprintf("%s %s -> %s\n", ret, parent, me) - } + var ret string + if n.values != nil { + ret = fmt.Sprintf("leaf%s [label=\"L: %x\nC: %x\nC₁: %x\nC₂:%x\"]\n%s -> leaf%s\n", path, n.hash.Bytes(), n.commitment.Bytes(), n.c1.Bytes(), n.c2.Bytes(), parent, path) + for i, v := range n.values { + if v != nil { + ret = fmt.Sprintf("%sval%s%x [label=\"%x\"]\nleaf%s -> val%s%x\n", ret, path, i, v, path, path, i) + } + } + } else { + ret = fmt.Sprintf("%s [label=\"I: %x\"]\n", me, n.hash.Bytes()) + if len(parent) > 0 { + ret = fmt.Sprintf("%s %s -> %s\n", ret, parent, me) + } - for i, child := range n.children { - ret = fmt.Sprintf("%s%s", ret, child.toDot(me, fmt.Sprintf("%s%02x", path, i))) + for i, child := range n.children { + ret = fmt.Sprintf("%s%s", ret, child.toDot(me, fmt.Sprintf("%s%02x", path, i))) + } } return ret diff --git a/stateless_test.go b/stateless_test.go new file mode 100644 index 00000000..47750c3e --- /dev/null +++ b/stateless_test.go @@ -0,0 +1,53 @@ +// This is free and unencumbered software released into the public domain. +// +// Anyone is free to copy, modify, publish, use, compile, sell, or +// distribute this software, either in source code form or as a compiled +// binary, for any purpose, commercial or non-commercial, and by any +// means. +// +// In jurisdictions that recognize copyright laws, the author or authors +// of this software dedicate any and all copyright interest in the +// software to the public domain. We make this dedication for the benefit +// of the public at large and to the detriment of our heirs and +// successors. We intend this dedication to be an overt act of +// relinquishment in perpetuity of all present and future rights to this +// software under copyright law. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// +// For more information, please refer to + +package verkle + +import "testing" + +func TestStatelessInsertLeafIntoRoot(t *testing.T) { + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + + rootRef := New() + rootRef.Insert(zeroKeyTest, fourtyKeyTest, nil) + hash := rootRef.ComputeCommitment() + + if !Equal(hash, root.hash) { + t.Fatalf("hashes differ after insertion %v %v", hash, root.hash) + } + + // Overwrite one leaf and check that the update + // is what is expected. + rootRef = New() + rootRef.Insert(zeroKeyTest, oneKeyTest, nil) + hash = rootRef.ComputeCommitment() + + root.Insert(zeroKeyTest, oneKeyTest, nil) + + if !Equal(hash, root.hash) { + t.Fatalf("hashes differ after update %v %v", hash, root.hash) + } +} diff --git a/tree.go b/tree.go index bb47db76..6788e191 100644 --- a/tree.go +++ b/tree.go @@ -458,11 +458,14 @@ func (n *InternalNode) ComputeCommitment() *Fr { return n.hash } + // Special cases of a node with no children: either it's + // an empty root, or it's an invalid node. if n.count == 0 { if n.depth != 0 { panic("internal node should be empty node") } + // case of an empty root n.commitment = new(Point) n.commitment.Identity() n.hash = new(Fr) From 38c81883d073537abb576832056d782e3cd9563a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 8 Dec 2021 00:11:42 +0100 Subject: [PATCH 03/14] fix the update calculation in stateless leaf insert --- stateless.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/stateless.go b/stateless.go index de97b8f9..0b16b568 100644 --- a/stateless.go +++ b/stateless.go @@ -90,10 +90,6 @@ func (n *StatelessNode) SetChild(i int, v VerkleNode) error { } func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn) error { - // Save the value of the initial commitment - var pre Point - //CopyPoint(&pre, n.commitment) - // if this is a leaf value and the stems are different, intermediate // nodes need to be inserted. if n.values != nil { @@ -174,13 +170,20 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn return nil } child := n.children[nChild] + + // Save the value of the initial child commitment + var pre Fr + CopyFr(&pre, child.hash) + // node is an internal node, pick the child and insert if err := child.Insert(key, value, resolver); err != nil { return err } // update the commitment - n.commitment.Add(n.commitment, pre.Sub(n.children[nChild].commitment, &pre)) + var diff Point + diff.ScalarMul(&GetConfig().conf.SRS[nChild], pre.Sub(child.hash, &pre)) + n.commitment.Add(n.commitment, &diff) } toFr(n.hash, n.commitment) From 42ec7fc196723cb84929b830900d57144a7ec8f0 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 8 Dec 2021 12:29:58 +0100 Subject: [PATCH 04/14] Add test for the insertion of a different value --- stateless_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/stateless_test.go b/stateless_test.go index 47750c3e..dd286db3 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -51,3 +51,29 @@ func TestStatelessInsertLeafIntoRoot(t *testing.T) { t.Fatalf("hashes differ after update %v %v", hash, root.hash) } } + +func TestStatelessInsertLeafIntoLeaf(t *testing.T) { + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + root.Insert(oneKeyTest, fourtyKeyTest, nil) + + rootRef := New() + rootRef.Insert(zeroKeyTest, fourtyKeyTest, nil) + rootRef.Insert(oneKeyTest, fourtyKeyTest, nil) + hash := rootRef.ComputeCommitment() + + if !Equal(hash, root.hash) { + t.Fatalf("hashes differ after insertion %v %v", hash, root.hash) + } + + rootRef = New() + rootRef.Insert(zeroKeyTest, fourtyKeyTest, nil) + rootRef.Insert(oneKeyTest, oneKeyTest, nil) + hash = rootRef.ComputeCommitment() + + root.Insert(oneKeyTest, oneKeyTest, nil) + + if !Equal(hash, root.hash) { + t.Fatalf("hashes differ after update %v %v", hash, root.hash) + } +} From 4081c676e991916d4b0da394a70aba2b3ddabeab Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 10 Dec 2021 15:25:29 +0100 Subject: [PATCH 05/14] add commitment calculation when inserting into leaf (#145) * fix commitment calculation in insert into leaf * fix insertion in leaf --- stateless.go | 130 +++++++--------------------------------------- stateless_test.go | 19 ++++++- 2 files changed, 38 insertions(+), 111 deletions(-) diff --git a/stateless.go b/stateless.go index 0b16b568..7c05edf9 100644 --- a/stateless.go +++ b/stateless.go @@ -111,7 +111,7 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn // A new branch node has to be inserted. Depending // on the next word in both keys, a recursion into // the moved leaf node can occur. - nextWordInExistingKey := offset2key(n.stem, n.depth+NodeBitWidth) + nextWordInExistingKey := offset2key(n.stem, n.depth) oldExtNode := &StatelessNode{ depth: n.depth + NodeBitWidth, committer: n.committer, @@ -120,33 +120,44 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn stem: n.stem, commitment: n.commitment, hash: n.hash, + c1: n.c1, + c2: n.c2, } n.children = map[byte]*StatelessNode{ nextWordInExistingKey: oldExtNode, } n.values = nil n.stem = nil + n.c1 = nil + n.c2 = nil + n.count++ - nextWordInInsertedKey := offset2key(key, n.depth+NodeBitWidth) + nextWordInInsertedKey := offset2key(key, n.depth) if nextWordInInsertedKey != nextWordInExistingKey { // Next word differs, so the branching point // has been reached. Create the "new" child. n.children[nextWordInInsertedKey] = &StatelessNode{ + depth: n.depth + NodeBitWidth, stem: key[:31], - values: make(map[byte][]byte, NodeWidth), + values: map[byte][]byte{key[31]: value}, committer: n.committer, count: 1, } - n.count++ + n.children[nextWordInInsertedKey].ComputeCommitment() } // recurse into the newly created child if err := n.children[nextWordInInsertedKey].Insert(key, value, resolver); err != nil { return err } - n.children[nextWordInInsertedKey].ComputeCommitment() - n.commitment.Add(n.commitment, n.children[nextWordInInsertedKey].commitment.ScalarMul(&GetConfig().conf.SRS[nextWordInInsertedKey], n.children[nextWordInInsertedKey].hash)) + var poly [NodeWidth]Fr + CopyFr(&poly[nextWordInExistingKey], oldExtNode.hash) + if nextWordInExistingKey != nextWordInInsertedKey { + CopyFr(&poly[nextWordInInsertedKey], n.children[nextWordInInsertedKey].hash) + } + n.commitment = n.committer.CommitToPoly(poly[:], NodeWidth-2) + toFr(n.hash, n.commitment) } } else { // internal node @@ -169,20 +180,18 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn toFr(n.hash, n.commitment) return nil } - child := n.children[nChild] // Save the value of the initial child commitment var pre Fr - CopyFr(&pre, child.hash) + CopyFr(&pre, n.children[nChild].hash) - // node is an internal node, pick the child and insert - if err := child.Insert(key, value, resolver); err != nil { + if err := n.children[nChild].Insert(key, value, resolver); err != nil { return err } // update the commitment var diff Point - diff.ScalarMul(&GetConfig().conf.SRS[nChild], pre.Sub(child.hash, &pre)) + diff.ScalarMul(&GetConfig().conf.SRS[nChild], pre.Sub(n.children[nChild].hash, &pre)) n.commitment.Add(n.commitment, &diff) } @@ -194,105 +203,6 @@ func (n *StatelessNode) toHashedNode() *HashedNode { return &HashedNode{n.hash, n.commitment} } - // Clear cached commitment on modification - //if n.commitment != nil { - //n.commitment = nil - //n.hash = nil - //} - - //nChild := offset2key(key, n.depth) - - //switch child := n.children[nChild].(type) { - //case Empty: - // Insert into a new subtrie, which means that the - // subtree directly preceding this new one, can - // safely be calculated. - //searchFirstNonEmptyChild: - //for i := int(nChild) - 1; i >= 0; i-- { - //switch child := n.children[i].(type) { - //case Empty: - //continue - //case *LeafNode: - //child.ComputeCommitment() - //if flush != nil { - //flush(child) - //} - //n.children[i] = child.toHashedNode() - //break searchFirstNonEmptyChild - //case *HashedNode: - //break searchFirstNonEmptyChild - //case *InternalNode: - //n.children[i].ComputeCommitment() - //if flush != nil { - //child.Flush(flush) - //} - //n.children[i] = child.toHashedNode() - //break searchFirstNonEmptyChild - //} - //} - - // NOTE: these allocations are inducing a noticeable slowdown - //lastNode := &LeafNode{ - //stem: key[:31], - //values: make([][]byte, NodeWidth), - //committer: n.committer, - //} - //lastNode.values[key[31]] = value - //n.children[nChild] = lastNode - //n.count++ - - // If the node was already created, then there was at least one - // child. As a result, inserting this new leaf means there are - // now more than one child in this node. - //case *HashedNode: - //return errInsertIntoHash - //case *LeafNode: - // Need to add a new branch node to differentiate - // between two keys, if the keys are different. - // Otherwise, just update the key. - //if equalPaths(child.stem, key) { - //child.values[key[31]] = value - //} else { - // A new branch node has to be inserted. Depending - // on the next word in both keys, a recursion into - // the moved leaf node can occur. - //nextWordInExistingKey := offset2key(child.stem, n.depth+NodeBitWidth) - //newBranch := newInternalNode(n.depth+NodeBitWidth, n.committer).(*InternalNodeStateless) - //newBranch.count = 1 - //n.children[nChild] = newBranch - - //nextWordInInsertedKey := offset2key(key, n.depth+NodeBitWidth) - //if nextWordInInsertedKey != nextWordInExistingKey { - // Directly hash the (left) node that was already - // inserted. - //child.ComputeCommitment() - //if flush != nil { - //flush(child) - //} - //newBranch.children[nextWordInExistingKey] = child.toHashedNode() - // Next word differs, so this was the last level. - // Insert it directly into its final slot. - //lastNode := &LeafNode{ - //stem: key[:31], - //values: make([][]byte, NodeWidth), - //committer: n.committer, - //} - //lastNode.values[key[31]] = value - //newBranch.children[nextWordInInsertedKey] = lastNode - //newBranch.count++ - //} else { - // Reinsert the leaf in order to recurse - //newBranch.children[nextWordInExistingKey] = child - //if err := newBranch.InsertOrdered(key, value, flush); err != nil { - //return err - //} - //} - //} - //default: // InternalNode - //return child.InsertOrdered(key, value, flush) - //} - //return nil - return errors.New("not implemented yet") func (n *StatelessNode) InsertOrdered(key []byte, value []byte, flush NodeFlushFn) error { return errors.New("not implemented") } diff --git a/stateless_test.go b/stateless_test.go index dd286db3..ae399f98 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -25,7 +25,10 @@ package verkle -import "testing" +import ( + "encoding/hex" + "testing" +) func TestStatelessInsertLeafIntoRoot(t *testing.T) { root := NewStateless() @@ -77,3 +80,17 @@ func TestStatelessInsertLeafIntoLeaf(t *testing.T) { t.Fatalf("hashes differ after update %v %v", hash, root.hash) } } +func TestStatelessInsertLeafIntoInternal(t *testing.T) { + key1, _ := hex.DecodeString("0000100000000000000000000000000000000000000000000000000000000000") + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + root.Insert(key1, fourtyKeyTest, nil) + rootRef := New() + rootRef.Insert(zeroKeyTest, fourtyKeyTest, nil) + rootRef.Insert(key1, fourtyKeyTest, nil) + hash := rootRef.ComputeCommitment() + + if !Equal(hash, root.hash) { + t.Fatalf("hashes differ after insertion %v %v", hash, root.hash) + } +} From 1729d420f7a4267f151b3978edeeb771b2f92d2d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 10 Dec 2021 15:31:36 +0100 Subject: [PATCH 06/14] deepsource fixes (#146) --- stateless.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/stateless.go b/stateless.go index 7c05edf9..d195addd 100644 --- a/stateless.go +++ b/stateless.go @@ -199,11 +199,7 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn return nil } -func (n *StatelessNode) toHashedNode() *HashedNode { - return &HashedNode{n.hash, n.commitment} -} - -func (n *StatelessNode) InsertOrdered(key []byte, value []byte, flush NodeFlushFn) error { +func (*StatelessNode) InsertOrdered([]byte, []byte, NodeFlushFn) error { return errors.New("not implemented") } @@ -336,7 +332,7 @@ func (n *StatelessNode) GetCommitmentsAlongPath(key []byte) *ProofElements { return pe } -func (n *StatelessNode) Serialize() ([]byte, error) { +func (*StatelessNode) Serialize() ([]byte, error) { return nil, errors.New("not supported") } From 1af43374d00331b31e43c104f530036e658ad061 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 13 Dec 2021 16:40:03 +0100 Subject: [PATCH 07/14] Do not use an overwritten FrZero in tests (#147) * Do not use an overwritten FrZero * a more thorough fix: instantiate a new zero --- stateless.go | 2 +- tree_ipa_test.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/stateless.go b/stateless.go index d195addd..1b604cee 100644 --- a/stateless.go +++ b/stateless.go @@ -63,7 +63,7 @@ type StatelessNode struct { func NewStateless() *StatelessNode { return &StatelessNode{ children: make(map[byte]*StatelessNode), - hash: &FrZero, + hash: new(Fr).SetZero(), committer: GetConfig(), commitment: Generator(), } diff --git a/tree_ipa_test.go b/tree_ipa_test.go index e6f72aaa..a15cf8bf 100644 --- a/tree_ipa_test.go +++ b/tree_ipa_test.go @@ -45,7 +45,7 @@ func extensionAndSuffixOneKey(key, value []byte, ret *Point) { vs [2]Fr srs = GetConfig().conf.SRS stemComm1, stemComm3, stemComm2 Point - zero, t1, t2, c1 Point + t1, t2, c1 Point ) stemComm0 := srs[0] fromBytes(&v, key[:31]) @@ -56,9 +56,8 @@ func extensionAndSuffixOneKey(key, value []byte, ret *Point) { toFr(&v, &c1) stemComm2.ScalarMul(&srs[2], &v) - (&zero).Identity() - toFr(&v, &zero) - stemComm3.ScalarMul(&srs[3], &FrZero) + v.SetZero() + stemComm3.ScalarMul(&srs[3], &v) t1.Add(&stemComm0, &stemComm1) t2.Add(&stemComm2, &stemComm3) From 0a7bd7dbc26a6eddcab9b39cc9142e54ea61091f Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 14 Dec 2021 19:54:50 +0100 Subject: [PATCH 08/14] fix delete + add test (#149) --- stateless.go | 12 +++++++++--- stateless_test.go | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/stateless.go b/stateless.go index 1b604cee..aae014ea 100644 --- a/stateless.go +++ b/stateless.go @@ -209,19 +209,25 @@ func (n *StatelessNode) Delete(key []byte) error { var zero [32]byte // Set the value to 0, data can not be deleted n.values[key[31]] = zero[:] + n.hash = nil n.ComputeCommitment() return nil } nChild := offset2key(key, n.depth) child := n.children[nChild] - var pre Point - CopyPoint(&pre, child.commitment) + var pre Fr + CopyFr(&pre, child.hash) if err := child.Delete(key); err != nil { return err } - n.commitment.Add(n.commitment, pre.Sub(&pre, child.commitment)) + pre.Sub(child.hash, &pre) + + var tmp Point + tmp.ScalarMul(&GetConfig().conf.SRS[nChild], &pre) + n.commitment.Add(n.commitment, &tmp) + toFr(n.hash, n.commitment) return nil } diff --git a/stateless_test.go b/stateless_test.go index ae399f98..fed1a3dd 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -30,6 +30,29 @@ import ( "testing" ) +func TestStatelessDelete(t *testing.T) { + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + var single Point + CopyPoint(&single, root.commitment) + + root.Insert(oneKeyTest, fourtyKeyTest, nil) + if root.commitment.Equal(&single) { + t.Fatal("second insert didn't update") + } + + root.Delete(oneKeyTest) + + rootRef := New() + rootRef.Insert(zeroKeyTest, fourtyKeyTest, nil) + rootRef.Insert(oneKeyTest, fourtyKeyTest, nil) + rootRef.Delete(oneKeyTest) + + if !Equal(rootRef.ComputeCommitment(), root.hash) { + t.Fatal("error in delete", rootRef.ComputeCommitment(), root.hash) + } +} + func TestStatelessInsertLeafIntoRoot(t *testing.T) { root := NewStateless() root.Insert(zeroKeyTest, fourtyKeyTest, nil) From baca15c8d53388324fb5a66c28210f67220b403f Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 15 Dec 2021 11:18:24 +0100 Subject: [PATCH 09/14] Add C1 and C2 in dot representation of trees (#148) --- tree.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tree.go b/tree.go index 6788e191..93af0749 100644 --- a/tree.go +++ b/tree.go @@ -841,7 +841,7 @@ func (n *LeafNode) Value(i int) []byte { } func (n *LeafNode) toDot(parent, path string) string { - ret := fmt.Sprintf("leaf%s [label=\"L: %x\nC: %x\"]\n%s -> leaf%s\n", path, n.hash.Bytes(), n.commitment.Bytes(), parent, path) + ret := fmt.Sprintf("leaf%s [label=\"L: %x\nC: %x\nC₁: %x\nC₂:%x\"]\n%s -> leaf%s\n", path, n.hash.Bytes(), n.commitment.Bytes(), n.c1.Bytes(), n.c2.Bytes(), parent, path) for i, v := range n.values { if v != nil { ret = fmt.Sprintf("%sval%s%x [label=\"%x\"]\nleaf%s -> val%s%x\n", ret, path, i, v, path, path, i) From b6f8c767c74d66518e547f13e27b8d31bcf3aaea Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 15 Dec 2021 11:32:40 +0100 Subject: [PATCH 10/14] stateless: coverage for InsertOrdered (#151) --- stateless.go | 4 +++- stateless_test.go | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/stateless.go b/stateless.go index aae014ea..56e8de77 100644 --- a/stateless.go +++ b/stateless.go @@ -60,6 +60,8 @@ type StatelessNode struct { committer Committer } +var errNotSupportedInStateless = errors.New("not implemented in stateless") + func NewStateless() *StatelessNode { return &StatelessNode{ children: make(map[byte]*StatelessNode), @@ -200,7 +202,7 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn } func (*StatelessNode) InsertOrdered([]byte, []byte, NodeFlushFn) error { - return errors.New("not implemented") + return errNotSupportedInStateless } func (n *StatelessNode) Delete(key []byte) error { diff --git a/stateless_test.go b/stateless_test.go index fed1a3dd..5523f2d5 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -103,6 +103,7 @@ func TestStatelessInsertLeafIntoLeaf(t *testing.T) { t.Fatalf("hashes differ after update %v %v", hash, root.hash) } } + func TestStatelessInsertLeafIntoInternal(t *testing.T) { key1, _ := hex.DecodeString("0000100000000000000000000000000000000000000000000000000000000000") root := NewStateless() @@ -117,3 +118,11 @@ func TestStatelessInsertLeafIntoInternal(t *testing.T) { t.Fatalf("hashes differ after insertion %v %v", hash, root.hash) } } + +func TestStatelessInsertOrdered(t *testing.T) { + root := NewStateless() + err := root.InsertOrdered(zeroKeyTest, fourtyKeyTest, nil) + if err != errNotSupportedInStateless { + t.Fatalf("got the wrong error: expected %v, got %v", errNotSupportedInStateless, err) + } +} From eccd476d1aee14847935cff8f5a38585b658f5e6 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 15 Dec 2021 12:04:33 +0100 Subject: [PATCH 11/14] stateless: add a test for copy (#152) --- stateless_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/stateless_test.go b/stateless_test.go index 5523f2d5..bf880ea2 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -126,3 +126,16 @@ func TestStatelessInsertOrdered(t *testing.T) { t.Fatalf("got the wrong error: expected %v, got %v", errNotSupportedInStateless, err) } } + +func TestStatelessCopy(t *testing.T) { + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + rootCopy := root.Copy() + if !Equal(rootCopy.ComputeCommitment(), root.hash) { + t.Fatal("copy produced the wrong hash") + } + root.Insert(oneKeyTest, fourtyKeyTest, nil) + if Equal(rootCopy.ComputeCommitment(), root.hash) { + t.Fatal("copy did not update the hash") + } +} From 137c833304865c92272430b10a92a891f1fcdfec Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 15 Dec 2021 13:17:30 +0100 Subject: [PATCH 12/14] stateless: add test for Get (#153) --- stateless_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/stateless_test.go b/stateless_test.go index bf880ea2..bdec9780 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -26,6 +26,7 @@ package verkle import ( + "bytes" "encoding/hex" "testing" ) @@ -139,3 +140,23 @@ func TestStatelessCopy(t *testing.T) { t.Fatal("copy did not update the hash") } } + +func TestStatelessGet(t *testing.T) { + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + data, err := root.Get(zeroKeyTest, nil) + if err != nil { + t.Fatalf("error while getting existing value %v", err) + } + if !bytes.Equal(data, fourtyKeyTest) { + t.Fatalf("error getting value, expected %x, got %x", fourtyKeyTest, data) + } + + data, err = root.Get(oneKeyTest, nil) + if err != nil { + t.Fatalf("error while getting non-existing value %v", err) + } + if data != nil { + t.Fatalf("error: got value %x, expected nil", data) + } +} From bd1712f39251ed5a977f28f1e0f2ae3183fbbd90 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 15 Dec 2021 17:30:16 +0100 Subject: [PATCH 13/14] stateless: add a test for toDot and fix a commitment copy issue (#154) --- stateless.go | 4 +++- stateless_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/stateless.go b/stateless.go index 56e8de77..e0a6bb89 100644 --- a/stateless.go +++ b/stateless.go @@ -120,7 +120,7 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn count: n.count, values: n.values, stem: n.stem, - commitment: n.commitment, + commitment: new(Point), hash: n.hash, c1: n.c1, c2: n.c2, @@ -133,6 +133,8 @@ func (n *StatelessNode) Insert(key []byte, value []byte, resolver NodeResolverFn n.c1 = nil n.c2 = nil n.count++ + CopyPoint(oldExtNode.commitment, n.commitment) + n.hash = new(Fr) nextWordInInsertedKey := offset2key(key, n.depth) if nextWordInInsertedKey != nextWordInExistingKey { diff --git a/stateless_test.go b/stateless_test.go index bdec9780..f4e57166 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -28,6 +28,8 @@ package verkle import ( "bytes" "encoding/hex" + "sort" + "strings" "testing" ) @@ -160,3 +162,27 @@ func TestStatelessGet(t *testing.T) { t.Fatalf("error: got value %x, expected nil", data) } } + +func TestStatelessToDot(t *testing.T) { + key1, _ := hex.DecodeString("0000100000000000000000000000000000000000000000000000000000000000") + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + root.Insert(oneKeyTest, fourtyKeyTest, nil) + root.Insert(key1, fourtyKeyTest, nil) + rootRef := New() + rootRef.Insert(zeroKeyTest, fourtyKeyTest, nil) + rootRef.Insert(oneKeyTest, fourtyKeyTest, nil) + rootRef.Insert(key1, fourtyKeyTest, nil) + rootRef.ComputeCommitment() + + stl := strings.Split(root.toDot("", ""), "\n") + stf := strings.Split(rootRef.toDot("", ""), "\n") + sort.Strings(stl) + sort.Strings(stf) + stfJ := strings.Join(stf, "\n") + stlJ := strings.Join(stl, "\n") + + if stfJ != stlJ { + t.Fatalf("hashes differ after insertion %v %v", stf, stl) + } +} From d1f9a2e880b919d25c289e68848821afcd569d7c Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 16 Dec 2021 11:49:10 +0100 Subject: [PATCH 14/14] stateless: add a test for GetCommitmentsAlongPath --- stateless_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/stateless_test.go b/stateless_test.go index f4e57166..a672d3dd 100644 --- a/stateless_test.go +++ b/stateless_test.go @@ -28,6 +28,7 @@ package verkle import ( "bytes" "encoding/hex" + "reflect" "sort" "strings" "testing" @@ -130,6 +131,26 @@ func TestStatelessInsertOrdered(t *testing.T) { } } +func TestStatelessGetCommitmentsAlongPath(t *testing.T) { + root := NewStateless() + root.Insert(zeroKeyTest, fourtyKeyTest, nil) + root.Insert(oneKeyTest, fourtyKeyTest, nil) + root.Insert(fourtyKeyTest, fourtyKeyTest, nil) + + rootRef := New() + rootRef.Insert(zeroKeyTest, fourtyKeyTest, nil) + rootRef.Insert(oneKeyTest, fourtyKeyTest, nil) + rootRef.Insert(fourtyKeyTest, fourtyKeyTest, nil) + rootRef.ComputeCommitment() + + stf := rootRef.GetCommitmentsAlongPath(oneKeyTest) + stl := root.GetCommitmentsAlongPath(oneKeyTest) + + if !reflect.DeepEqual(stf, stl) { + t.Fatalf("commitments along path differ %v != %v", stf, stl) + } +} + func TestStatelessCopy(t *testing.T) { root := NewStateless() root.Insert(zeroKeyTest, fourtyKeyTest, nil)