use UnknownNode in InternalNode to turn it into a stateless node (#345)

* use nil in InternalNode to turn it into a stateless node

* move stateless proofs check from stateless_test to proof

* safety improvement: use Unknown instead of nil (#346)

* safety improvment: use Unknown instead of nil

* fix tests

* rename Unknown to the clearer UnknownNode

* return an error in GetProofItems and forbid calling it on incomplete trees

* remove unused functions

* remove all references to stateless (#347)

* remove all references to stateless

* remove unused error

* fix linter error

* Update proof_ipa.go

doc.go

@@ -33,9 +33,9 @@ var (
 	errDeleteHash              = errors.New("trying to delete from a hashed subtree")
 	errReadFromInvalid         = errors.New("trying to read from an invalid child")
 	errSerializeHashedNode     = errors.New("trying to serialize a hashed internal node")
-	errNotSupportedInStateless = errors.New("not implemented in stateless")
 	errInsertIntoOtherStem     = errors.New("insert splits a stem where it should not happen")
 	errStatelessAndStatefulMix = errors.New("a stateless node should not be found in a stateful tree")
+	errMissingNodeInStateless  = errors.New("trying to access a node that is missing from the stateless view")
 )
 
 const (

empty.go

@@ -53,8 +53,8 @@ func (Empty) Commitment() *Point {
 	return &id
 }
 
-func (Empty) GetProofItems(keylist) (*ProofElements, []byte, [][]byte) {
-	panic("trying to produce a commitment for an empty subtree")
+func (Empty) GetProofItems(keylist) (*ProofElements, []byte, [][]byte, error) {
+	return nil, nil, nil, errors.New("trying to produce a commitment for an empty subtree")
 }
 
 func (Empty) Serialize() ([]byte, error) {

encoding.go

@@ -86,20 +86,6 @@ func ParseNode(serializedNode []byte, depth byte, comm SerializedPointCompressed
 	}
 }
 
-func ParseStatelessNode(serialized []byte, depth byte, comm SerializedPointCompressed) (VerkleNode, error) {
-	if len(serialized) < 1+StemSize+SerializedPointCompressedSize {
-		return nil, errSerializedPayloadTooShort
-	}
-	switch serialized[0] {
-	case leafRLPType:
-		return parseLeafNode(serialized, depth, comm)
-	case internalRLPType:
-		return deserializeIntoStateless(serialized[1:33], serialized[33:], depth, comm)
-	default:
-		return nil, ErrInvalidNodeEncoding
-	}
-}
-
 func parseLeafNode(serialized []byte, depth byte, comm SerializedPointCompressed) (VerkleNode, error) {
 	bitlist := serialized[leafBitlistOffset : leafBitlistOffset+bitlistSize]
 	var values [NodeWidth][]byte
@@ -124,23 +110,6 @@ func parseLeafNode(serialized []byte, depth byte, comm SerializedPointCompressed
 	return ln, nil
 }
 
-func deserializeIntoStateless(bitlist []byte, raw []byte, depth byte, comm SerializedPointCompressed) (*StatelessNode, error) {
-	// GetTreeConfig caches computation result, hence
-	// this op has low overhead
-	n := NewStateless()
-	n.setDepth(depth)
-	indices := indicesFromBitlist(bitlist)
-	if len(raw)/SerializedPointCompressedSize != len(indices) {
-		return nil, ErrInvalidNodeEncoding
-	}
-	for i, index := range indices {
-		n.unresolved[byte(index)] = raw[i*SerializedPointCompressedSize : (i+1)*SerializedPointCompressedSize]
-	}
-	n.commitment = new(Point)
-	n.commitment.SetBytesTrusted(comm)
-	return n, nil
-}
-
 func CreateInternalNode(bitlist []byte, raw []byte, depth byte, comm SerializedPointCompressed) (*InternalNode, error) {
 	// GetTreeConfig caches computation result, hence
 	// this op has low overhead

hashednode.go

@@ -65,8 +65,8 @@ func (n *HashedNode) Commitment() *Point {
 	return n.Commit()
 }
 
-func (*HashedNode) GetProofItems(keylist) (*ProofElements, []byte, [][]byte) {
-	panic("can not get the full path, and there is no proof of absence")
+func (*HashedNode) GetProofItems(keylist) (*ProofElements, []byte, [][]byte, error) {
+	return nil, nil, nil, errors.New("can not get the full path, and there is no proof of absence")
 }
 
 func (*HashedNode) Serialize() ([]byte, error) {

proof_ipa.go

@@ -74,7 +74,7 @@ type StemStateDiff struct {
 
 type StateDiff []StemStateDiff
 
-func GetCommitmentsForMultiproof(root VerkleNode, keys [][]byte) (*ProofElements, []byte, [][]byte) {
+func GetCommitmentsForMultiproof(root VerkleNode, keys [][]byte) (*ProofElements, []byte, [][]byte, error) {
 	sort.Sort(keylist(keys))
 	return root.GetProofItems(keylist(keys))
 }
@@ -89,7 +89,10 @@ func MakeVerkleMultiProof(root VerkleNode, keys [][]byte, keyvals map[string][]b
 	tr := common.NewTranscript("vt")
 	root.Commit()
 
-	pe, es, poas := GetCommitmentsForMultiproof(root, keys)
+	pe, es, poas, err := GetCommitmentsForMultiproof(root, keys)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
 
 	var vals [][]byte
 	for _, k := range keys {
@@ -335,23 +338,26 @@ func TreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) {
 		}
 	}
 
-	root := NewStatelessWithCommitment(rootC)
+	root := NewStatelessInternal(0, rootC).(*InternalNode)
 	comms := proof.Cs
 	for _, p := range paths {
-		comms, err = root.insertStem(p, info[string(p)], comms)
-		if err != nil {
-			return nil, err
-		}
-	}
+		// NOTE: the reconstructed tree won't tell the
+		// difference between leaves missing from view
+		// and absent leaves. This is enough for verification
+		// but not for block validation.
+		values := make([][]byte, NodeWidth)
+		for i, k := range proof.Keys {
+			if len(proof.Values[i]) == 0 {
+				// Skip the nil keys, they are here to prove
+				// an absence.
+				continue
+			}
 
-	for i, k := range proof.Keys {
-		if len(proof.Values[i]) == 0 {
-			// Skip the nil keys, they are here to prove
-			// an absence.
-			continue
+			if bytes.Equal(k, info[string(p)].stem) {
+				values[k[31]] = proof.Values[i]
+			}
 		}
-
-		err = root.insertValue(k, proof.Values[i])
+		comms, err = root.CreatePath(p, info[string(p)], comms, values)
 		if err != nil {
 			return nil, err
 		}

proof_test.go

@@ -85,7 +85,10 @@ func TestMultiProofVerifyMultipleLeaves(t *testing.T) {
 
 	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys[0:2], kv)
 
-	pe, _, _ := GetCommitmentsForMultiproof(root, keys[0:2])
+	pe, _, _, err := GetCommitmentsForMultiproof(root, keys[0:2])
+	if err != nil {
+		t.Fatal(err)
+	}
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, pe.Cis, pe.Zis, pe.Yis, cfg) {
 		t.Fatal("could not verify verkle proof")
@@ -118,7 +121,10 @@ func TestMultiProofVerifyMultipleLeavesWithAbsentStem(t *testing.T) {
 
 	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys, kv)
 
-	pe, _, isabsent := GetCommitmentsForMultiproof(root, keys)
+	pe, _, isabsent, err := GetCommitmentsForMultiproof(root, keys)
+	if err != nil {
+		t.Fatal(err)
+	}
 	if len(isabsent) == 0 {
 		t.Fatal("should have detected an absent stem")
 	}
@@ -145,7 +151,10 @@ func TestMultiProofVerifyMultipleLeavesCommitmentRedundancy(t *testing.T) {
 
 	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys, kv)
 
-	pe, _, _ := GetCommitmentsForMultiproof(root, keys)
+	pe, _, _, err := GetCommitmentsForMultiproof(root, keys)
+	if err != nil {
+		t.Fatal(err)
+	}
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, pe.Cis, pe.Zis, pe.Yis, cfg) {
 		t.Fatal("could not verify verkle proof")
@@ -347,7 +356,10 @@ func TestProofDeserialize(t *testing.T) {
 	}
 	_ = deserialized
 
-	pe, _, _ := root.GetProofItems(keylist{absentkey[:]})
+	pe, _, _, err := root.GetProofItems(keylist{absentkey[:]})
+	if err != nil {
+		t.Fatal(err)
+	}
 	cfg := GetConfig()
 	if !VerifyVerkleProof(deserialized, pe.Cis, pe.Zis, pe.Yis, cfg) {
 		t.Fatal("could not verify verkle proof")
@@ -557,3 +569,108 @@ func TestVerkleProofMarshalUnmarshalJSON(t *testing.T) {
 		t.Errorf("expected %v, got %v", vp1, vp2)
 	}
 }
+
+func TestStatelessDeserialize(t *testing.T) {
+	root := New()
+	for _, k := range [][]byte{zeroKeyTest, oneKeyTest, fourtyKeyTest, ffx32KeyTest} {
+		root.Insert(k, fourtyKeyTest, nil)
+	}
+
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, fourtyKeyTest}, map[string][]byte{string(zeroKeyTest): fourtyKeyTest, string(fourtyKeyTest): fourtyKeyTest})
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := TreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !Equal(droot.Commit(), root.Commitment()) {
+		t.Log(ToDot(droot), ToDot(root))
+		t.Fatalf("differing root commitments %x != %x", droot.Commitment().Bytes(), root.Commitment().Bytes())
+	}
+
+	if !Equal(droot.(*InternalNode).children[0].(*LeafNode).commitment, root.(*InternalNode).children[0].Commit()) {
+		t.Fatal("differing commitment for child #0")
+	}
+
+	if !Equal(droot.(*InternalNode).children[64].Commit(), root.(*InternalNode).children[64].Commit()) {
+		t.Fatal("differing commitment for child #64")
+	}
+}
+
+func TestStatelessDeserializeMissginChildNode(t *testing.T) {
+	root := New()
+	for _, k := range [][]byte{zeroKeyTest, oneKeyTest, ffx32KeyTest} {
+		root.Insert(k, fourtyKeyTest, nil)
+	}
+
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, fourtyKeyTest}, map[string][]byte{string(zeroKeyTest): fourtyKeyTest, string(fourtyKeyTest): nil})
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := TreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !Equal(droot.Commit(), root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+	if !Equal(droot.(*InternalNode).children[0].Commit(), root.(*InternalNode).children[0].Commit()) {
+		t.Fatal("differing commitment for child #0")
+	}
+
+	if droot.(*InternalNode).children[64] != UnknownNode(struct{}{}) {
+		t.Fatalf("non-nil child #64: %v", droot.(*InternalNode).children[64])
+	}
+}
+
+func TestStatelessDeserializeDepth2(t *testing.T) {
+	root := New()
+	key1, _ := hex.DecodeString("0000010000000000000000000000000000000000000000000000000000000000")
+	for _, k := range [][]byte{zeroKeyTest, key1} {
+		root.Insert(k, fourtyKeyTest, nil)
+	}
+
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, key1}, map[string][]byte{string(zeroKeyTest): fourtyKeyTest, string(key1): nil})
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := TreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !Equal(droot.Commit(), root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	if !Equal(droot.(*InternalNode).children[0].Commit(), root.(*InternalNode).children[0].Commit()) {
+		t.Fatal("differing commitment for child #0")
+	}
+}