TreeHash.EmptyHash -> TreeHash.EmptyHash(height int) (#685)

* MapHasher interface

Replaces a private [][]byte cache of empty nodes with an interface
`EmptyHash(depth int)` which can have multiple implementations.

Moves the default HStar2 empty hash algorithm and tests into a separate
package which implements the MapHasher interface.

* Make TreeHasher generic

- Use height rather than depth for HashEmpty to simplify math.
- Move emtpy hash logic into rfc6962 hasher.
- Create LogHasher to support special casing an empty tree.

* comments and spelling

* Separate map hasher from rfc6962 hasher

Logs do not need to compute empty branches.
Rather than creating a unified hasher for logs and maps, create separate
hashers that will panic when used in the wrong setting.

* Add MAP_HASHER to createtree

* Reviewer guided cleanup

- Removed map references from rfc6962
- Removed generic references from maphasher
- Comment cleanup

* Separate LogHasher and MapHasher

It turns out that having two separate interfaces for Log and Map hashing
is both feasible and suports better code design.

The inner merkle tree computations for the log and map do not actually
share code, making this possible.

* Review modifications

- Spelling and comments
- MAP_HASHER -> TEST_MAP_HASHER

client/client.go

@@ -41,7 +41,7 @@ type LogClient struct {
 }
 
 // New returns a new LogClient.
-func New(logID int64, client trillian.TrillianLogClient, hasher merkle.TreeHasher, pubKey crypto.PublicKey) *LogClient {
+func New(logID int64, client trillian.TrillianLogClient, hasher merkle.LogHasher, pubKey crypto.PublicKey) *LogClient {
 	return &LogClient{
 		LogID:  logID,
 		client: client,

client/offline.go

@@ -25,13 +25,13 @@ import (
 
 // logVerifier contains state needed to verify output from Trillian Logs.
 type logVerifier struct {
-	hasher merkle.TreeHasher
+	hasher merkle.LogHasher
 	pubKey crypto.PublicKey
 	v      merkle.LogVerifier
 }
 
 // NewLogVerifier returns an object that can verify output from Trillian Logs.
-func NewLogVerifier(hasher merkle.TreeHasher, pubKey crypto.PublicKey) LogVerifier {
+func NewLogVerifier(hasher merkle.LogHasher, pubKey crypto.PublicKey) LogVerifier {
 	return &logVerifier{
 		hasher: hasher,
 		pubKey: pubKey,

integration/map.go

@@ -24,6 +24,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/google/trillian"
 	"github.com/google/trillian/merkle"
+	"github.com/google/trillian/merkle/maphasher"
 	"github.com/google/trillian/testonly"
 )
 
@@ -85,7 +86,7 @@ func RunMapIntegration(ctx context.Context, mapID int64, client trillian.Trillia
 
 	// Check values
 	// Mix up the ordering of requests
-	h := merkle.NewMapHasher(testonly.Hasher)
+	h := maphasher.Default
 	randIndexes := make([][]byte, len(tests))
 	for i, r := range rand.Perm(len(tests)) {
 		randIndexes[i] = tests[r].Index
@@ -131,7 +132,7 @@ func RunMapIntegration(ctx context.Context, mapID int64, client trillian.Trillia
 // Ensure that a query for a leaf that does not exist results in a valid inclusion proof.
 func testForNonExistentLeaf(ctx context.Context, mapID int64,
 	client trillian.TrillianMapClient, latestRoot trillian.SignedMapRoot) error {
-	h := merkle.NewMapHasher(testonly.Hasher)
+	h := maphasher.Default
 	index1 := []byte("doesnotexist....................")
 	r, err := client.GetLeaves(ctx, &trillian.GetMapLeavesRequest{
 		MapId:    mapID,

integration/map_integration_test.sh

@@ -23,6 +23,7 @@ TEST_TREE_ID=$(./createtree \
   --tree_type=MAP \
   --pem_key_path=${TRILLIAN_PATH}/testdata/map-rpc-server.privkey.pem \
   --pem_key_password=towel \
+  --hash_strategy=TEST_MAP_HASHER \
   --signature_algorithm=ECDSA)
 echo "Created tree ${TEST_TREE_ID}"
 

log/sequencer.go

@@ -78,7 +78,7 @@ func createMetrics(mf monitoring.MetricFactory) {
 // There is no strong ordering guarantee but in general entries will be processed
 // in order of submission to the log.
 type Sequencer struct {
-	hasher     merkle.TreeHasher
+	hasher     merkle.LogHasher
 	timeSource util.TimeSource
 	logStorage storage.LogStorage
 	signer     *crypto.Signer
@@ -93,7 +93,7 @@ const maxTreeDepth = 64
 
 // NewSequencer creates a new Sequencer instance for the specified inputs.
 func NewSequencer(
-	hasher merkle.TreeHasher,
+	hasher merkle.LogHasher,
 	timeSource util.TimeSource,
 	logStorage storage.LogStorage,
 	signer *crypto.Signer,

merkle/compact_merkle_tree.go

@@ -36,7 +36,7 @@ func (r RootHashMismatchError) Error() string {
 // CompactMerkleTree is a compact Merkle tree representation.
 // Uses log(n) nodes to represent the current on-disk tree.
 type CompactMerkleTree struct {
-	hasher TreeHasher
+	hasher LogHasher
 	root   []byte
 	// the list of "dangling" left-hand nodes, NOTE: index 0 is the leaf, not the root.
 	nodes [][]byte
@@ -66,13 +66,13 @@ type GetNodeFunc func(depth int, index int64) ([]byte, error)
 // |f| will be called a number of times with the co-ordinates of internal MerkleTree nodes whose hash values are
 // required to initialize the internal state of the CompactMerkleTree.  |expectedRoot| is the known-good tree root
 // of the tree at |size|, and is used to verify the correct initial state of the CompactMerkleTree after initialisation.
-func NewCompactMerkleTreeWithState(hasher TreeHasher, size int64, f GetNodeFunc, expectedRoot []byte) (*CompactMerkleTree, error) {
+func NewCompactMerkleTreeWithState(hasher LogHasher, size int64, f GetNodeFunc, expectedRoot []byte) (*CompactMerkleTree, error) {
 	sizeBits := bitLen(size)
 
 	r := CompactMerkleTree{
 		hasher: hasher,
 		nodes:  make([][]byte, sizeBits),
-		root:   hasher.HashEmpty(),
+		root:   hasher.EmptyRoot(),
 		size:   size,
 	}
 
@@ -108,10 +108,10 @@ func NewCompactMerkleTreeWithState(hasher TreeHasher, size int64, f GetNodeFunc,
 }
 
 // NewCompactMerkleTree creates a new CompactMerkleTree with size zero. This always succeeds.
-func NewCompactMerkleTree(hasher TreeHasher) *CompactMerkleTree {
+func NewCompactMerkleTree(hasher LogHasher) *CompactMerkleTree {
 	r := CompactMerkleTree{
 		hasher: hasher,
-		root:   hasher.HashEmpty(),
+		root:   hasher.EmptyRoot(),
 		nodes:  make([][]byte, 0),
 		size:   0,
 	}

merkle/compact_merkle_tree_test.go

@@ -76,7 +76,7 @@ func TestAddingLeaves(t *testing.T) {
 			t.Errorf("Size()=%d, want %d", got, want)
 		}
 		if got, want := tree.CurrentRoot(), testonly.EmptyMerkleTreeRootHash(); !bytes.Equal(got, want) {
-			t.Errorf("CurrentRoot()=%v, want %v", got, want)
+			t.Errorf("CurrentRoot()=%x, want %x", got, want)
 		}
 
 		for i := 0; i < 8; i++ {

merkle/hstar2.go

@@ -37,16 +37,13 @@ type HStar2LeafHash struct {
 // HStar2 is a recursive implementation for calculating the root hash of a sparse
 // Merkle tree.
 type HStar2 struct {
-	hasher          TreeHasher
-	hStarEmptyCache [][]byte
+	hasher MapHasher
 }
 
-// NewHStar2 creates a new HStar2 tree calculator based on the passed in
-// TreeHasher.
-func NewHStar2(treeHasher TreeHasher) HStar2 {
+// NewHStar2 creates a new HStar2 tree calculator based on the passed in MapHasher.
+func NewHStar2(hasher MapHasher) HStar2 {
 	return HStar2{
-		hasher:          treeHasher,
-		hStarEmptyCache: [][]byte{treeHasher.HashLeaf([]byte(""))},
+		hasher: hasher,
 	}
 }
 
@@ -56,7 +53,9 @@ func (s *HStar2) HStar2Root(n int, values []HStar2LeafHash) ([]byte, error) {
 	by(indexLess).Sort(values)
 	offset := big.NewInt(0)
 	return s.hStar2b(n, values, offset,
-		func(depth int, index *big.Int) ([]byte, error) { return s.hStarEmpty(depth) },
+		func(depth int, index *big.Int) ([]byte, error) {
+			return s.hasher.HashEmpty(depth), nil
+		},
 		func(int, *big.Int, []byte) error { return nil })
 }
 
@@ -97,33 +96,13 @@ func (s *HStar2) HStar2Nodes(treeDepth, treeLevelOffset int, values []HStar2Leaf
 				return h, nil
 			}
 			// otherwise just return the null hash for this level
-			return s.hStarEmpty(depth + treeLevelOffset)
+			return s.hasher.HashEmpty(depth + treeLevelOffset), nil
 		},
 		func(depth int, index *big.Int, hash []byte) error {
 			return set(treeDepth-depth, index, hash)
 		})
 }
 
-// hStarEmpty calculates (and caches) the "null-hash" for the requested tree
-// level.
-func (s *HStar2) hStarEmpty(n int) ([]byte, error) {
-	if len(s.hStarEmptyCache) <= n {
-		emptyRoot, err := s.hStarEmpty(n - 1)
-		if err != nil {
-			return nil, err
-		}
-		h := s.hasher.HashChildren(emptyRoot, emptyRoot)
-		if len(s.hStarEmptyCache) != n {
-			return nil, fmt.Errorf("cache wrong size - expected %d, but cache contains %d entries", n, len(s.hStarEmptyCache))
-		}
-		s.hStarEmptyCache = append(s.hStarEmptyCache, h)
-	}
-	if n >= len(s.hStarEmptyCache) {
-		return nil, fmt.Errorf("cache too small - want level %d, but cache only contains %d entries", n, len(s.hStarEmptyCache))
-	}
-	return s.hStarEmptyCache[n], nil
-}
-
 var (
 	smtOne = big.NewInt(1)
 )

merkle/hstar2_test.go

@@ -17,11 +17,11 @@ package merkle
 import (
 	"bytes"
 	"crypto/sha256"
-	"encoding/base64"
 	"fmt"
 	"math/big"
 	"testing"
 
+	"github.com/google/trillian/merkle/maphasher"
 	"github.com/google/trillian/testonly"
 )
 
@@ -34,11 +34,11 @@ var sparseEmptyRootHashB64 = testonly.MustDecodeBase64("xmifEIEqCYCXbZUz2Dh1KCFm
 // passing into a the HStar2 sparse Merkle tree implementation.
 // The map keys will be SHA256 hashed before being added to the returned
 // structs.
-func createHStar2Leaves(th TreeHasher, values map[string]string) []HStar2LeafHash {
+func createHStar2Leaves(hasher MapHasher, values map[string]string) []HStar2LeafHash {
 	r := []HStar2LeafHash{}
 	for k := range values {
 		khash := sha256.Sum256([]byte(k))
-		vhash := th.HashLeaf([]byte(values[k]))
+		vhash := hasher.HashLeaf([]byte(values[k]))
 		r = append(r, HStar2LeafHash{
 			Index:    new(big.Int).SetBytes(khash[:]),
 			LeafHash: vhash[:],
@@ -48,7 +48,7 @@ func createHStar2Leaves(th TreeHasher, values map[string]string) []HStar2LeafHas
 }
 
 func TestHStar2EmptyRootKAT(t *testing.T) {
-	s := NewHStar2(testonly.Hasher)
+	s := NewHStar2(maphasher.Default)
 	root, err := s.HStar2Root(s.hasher.Size()*8, []HStar2LeafHash{})
 	if err != nil {
 		t.Fatalf("Failed to calculate root: %v", err)
@@ -71,18 +71,19 @@ var simpleTestVector = []struct {
 }
 
 func TestHStar2SimpleDataSetKAT(t *testing.T) {
-	s := NewHStar2(testonly.Hasher)
+	s := NewHStar2(maphasher.Default)
 
 	m := make(map[string]string)
 	for i, x := range simpleTestVector {
 		m[x.k] = x.v
-		values := createHStar2Leaves(testonly.Hasher, m)
+		values := createHStar2Leaves(maphasher.Default, m)
 		root, err := s.HStar2Root(s.hasher.Size()*8, values)
 		if err != nil {
-			t.Fatalf("Failed to calculate root at iteration %d: %v", i, err)
+			t.Errorf("Failed to calculate root at iteration %d: %v", i, err)
+			continue
 		}
-		if expected, got := x.root, root; !bytes.Equal(expected, got) {
-			t.Fatalf("Expected root:\n%v\nGot:\n%v", base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(got))
+		if got, want := root, x.root; !bytes.Equal(got, want) {
+			t.Errorf("Root: \n%x, want:\n%x", got, want)
 		}
 	}
 }
@@ -95,10 +96,10 @@ func TestHStar2GetSet(t *testing.T) {
 	cache := make(map[string][]byte)
 
 	for i, x := range simpleTestVector {
-		s := NewHStar2(testonly.Hasher)
+		s := NewHStar2(maphasher.Default)
 		m := make(map[string]string)
 		m[x.k] = x.v
-		values := createHStar2Leaves(testonly.Hasher, m)
+		values := createHStar2Leaves(maphasher.Default, m)
 		// ensure we're going incrementally, one leaf at a time.
 		if len(values) != 1 {
 			t.Fatalf("Should only have 1 leaf per run, got %d", len(values))
@@ -112,18 +113,19 @@ func TestHStar2GetSet(t *testing.T) {
 				return nil
 			})
 		if err != nil {
-			t.Fatalf("Failed to calculate root at iteration %d: %v", i, err)
+			t.Errorf("Failed to calculate root at iteration %d: %v", i, err)
+			continue
 		}
-		if expected, got := x.root, root; !bytes.Equal(expected, got) {
-			t.Fatalf("Expected root:\n%v\nGot:\n%v", base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(got))
+		if got, want := root, x.root; !bytes.Equal(got, want) {
+			t.Errorf("Root:\n%x\n, want:\n%x", got, want)
 		}
 	}
 }
 
 // Checks that we calculate the same empty root hash as a 256-level tree has
 // when calculating top subtrees using an appropriate offset.
 func TestHStar2OffsetEmptyRootKAT(t *testing.T) {
-	s := NewHStar2(testonly.Hasher)
+	s := NewHStar2(maphasher.Default)
 
 	for size := 1; size < 255; size++ {
 		root, err := s.HStar2Nodes(size, s.hasher.Size()*8-size, []HStar2LeafHash{},
@@ -132,8 +134,8 @@ func TestHStar2OffsetEmptyRootKAT(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to calculate root %v", err)
 		}
-		if expected, got := sparseEmptyRootHashB64, root; !bytes.Equal(expected, got) {
-			t.Fatalf("Expected root:\n%v\nGot:\n%v", base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(got))
+		if got, want := root, sparseEmptyRootHashB64; !bytes.Equal(got, want) {
+			t.Fatalf("Got root:\n%x, want:\n%x", got, want)
 		}
 	}
 }
@@ -143,7 +145,7 @@ func TestHStar2OffsetEmptyRootKAT(t *testing.T) {
 // 256-prefixSize, and can be passed in as leaves to top-subtree calculation.
 func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HStar2LeafHash {
 	var ret []HStar2LeafHash
-	s := NewHStar2(testonly.Hasher)
+	s := NewHStar2(maphasher.Default)
 	for i := range lh {
 		prefix := new(big.Int).Rsh(lh[i].Index, uint(s.hasher.Size()*8-prefixSize))
 		b := lh[i].Index.Bytes()
@@ -165,7 +167,7 @@ func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HS
 // (single top subtree of size n, and multipl bottom subtrees of size 256-n)
 // still arrives at the same Known Answers for root hash.
 func TestHStar2OffsetRootKAT(t *testing.T) {
-	s := NewHStar2(testonly.Hasher)
+	s := NewHStar2(maphasher.Default)
 
 	m := make(map[string]string)
 
@@ -175,28 +177,29 @@ func TestHStar2OffsetRootKAT(t *testing.T) {
 		// requirement.
 		for size := 24; size < 256; size += 8 {
 			m[x.k] = x.v
-			intermediates := rootsForTrimmedKeys(t, size, createHStar2Leaves(testonly.Hasher, m))
+			intermediates := rootsForTrimmedKeys(t, size, createHStar2Leaves(maphasher.Default, m))
 
 			root, err := s.HStar2Nodes(size, s.hasher.Size()*8-size, intermediates,
 				func(int, *big.Int) ([]byte, error) { return nil, nil },
 				func(int, *big.Int, []byte) error { return nil })
 			if err != nil {
-				t.Fatalf("Failed to calculate root at iteration %d: %v", i, err)
+				t.Errorf("Failed to calculate root at iteration %d: %v", i, err)
+				continue
 			}
-			if expected, got := x.root, root; !bytes.Equal(expected, got) {
-				t.Fatalf("Expected root:\n%v\nGot:\n%v", base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(got))
+			if got, want := root, x.root; !bytes.Equal(got, want) {
+				t.Errorf("Root:\n%x\n, want:\n%x", got, want)
 			}
 		}
 	}
 }
 
 func TestHStar2NegativeTreeLevelOffset(t *testing.T) {
-	s := NewHStar2(testonly.Hasher)
+	s := NewHStar2(maphasher.Default)
 
 	_, err := s.HStar2Nodes(32, -1, []HStar2LeafHash{},
 		func(int, *big.Int) ([]byte, error) { return nil, nil },
 		func(int, *big.Int, []byte) error { return nil })
-	if expected, got := ErrNegativeTreeLevelOffset, err; expected != got {
-		t.Fatalf("expected %v, but got %v", expected, got)
+	if got, want := err, ErrNegativeTreeLevelOffset; got != want {
+		t.Fatalf("Hstar2Nodes(): %v, want %v", got, want)
 	}
 }

merkle/log_verifier.go

@@ -32,11 +32,11 @@ func (e RootMismatchError) Error() string {
 
 // LogVerifier verifies inclusion and consistency proofs for append only logs.
 type LogVerifier struct {
-	hasher TreeHasher
+	hasher LogHasher
 }
 
 // NewLogVerifier returns a new LogVerifier for a tree.
-func NewLogVerifier(hasher TreeHasher) LogVerifier {
+func NewLogVerifier(hasher LogHasher) LogVerifier {
 	return LogVerifier{
 		hasher: hasher,
 	}