Add Browser Support

bamboozle_unit_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcwallet/walletdb"
 	"github.com/lightninglabs/neutrino/headerfs"
+	_ "github.com/linden/tempdb"
 )
 
 func decodeHashNoError(str string) *chainhash.Hash {
@@ -525,9 +526,7 @@ func runCheckCFCheckptSanityTestCase(t *testing.T, testCase *cfCheckptTestCase)
 	}
 	defer os.RemoveAll(tempDir)
 
-	db, err := walletdb.Create(
-		"bdb", tempDir+"/weks.db", true, dbOpenTimeout,
-	)
+	db, err := walletdb.Create("tempdb", "weks.db")
 	if err != nil {
 		t.Fatalf("Error opening DB: %s", err)
 	}

banman/store_test.go

@@ -1,46 +1,30 @@
 package banman_test
 
 import (
-	"io/ioutil"
 	"net"
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/btcsuite/btcwallet/walletdb"
-	_ "github.com/btcsuite/btcwallet/walletdb/bdb"
 	"github.com/lightninglabs/neutrino/banman"
+	_ "github.com/linden/tempdb"
 )
 
 // createTestBanStore creates a test Store backed by a boltdb instance.
-func createTestBanStore(t *testing.T) (banman.Store, func()) {
+func createTestBanStore(t *testing.T) banman.Store {
 	t.Helper()
 
-	dbDir, err := ioutil.TempDir("", "")
+	db, err := walletdb.Create("tempdb", "test.db")
 	if err != nil {
-		t.Fatalf("unable to create db dir: %v", err)
-	}
-	dbPath := filepath.Join(dbDir, "test.db")
-
-	db, err := walletdb.Create("bdb", dbPath, true, time.Second*10)
-	if err != nil {
-		os.RemoveAll(dbDir)
 		t.Fatalf("unable to create db: %v", err)
 	}
 
-	cleanUp := func() {
-		db.Close()
-		os.RemoveAll(dbDir)
-	}
-
 	banStore, err := banman.NewStore(db)
 	if err != nil {
-		cleanUp()
 		t.Fatalf("unable to create ban store: %v", err)
 	}
 
-	return banStore, cleanUp
+	return banStore
 }
 
 // TestBanStore ensures that the BanStore's state correctly reflects the
@@ -50,8 +34,7 @@ func TestBanStore(t *testing.T) {
 
 	// We'll start by creating our test BanStore backed by a boltdb
 	// instance.
-	banStore, cleanUp := createTestBanStore(t)
-	defer cleanUp()
+	banStore := createTestBanStore(t)
 
 	// checkBanStore is a helper closure to ensure to the IP network's ban
 	// status is correctly reflected within the BanStore.

blockmanager_test.go

@@ -59,9 +59,7 @@ func setupBlockManager(t *testing.T) (*blockManager, headerfs.BlockHeaderStore,
 
 	// Set up the block and filter header stores.
 	tempDir := t.TempDir()
-	db, err := walletdb.Create(
-		"bdb", tempDir+"/weks.db", true, dbOpenTimeout,
-	)
+	db, err := walletdb.Create("tempdb", "weks.db")
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("error opening DB: %s", err)
 	}

filterdb/db_test.go

@@ -3,27 +3,19 @@ package filterdb
 import (
 	"math/rand"
 	"testing"
-	"time"
 
 	"github.com/btcsuite/btcd/btcutil/gcs"
 	"github.com/btcsuite/btcd/btcutil/gcs/builder"
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcwallet/walletdb"
-	_ "github.com/btcsuite/btcwallet/walletdb/bdb"
+	_ "github.com/linden/tempdb"
 	"github.com/stretchr/testify/require"
 )
 
 func createTestDatabase(t *testing.T) FilterDatabase {
-	tempDir := t.TempDir()
-
-	db, err := walletdb.Create(
-		"bdb", tempDir+"/test.db", true, time.Second*10,
-	)
+	db, err := walletdb.Create("tempdb", "test.db")
 	require.NoError(t, err)
-	t.Cleanup(func() {
-		require.NoError(t, db.Close())
-	})
 
 	filterDB, err := New(db, chaincfg.SimNetParams)
 	require.NoError(t, err)

go.mod

@@ -7,11 +7,13 @@ require (
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3
-	github.com/btcsuite/btcwallet/walletdb v1.3.5
+	github.com/btcsuite/btcwallet/walletdb v1.4.0
 	github.com/btcsuite/btcwallet/wtxmgr v1.5.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/lightninglabs/neutrino/cache v1.1.0
 	github.com/lightningnetwork/lnd/queue v1.0.1
+	github.com/linden/indexeddb v0.0.0-20240218035359-81389d584a5e
+	github.com/linden/tempdb v0.0.0-20240218031655-83bc03e79f51
 	github.com/stretchr/testify v1.8.1
 )
 
@@ -28,10 +30,11 @@ require (
 	github.com/lightningnetwork/lnd/clock v1.0.1 // indirect
 	github.com/lightningnetwork/lnd/ticker v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/sys v0.1.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-go 1.18
+go 1.21.2
+
+toolchain go1.21.5

go.sum

@@ -27,8 +27,9 @@ github.com/btcsuite/btcwallet/wallet/txrules v1.2.0 h1:BtEN5Empw62/RVnZ0VcJaVtVl
 github.com/btcsuite/btcwallet/wallet/txrules v1.2.0/go.mod h1:AtkqiL7ccKWxuLYtZm8Bu8G6q82w4yIZdgq6riy60z0=
 github.com/btcsuite/btcwallet/wallet/txsizes v1.1.0 h1:wZnOolEAeNOHzHTnznw/wQv+j35ftCIokNrnOTOU5o8=
 github.com/btcsuite/btcwallet/wallet/txsizes v1.1.0/go.mod h1:pauEU8UuMFiThe5PB3EO+gO5kx87Me5NvdQDsTuq6cs=
-github.com/btcsuite/btcwallet/walletdb v1.3.5 h1:SoxUPLgJUkyO1XqON6X7x+rjHJoIpRQov8o8X6gNoz8=
 github.com/btcsuite/btcwallet/walletdb v1.3.5/go.mod h1:oJDxAEUHVtnmIIBaa22wSBPTVcs6hUp5NKWmI8xDwwU=
+github.com/btcsuite/btcwallet/walletdb v1.4.0 h1:/C5JRF+dTuE2CNMCO/or5N8epsrhmSM4710uBQoYPTQ=
+github.com/btcsuite/btcwallet/walletdb v1.4.0/go.mod h1:oJDxAEUHVtnmIIBaa22wSBPTVcs6hUp5NKWmI8xDwwU=
 github.com/btcsuite/btcwallet/wtxmgr v1.5.0 h1:WO0KyN4l6H3JWnlFxfGR7r3gDnlGT7W2cL8vl6av4SU=
 github.com/btcsuite/btcwallet/wtxmgr v1.5.0/go.mod h1:TQVDhFxseiGtZwEPvLgtfyxuNUDsIdaJdshvWzR0HJ4=
 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
@@ -83,6 +84,10 @@ github.com/lightningnetwork/lnd/queue v1.0.1 h1:jzJKcTy3Nj5lQrooJ3aaw9Lau3I0IwvQ
 github.com/lightningnetwork/lnd/queue v1.0.1/go.mod h1:vaQwexir73flPW43Mrm7JOgJHmcEFBWWSl9HlyASoms=
 github.com/lightningnetwork/lnd/ticker v1.0.0 h1:S1b60TEGoTtCe2A0yeB+ecoj/kkS4qpwh6l+AkQEZwU=
 github.com/lightningnetwork/lnd/ticker v1.0.0/go.mod h1:iaLXJiVgI1sPANIF2qYYUJXjoksPNvGNYowB8aRbpX0=
+github.com/linden/indexeddb v0.0.0-20240218035359-81389d584a5e h1:6FTMUiW0wI+my/+7w4CEEYt5zvGmbAjugt2q/fCUyBM=
+github.com/linden/indexeddb v0.0.0-20240218035359-81389d584a5e/go.mod h1:2AP38q2ks+pRwMc2EhmCexlyri456KCewavN30MZQ8Y=
+github.com/linden/tempdb v0.0.0-20240218031655-83bc03e79f51 h1:RfZREkHD3XpItaIN2I1/tb2hCzE2TN5e14OkTH6Sv74=
+github.com/linden/tempdb v0.0.0-20240218031655-83bc03e79f51/go.mod h1:xR9HUmc4girdp/lNzw1jOt53GaCSmctyB8t+Q6EkWp8=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

headerfs/file.go

@@ -1,18 +1,211 @@
+//go:build !windows && !js && !wasm
+
 package headerfs
 
 import (
 	"bytes"
 	"fmt"
 	"os"
+	"path/filepath"
+	"sync"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcwallet/walletdb"
 )
 
-// ErrHeaderNotFound is returned when a target header on disk (flat file) can't
-// be found.
-type ErrHeaderNotFound struct {
-	error
+// headerBufPool is a pool of bytes.Buffer that will be re-used by the various
+// headerStore implementations to batch their header writes to disk. By
+// utilizing this variable we can minimize the total number of allocations when
+// writing headers to disk.
+var headerBufPool = sync.Pool{
+	New: func() interface{} { return new(bytes.Buffer) },
+}
+
+// headerStore combines a on-disk set of headers within a flat file in addition
+// to a database which indexes that flat file. Together, these two abstractions
+// can be used in order to build an indexed header store for any type of
+// "header" as it deals only with raw bytes, and leaves it to a higher layer to
+// interpret those raw bytes accordingly.
+//
+// TODO(roasbeef): quickcheck coverage.
+type headerStore struct {
+	mtx sync.RWMutex // nolint:structcheck // false positive because used as embedded struct only
+
+	fileName string
+
+	file *os.File
+
+	hType HeaderType
+
+	*headerIndex
+}
+
+// newHeaderStore creates a new headerStore given an already open database, a
+// target file path for the flat-file and a particular header type. The target
+// file will be created as necessary.
+func newHeaderStore(db walletdb.DB, filePath string,
+	hType HeaderType) (*headerStore, error) {
+
+	var flatFileName string
+	switch hType {
+	case Block:
+		flatFileName = "block_headers.bin"
+	case RegularFilter:
+		flatFileName = "reg_filter_headers.bin"
+	default:
+		return nil, fmt.Errorf("unrecognized filter type: %v", hType)
+	}
+
+	flatFileName = filepath.Join(filePath, flatFileName)
+
+	// We'll open the file, creating it if necessary and ensuring that all
+	// writes are actually appends to the end of the file.
+	fileFlags := os.O_RDWR | os.O_APPEND | os.O_CREATE
+	headerFile, err := os.OpenFile(flatFileName, fileFlags, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	// With the file open, we'll then create the header index so we can
+	// have random access into the flat files.
+	index, err := newHeaderIndex(db, hType)
+	if err != nil {
+		return nil, err
+	}
+
+	return &headerStore{
+		fileName:    flatFileName,
+		file:        headerFile,
+		hType:       hType,
+		headerIndex: index,
+	}, nil
+}
+
+// WriteHeaders writes a set of headers to disk and updates the index in a
+// single atomic transaction.
+//
+// NOTE: Part of the BlockHeaderStore interface.
+func (h *blockHeaderStore) WriteHeaders(hdrs ...BlockHeader) error {
+	// Lock store for write.
+	h.mtx.Lock()
+	defer h.mtx.Unlock()
+
+	// First, we'll grab a buffer from the write buffer pool so we an
+	// reduce our total number of allocations, and also write the headers
+	// in a single swoop.
+	headerBuf := headerBufPool.Get().(*bytes.Buffer)
+	headerBuf.Reset()
+	defer headerBufPool.Put(headerBuf)
+
+	// Next, we'll write out all the passed headers in series into the
+	// buffer we just extracted from the pool.
+	for _, header := range hdrs {
+		if err := header.Serialize(headerBuf); err != nil {
+			return err
+		}
+	}
+
+	// With all the headers written to the buffer, we'll now write out the
+	// entire batch in a single write call.
+	if err := h.appendRaw(headerBuf.Bytes()); err != nil {
+		return err
+	}
+
+	// Once those are written, we'll then collate all the headers into
+	// headerEntry instances so we can write them all into the index in a
+	// single atomic batch.
+	headerLocs := make([]headerEntry, len(hdrs))
+	for i, header := range hdrs {
+		headerLocs[i] = header.toIndexEntry()
+	}
+
+	return h.addHeaders(headerLocs)
+}
+
+// WriteHeaders writes a batch of filter headers to persistent storage. The
+// headers themselves are appended to the flat file, and then the index updated
+// to reflect the new entires.
+func (f *FilterHeaderStore) WriteHeaders(hdrs ...FilterHeader) error {
+	// Lock store for write.
+	f.mtx.Lock()
+	defer f.mtx.Unlock()
+
+	// If there are 0 headers to be written, return immediately. This
+	// prevents the newTip assignment from panicking because of an index
+	// of -1.
+	if len(hdrs) == 0 {
+		return nil
+	}
+
+	// First, we'll grab a buffer from the write buffer pool so we an
+	// reduce our total number of allocations, and also write the headers
+	// in a single swoop.
+	headerBuf := headerBufPool.Get().(*bytes.Buffer)
+	headerBuf.Reset()
+	defer headerBufPool.Put(headerBuf)
+
+	// Next, we'll write out all the passed headers in series into the
+	// buffer we just extracted from the pool.
+	for _, header := range hdrs {
+		if _, err := headerBuf.Write(header.FilterHash[:]); err != nil {
+			return err
+		}
+	}
+
+	// With all the headers written to the buffer, we'll now write out the
+	// entire batch in a single write call.
+	if err := f.appendRaw(headerBuf.Bytes()); err != nil {
+		return err
+	}
+
+	// As the block headers should already be written, we only need to
+	// update the tip pointer for this particular header type.
+	newTip := hdrs[len(hdrs)-1].toIndexEntry().hash
+	return f.truncateIndex(&newTip, false)
+}
+
+// Remove the file.
+func (h *headerStore) Remove() error {
+	// Close the file before removing it. This is required by some
+	// OS, e.g., Windows.
+	if err := h.file.Close(); err != nil {
+		return err
+	}
+	if err := os.Remove(h.fileName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Calculate the current height.
+func (h *headerStore) height() (uint32, bool, error) {
+	fileInfo, err := h.file.Stat()
+	if err != nil {
+		return 0, false, err
+	}
+
+	size := fileInfo.Size()
+
+	// Check if the file is empty. Fallback to a height of zero.
+	if size == 0 {
+		return 0, true, nil
+	}
+
+	var fileHeight uint32
+
+	// Compute the size of the current file so we can
+	// calculate the latest header written to disk.
+	switch h.hType {
+	case Block:
+		fileHeight = uint32(size/80) - 1
+
+	case RegularFilter:
+		fileHeight = uint32(size/32) - 1
+	}
+
+	return fileHeight, false, nil
 }
 
 // appendRaw appends a new raw header to the end of the flat file.