Merge pull request #73 from ipfs/fix/skip-non-directories

Windows CI + Fixes

.circleci/config.yml

@@ -1,6 +1,16 @@
 version: 2.1
 orbs:
   ci-go: ipfs/[email protected]
+  win: circleci/[email protected]
+
+jobs:
+  "windows-test":
+    executor: win/default
+    steps:
+      - checkout
+      - run:
+          name: "Go test"
+          command: go test .
 
 workflows:
   version: 2
@@ -9,3 +19,4 @@ workflows:
       - ci-go/build
       - ci-go/lint
       - ci-go/test
+      - windows-test

convert_test.go

@@ -7,11 +7,12 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
 	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-ds-flatfs"
+	flatfs "github.com/ipfs/go-ds-flatfs"
 )
 
 func TestMove(t *testing.T) {
@@ -57,6 +58,9 @@ func TestMove(t *testing.T) {
 }
 
 func TestMoveRestart(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	tempdir, cleanup := tempdir(t)
 	defer cleanup()
 

flatfs.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math"
 	"math/rand"
 	"os"
@@ -241,12 +240,12 @@ func Open(path string, syncFiles bool) (*Datastore, error) {
 	tempPath := filepath.Join(path, ".temp")
 	err = os.RemoveAll(tempPath)
 	if err != nil && !os.IsNotExist(err) {
-		return nil, fmt.Errorf("failed to remove temporary directory: %w", err)
+		return nil, fmt.Errorf("failed to remove temporary directory: %v", err)
 	}
 
 	err = os.Mkdir(tempPath, 0755)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create temporary directory: %w", err)
+		return nil, fmt.Errorf("failed to create temporary directory: %v", err)
 	}
 
 	shardId, err := ReadShardFunc(path)
@@ -380,7 +379,7 @@ var putMaxRetries = 6
 // will win.
 func (fs *Datastore) Put(key datastore.Key, value []byte) error {
 	if !keyIsValid(key) {
-		return fmt.Errorf("when putting '%q': %w", key, ErrInvalidKey)
+		return fmt.Errorf("when putting '%q': %v", key, ErrInvalidKey)
 	}
 
 	fs.shutdownLock.RLock()
@@ -608,7 +607,7 @@ func (fs *Datastore) Get(key datastore.Key) (value []byte, err error) {
 	}
 
 	_, path := fs.encode(key)
-	data, err := ioutil.ReadFile(path)
+	data, err := readFile(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, datastore.ErrNotFound
@@ -734,11 +733,15 @@ func (fs *Datastore) walkTopLevel(path string, result *query.ResultBuilder) erro
 		return err
 	}
 	defer dir.Close()
-	names, err := dir.Readdirnames(-1)
+	entries, err := dir.Readdir(-1)
 	if err != nil {
 		return err
 	}
-	for _, dir := range names {
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		dir := entry.Name()
 		if len(dir) == 0 || dir[0] == '.' {
 			continue
 		}
@@ -1019,7 +1022,7 @@ func (fs *Datastore) writeDiskUsageFile(du int64, doSync bool) {
 // readDiskUsageFile is only safe to call in Open()
 func (fs *Datastore) readDiskUsageFile() int64 {
 	fpath := filepath.Join(fs.path, DiskUsageFile)
-	duB, err := ioutil.ReadFile(fpath)
+	duB, err := readFile(fpath)
 	if err != nil {
 		return 0
 	}
@@ -1056,10 +1059,11 @@ func (fs *Datastore) Accuracy() string {
 }
 
 func (fs *Datastore) tempFile() (*os.File, error) {
-	file, err := ioutil.TempFile(fs.tempPath, "temp-")
+	file, err := tempFile(fs.tempPath, "temp-")
 	return file, err
 }
 
+// only call this on directories.
 func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error {
 	dir, err := os.Open(path)
 	if err != nil {
@@ -1071,15 +1075,6 @@ func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error {
 	}
 	defer dir.Close()
 
-	// ignore non-directories
-	fileInfo, err := dir.Stat()
-	if err != nil {
-		return err
-	}
-	if !fileInfo.IsDir() {
-		return nil
-	}
-
 	names, err := dir.Readdirnames(-1)
 	if err != nil {
 		return err
@@ -1099,7 +1094,7 @@ func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error {
 		var result query.Result
 		result.Key = key.String()
 		if !qrb.Query.KeysOnly {
-			value, err := ioutil.ReadFile(filepath.Join(path, fn))
+			value, err := readFile(filepath.Join(path, fn))
 			if err != nil {
 				result.Error = err
 			} else {
@@ -1163,7 +1158,7 @@ func (fs *Datastore) Batch() (datastore.Batch, error) {
 
 func (bt *flatfsBatch) Put(key datastore.Key, val []byte) error {
 	if !keyIsValid(key) {
-		return fmt.Errorf("when putting '%q': %w", key, ErrInvalidKey)
+		return fmt.Errorf("when putting '%q': %v", key, ErrInvalidKey)
 	}
 	bt.puts[key] = val
 	return nil

go.mod

@@ -1,6 +1,7 @@
 module github.com/ipfs/go-ds-flatfs
 
 require (
+	github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
 	github.com/ipfs/go-datastore v0.4.4
 	github.com/ipfs/go-log v1.0.3
 	github.com/jbenet/goprocess v0.1.4

go.sum

@@ -1,3 +1,5 @@
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=

util_unix.go

@@ -0,0 +1,16 @@
+// +build !windows
+
+package flatfs
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+func tempFile(dir, pattern string) (*os.File, error) {
+	return ioutil.TempFile(dir, pattern)
+}
+
+func readFile(filename string) ([]byte, error) {
+	return ioutil.ReadFile(filename)
+}

util_windows.go

@@ -0,0 +1,101 @@
+// +build windows
+
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Note: This file is a variant of a subset of the golang standard library
+// src/io/ioutil/tempfile.go
+// with calls to os.Open replaced with the goissue34681.Open variant.
+
+package flatfs
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	goissue34681 "github.com/alexbrainman/goissue34681"
+)
+
+var tmpRand uint32
+var randmu sync.Mutex
+
+func reseed() uint32 {
+	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
+}
+
+func nextRandom() string {
+	randmu.Lock()
+	r := tmpRand
+	if r == 0 {
+		r = reseed()
+	}
+	r = r*1664525 + 1013904223 // constants from Numerical Recipes
+	tmpRand = r
+	randmu.Unlock()
+	return strconv.Itoa(int(1e9 + r%1e9))[1:]
+}
+
+func prefixAndSuffix(pattern string) (prefix, suffix string) {
+	if pos := strings.LastIndex(pattern, "*"); pos != -1 {
+		prefix, suffix = pattern[:pos], pattern[pos+1:]
+	} else {
+		prefix = pattern
+	}
+	return
+}
+
+func tempFile(dir, pattern string) (f *os.File, err error) {
+	if dir == "" {
+		dir = os.TempDir()
+	}
+
+	prefix, suffix := prefixAndSuffix(pattern)
+
+	nconflict := 0
+	for i := 0; i < 10000; i++ {
+		name := filepath.Join(dir, prefix+nextRandom()+suffix)
+		f, err = goissue34681.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
+		if os.IsExist(err) {
+			if nconflict++; nconflict > 10 {
+				randmu.Lock()
+				tmpRand = reseed()
+				randmu.Unlock()
+			}
+			continue
+		}
+		break
+	}
+	return
+}
+
+func readFile(filename string) ([]byte, error) {
+	f, err := goissue34681.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	// It's a good but not certain bet that FileInfo will tell us exactly how much to
+	// read, so let's try it but be prepared for the answer to be wrong.
+	var n int64 = bytes.MinRead
+
+	if fi, err := f.Stat(); err == nil {
+		// As initial capacity for readAll, use Size + a little extra in case Size
+		// is zero, and to avoid another allocation after Read has filled the
+		// buffer. The readAll call will read into its allocated internal buffer
+		// cheaply. If the size was wrong, we'll either waste some space off the end
+		// or reallocate as needed, but in the overwhelmingly common case we'll get
+		// it just right.
+		if size := fi.Size() + bytes.MinRead; size > n {
+			n = size
+		}
+	}
+
+	return ioutil.ReadAll(f)
+}