Merge pull request #141 from CortexFoundation/dev

sqlite storage

common.go

@@ -21,6 +21,9 @@ import (
 )
 
 func ProgressBar(x, y int64, desc string) string {
+	if y == 0 {
+		return "[            ] 0%"
+	}
 	progress := ""
 	for i := 10; i > 0; i-- {
 		if int64(i) > (10*x)/y {
@@ -31,6 +34,6 @@ func ProgressBar(x, y int64, desc string) string {
 	}
 
 	prog := float64(x*100) / float64(y)
-	f := strconv.FormatFloat(prog, 'f', 2, 64)
+	f := strconv.FormatFloat(prog, 'f', 4, 64)
 	return "[ " + progress + " ] " + f + "% " + desc
 }

go.mod

@@ -8,10 +8,10 @@ require (
 	github.com/CortexFoundation/merkletree v0.0.0-20210226031755-94f6423532ee
 	github.com/allegro/bigcache/v3 v3.0.0
 	github.com/anacrolix/envpprof v1.1.1
-	github.com/anacrolix/log v0.8.0
+	github.com/anacrolix/log v0.9.0
 	github.com/anacrolix/missinggo/v2 v2.5.0
-	github.com/anacrolix/tagflag v1.2.0
-	github.com/anacrolix/torrent v1.26.0-alpha.1.0.20210324225604-615ac41ae3e0
+	github.com/anacrolix/tagflag v1.3.0
+	github.com/anacrolix/torrent v1.28.1-0.20210517015610-b21aebeaae6a
 	github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dustin/go-humanize v1.0.0

handler.go

@@ -322,6 +322,19 @@ func (tm *TorrentManager) addInfoHash(ih string, BytesRequested int64, ch chan b
 		spec = tm.loadSpec(ih, tmpTorrentPath)
 	}
 
+	/*if _, err := os.Stat(filepath.Join(tm.TmpDataDir, ih, ".torrent.db")); err != nil {
+		os.Symlink(
+			filepath.Join(tm.DataDir, ".torrent.db"),
+			filepath.Join(tm.TmpDataDir, ih, ".torrent.db"),
+		)
+	}
+	if _, err := os.Stat(filepath.Join(tm.TmpDataDir, ih, ".torrent.bolt.db")); err != nil {
+		os.Symlink(
+			filepath.Join(tm.DataDir, ".torrent.bolt.db"),
+			filepath.Join(tm.TmpDataDir, ih, ".torrent.bolt.db"),
+		)
+	}*/
+
 	//if spec == nil {
 	/*if tm.boost {
 		if data, err := tm.boostFetcher.FetchTorrent(ih.String()); err == nil {
@@ -371,6 +384,29 @@ func (tm *TorrentManager) addInfoHash(ih string, BytesRequested int64, ch chan b
 
 	if spec == nil {
 		tmpDataPath := filepath.Join(tm.TmpDataDir, ih)
+
+		//if _, err := os.Stat(filepath.Join(tmpDataPath, ".torrent.db")); err != nil {
+		if _, err := os.Stat(tmpDataPath); err != nil {
+			if err := os.MkdirAll(tmpDataPath, 0600); err != nil {
+				log.Warn("torrent path create failed", "err", err)
+				return nil
+			}
+		}
+
+		//if _, err := os.Stat(filepath.Join(tmpDataPath, ".torrent.db")); err != nil {
+		//	os.Symlink(
+		//		filepath.Join(tm.DataDir, ".torrent.db"),
+		//		filepath.Join(tmpDataPath, ".torrent.db"),
+		//	)
+		//}
+
+		//if _, err := os.Stat(filepath.Join(tmpDataPath, ".torrent.bolt.db")); err != nil {
+		//	os.Symlink(
+		//		filepath.Join(tm.DataDir, ".torrent.bolt.db"),
+		//		filepath.Join(tmpDataPath, ".torrent.bolt.db"),
+		//	)
+		//}
+
 		spec = &torrent.TorrentSpec{
 			InfoHash: metainfo.NewHashFromHex(ih),
 			Storage:  storage.NewFile(tmpDataPath),
@@ -439,7 +475,7 @@ func NewTorrentManager(config *Config, fsid uint64, cache, compress bool) (*Torr
 	tmpFilePath := filepath.Join(config.DataDir, defaultTmpPath)
 
 	if _, err := os.Stat(tmpFilePath); err != nil {
-		err = os.MkdirAll(filepath.Dir(tmpFilePath), 0770) //os.FileMode(os.ModePerm))
+		err = os.MkdirAll(filepath.Dir(tmpFilePath), 0600) //os.FileMode(os.ModePerm))
 		if err != nil {
 			log.Error("Mkdir failed", "path", tmpFilePath)
 			return nil, err

params/chain.go

@@ -44,7 +44,7 @@ var (
 
 		TfsRoot: common.HexToHash("0xe78706dbcc1f853336a31a3e3f55dcb3d0d082fb8fd4b4b273fe859d657e5dcc"),
 		Skips: []Skip{
-			{From: 1039, To: 4468}, {From: 5515, To: 10888}, {From: 10888, To: 137784}, {From: 137790, To: 156609}, {From: 160264, To: 395088}, {From: 395964, To: 1261969}, {From: 1261969, To: 1988320}, {From: 1988320, To: 2107911}, {From: 2108043, To: 2270725}, {From: 2270777, To: 2754916}, {From: 2755628, To: 3211264},
+			{From: 1039, To: 4468}, {From: 5515, To: 10888}, {From: 10888, To: 137784}, {From: 137790, To: 156609}, {From: 160264, To: 395088}, {From: 395964, To: 1261969}, {From: 1261969, To: 1988320}, {From: 1988320, To: 2107911}, {From: 2108043, To: 2270725}, {From: 2270777, To: 2754916}, {From: 2755628, To: 3801088},
 		},
 	}
 )

sync.go

@@ -577,27 +577,40 @@ func (m *Monitor) syncLatestBlock() {
 	timer := time.NewTimer(time.Second * queryTimeInterval)
 	defer timer.Stop()
 	progress := uint64(0)
+	end := false
 	for {
 		select {
 		case <-timer.C:
 			progress = m.syncLastBlock()
 			// Avoid sync in full mode, fresh interval may be less.
 			if progress >= delay {
-				timer.Reset(0)
-			} else if progress > 1 {
-				timer.Reset(time.Millisecond * 1000)
+				//timer.Reset(0)
+				end = false
+				timer.Reset(time.Millisecond * 2000)
+			} else if progress >= 1 {
+				end = false
+				timer.Reset(time.Millisecond * 3000)
 			} else {
 				if !m.listen {
 					if (m.ckp != nil && m.currentNumber >= m.ckp.TfsCheckPoint) || (m.ckp == nil && m.currentNumber > 0) {
+						if !end {
+							end = true
+							timer.Reset(time.Millisecond * 6000)
+							continue
+						}
 						m.fs.Flush()
-						go m.exit()
+						//go m.exit()
 						elapsed := time.Duration(mclock.Now()) - time.Duration(m.start)
-						log.Warn("Finish sync, listener will be stopped", "current", m.currentNumber, "elapsed", common.PrettyDuration(elapsed))
-						return
+						log.Warn("Finish sync, listener will be paused", "current", m.currentNumber, "elapsed", common.PrettyDuration(elapsed), "ckp", m.ckp.TfsCheckPoint, "progress", progress, "end", end)
+						//return
+						timer.Reset(time.Millisecond * 1000 * 300)
+						end = false
+						continue
 					}
 				}
-				timer.Reset(time.Millisecond * 2000)
+				timer.Reset(time.Millisecond * 6000)
 			}
+			log.Info(ProgressBar(int64(m.lastNumber), int64(m.currentNumber), ""), "blocks", progress, "current", m.currentNumber, "latest", m.lastNumber, "end", end)
 			m.fs.Flush()
 		case <-m.exitCh:
 			log.Debug("Block syncer stopped")

torrent.go

@@ -65,14 +65,14 @@ func (t *Torrent) InfoHash() string {
 
 func (t *Torrent) ReloadFile(files []string, datas [][]byte, tm *TorrentManager) {
 	if len(files) > 1 {
-		err := os.MkdirAll(filepath.Dir(filepath.Join(t.filepath, "data")), 0750) //os.ModePerm)
+		err := os.MkdirAll(filepath.Dir(filepath.Join(t.filepath, "data")), 0600) //os.ModePerm)
 		if err != nil {
 			return
 		}
 	}
 	for i, filename := range files {
 		filePath := filepath.Join(t.filepath, filename)
-		f, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0660)
+		f, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
 		if err != nil {
 			return
 		}
@@ -138,7 +138,7 @@ func (t *Torrent) WriteTorrent() error {
 		return nil
 	}
 
-	if f, err := os.OpenFile(filepath.Join(t.filepath, "torrent"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0660); err == nil {
+	if f, err := os.OpenFile(filepath.Join(t.filepath, "torrent"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600); err == nil {
 		defer f.Close()
 		log.Debug("Write seed file", "path", t.filepath)
 		if err := t.Metainfo().Write(f); err == nil {