From 1df6564baf5dabeaef67fc55babf2f7d5aa5d906 Mon Sep 17 00:00:00 2001 From: Tug Witt Date: Sun, 12 Feb 2023 23:22:58 -0800 Subject: [PATCH] remove verthash --- README.md | 9 +- verthash/README.md | 3 - verthash/client.go | 57 ---- verthash/graph.go | 598 -------------------------------------- verthash/verthash.go | 63 ---- verthash/verthash_test.go | 33 --- 6 files changed, 3 insertions(+), 760 deletions(-) delete mode 100644 verthash/README.md delete mode 100644 verthash/client.go delete mode 100644 verthash/graph.go delete mode 100644 verthash/verthash.go delete mode 100644 verthash/verthash_test.go diff --git a/README.md b/README.md index e83812f..ec1b60b 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,9 @@ but finding the exact differences is painful. This is meant to be a unified libr make the specification of existing Proof of Work algorithms more standardized. All DAG-based algorithms only implement a light DAG, which is sufficient for validation -but not full nodes or miners. For the DAG-based algorithms and verthash, data is cached in `~/.powcache`. -Ethash will generally be between 40-80Mb per epoch (and generally 3 caches are stored), but verthash -requires a strict 1.2Gb, so be careful if you're using verthash in memory. At the time of writing, running -`make test` will throw about 3.3Gb of data into `~/.powcache` due to the variety and breadth of tests. +but not full nodes or miners. For the DAG-based algorithms, data is cached in `~/.powcache`. +Ethash will generally be between 40-80Mb per epoch (and generally 3 caches are stored). At the time of writing, running +`make test` will throw about 800Mb of data into `~/.powcache` due to the variety and breadth of tests. # Algorithms @@ -28,7 +27,6 @@ requires a strict 1.2Gb, so be careful if you're using verthash in memory. At th | Kawpow | yes | yes | Firopow | yes | yes | Octopus | yes | yes -| Verthash | yes | yes | Equihash | no | yes | HeavyHash | no | yes | Autolykos2 | no | yes @@ -62,7 +60,6 @@ will probably be pretty set in stone. - [Ethereum Classic Labs: go-etchash](https://github.com/etclabscore/go-etchash) - [RavencoinCommunity: cpp-kawpow](https://github.com/RavenCommunity/cpp-kawpow/) - [Zcash: librustzcash (equihash)](https://github.com/zcash/librustzcash/tree/master/components/equihash) - - [Gert-Jaap Glasbergen: verthash-go](https://github.com/gertjaap/verthash-go/) - [Firo: firo](https://github.com/firoorg/firo/tree/master/src/crypto/progpow) - [Ergo: ergo](https://github.com/ergoplatform/ergo/blob/0af9dd9d8846d672c1e2a77f8ab29963fa5acd1e/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala) - [leifjacky: erg-gominer-demo](https://github.com/leifjacky/erg-gominer-demo) diff --git a/verthash/README.md b/verthash/README.md deleted file mode 100644 index 4885d39..0000000 --- a/verthash/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Verthash - -Most of this was borrowed from [verthash-go](https://github.com/gertjaap/verthash-go). Verthash requires a 1.2Gb file to be generated (in `~/.powcache`). \ No newline at end of file diff --git a/verthash/client.go b/verthash/client.go deleted file mode 100644 index 49becfe..0000000 --- a/verthash/client.go +++ /dev/null @@ -1,57 +0,0 @@ -package verthash - -import ( - "io/ioutil" - "os" - "path/filepath" - - "github.com/sencha-dev/powkit/internal/common" -) - -const ( - // graph constants - hashSize = 32 - nodeSize = hashSize - datasetSize = 1283457024 - - // verthash constants - verthashHeaderSize uint32 = 80 - verthashHashOutSize uint32 = 32 - verthashP0Size uint32 = 64 - verthashIter uint32 = 8 - verthashSubset uint32 = verthashP0Size * verthashIter - verthashRotations uint32 = 32 - verthashIndexes uint32 = 4096 - verthashByteAlignment uint32 = 16 -) - -type Client struct { - data []byte -} - -func New() (*Client, error) { - path := filepath.Join(common.DefaultDir(".powcache"), "verthash.dat") - if _, err := os.Stat(path); os.IsNotExist(err) { - writeGraph(path) - } - - err := validateGraph(path) - if err != nil { - return nil, err - } - - data, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - c := &Client{ - data: data, - } - - return c, nil -} - -func (c *Client) Compute(input []byte) []byte { - return verthash(c.data, input) -} diff --git a/verthash/graph.go b/verthash/graph.go deleted file mode 100644 index 1ee0fd6..0000000 --- a/verthash/graph.go +++ /dev/null @@ -1,598 +0,0 @@ -package verthash - -import ( - "bytes" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "fmt" - "io/ioutil" - "os" - "time" - - "golang.org/x/crypto/sha3" -) - -func log2(x int64) int64 { - r := int64(0) - for ; x > 1; x >>= 1 { - r++ - } - - return r -} - -type node struct { - H []byte // hash at the file -} - -func (n *node) MarshalBinary() ([]byte, error) { - return n.H, nil -} - -func (n *node) UnmarshalBinary(data []byte) error { - n.H = data - return nil -} - -type graph struct { - pk []byte - fn string - db *os.File - index int64 - log2 int64 - pow2 int64 - size int64 -} - -func writeGraph(path string) { - pk := sha3.Sum256([]byte("Verthash Proof-of-Space Datafile")) - newGraph(17, path, pk[:], nil) -} - -func validateGraph(path string) error { - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - if size := len(b); size != datasetSize { - return fmt.Errorf("verthash graph size mismatch: have %d want %d", size, datasetSize) - } - - hash := sha256.Sum256(b) - expectedHash, _ := hex.DecodeString("a55531e843cd56b010114aaf6325b0d529ecf88f8ad47639b6ededafd721aa48") - if !bytes.Equal(hash[:], expectedHash) { - return fmt.Errorf("verthash graph checksum hash mismatch: have %x want %x", hash, expectedHash) - } - - return nil -} - -// Generate a new PoS graph of index -// Currently only supports the weaker PoS graph -// Note that this graph will have O(2^index) nodes -func newGraph(index int64, fn string, pk []byte, progress chan float64) *graph { - var db *os.File - _, err := os.Stat(fn) - fileExists := err == nil - if fileExists { //file exists - db, err = os.OpenFile(fn, os.O_RDWR, 0666) - if err != nil { - panic(err) - } - } else { - db, err = os.Create(fn) - if err != nil { - panic(err) - } - } - - size := numXi(index) - log2 := log2(size) + 1 - pow2 := int64(1 << log2) - - g := &graph{ - pk: pk, - fn: fn, - db: db, - index: index, - log2: log2, - size: size, - pow2: pow2, - } - - done := make(chan bool, 1) - if !fileExists { - go func() { - g.xiGraphIter(index) - done <- true - }() - - for { - genDone := false - select { - case genDone = <-done: - case <-time.After(1 * time.Second): - } - - if genDone { - break - } - if progress != nil { - s, err := g.db.Stat() - if err == nil { - progress <- float64(s.Size()) / float64(1283457024) - } - } - } - - } - - return g -} - -// compute parents of nodes -func (g *graph) getParents(node, index int64) []int64 { - if node < int64(1< (index - 1) { - shift = level - (index - 1) - } - i := (node - begin) % pow2index_1 - if (i>>uint64(shift))&1 == 0 { - prev = i + (1 << uint64(shift)) - } else { - prev = i - (1 << uint64(shift)) - } - parent0 := begin + (level-1)*pow2index_1 + prev - parent1 := node - pow2index_1 - return parent0, parent1 -} - -// get graph that node belongs to, so i can find the parents -func (g *graph) getGraph(node, index int64) (int64, int64) { - if index == 1 { - if node < 2 { - return 2, 0 - } else if node == 2 { - return 1, 2 - } else if node == 3 { - return 3, 2 - } - } - - pow2index := int64(1 << uint64(index)) - pow2index_1 := int64(1 << uint64(index-1)) - sources := pow2index - firstButter := sources + numButterfly(index-1) - firstXi := firstButter + numXi(index-1) - secondXi := firstXi + numXi(index-1) - secondButter := secondXi + numButterfly(index-1) - sinks := secondButter + sources - - if node < sources { - return pow2index, 0 - } else if node >= sources && node < firstButter { - if node < sources+pow2index_1 { - return pow2index, pow2index_1 - } else { - parent0, parent1 := g.butterflyParents(sources, node, index) - return node - parent0, node - parent1 - } - } else if node >= firstButter && node < firstXi { - node = node - firstButter - return g.getGraph(node, index-1) - } else if node >= firstXi && node < secondXi { - node = node - firstXi - return g.getGraph(node, index-1) - } else if node >= secondXi && node < secondButter { - if node < secondXi+pow2index_1 { - return pow2index_1, 0 - } else { - parent0, parent1 := g.butterflyParents(secondXi, node, index) - return node - parent0, node - parent1 - } - } else if node >= secondButter && node < sinks { - offset := (node - secondButter) % pow2index_1 - parent1 := sinks - numXi(index) + offset - if offset+secondButter == node { - return pow2index_1, node - parent1 - } else { - return pow2index, node - parent1 - pow2index_1 - } - } else { - return 0, 0 - } -} - -func (g *graph) newNodeById(id int64, hash []byte) { - node := &node{ - H: hash, - } - g.writeId(node, id) -} - -func (g *graph) newNode(id int64, hash []byte) { - node := &node{ - H: hash, - } - g.writeNode(node, id) -} - -func (g *graph) getId(id int64) *node { - //fmt.Println("read id", id) - node := new(node) - data := make([]byte, nodeSize) - num, err := g.db.ReadAt(data, id*nodeSize) - if err != nil || num != nodeSize { - panic(err) - } - node.H = data - return node -} - -func (g *graph) writeId(node *node, id int64) { - //fmt.Println("write id", id) - num, err := g.db.WriteAt(node.H, id*nodeSize) - if err != nil || num != nodeSize { - panic(err) - } -} - -func (g *graph) getNode(id int64) *node { - idx := g.bfsToPost(id) - //fmt.Println("read", idx) - return g.getId(idx) -} - -func (g *graph) writeNode(node *node, id int64) { - idx := g.bfsToPost(id) - //fmt.Println("write", idx) - g.writeId(node, idx) -} - -func (g *graph) close() { - g.db.Close() -} - -// post-order is better for disk than bfs~ -func (g *graph) bfsToPost(node int64) int64 { - return node & (^(g.pow2)) -} - -func numXi(index int64) int64 { - return (1 << uint64(index)) * (index + 1) * index -} - -func numButterfly(index int64) int64 { - return 2 * (1 << uint64(index)) * index -} - -func (g *graph) butterflyGraph(index int64, count *int64) { - if index == 0 { - index = 1 - } - numLevel := 2 * index - perLevel := int64(1 << uint64(index)) - begin := *count - perLevel // level 0 created outside - // no parents at level 0 - var level, i int64 - for level = 1; level < numLevel; level++ { - for i = 0; i < perLevel; i++ { - var prev int64 - shift := index - level - if level > numLevel/2 { - shift = level - numLevel/2 - } - if (i>>uint64(shift))&1 == 0 { - prev = i + (1 << uint64(shift)) - } else { - prev = i - (1 << uint64(shift)) - } - parent0 := g.getNode(begin + (level-1)*perLevel + prev) - parent1 := g.getNode(*count - perLevel) - - ph := append(parent0.H, parent1.H...) - buf := make([]byte, hashSize) - binary.PutVarint(buf, *count) - val := append(g.pk, buf...) - val = append(val, ph...) - hash := sha3.Sum256(val) - - g.newNode(*count, hash[:]) - *count++ - } - } -} - -// Iterative generation of the graph -func (g *graph) xiGraphIter(index int64) { - count := g.pow2 - - stack := []int64{index, index, index, index, index} - graphStack := []int{4, 3, 2, 1, 0} - - var i int64 - graph := 0 - pow2index := int64(1 << uint64(index)) - for i = 0; i < pow2index; i++ { //sources at this level - buf := make([]byte, hashSize) - binary.PutVarint(buf, count) - val := append(g.pk, buf...) - hash := sha3.Sum256(val) - - g.newNode(count, hash[:]) - count++ - } - - if index == 1 { - g.butterflyGraph(index, &count) - return - } - - for len(stack) != 0 && len(graphStack) != 0 { - index, stack = stack[len(stack)-1], stack[:len(stack)-1] - graph, graphStack = graphStack[len(graphStack)-1], graphStack[:len(graphStack)-1] - - indices := []int64{index - 1, index - 1, index - 1, index - 1, index - 1} - graphs := []int{4, 3, 2, 1, 0} - - pow2index := int64(1 << uint64(index)) - pow2index_1 := int64(1 << uint64(index-1)) - - if graph == 0 { - sources := count - pow2index - // sources to sources of first butterfly - // create sources of first butterly - for i = 0; i < pow2index_1; i++ { - parent0 := g.getNode(sources + i) - parent1 := g.getNode(sources + i + pow2index_1) - - ph := append(parent0.H, parent1.H...) - buf := make([]byte, hashSize) - binary.PutVarint(buf, count) - val := append(g.pk, buf...) - val = append(val, ph...) - hash := sha3.Sum256(val) - - g.newNode(count, hash[:]) - count++ - } - } else if graph == 1 { - firstXi := count - // sinks of first butterfly to sources of first xi graph - for i = 0; i < pow2index_1; i++ { - nodeId := firstXi + i - // index is the last level; i.e., sinks - parent := g.getNode(firstXi - pow2index_1 + i) - - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId) - val := append(g.pk, buf...) - val = append(val, parent.H...) - hash := sha3.Sum256(val) - - g.newNode(nodeId, hash[:]) - count++ - } - } else if graph == 2 { - secondXi := count - // sinks of first xi to sources of second xi - for i = 0; i < pow2index_1; i++ { - nodeId := secondXi + i - parent := g.getNode(secondXi - pow2index_1 + i) - - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId) - val := append(g.pk, buf...) - val = append(val, parent.H...) - hash := sha3.Sum256(val) - - g.newNode(nodeId, hash[:]) - count++ - } - } else if graph == 3 { - secondButter := count - // sinks of second xi to sources of second butterfly - for i = 0; i < pow2index_1; i++ { - nodeId := secondButter + i - parent := g.getNode(secondButter - pow2index_1 + i) - - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId) - val := append(g.pk, buf...) - val = append(val, parent.H...) - hash := sha3.Sum256(val) - - g.newNode(nodeId, hash[:]) - count++ - } - } else { - sinks := count - sources := sinks + pow2index - numXi(index) - for i = 0; i < pow2index_1; i++ { - nodeId0 := sinks + i - nodeId1 := sinks + i + pow2index_1 - parent0 := g.getNode(sinks - pow2index_1 + i) - parent1_0 := g.getNode(sources + i) - parent1_1 := g.getNode(sources + i + pow2index_1) - - ph := append(parent0.H, parent1_0.H...) - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId0) - val := append(g.pk, buf...) - val = append(val, ph...) - hash1 := sha3.Sum256(val) - - ph = append(parent0.H, parent1_1.H...) - binary.PutVarint(buf, nodeId1) - val = append(g.pk, buf...) - val = append(val, ph...) - hash2 := sha3.Sum256(val) - - g.newNode(nodeId0, hash1[:]) - g.newNode(nodeId1, hash2[:]) - count += 2 - } - } - - if (graph == 0 || graph == 3) || - ((graph == 1 || graph == 2) && index == 2) { - g.butterflyGraph(index-1, &count) - } else if graph == 1 || graph == 2 { - stack = append(stack, indices...) - graphStack = append(graphStack, graphs...) - } - } -} - -func (g *graph) xiGraph(index int64, count *int64) { - // recursively generate graphs - // compute hashes along the way - - pow2index := int64(1 << uint64(index)) - - // the first sources - // if index == 1, then this will generate level 0 of the butterfly - var i int64 - - if *count == g.pow2 { - for i = 0; i < pow2index; i++ { - buf := make([]byte, hashSize) - binary.PutVarint(buf, *count) - val := append(g.pk, buf...) - hash := sha3.Sum256(val) - - g.newNode(*count, hash[:]) - *count++ - } - } - - if index == 1 { - g.butterflyGraph(index, count) - return - } - - sources := *count - pow2index - firstButter := sources + pow2index - firstXi := firstButter + numButterfly(index-1) - secondXi := firstXi + numXi(index-1) - secondButter := secondXi + numXi(index-1) - sinks := secondButter + numButterfly(index-1) - pow2index_1 := int64(1 << uint64(index-1)) - - // sources to sources of first butterfly - // create sources of first butterly - for i = 0; i < pow2index_1; i++ { - parent0 := g.getNode(sources + i) - parent1 := g.getNode(sources + i + pow2index_1) - - ph := append(parent0.H, parent1.H...) - buf := make([]byte, hashSize) - binary.PutVarint(buf, *count) - val := append(g.pk, buf...) - val = append(val, ph...) - hash := sha3.Sum256(val) - - g.newNode(*count, hash[:]) - *count++ - } - - g.butterflyGraph(index-1, count) - // sinks of first butterfly to sources of first xi graph - for i = 0; i < pow2index_1; i++ { - nodeId := firstXi + i - parent := g.getNode(firstXi - pow2index_1 + i) - - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId) - val := append(g.pk, buf...) - val = append(val, parent.H...) - hash := sha3.Sum256(val) - - g.newNode(nodeId, hash[:]) - *count++ - } - - g.xiGraph(index-1, count) - // sinks of first xi to sources of second xi - for i = 0; i < pow2index_1; i++ { - nodeId := secondXi + i - parent := g.getNode(secondXi - pow2index_1 + i) - - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId) - val := append(g.pk, buf...) - val = append(val, parent.H...) - hash := sha3.Sum256(val) - - g.newNode(nodeId, hash[:]) - *count++ - } - - g.xiGraph(index-1, count) - // sinks of second xi to sources of second butterfly - for i = 0; i < pow2index_1; i++ { - nodeId := secondButter + i - parent := g.getNode(secondButter - pow2index_1 + i) - - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId) - val := append(g.pk, buf...) - val = append(val, parent.H...) - hash := sha3.Sum256(val) - - g.newNode(nodeId, hash[:]) - *count++ - } - - // generate sinks - // sinks of second butterfly to sinks - // and sources to sinks directly - g.butterflyGraph(index-1, count) - for i = 0; i < pow2index_1; i++ { - nodeId0 := sinks + i - nodeId1 := sinks + i + pow2index_1 - parent0 := g.getNode(sinks - pow2index_1 + i) - parent1_0 := g.getNode(sources + i) - parent1_1 := g.getNode(sources + i + pow2index_1) - - ph := append(parent0.H, parent1_0.H...) - buf := make([]byte, hashSize) - binary.PutVarint(buf, nodeId0) - val := append(g.pk, buf...) - val = append(val, ph...) - hash1 := sha3.Sum256(val) - - ph = append(parent0.H, parent1_1.H...) - binary.PutVarint(buf, nodeId1) - val = append(g.pk, buf...) - val = append(val, ph...) - hash2 := sha3.Sum256(val) - - g.newNode(nodeId0, hash1[:]) - g.newNode(nodeId1, hash2[:]) - *count += 2 - } -} diff --git a/verthash/verthash.go b/verthash/verthash.go deleted file mode 100644 index 42d4f06..0000000 --- a/verthash/verthash.go +++ /dev/null @@ -1,63 +0,0 @@ -package verthash - -import ( - "bytes" - "encoding/binary" - - "github.com/sencha-dev/powkit/internal/crypto" - - "golang.org/x/crypto/sha3" -) - -func verthash(data, input []byte) []byte { - p1 := [32]byte{} - - inputCopy := make([]byte, len(input)) - copy(inputCopy[:], input[:]) - sha3hash := sha3.Sum256(inputCopy) - - copy(p1[:], sha3hash[:]) - p0 := make([]byte, verthashSubset) - for i := uint32(0); i < verthashIter; i++ { - inputCopy[0] += 0x01 - digest64 := sha3.Sum512(inputCopy) - copy(p0[i*verthashP0Size:], digest64[:]) - } - - buf := bytes.NewBuffer(p0) - p0Index := make([]uint32, len(p0)/4) - for i := 0; i < len(p0Index); i++ { - binary.Read(buf, binary.LittleEndian, &p0Index[i]) - } - - seekIndexes := make([]uint32, verthashIndexes) - for x := uint32(0); x < verthashRotations; x++ { - copy(seekIndexes[x*verthashSubset/4:], p0Index) - for y := 0; y < len(p0Index); y++ { - p0Index[y] = (p0Index[y] << 1) | (1 & (p0Index[y] >> 31)) - } - } - - mdiv := ((uint32(datasetSize) - verthashHashOutSize) / verthashByteAlignment) + 1 - valueAccumulator := uint32(0x811c9dc5) - buf = bytes.NewBuffer(p1[:]) - p1Arr := make([]uint32, verthashHashOutSize/4) - for i := 0; i < len(p1Arr); i++ { - binary.Read(buf, binary.LittleEndian, &p1Arr[i]) - } - for i := uint32(0); i < verthashIndexes; i++ { - offset := (crypto.Fnv1a(seekIndexes[i], valueAccumulator) % mdiv) * verthashByteAlignment - data := data[offset : offset+verthashHashOutSize] - for i2 := uint32(0); i2 < verthashHashOutSize/4; i2++ { - value := binary.LittleEndian.Uint32(data[i2*4 : ((i2 + 1) * 4)]) - p1Arr[i2] = crypto.Fnv1a(p1Arr[i2], value) - valueAccumulator = crypto.Fnv1a(valueAccumulator, value) - } - } - - for i := uint32(0); i < verthashHashOutSize/4; i++ { - binary.LittleEndian.PutUint32(p1[i*4:], p1Arr[i]) - } - - return p1[:] -} diff --git a/verthash/verthash_test.go b/verthash/verthash_test.go deleted file mode 100644 index f97e731..0000000 --- a/verthash/verthash_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package verthash - -import ( - "bytes" - "testing" - - "github.com/sencha-dev/powkit/internal/common/testutil" -) - -func TestVerify(t *testing.T) { - tests := []struct { - input []byte - output []byte - }{ - { - input: testutil.MustDecodeHex("000000203a297b4b7685170d7644b43e5a6056234cc2414edde454a87580e1967d14c1078c13ea916117b0608732f3f65c2e03b81322efc0a62bcee77d8a9371261970a58a5a715da80e031b02560ad8"), - output: testutil.MustDecodeHex("E0F6C10B4A38F35A6CDCC26D32A7ED8C3BFC5D827A9BC72647AFA324B70D0463"), - }, - } - - verthash, err := New() - if err != nil { - t.Errorf("failed on new verthash: %v", err) - return - } - - for i, tt := range tests { - digest := verthash.Compute(tt.input) - if !bytes.Equal(digest, tt.output) { - t.Errorf("failed on %d: digest mismatch: have %x, want %x", i, digest, tt.output) - } - } -}