Speedup unmarshalling

.gitignore

@@ -8,3 +8,4 @@ tlb/testdata/*.output.json
 tlb/parser/testdata/*.output.out
 vendor/
 cover.out
+*.output.json

boc/cell.go

@@ -299,7 +299,8 @@ func (c *Cell) CopyRemaining() *Cell {
 	c2 := NewCellWithBits(c.bits.ReadRemainingBits())
 	c.bits.rCursor = rCursor
 	refCursor := c.refCursor
-	for c.RefsAvailableForRead() > 0 {
+	refsNums := c.RefsAvailableForRead()
+	for i := 0; i < refsNums; i++ {
 		ref, err := c.NextRef()
 		if err != nil {
 			// this should never happen but anyway

boc/immutable_cell.go

@@ -89,8 +89,20 @@ func newImmutableCell(c *Cell, cache map[*Cell]*immutableCell) (*immutableCell,
 		x := sha256.New()
 		if hashIndex == offset {
 			// either i=0 or cellType=PrunedBranchCell
-			cellRepr := c.bocReprWithoutRefs(c.mask.Apply(i))
-			x.Write(cellRepr)
+
+			// we want to avoid slice allocations,
+			// that's why we don't use c.bocReprWithoutRefs(c.mask.Apply(i))
+			x.Write([]byte{d1(c, c.mask.Apply(i)), d2(c)})
+			if c.BitSize()%8 == 0 {
+				x.Write(c.getBuffer())
+			} else {
+				// a trick to store info about the cell's length in the last byte.
+				buf := c.getBuffer()
+				x.Write(buf[:len(buf)-1])
+				z := buf[len(buf)-1]
+				z |= 1 << (7 - c.BitSize()%8)
+				x.Write([]byte{z})
+			}
 		} else {
 			// i>0
 			x.Write([]byte{d1(c, c.mask.Apply(i)), d2(c)})

tlb/benchmark_test.go

@@ -1,11 +1,7 @@
 package tlb
 
 import (
-	"bytes"
-	"encoding/hex"
-	"encoding/json"
 	"os"
-	"reflect"
 	"testing"
 
 	"github.com/tonkeeper/tongo/boc"
@@ -34,7 +30,7 @@ func Benchmark_Tlb_Unmarshal(b *testing.B) {
 }
 
 func Test_block(b *testing.T) {
-	data, err := os.ReadFile("../ton/testdata/raw-13516764.bin")
+	data, err := os.ReadFile("testdata/block-1/block.bin")
 	if err != nil {
 		b.Errorf("ReadFile() failed: %v", err)
 	}
@@ -49,88 +45,3 @@ func Test_block(b *testing.T) {
 		b.Errorf("Unmarshal() failed: %v", err)
 	}
 }
-
-func Test_tlb_Unmarshal(t *testing.T) {
-	type transaction struct {
-		AccountAddr   string
-		Lt            uint64
-		PrevTransHash string
-		PrevTransLt   uint64
-		Now           uint32
-		OutMsgCnt     Uint15
-		OrigStatus    AccountStatus
-		EndStatus     AccountStatus
-	}
-	type accountBlock struct {
-		Transactions map[uint64]transaction
-	}
-	data, err := os.ReadFile("testdata/raw-block.bin")
-	if err != nil {
-		t.Errorf("ReadFile() failed: %v", err)
-	}
-	cell, err := boc.DeserializeBoc(data)
-	if err != nil {
-		t.Errorf("boc.DeserializeBoc() failed: %v", err)
-	}
-	var block Block
-	err = Unmarshal(cell[0], &block)
-	if err != nil {
-		t.Errorf("Unmarshal() failed: %v", err)
-	}
-	accounts := map[string]*accountBlock{}
-	for _, account := range block.Extra.AccountBlocks.Values() {
-		accBlock, ok := accounts[hex.EncodeToString(account.AccountAddr[:])]
-		if !ok {
-			accBlock = &accountBlock{Transactions: map[uint64]transaction{}}
-			accounts[hex.EncodeToString(account.AccountAddr[:])] = accBlock
-		}
-		for _, txRef := range account.Transactions.Values() {
-			tx := txRef.Value
-			accBlock.Transactions[txRef.Value.Lt] = transaction{
-				AccountAddr:   hex.EncodeToString(tx.AccountAddr[:]),
-				Lt:            tx.Lt,
-				PrevTransHash: hex.EncodeToString(tx.PrevTransHash[:]),
-				PrevTransLt:   tx.PrevTransLt,
-				Now:           tx.Now,
-				OutMsgCnt:     tx.OutMsgCnt,
-				OrigStatus:    tx.OrigStatus,
-				EndStatus:     tx.EndStatus,
-			}
-		}
-	}
-
-	bs, err := json.MarshalIndent(accounts, " ", "  ")
-	if err != nil {
-		t.Errorf("json.MarshalIndent() failed: %v", err)
-	}
-	if err := os.WriteFile("testdata/raw-block.output.json", bs, 0644); err != nil {
-		t.Errorf("WriteFile() failed: %v", err)
-	}
-	content, err := os.ReadFile("testdata/raw-block.expected.json")
-	if err != nil {
-		t.Errorf("ReadFile() failed: %v", err)
-	}
-	expectedAccounts := map[string]*accountBlock{}
-	if err := json.Unmarshal(content, &expectedAccounts); err != nil {
-		t.Errorf("json.Unmarshal() failed: %v", err)
-	}
-	if !reflect.DeepEqual(accounts, expectedAccounts) {
-		t.Errorf("expectedAccounts differs from accounts")
-	}
-	bs, err = json.MarshalIndent(block.ValueFlow, " ", "  ")
-	if err != nil {
-		t.Fatalf("MarshalIndent() failed: %v", err)
-	}
-	if err := os.WriteFile("testdata/value-flow.output.json", bs, 0644); err != nil {
-		t.Fatalf("WriteFile() failed: %v", err)
-	}
-	data, err = os.ReadFile("testdata/value-flow.expected.json")
-	if err != nil {
-		t.Fatalf("ReadFile() failed: %v", err)
-	}
-	if bytes.Compare(bytes.Trim(bs, " \n"), bytes.Trim(data, " \n")) != 0 {
-		t.Errorf("ValueFlows differ")
-
-	}
-
-}

tlb/block_test.go

@@ -0,0 +1,144 @@
+package tlb
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"path"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/tonkeeper/tongo/boc"
+)
+
+func Test_tlb_Unmarshal(t *testing.T) {
+	type transaction struct {
+		AccountAddr   string
+		Lt            uint64
+		PrevTransHash string
+		PrevTransLt   uint64
+		Now           uint32
+		OutMsgCnt     Uint15
+		OrigStatus    AccountStatus
+		EndStatus     AccountStatus
+	}
+	type accountBlock struct {
+		Transactions map[uint64]transaction
+	}
+	testCases := []struct {
+		name   string
+		folder string
+	}{
+		{
+			name:   "all good",
+			folder: "testdata/block-1",
+		},
+		{
+			name:   "all good",
+			folder: "testdata/block-2",
+		},
+		{
+			name:   "all good",
+			folder: "testdata/block-3",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			inputFilename := path.Join(tc.folder, "block.bin")
+			data, err := os.ReadFile(inputFilename)
+			if err != nil {
+				t.Errorf("ReadFile() failed: %v", err)
+			}
+			cell, err := boc.DeserializeBoc(data)
+			if err != nil {
+				t.Errorf("boc.DeserializeBoc() failed: %v", err)
+			}
+			var block Block
+			err = Unmarshal(cell[0], &block)
+			if err != nil {
+				t.Errorf("Unmarshal() failed: %v", err)
+			}
+			accounts := map[string]*accountBlock{}
+			var txHashes []string
+			for _, account := range block.Extra.AccountBlocks.Values() {
+				accBlock, ok := accounts[hex.EncodeToString(account.AccountAddr[:])]
+				if !ok {
+					accBlock = &accountBlock{Transactions: map[uint64]transaction{}}
+					accounts[hex.EncodeToString(account.AccountAddr[:])] = accBlock
+				}
+				for _, txRef := range account.Transactions.Values() {
+					tx := txRef.Value
+					accBlock.Transactions[txRef.Value.Lt] = transaction{
+						AccountAddr:   hex.EncodeToString(tx.AccountAddr[:]),
+						Lt:            tx.Lt,
+						PrevTransHash: hex.EncodeToString(tx.PrevTransHash[:]),
+						PrevTransLt:   tx.PrevTransLt,
+						Now:           tx.Now,
+						OutMsgCnt:     tx.OutMsgCnt,
+						OrigStatus:    tx.OrigStatus,
+						EndStatus:     tx.EndStatus,
+					}
+					txHashes = append(txHashes, tx.Hash().Hex())
+				}
+			}
+			sort.Slice(txHashes, func(i, j int) bool {
+				return txHashes[i] < txHashes[j]
+			})
+			bs, err := json.MarshalIndent(accounts, " ", "  ")
+			if err != nil {
+				t.Errorf("json.MarshalIndent() failed: %v", err)
+			}
+			outputFilename := path.Join(tc.folder, "block.output.json")
+			if err := os.WriteFile(outputFilename, bs, 0644); err != nil {
+				t.Errorf("WriteFile() failed: %v", err)
+			}
+			expectedFilename := path.Join(tc.folder, "block.expected.json")
+			content, err := os.ReadFile(expectedFilename)
+			if err != nil {
+				t.Errorf("ReadFile() failed: %v", err)
+			}
+			expectedAccounts := map[string]*accountBlock{}
+			if err := json.Unmarshal(content, &expectedAccounts); err != nil {
+				t.Errorf("json.Unmarshal() failed: %v", err)
+			}
+			if !reflect.DeepEqual(accounts, expectedAccounts) {
+				t.Errorf("expectedAccounts differs from accounts")
+			}
+			bs, err = json.MarshalIndent(block.ValueFlow, " ", "  ")
+			if err != nil {
+				t.Fatalf("MarshalIndent() failed: %v", err)
+			}
+			valueFlowOutput := path.Join(tc.folder, "value-flow.output.json")
+			if err := os.WriteFile(valueFlowOutput, bs, 0644); err != nil {
+				t.Fatalf("WriteFile() failed: %v", err)
+			}
+			expectedValueFlowFilename := path.Join(tc.folder, "value-flow.expected.json")
+			data, err = os.ReadFile(expectedValueFlowFilename)
+			if err != nil {
+				t.Fatalf("ReadFile() failed: %v", err)
+			}
+			if bytes.Compare(bytes.Trim(bs, " \n"), bytes.Trim(data, " \n")) != 0 {
+				t.Errorf("ValueFlows differ")
+			}
+			// compare hashes
+			bs, err = json.MarshalIndent(txHashes, " ", "  ")
+			if err != nil {
+				t.Fatalf("MarshalIndent() failed: %v", err)
+			}
+			hashesOutputFilename := path.Join(tc.folder, "tx-hashes.output.json")
+			if err := os.WriteFile(hashesOutputFilename, bs, 0644); err != nil {
+				t.Fatalf("WriteFile() failed: %v", err)
+			}
+			expectedHashesFilename := path.Join(tc.folder, "tx-hashes.expected.json")
+			data, err = os.ReadFile(expectedHashesFilename)
+			if err != nil {
+				t.Fatalf("ReadFile() failed: %v", err)
+			}
+			if bytes.Compare(bytes.Trim(bs, " \n"), bytes.Trim(data, " \n")) != 0 {
+				t.Errorf("tx hashes differ")
+			}
+		})
+	}
+}

tlb/primitives.go

@@ -257,21 +257,7 @@ func (a Any) MarshalTLB(c *boc.Cell, encoder *Encoder) error {
 }
 
 func (a *Any) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error {
-	x := boc.NewCell()
-	err := x.WriteBitString(c.ReadRemainingBits())
-	if err != nil {
-		return err
-	}
-	for c.RefsAvailableForRead() > 0 {
-		ref, err := c.NextRef()
-		if err != nil {
-			return err
-		}
-		err = x.AddRef(ref)
-		if err != nil {
-			return err
-		}
-	}
+	x := c.CopyRemaining()
 	*a = Any(*x)
 	return nil
 }