Skip to content

Commit

Permalink
Merge pull request #236 from tonkeeper/optimize-block
Browse files Browse the repository at this point in the history 
Optimize block unmarshalling
  • Loading branch information
@mr-tron
mr-tron authored Feb 2, 2024
2 parents c11f314 + 94afbc5 commit 64ef975
Show file tree
Hide file tree
Showing 12 changed files with 15,673 additions and 15,562 deletions.
42 changes: 35 additions & 7 deletions tlb/block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,13 +349,41 @@ func (m *ValueFlow) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error {
// created_by:bits256
// custom:(Maybe ^McBlockExtra) = BlockExtra;
type BlockExtra struct {
Magic Magic `tlb:"block_extra#4a33f6fd"`
InMsgDescr HashmapAugE[Bits256, InMsg, ImportFees] `tlb:"^"` // tlb.Any `tlb:"^"`
OutMsgDescr HashmapAugE[Bits256, OutMsg, CurrencyCollection] `tlb:"^"` // tlb.Any `tlb:"^"`
AccountBlocks HashmapAugE[Bits256, AccountBlock, CurrencyCollection] `tlb:"^"` // tlb.Any `tlb:"^"` //
RandSeed Bits256
CreatedBy Bits256
Custom Maybe[Ref[McBlockExtra]]
Magic Magic `tlb:"block_extra#4a33f6fd"`
InMsgDescrCell boc.Cell `tlb:"^"`
OutMsgDescrCell boc.Cell `tlb:"^"`
AccountBlocks HashmapAugE[Bits256, AccountBlock, CurrencyCollection] `tlb:"^"`
RandSeed Bits256
CreatedBy Bits256
Custom Maybe[Ref[McBlockExtra]]
}

func (extra *BlockExtra) InMsgDescrLength() (int, error) {
cell := boc.Cell(extra.InMsgDescrCell)
cell.ResetCounters()
return hashmapAugExtraCountLeafs[Bits256](&cell)
}

func (extra *BlockExtra) InMsgDescr() (HashmapAugE[Bits256, InMsg, ImportFees], error) {
var hashmap HashmapAugE[Bits256, InMsg, ImportFees]
if err := Unmarshal(&extra.InMsgDescrCell, &hashmap); err != nil {
return HashmapAugE[Bits256, InMsg, ImportFees]{}, err
}
return hashmap, nil
}

func (extra *BlockExtra) OutMsgDescrLength() (int, error) {
cell := boc.Cell(extra.OutMsgDescrCell)
cell.ResetCounters()
return hashmapAugExtraCountLeafs[Bits256](&cell)
}

func (extra *BlockExtra) OutMsgDescr() (HashmapAugE[Bits256, OutMsg, CurrencyCollection], error) {
var hashmap HashmapAugE[Bits256, OutMsg, CurrencyCollection]
if err := Unmarshal(&extra.OutMsgDescrCell, &hashmap); err != nil {
return HashmapAugE[Bits256, OutMsg, CurrencyCollection]{}, err
}
return hashmap, nil
}

// masterchain_block_extra#cca5
Expand Down
78 changes: 31 additions & 47 deletions tlb/block_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ import (
"encoding/json"
"os"
"path"
"reflect"
"sort"
"testing"

"github.com/tonkeeper/tongo/boc"
)

func Test_tlb_Unmarshal(t *testing.T) {
type transaction struct {
type Transaction struct {
AccountAddr string
Lt uint64
PrevTransHash string
Expand All @@ -24,8 +23,15 @@ func Test_tlb_Unmarshal(t *testing.T) {
OrigStatus AccountStatus
EndStatus AccountStatus
}
type accountBlock struct {
Transactions map[uint64]transaction
type AccountBlock struct {
Transactions map[uint64]Transaction
}
type BlockContent struct {
Accounts map[string]*AccountBlock
TxHashes []string
ValueFlow ValueFlow
InMsgDescrLength int
OutMsgDescrLength int
}
testCases := []struct {
name string
Expand Down Expand Up @@ -60,17 +66,17 @@ func Test_tlb_Unmarshal(t *testing.T) {
if err != nil {
t.Errorf("Unmarshal() failed: %v", err)
}
accounts := map[string]*accountBlock{}
accounts := map[string]*AccountBlock{}
var txHashes []string
for _, account := range block.Extra.AccountBlocks.Values() {
accBlock, ok := accounts[hex.EncodeToString(account.AccountAddr[:])]
if !ok {
accBlock = &accountBlock{Transactions: map[uint64]transaction{}}
accBlock = &AccountBlock{Transactions: map[uint64]Transaction{}}
accounts[hex.EncodeToString(account.AccountAddr[:])] = accBlock
}
for _, txRef := range account.Transactions.Values() {
tx := txRef.Value
accBlock.Transactions[txRef.Value.Lt] = transaction{
accBlock.Transactions[txRef.Value.Lt] = Transaction{
AccountAddr: hex.EncodeToString(tx.AccountAddr[:]),
Lt: tx.Lt,
PrevTransHash: hex.EncodeToString(tx.PrevTransHash[:]),
Expand All @@ -86,7 +92,22 @@ func Test_tlb_Unmarshal(t *testing.T) {
sort.Slice(txHashes, func(i, j int) bool {
return txHashes[i] < txHashes[j]
})
bs, err := json.MarshalIndent(accounts, " ", " ")
inMsgLength, err := block.Extra.InMsgDescrLength()
if err != nil {
t.Errorf("InMsgDescrLength() failed: %v", err)
}
outMsgLength, err := block.Extra.OutMsgDescrLength()
if err != nil {
t.Errorf("InMsgDescrLength() failed: %v", err)
}
blk := BlockContent{
Accounts: accounts,
TxHashes: txHashes,
ValueFlow: block.ValueFlow,
InMsgDescrLength: inMsgLength,
OutMsgDescrLength: outMsgLength,
}
bs, err := json.MarshalIndent(blk, " ", " ")
if err != nil {
t.Errorf("json.MarshalIndent() failed: %v", err)
}
Expand All @@ -99,45 +120,8 @@ func Test_tlb_Unmarshal(t *testing.T) {
if err != nil {
t.Errorf("ReadFile() failed: %v", err)
}
expectedAccounts := map[string]*accountBlock{}
if err := json.Unmarshal(content, &expectedAccounts); err != nil {
t.Errorf("json.Unmarshal() failed: %v", err)
}
if !reflect.DeepEqual(accounts, expectedAccounts) {
t.Errorf("expectedAccounts differs from accounts")
}
bs, err = json.MarshalIndent(block.ValueFlow, " ", " ")
if err != nil {
t.Fatalf("MarshalIndent() failed: %v", err)
}
valueFlowOutput := path.Join(tc.folder, "value-flow.output.json")
if err := os.WriteFile(valueFlowOutput, bs, 0644); err != nil {
t.Fatalf("WriteFile() failed: %v", err)
}
expectedValueFlowFilename := path.Join(tc.folder, "value-flow.expected.json")
data, err = os.ReadFile(expectedValueFlowFilename)
if err != nil {
t.Fatalf("ReadFile() failed: %v", err)
}
if bytes.Compare(bytes.Trim(bs, " \n"), bytes.Trim(data, " \n")) != 0 {
t.Errorf("ValueFlows differ")
}
// compare hashes
bs, err = json.MarshalIndent(txHashes, " ", " ")
if err != nil {
t.Fatalf("MarshalIndent() failed: %v", err)
}
hashesOutputFilename := path.Join(tc.folder, "tx-hashes.output.json")
if err := os.WriteFile(hashesOutputFilename, bs, 0644); err != nil {
t.Fatalf("WriteFile() failed: %v", err)
}
expectedHashesFilename := path.Join(tc.folder, "tx-hashes.expected.json")
data, err = os.ReadFile(expectedHashesFilename)
if err != nil {
t.Fatalf("ReadFile() failed: %v", err)
}
if bytes.Compare(bytes.Trim(bs, " \n"), bytes.Trim(data, " \n")) != 0 {
t.Errorf("tx hashes differ")
if bytes.Compare(bytes.Trim(content, " \n"), bytes.Trim(bs, " \n")) != 0 {
t.Errorf("block content mismatch")
}
})
}
Expand Down
87 changes: 87 additions & 0 deletions tlb/hashmap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package tlb

import (
"encoding/json"
"errors"
"fmt"
"strings"

Expand Down Expand Up @@ -121,6 +122,55 @@ func (h *Hashmap[keyT, T]) UnmarshalTLB(c *boc.Cell, decoder *Decoder) error {
return nil
}

func hashmapAugExtraCountLeafs[keyT fixedSize](c *boc.Cell) (int, error) {
maybe, err := c.ReadBit()
if err != nil {
return 0, err
}
if !maybe {
return 0, nil
}
ref, err := c.NextRef()
if err != nil {
return 0, err
}
var s keyT
keySize := s.FixedSize()
return countLeafs(keySize, keySize, ref)
}

func countLeafs(keySize, leftKeySize int, c *boc.Cell) (int, error) {
if c.CellType() == boc.PrunedBranchCell {
return 0, errors.New("can't count leafs for hashmap with pruned branch cell")
}
size, err := loadLabelSize(leftKeySize, c)
if err != nil {
return 0, err
}
if keySize-leftKeySize+size < keySize {
// 0 bit branch
left, err := c.NextRef()
if err != nil {
return 0, err
}
leftCount, err := countLeafs(keySize, leftKeySize-(1+size), left)
if err != nil {
return 0, err
}
// 1 bit branch
right, err := c.NextRef()
if err != nil {
return 0, err
}
rightCount, err := countLeafs(keySize, leftKeySize-(1+size), right)
if err != nil {
return 0, err
}
return leftCount + rightCount, nil
}
return 1, nil
}

func (h *Hashmap[keyT, T]) mapInner(keySize, leftKeySize int, c *boc.Cell, keyPrefix *boc.BitString, decoder *Decoder) error {
var err error
var size int
Expand Down Expand Up @@ -456,6 +506,43 @@ func (h HashmapAugE[keyT, T1, T2]) Keys() []keyT {
return h.m.keys
}

func loadLabelSize(size int, c *boc.Cell) (int, error) {
first, err := c.ReadBit()
if err != nil {
return 0, err
}
// hml_short$0
if !first {
// Unary, while 1, add to ln
ln, err := c.ReadUnary()
if err != nil {
return 0, err
}
return int(ln), nil
}
second, err := c.ReadBit()
if err != nil {
return 0, err
}
// hml_long$10
if !second {
ln, err := c.ReadLimUint(size)
if err != nil {
return 0, err
}
return int(ln), nil
}
// hml_same$11
_, err = c.ReadBit()
if err != nil {
return 0, err
}
ln, err := c.ReadLimUint(size)
if err != nil {
return 0, err
}
return int(ln), nil
}
func loadLabel(size int, c *boc.Cell, key *boc.BitString) (int, *boc.BitString, error) {
first, err := c.ReadBit()
if err != nil {
Expand Down
Loading

0 comments on commit 64ef975

Please sign in to comment.