Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wip] SSZ-optimize proof (type 2) #443

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[wip] SSZ-optimize proof (type 2) #443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 143 additions & 64 deletions proof_ipa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"errors"
"fmt"
"sort"
"unsafe"

ipa "github.com/crate-crypto/go-ipa"
"github.com/crate-crypto/go-ipa/common"
Expand Down Expand Up @@ -83,17 +82,20 @@ type Proof struct {
PostValues [][]byte
}

type SuffixStateDiff struct {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Something we should remember, this kind of new format requires more integrity checks to see if same suffixes aren't claimed to be new, updated and/or untouched at the same time.

Suffix byte `json:"suffix"`
CurrentValue *[32]byte `json:"currentValue"`
NewValue *[32]byte `json:"newValue"`
}
type StemStateDiff struct {
Stem [StemSize]byte `json:"stem"`

type SuffixStateDiffs []SuffixStateDiff
UpdatedSuffixes []byte `json:"updatedsuffixes"`
UpdatedCurrent [][]byte `json:"updatedcurrent"`
UpdatedNew [][]byte `json:"updatednew"`

type StemStateDiff struct {
Stem [StemSize]byte `json:"stem"`
SuffixDiffs SuffixStateDiffs `json:"suffixDiffs"`
ReadSuffixes []byte `json:"readsuffixes"`
ReadCurrent [][]byte `json:"readcurrent"`

InsertedSuffixes []byte `json:"insertedsuffixes"`
InsertedNew [][]byte `json:"insertednew"`

UntouchedSuffixes []byte `json:"untouchedsuffixes"`
}
Comment on lines -92 to 99
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: the json tags must be camel case, as usual.


type StateDiff []StemStateDiff
Expand All @@ -102,16 +104,47 @@ func (sd StateDiff) Copy() StateDiff {
ret := make(StateDiff, len(sd))
for i := range sd {
copy(ret[i].Stem[:], sd[i].Stem[:])
ret[i].SuffixDiffs = make([]SuffixStateDiff, len(sd[i].SuffixDiffs))
for j := range sd[i].SuffixDiffs {
ret[i].SuffixDiffs[j].Suffix = sd[i].SuffixDiffs[j].Suffix
if sd[i].SuffixDiffs[j].CurrentValue != nil {
ret[i].SuffixDiffs[j].CurrentValue = &[32]byte{}
copy((*ret[i].SuffixDiffs[j].CurrentValue)[:], (*sd[i].SuffixDiffs[j].CurrentValue)[:])

ret[i].UpdatedSuffixes = make([]byte, len(sd[i].UpdatedSuffixes))
copy(ret[i].UpdatedSuffixes, sd[i].UpdatedSuffixes)
ret[i].ReadSuffixes = make([]byte, len(sd[i].ReadSuffixes))
copy(ret[i].ReadSuffixes, sd[i].ReadSuffixes)
ret[i].InsertedSuffixes = make([]byte, len(sd[i].InsertedSuffixes))
copy(ret[i].InsertedSuffixes, sd[i].InsertedSuffixes)
ret[i].UntouchedSuffixes = make([]byte, len(sd[i].UntouchedSuffixes))
copy(ret[i].UntouchedSuffixes, sd[i].UntouchedSuffixes)

ret[i].UpdatedCurrent = make([][]byte, len(sd[i].UpdatedCurrent))
for j := range sd[i].UpdatedCurrent {
if len(sd[i].UpdatedCurrent[j]) == 0 {

Comment on lines +119 to +120
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: remove empty if. Same applies in below cases.

} else {
copy(ret[i].UpdatedCurrent[j], sd[i].UpdatedCurrent[j])
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comments as mentioned in this other PR. Same applies in below cases in this method.

}
if sd[i].SuffixDiffs[j].NewValue != nil {
ret[i].SuffixDiffs[j].NewValue = &[32]byte{}
copy((*ret[i].SuffixDiffs[j].NewValue)[:], (*sd[i].SuffixDiffs[j].NewValue)[:])
}
ret[i].ReadCurrent = make([][]byte, len(sd[i].ReadCurrent))
for j := range sd[i].ReadCurrent {
if len(sd[i].ReadCurrent[j]) == 0 {

} else {
copy(ret[i].ReadCurrent[j], sd[i].ReadCurrent[j])
}
}

ret[i].UpdatedNew = make([][]byte, len(sd[i].UpdatedNew))
for j := range sd[i].UpdatedNew {
if len(sd[i].UpdatedNew[j]) == 0 {

} else {
copy(ret[i].UpdatedNew[j], sd[i].UpdatedNew[j])
}
}
ret[i].InsertedNew = make([][]byte, len(sd[i].InsertedNew))
for j := range sd[i].InsertedNew {
if len(sd[i].InsertedNew[j]) == 0 {

} else {
copy(ret[i].InsertedNew[j], sd[i].InsertedNew[j])
}
}
}
Expand Down Expand Up @@ -221,6 +254,16 @@ func VerifyVerkleProof(proof *Proof, Cs []*Point, indices []uint8, ys []*Fr, tc
return ipa.CheckMultiProof(tr, tc.conf, proof.Multipoint, Cs, ys, indices)
}

func isInsertion(preLen, postLen int) bool {
return preLen == 0 && postLen != 0
}
func isRead(preLen, postLen int) bool {
return preLen != 0 && postLen == 0
}
func isUpdate(preLen, postLen int) bool {
return preLen != 0 && postLen != 0
}

// SerializeProof serializes the proof in the rust-verkle format:
// * len(Proof of absence stem) || Proof of absence stems
// * len(depths) || serialize(depth || ext statusi)
Expand Down Expand Up @@ -257,32 +300,45 @@ func SerializeProof(proof *Proof) (*VerkleProof, StateDiff, error) {
stemdiff = &statediff[len(statediff)-1]
copy(stemdiff.Stem[:], stem)
}
stemdiff.SuffixDiffs = append(stemdiff.SuffixDiffs, SuffixStateDiff{Suffix: key[StemSize]})
newsd := &stemdiff.SuffixDiffs[len(stemdiff.SuffixDiffs)-1]

var valueLen = len(proof.PreValues[i])
switch valueLen {
case 0:
// null value
case 32:
newsd.CurrentValue = (*[32]byte)(proof.PreValues[i])
default:
var aligned [32]byte
copy(aligned[:valueLen], proof.PreValues[i])
newsd.CurrentValue = (*[32]byte)(unsafe.Pointer(&aligned[0]))
}

valueLen = len(proof.PostValues[i])
switch valueLen {
case 0:
// null value
case 32:
newsd.NewValue = (*[32]byte)(proof.PostValues[i])
preLen := len(proof.PreValues[i])
postLen := len(proof.PostValues[i])
switch {
case isInsertion(preLen, postLen):
stemdiff.InsertedSuffixes = append(stemdiff.InsertedSuffixes, key[StemSize])
if postLen == 0 {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you meant != 32?

If we're in an isInsertion then postLen can't be 0.

stemdiff.InsertedNew = append(stemdiff.InsertedNew, proof.PostValues[i])
} else {
var aligned [32]byte
copy(aligned[:postLen], proof.PostValues[i])
stemdiff.InsertedNew = append(stemdiff.InsertedNew, aligned[:])
}
case isRead(preLen, postLen):
stemdiff.ReadSuffixes = append(stemdiff.ReadSuffixes, key[StemSize])
if preLen == 0 {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ditto.

stemdiff.ReadCurrent = append(stemdiff.ReadCurrent, proof.PreValues[i])
} else {
var aligned [32]byte
copy(aligned[:preLen], proof.PreValues[i])
stemdiff.ReadCurrent = append(stemdiff.ReadCurrent, aligned[:])
}
case isUpdate(preLen, postLen):
stemdiff.UpdatedSuffixes = append(stemdiff.UpdatedSuffixes, key[StemSize])
if preLen == 0 {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ditto.

stemdiff.UpdatedCurrent = append(stemdiff.UpdatedCurrent, proof.PreValues[i])
} else {
var aligned [32]byte
copy(aligned[:preLen], proof.PreValues[i])
stemdiff.UpdatedCurrent = append(stemdiff.UpdatedCurrent, aligned[:])
}
if postLen == 0 {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ditto.

stemdiff.UpdatedNew = append(stemdiff.UpdatedNew, proof.PostValues[i])
} else {
var aligned [32]byte
copy(aligned[:postLen], proof.PostValues[i])
stemdiff.UpdatedNew = append(stemdiff.UpdatedNew, aligned[:])
}
default:
// TODO remove usage of unsafe
var aligned [32]byte
copy(aligned[:valueLen], proof.PostValues[i])
newsd.NewValue = (*[32]byte)(unsafe.Pointer(&aligned[0]))
stemdiff.UntouchedSuffixes = append(stemdiff.UntouchedSuffixes, key[StemSize])
}
}

Expand Down Expand Up @@ -347,22 +403,37 @@ func DeserializeProof(vp *VerkleProof, statediff StateDiff) (*Proof, error) {

// turn statediff into keys and values
for _, stemdiff := range statediff {
for _, suffixdiff := range stemdiff.SuffixDiffs {
for i, suffix := range stemdiff.UpdatedSuffixes {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = suffixdiff.Suffix
k[StemSize] = suffix
keys = append(keys, k[:])
if suffixdiff.CurrentValue != nil {
prevalues = append(prevalues, suffixdiff.CurrentValue[:])
} else {
prevalues = append(prevalues, nil)
}

if suffixdiff.NewValue != nil {
postvalues = append(postvalues, suffixdiff.NewValue[:])
} else {
postvalues = append(postvalues, nil)
}
prevalues = append(prevalues, stemdiff.UpdatedCurrent[i])
postvalues = append(postvalues, stemdiff.UpdatedNew[i])
}
for i, suffix := range stemdiff.InsertedSuffixes {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = suffix
keys = append(keys, k[:])
prevalues = append(prevalues, nil)
postvalues = append(postvalues, stemdiff.InsertedNew[i])
}
for i, suffix := range stemdiff.ReadSuffixes {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = suffix
keys = append(keys, k[:])
prevalues = append(prevalues, stemdiff.ReadCurrent[i])
postvalues = append(postvalues, nil)
}
for _, suffix := range stemdiff.UntouchedSuffixes {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = suffix
keys = append(keys, k[:])
prevalues = append(prevalues, nil)
postvalues = append(postvalues, nil)
}
}

Expand Down Expand Up @@ -524,20 +595,28 @@ func PostStateTreeFromStateDiff(preroot VerkleNode, statediff StateDiff) (Verkle

for _, stemstatediff := range statediff {
var (
values = make([][]byte, NodeWidth)
overwrites bool
values = make([][]byte, NodeWidth)
overwrite bool
)

for _, suffixdiff := range stemstatediff.SuffixDiffs {
if /* len(suffixdiff.NewValue) > 0 - this only works for a slice */ suffixdiff.NewValue != nil {
for i, suffix := range stemstatediff.InsertedSuffixes {
if /* len(suffixdiff.NewValue) > 0 - this only works for a slice */ stemstatediff.InsertedNew[i] != nil {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm a bit lost here. Shouldn't stemstatediff.InsertedNew items always be not nil?
If was an inserted value, we need a value.

// if this value is non-nil, it means InsertValuesAtStem should be
// called, otherwise, skip updating the tree.
values[suffix] = stemstatediff.InsertedNew[i]
overwrite = true
}
}
for i, suffix := range stemstatediff.UpdatedSuffixes {
if /* len(suffixdiff.NewValue) > 0 - this only works for a slice */ stemstatediff.UpdatedNew[i] != nil {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ditto.

// if this value is non-nil, it means InsertValuesAtStem should be
// called, otherwise, skip updating the tree.
overwrites = true
values[suffixdiff.Suffix] = suffixdiff.NewValue[:]
values[suffix] = stemstatediff.UpdatedNew[i]
overwrite = true
}
}

if overwrites {
if overwrite {
var stem [StemSize]byte
copy(stem[:StemSize], stemstatediff.Stem[:])
if err := postroot.(*InternalNode).InsertValuesAtStem(stem[:], values, nil); err != nil {
Expand Down
95 changes: 23 additions & 72 deletions proof_json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,92 +183,43 @@
return nil
}

// TODO use gencodec if that works
type stemStateDiffMarshaller struct {
Stem string `json:"stem"`
SuffixDiffs SuffixStateDiffs `json:"suffixDiffs"`
Stem string `json:"stem"`
Suffixes string `json:"suffixes"`
Current []string `json:"current"`
New []string `json:"new"`
}

func (ssd StemStateDiff) MarshalJSON() ([]byte, error) {
return json.Marshal(&stemStateDiffMarshaller{
Stem: HexToPrefixedString(ssd.Stem[:]),
SuffixDiffs: ssd.SuffixDiffs,
Stem: HexToPrefixedString(ssd.Stem[:]),
Suffixes: HexToPrefixedString(ssd.UpdatedSuffixes),
// Current:
// TODO implement if it makes sense.
})
}

func (ssd *StemStateDiff) UnmarshalJSON(data []byte) error {
var aux stemStateDiffMarshaller
if err := json.Unmarshal(data, &aux); err != nil {
return fmt.Errorf("stemdiff unmarshal error: %w", err)
}

stem, err := PrefixedHexStringToBytes(aux.Stem)
if err != nil {
return fmt.Errorf("invalid hex string for stem: %w", err)
}
*ssd = StemStateDiff{
SuffixDiffs: aux.SuffixDiffs,
}
copy(ssd.Stem[:], stem)
// var aux stemStateDiffMarshaller
// if err := json.Unmarshal(data, &aux); err != nil {
// return fmt.Errorf("stemdiff unmarshal error: %w", err)
// }

// stem, err := PrefixedHexStringToBytes(aux.Stem)
// if err != nil {
// return fmt.Errorf("invalid hex string for stem: %w", err)
// }
// *ssd = StemStateDiff{
// SuffixDiffs: aux.SuffixDiffs,
// }
// copy(ssd.Stem[:], stem)
// TODO implement if it makes sense
return nil
}

type suffixStateDiffMarshaller struct {

Check failure on line 221 in proof_json.go

View workflow job for this annotation

GitHub Actions / lint 

type `suffixStateDiffMarshaller` is unused (unused)
Suffix byte `json:"suffix"`
CurrentValue *string `json:"currentValue"`
NewValue *string `json:"newValue"`
}

func (ssd SuffixStateDiff) MarshalJSON() ([]byte, error) {
var cvstr, nvstr *string
if ssd.CurrentValue != nil {
tempstr := HexToPrefixedString(ssd.CurrentValue[:])
cvstr = &tempstr
}
if ssd.NewValue != nil {
tempstr := HexToPrefixedString(ssd.NewValue[:])
nvstr = &tempstr
}
return json.Marshal(&suffixStateDiffMarshaller{
Suffix: ssd.Suffix,
CurrentValue: cvstr,
NewValue: nvstr,
})
}

func (ssd *SuffixStateDiff) UnmarshalJSON(data []byte) error {
aux := &suffixStateDiffMarshaller{}

if err := json.Unmarshal(data, &aux); err != nil {
return fmt.Errorf("suffix diff unmarshal error: %w", err)
}

if aux.CurrentValue != nil && len(*aux.CurrentValue) != 64 && len(*aux.CurrentValue) != 0 && len(*aux.CurrentValue) != 66 {
return fmt.Errorf("invalid hex string for current value: %s", *aux.CurrentValue)
}

*ssd = SuffixStateDiff{
Suffix: aux.Suffix,
}

if aux.CurrentValue != nil && len(*aux.CurrentValue) != 0 {
currentValueBytes, err := PrefixedHexStringToBytes(*aux.CurrentValue)
if err != nil {
return fmt.Errorf("error decoding hex string for current value: %v", err)
}

ssd.CurrentValue = &[32]byte{}
copy(ssd.CurrentValue[:], currentValueBytes)
}

if aux.NewValue != nil && len(*aux.NewValue) != 0 {
newValueBytes, err := PrefixedHexStringToBytes(*aux.NewValue)
if err != nil {
return fmt.Errorf("error decoding hex string for current value: %v", err)
}

ssd.NewValue = &[32]byte{}
copy(ssd.NewValue[:], newValueBytes)
}

return nil
}
Loading
Loading