diff --git a/.gitignore b/.gitignore index e18ee446..d9e114b6 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ __pycache__ .client_cache/ .utxo_data/ +.utreexo_data .timestamps_data/ .arguments*.json diff --git a/README.md b/README.md index 4867202d..618a6f36 100644 --- a/README.md +++ b/README.md @@ -96,18 +96,21 @@ Tasks: ### Milestone 5 - Full consensus validation -Validate full block execution, including the Bitcoin scripts checks and Utreexo. +Validate full block execution over large number of blocks, including the Bitcoin scripts checks and Utreexo proofs. -### Milestone 6 - Optimizations +* [x] consensus logic +* [ ] consensus logic + utreexo proofs +* [ ] consensus logic + utreexo proofs + scripts -* [ ] identify Cairo code botlenecks -* [ ] consider using garaga msm to batch signature verifications - -### Milestone 7 - Proving the validation +### Milestone 6 - Proving -Recursively verify STARK proofs of chain state updates. Still largely tbd. +Recursively verify STARK proofs of chain state updates. Still largely tbd. From initial observations it is clear that a series of optimizations will be necessary. -* [ ] use garaga to implement signature verifications +* [ ] sha256 optimization +* [ ] don't use ByteArray when serializing data +* [ ] blocklevel recursion +* [ ] consider using garaga msm to batch signature verifications +* [ ] identify other Cairo code botlenecks # Contact @@ -139,6 +142,7 @@ pip install -r scripts/data/requirements.txt ## References * [Data processing notes](./docs/data.md) +* [Utreexo implementation notes](./docs/utreexo.md) * [ZeroSync](https://github.com/ZeroSync/ZeroSync) * [Shinigami Script](https://github.com/keep-starknet-strange/shinigami) * [STWO](https://github.com/starkware-libs/stwo) diff --git a/docs/data.md b/docs/data.md index ff53a929..b588d8e5 100644 --- a/docs/data.md +++ b/docs/data.md @@ -16,4 +16,36 @@ Input data is processed in multiple steps: 5. Script [generate_data](../scripts/data/generate_data.py) generates data that can be consumed by the `validate_and_apply` function. ## UtxoSet -tbd \ No newline at end of file +tbd + +## Utreexo data + +In order to generate Utreexo states and batch proofs for every block in the Bitcoin history we need to use a special [Bridge node](https://github.com/Davidson-Souza/bridge). + +### Install + +```sh +cargo install --git https://github.com/Davidson-Souza/bridge.git --no-default-features --features shinigami +``` + +### Configure + +You need to configure connection to the Bitcoin RPC via environment variables: + +```sh +export BITCOIN_CORE_RPC_URL=http://localhost:8332 +export BITCOIN_CORE_RPC_USER=username +export BITCOIN_CORE_RPC_PASSWORD=password +``` + +### Run + +Run via `screen`, `nohup`, or set up a systemd service: + +```sh +~/.cargo/bin/bridge +``` + +You can access per-block data at `~/.bridge/blocks//.json`: +- Bucket size is 10k blocks; +- The data directory can be changed by setting the `DATA_DIR` environment variable. diff --git a/packages/client/src/test.cairo b/packages/client/src/test.cairo index 3ab773cc..5607dde0 100644 --- a/packages/client/src/test.cairo +++ b/packages/client/src/test.cairo @@ -3,8 +3,9 @@ use core::testing::get_available_gas; use consensus::types::block::Block; use consensus::types::chain_state::{ChainState, BlockValidatorImpl}; use consensus::types::utxo_set::{UtxoSet, UtxoSetTrait}; -use utreexo::vanilla::proof::UtreexoProof; -use utreexo::vanilla::state::{UtreexoState, UtreexoStateTrait}; +use utreexo::stump::accumulator::StumpUtreexoAccumulator; +use utreexo::stump::state::UtreexoStumpState; +use utreexo::stump::proof::UtreexoBatchProof; /// Integration testing program arguments. #[derive(Drop)] @@ -17,17 +18,21 @@ struct Args { expected_chain_state: ChainState, /// Optional Utreexo arguments. utreexo_args: Option, + /// If this flag is set, locking scripts will be executed + execute_script: bool, } /// Utreexo arguments necessary for constraining the UTXO set. #[derive(Drop, Serde)] struct UtreexoArgs { /// Current (initial) accumulator state. - state: UtreexoState, - /// Inclusion proofs for TXOs spent during program run. - proofs: Array, + state: UtreexoStumpState, + /// Batch inclusion proof for TXOs spent during the current block. + /// Note that it doesn't support flow with multiple blocks applied + /// in a single program run. + proof: UtreexoBatchProof, /// Expected accumulator state at the end of the execution. - expected_state: UtreexoState, + expected_state: UtreexoStumpState, } /// Integration testing program entrypoint. @@ -35,12 +40,14 @@ struct UtreexoArgs { /// Receives arguments in a serialized format (Cairo serde). /// Panics in case of a validation error or chain state mismatch. /// Prints result to the stdout. -pub(crate) fn main(mut arguments: Span, execute_script: bool) { +fn main(arguments: Array) -> Array { println!("Running integration test... "); let mut gas_before = get_available_gas(); + let mut args = arguments.span(); - let Args { mut chain_state, blocks, expected_chain_state, utreexo_args } = Serde::deserialize( - ref arguments + let Args { mut chain_state, blocks, expected_chain_state, utreexo_args, execute_script } = + Serde::deserialize( + ref args ) .expect('Failed to deserialize'); @@ -71,11 +78,8 @@ pub(crate) fn main(mut arguments: Span, execute_script: bool) { panic!(); } - if let Option::Some(UtreexoArgs { mut state, proofs, expected_state }) = utreexo_args { - match state - .validate_and_apply( - utxo_set.leaves_to_add.span(), utxo_set.leaves_to_delete.span(), proofs.span(), - ) { + if let Option::Some(UtreexoArgs { mut state, proof, expected_state }) = utreexo_args { + match state.verify_and_delete(@proof, utxo_set.leaves_to_delete.span()) { Result::Ok(new_state) => { state = new_state; }, Result::Err(err) => { println!("FAIL: gas_spent={} error='{:?}'", gas_before - get_available_gas(), err); @@ -83,6 +87,8 @@ pub(crate) fn main(mut arguments: Span, execute_script: bool) { } } + state = state.add(utxo_set.leaves_to_add.span()); + if state != expected_state { println!( "FAIL: gas_spent={} error='expected utreexo state {:?}, actual {:?}'", @@ -95,6 +101,7 @@ pub(crate) fn main(mut arguments: Span, execute_script: bool) { } println!("OK: gas_spent={}", gas_before - get_available_gas()); + array![] } /// Workaround for handling missing `utreexo_args` field. @@ -109,11 +116,14 @@ impl ArgsSerde of Serde { let blocks: Array = Serde::deserialize(ref serialized).expect('blocks'); let expected_chain_state: ChainState = Serde::deserialize(ref serialized) .expect('expected_chain_state'); - let utreexo_args: Option = if serialized.len() > 0 { + let utreexo_args: Option = if serialized.len() > 1 { Option::Some(Serde::deserialize(ref serialized).expect('utreexo_args')) } else { Option::None }; - Option::Some(Args { chain_state, blocks, expected_chain_state, utreexo_args, }) + let execute_script: bool = Serde::deserialize(ref serialized).expect('execute_script'); + Option::Some( + Args { chain_state, blocks, expected_chain_state, utreexo_args, execute_script, } + ) } } diff --git a/packages/client/tests/data/utreexo_0.json b/packages/client/tests/data/utreexo_0.json new file mode 100644 index 00000000..3949d462 --- /dev/null +++ b/packages/client/tests/data/utreexo_0.json @@ -0,0 +1,90 @@ +{ + "chain_state": { + "block_height": 0, + "total_work": "4295032833", + "best_block_hash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505 + ] + }, + "blocks": [ + { + "header": { + "version": 1, + "time": 1231469665, + "bits": 486604799, + "nonce": 2573394689 + }, + "data": { + "variant_id": 1, + "transactions": [ + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x04ffff001d0104", + "sequence": 4294967295, + "previous_output": { + "txid": "0000000000000000000000000000000000000000000000000000000000000000", + "vout": 4294967295, + "data": { + "value": 0, + "pk_script": "0x", + "cached": false + }, + "block_height": 0, + "median_time_past": 0, + "is_coinbase": false + }, + "witness": [ + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "outputs": [ + { + "value": 5000000000, + "pk_script": "0x410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac", + "cached": false + } + ], + "lock_time": 0 + } + ] + } + } + ], + "expected": { + "block_height": 1, + "total_work": "8590065666", + "best_block_hash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505, + 1231469665 + ] + }, + "utreexo": { + "state": { + "roots": [], + "num_leaves": 0 + }, + "proof": { + "proof": [], + "targets": [] + }, + "expected_state": { + "roots": [ + { + "variant_id": 0, + "value": 49459078824306138476779209834441505868925737545954320330266544605873965565 + } + ], + "num_leaves": 1 + } + } +} \ No newline at end of file diff --git a/packages/client/tests/data/utreexo_1.json b/packages/client/tests/data/utreexo_1.json new file mode 100644 index 00000000..395a5d62 --- /dev/null +++ b/packages/client/tests/data/utreexo_1.json @@ -0,0 +1,97 @@ +{ + "chain_state": { + "block_height": 1, + "total_work": "8590065666", + "best_block_hash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505, + 1231469665 + ] + }, + "blocks": [ + { + "header": { + "version": 1, + "time": 1231469744, + "bits": 486604799, + "nonce": 1639830024 + }, + "data": { + "variant_id": 1, + "transactions": [ + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x04ffff001d010b", + "sequence": 4294967295, + "previous_output": { + "txid": "0000000000000000000000000000000000000000000000000000000000000000", + "vout": 4294967295, + "data": { + "value": 0, + "pk_script": "0x", + "cached": false + }, + "block_height": 0, + "median_time_past": 0, + "is_coinbase": false + }, + "witness": [ + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "outputs": [ + { + "value": 5000000000, + "pk_script": "0x41047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac", + "cached": false + } + ], + "lock_time": 0 + } + ] + } + } + ], + "expected": { + "block_height": 2, + "total_work": "12885098499", + "best_block_hash": "000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231006505, + 1231469665, + 1231469744 + ] + }, + "utreexo": { + "state": { + "roots": [ + { + "variant_id": 0, + "value": 49459078824306138476779209834441505868925737545954320330266544605873965565 + } + ], + "num_leaves": 1 + }, + "proof": { + "proof": [], + "targets": [] + }, + "expected_state": { + "roots": [ + { + "variant_id": 0, + "value": 2915651996892698521196383484133475266682410400159810761618404984775762993564 + } + ], + "num_leaves": 2 + } + } +} \ No newline at end of file diff --git a/packages/client/tests/data/utreexo_169.json b/packages/client/tests/data/utreexo_169.json index 76b4be3e..83ec15f0 100644 --- a/packages/client/tests/data/utreexo_169.json +++ b/packages/client/tests/data/utreexo_169.json @@ -128,68 +128,57 @@ "roots": [ { "variant_id": 0, - "value": 1075667171590848154982556435125102517390553068689710620455153660349876510186 + "value": 1502558161404968896970257403264055788192189695540729001360864566276543485500 }, - null, - null, { "variant_id": 0, - "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171 + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 }, - null, { "variant_id": 0, - "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 + "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171 }, - null, { "variant_id": 0, - "value": 2789527937826812553983355805324006356113540169680511206602120580468925994144 - }, - null + "value": 1075667171590848154982556435125102517390553068689710620455153660349876510186 + } + ], + "num_leaves": 169 + }, + "proof": { + "proof": [ + 1341831566043387106325844845909158799402866001662974927108069625603583971480, + 1443675700228503369543683218943466499245617322985505154945738943716743694265, + 519646392377825945466547611274092870030802470736981675895864465165188803020, + 1080650929466224224603091215856096240942830750577279058777853241919006292201, + 3429608730564493038725327726731886582242605618763607208860625722555112702870, + 1464346753344216951108436013919475968846135803696833681088250234646995926893, + 2059532798546650597701371553400476891060690283065963625811202716166665983015 + ], + "targets": [ + 8 ] }, - "proofs": [ - { - "proof": [ - 1341831566043387106325844845909158799402866001662974927108069625603583971480, - 1443675700228503369543683218943466499245617322985505154945738943716743694265, - 519646392377825945466547611274092870030802470736981675895864465165188803020, - 3219090166904938598688939525052593187314786152670445479780817128254606574061, - 3429608730564493038725327726731886582242605618763607208860625722555112702870, - 1464346753344216951108436013919475968846135803696833681088250234646995926893, - 2059532798546650597701371553400476891060690283065963625811202716166665983015 - ], - "leaf_index": 8 - } - ], - "expected": { + "expected_state": { "roots": [ { "variant_id": 0, - "value": 2798110709967837858515372760817606172577565229842771835911030873647893889774 + "value": 472495115707702931286975202981613081341496930913817199425502977245962405497 }, { "variant_id": 0, - "value": 143588888139305297948079116321466646636133238970252601366055285460885727445 + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 }, - null, { "variant_id": 0, "value": 1545231468516220002373273797559461807914970524111940849750328803582855936171 }, - null, - { - "variant_id": 0, - "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 - }, - null, { "variant_id": 0, - "value": 72427602467111712670190708647558062003041829148690045408562661193902370953 - }, - null - ] + "value": 940126151801873300266416875114055738009783862579557360123104711005733824770 + } + ], + "num_leaves": 172 } } } \ No newline at end of file diff --git a/packages/client/tests/data/utreexo_182.json b/packages/client/tests/data/utreexo_182.json new file mode 100644 index 00000000..929c425d --- /dev/null +++ b/packages/client/tests/data/utreexo_182.json @@ -0,0 +1,195 @@ +{ + "chain_state": { + "block_height": 182, + "total_work": "785991008439", + "best_block_hash": "0000000054487811fc4ff7a95be738aa5ad9320c394c482b27c0da28b227ad5d", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231731853, + 1231732245, + 1231732861, + 1231733876, + 1231735142, + 1231736557, + 1231737131, + 1231738005, + 1231739452, + 1231740133, + 1231740736 + ] + }, + "blocks": [ + { + "header": { + "version": 1, + "time": 1231742062, + "bits": 486604799, + "nonce": 3235118355 + }, + "data": { + "variant_id": 1, + "transactions": [ + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x04ffff001d012a", + "sequence": 4294967295, + "previous_output": { + "txid": "0000000000000000000000000000000000000000000000000000000000000000", + "vout": 4294967295, + "data": { + "value": 0, + "pk_script": "0x", + "cached": false + }, + "block_height": 0, + "median_time_past": 0, + "is_coinbase": false + }, + "witness": [ + "0x0000000000000000000000000000000000000000000000000000000000000000" + ] + } + ], + "outputs": [ + { + "value": 5000000000, + "pk_script": "0x41041915d670aa55621ed1c438477c5da654344e10eecea90ede3a048103fc3cf4beccda70ddf71ce55a5083b7bade4319d159b44374234590e9296cbe08c67774b2ac", + "cached": false + } + ], + "lock_time": 0 + }, + { + "version": 1, + "is_segwit": false, + "inputs": [ + { + "script": "0x483045022052ffc1929a2d8bd365c6a2a4e3421711b4b1e1b8781698ca9075807b4227abcb0221009984107ddb9e3813782b095d0d84361ed4c76e5edaf6561d252ae162c2341cfb01", + "sequence": 4294967295, + "previous_output": { + "txid": "591e91f809d716912ca1d4a9295e70c3e78bab077683f79350f101da64588073", + "vout": 1, + "data": { + "value": 2900000000, + "pk_script": "0x410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac", + "cached": false + }, + "block_height": 182, + "median_time_past": 1231735142, + "is_coinbase": false + }, + "witness": [] + } + ], + "outputs": [ + { + "value": 100000000, + "pk_script": "0x4104baa9d36653155627c740b3409a734d4eaf5dcca9fb4f736622ee18efcf0aec2b758b2ec40db18fbae708f691edb2d4a2a3775eb413d16e2e3c0f8d4c69119fd1ac", + "cached": false + }, + { + "value": 2800000000, + "pk_script": "0x410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac", + "cached": false + } + ], + "lock_time": 0 + } + ] + } + } + ], + "expected": { + "block_height": 183, + "total_work": "790286041272", + "best_block_hash": "00000000f46e513f038baf6f2d9a95b2a28d8a6c985bcf24b9e07f0f63a29888", + "current_target": "26959535291011309493156476344723991336010898738574164086137773096960", + "epoch_start_time": 1231006505, + "prev_timestamps": [ + 1231732245, + 1231732861, + 1231733876, + 1231735142, + 1231736557, + 1231737131, + 1231738005, + 1231739452, + 1231740133, + 1231740736, + 1231742062 + ] + }, + "utreexo": { + "state": { + "roots": [ + { + "variant_id": 0, + "value": 472495115707702931286975202981613081341496930913817199425502977245962405497 + }, + { + "variant_id": 0, + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 + }, + { + "variant_id": 0, + "value": 3379475774617422517749239484786779535800164983837536926832995269432589051493 + }, + { + "variant_id": 0, + "value": 2960972179536349754495570588856684293379166070401208198804481828943354090177 + }, + { + "variant_id": 0, + "value": 1027525398361326601103535357192346699264220866701033959615605604739328879107 + } + ], + "num_leaves": 188 + }, + "proof": { + "proof": [ + 471794939040413771741937605933609163704938509795149935005523699179740318544, + 963498998615873495259761020963144237838429346272910499751572193538456625178 + ], + "targets": [ + 187 + ] + }, + "expected_state": { + "roots": [ + { + "variant_id": 0, + "value": 472495115707702931286975202981613081341496930913817199425502977245962405497 + }, + { + "variant_id": 0, + "value": 1377893971144855926456275647843230957136416614121223440964512004900657355746 + }, + { + "variant_id": 0, + "value": 3379475774617422517749239484786779535800164983837536926832995269432589051493 + }, + { + "variant_id": 0, + "value": 2960972179536349754495570588856684293379166070401208198804481828943354090177 + }, + { + "variant_id": 0, + "value": 1393936218426697313222000203973633691677533429852178919818058911865240713624 + }, + { + "variant_id": 0, + "value": 388190300499697056611762555012906818486802305728968255357693461417536026290 + }, + { + "variant_id": 0, + "value": 493770350110029404732941515600367633414267409352068154843236614730043385883 + } + ], + "num_leaves": 191 + } + } +} \ No newline at end of file diff --git a/packages/consensus/src/types/chain_state.cairo b/packages/consensus/src/types/chain_state.cairo index 0f740951..e31a5e17 100644 --- a/packages/consensus/src/types/chain_state.cairo +++ b/packages/consensus/src/types/chain_state.cairo @@ -73,11 +73,11 @@ pub impl BlockValidatorImpl of BlockValidator { TransactionData::MerkleRoot(root) => root, TransactionData::Transactions(txs) => { let (total_fees, txid_root, wtxid_root) = compute_and_validate_tx_data( - txs, block_height, median_time_past, ref utxo_set + txs, block_height, block.header.time, median_time_past, ref utxo_set )?; validate_coinbase(txs[0], total_fees, block_height, wtxid_root)?; if execute_script { - validate_scripts(@block.header, txs)?; + validate_scripts(@block.header, txs.slice(1, txs.len() - 1))?; } txid_root } diff --git a/packages/consensus/src/types/transaction.cairo b/packages/consensus/src/types/transaction.cairo index 99bcc4dd..d18eea74 100644 --- a/packages/consensus/src/types/transaction.cairo +++ b/packages/consensus/src/types/transaction.cairo @@ -276,7 +276,7 @@ mod tests { } #[test] - pub fn test_outpoint_poseidon_hash() { + pub fn test_outpoint_poseidon_hash_cb9() { let mut coinbase_9_utxo = OutPoint { txid: hex_to_hash_rev( "0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9" @@ -318,4 +318,48 @@ mod tests { 761592244424273723796345514960638980240531938129162865626185984897576522513, hash ); } + + #[test] + pub fn test_outpoint_poseidon_hash_cb1() { + let mut coinbase_9_utxo = OutPoint { + txid: hex_to_hash_rev( + "0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098" + ), + vout: 0, + data: TxOut { + value: 5000000000, + pk_script: @from_hex( + "410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac" + ), + cached: false, + }, + block_height: 1, + median_time_past: 1231006505, + is_coinbase: true, + }; + + let mut state: HashState = Default::default(); + state = state.update_with(coinbase_9_utxo); + + let expected: Array = array![ + 18931831195212887181660290436187791739, + 137019159177035157628276746705882390680, + 0, + 5000000000, + 2, + 114876729272917404712191936498804624660105992100397383656070609774475449467, + 406791163401893627439198994794895943141891052128672824792182596804809637667, + 286829113004, + 5, + 1, + 1231006505, + 1 + ]; + assert_eq!(expected, state.value); + + let hash = coinbase_9_utxo.hash(); + assert_eq!( + 49459078824306138476779209834441505868925737545954320330266544605873965565, hash + ); + } } diff --git a/packages/consensus/src/validation/block.cairo b/packages/consensus/src/validation/block.cairo index 550e7815..da088d4e 100644 --- a/packages/consensus/src/validation/block.cairo +++ b/packages/consensus/src/validation/block.cairo @@ -31,7 +31,11 @@ pub fn validate_block_weight(weight: usize) -> Result<(), ByteArray> { /// - wTXID commitment (only for blocks after Segwit upgrade, otherwise returns zero hash) /// - Block weight pub fn compute_and_validate_tx_data( - txs: Span, block_height: u32, median_time_past: u32, ref utxo_set: UtxoSet + txs: Span, + block_height: u32, + block_time: u32, + median_time_past: u32, + ref utxo_set: UtxoSet ) -> Result<(u64, Digest, Digest), ByteArray> { let mut txids: Array = array![]; let mut wtxids: Array = array![]; @@ -94,7 +98,9 @@ pub fn compute_and_validate_tx_data( is_coinbase = false; } else { let fee = - match validate_transaction(tx, block_height, median_time_past, txid, ref utxo_set) { + match validate_transaction( + tx, block_height, block_time, median_time_past, txid, ref utxo_set + ) { Result::Ok(fee) => fee, Result::Err(err) => { inner_result = Result::Err(err); diff --git a/packages/consensus/src/validation/locktime.cairo b/packages/consensus/src/validation/locktime.cairo index 8e240cbb..eb2fecfa 100644 --- a/packages/consensus/src/validation/locktime.cairo +++ b/packages/consensus/src/validation/locktime.cairo @@ -82,7 +82,7 @@ pub fn validate_relative_locktime( if absolute_lock_time > median_time_past { return Result::Err( format!( - "Relative time-based lock time is not respected: current MTP {}, outpoint MTP {}, lock time {} seconds", + "Relative time-based lock time is not respected: current MTP: {}, outpoint MTP: {}, lock time: {} seconds", median_time_past, *input.previous_output.median_time_past, lock_time @@ -94,7 +94,7 @@ pub fn validate_relative_locktime( if absolute_lock_time > block_height { return Result::Err( format!( - "Relative block-based lock time is not respected: current height {}, outpoint height {}, lock time {} blocks", + "Relative block-based lock time is not respected: current height: {}, outpoint height: {}, lock time: {} blocks", block_height, *input.previous_output.block_height, value diff --git a/packages/consensus/src/validation/script.cairo b/packages/consensus/src/validation/script.cairo index 0d6f5e38..e8df49ed 100644 --- a/packages/consensus/src/validation/script.cairo +++ b/packages/consensus/src/validation/script.cairo @@ -190,7 +190,7 @@ fn validate_script(header: @Header, tx: @Transaction, tx_idx: u32) -> Result<(), /// Validates scripts for one or multiple transactions. pub fn validate_scripts(header: @Header, txs: Span) -> Result<(), ByteArray> { let mut r = Result::Ok(()); - let mut i = 0; + let mut i = 1; for tx in txs { r = validate_script(header, tx, i); if r.is_err() { diff --git a/packages/consensus/src/validation/timestamp.cairo b/packages/consensus/src/validation/timestamp.cairo index 53870dac..5accee41 100644 --- a/packages/consensus/src/validation/timestamp.cairo +++ b/packages/consensus/src/validation/timestamp.cairo @@ -37,7 +37,7 @@ pub fn compute_median_time_past(prev_timestamps: Span) -> u32 { }; }; - *sorted_prev_timestamps.at(sorted_prev_timestamps.len() - 6) + *sorted_prev_timestamps.at(sorted_prev_timestamps.len() / 2) } /// Checks that the block time is greater than the Median Time Past (MTP). @@ -55,8 +55,11 @@ pub fn validate_timestamp(median_time_past: u32, block_time: u32) -> Result<(), /// one. pub fn next_prev_timestamps(prev_timestamps: Span, block_time: u32) -> Span { let mut timestamps: Array = prev_timestamps.into(); - timestamps.pop_front().unwrap(); // Remove the oldest timestamp (not necessarily the min) - timestamps.append(block_time); // Append the most recent timestamp (not necessarily the max) + if timestamps.len() == 11 { + timestamps.pop_front().unwrap(); // remove the oldest timestamp (not necessarily the min) + } + timestamps.append(block_time); // append the most recent timestamp (not necessarily the max) + timestamps.span() } @@ -125,4 +128,13 @@ mod tests { let result = validate_timestamp(mtp, block_time); assert!(result.is_err(), "MTP is greater than block's timestamp"); } + + #[test] + fn test_few_prev_timestamps() { + assert_eq!(1, compute_median_time_past(array![1].span())); + assert_eq!(2, compute_median_time_past(array![1, 2].span())); + assert_eq!(2, compute_median_time_past(array![1, 2, 3].span())); + assert_eq!(3, compute_median_time_past(array![1, 2, 3, 4].span())); + assert_eq!(3, compute_median_time_past(array![1, 2, 3, 4, 5].span())); + } } diff --git a/packages/consensus/src/validation/transaction.cairo b/packages/consensus/src/validation/transaction.cairo index ff059e38..1dfb4e8d 100644 --- a/packages/consensus/src/validation/transaction.cairo +++ b/packages/consensus/src/validation/transaction.cairo @@ -14,7 +14,12 @@ const MAX_SCRIPT_SIZE: u32 = 10000; /// /// This does not include script checks and outpoint inclusion verification. pub fn validate_transaction( - tx: @Transaction, block_height: u32, median_time_past: u32, txid: Digest, ref utxo_set: UtxoSet + tx: @Transaction, + block_height: u32, + block_time: u32, + median_time_past: u32, + txid: Digest, + ref utxo_set: UtxoSet ) -> Result { if (*tx.inputs).is_empty() { return Result::Err("transaction inputs are empty"); @@ -48,7 +53,7 @@ pub fn validate_transaction( } } - if !is_input_final(*input.sequence) { + if *tx.version >= 2 && !is_input_final(*input.sequence) { inner_result = validate_relative_locktime(input, block_height, median_time_past); if inner_result.is_err() { break; @@ -65,7 +70,7 @@ pub fn validate_transaction( if !is_tx_final { // If at least one input is not final - validate_absolute_locktime(*tx.lock_time, block_height, median_time_past)?; + validate_absolute_locktime(*tx.lock_time, block_height, block_time)?; } // Validate and process transaction outputs @@ -181,11 +186,11 @@ mod tests { let txid = double_sha256_byte_array(tx_bytes_legacy); let mut utxo_set: UtxoSet = Default::default(); - assert!(validate_transaction(@tx, 0, 0, txid, ref utxo_set).is_err()); + assert!(validate_transaction(@tx, 0, 0, 0, txid, ref utxo_set).is_err()); utxo_set = Default::default(); - let fee = validate_transaction(@tx, 101, 0, txid, ref utxo_set).unwrap(); + let fee = validate_transaction(@tx, 101, 0, 0, txid, ref utxo_set).unwrap(); assert_eq!(fee, 10); } @@ -210,7 +215,7 @@ mod tests { let txid = double_sha256_byte_array(tx_bytes_legacy); let mut utxo_set: UtxoSet = Default::default(); - let result = validate_transaction(@tx, 0, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, 0, 0, 0, txid, ref utxo_set); assert_eq!(result.unwrap_err(), "transaction inputs are empty"); } @@ -245,14 +250,14 @@ mod tests { let txid = double_sha256_byte_array(tx_bytes_legacy); let mut utxo_set: UtxoSet = Default::default(); - let result = validate_transaction(@tx, 0, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, 0, 0, 0, txid, ref utxo_set); assert_eq!(result.unwrap_err(), "transaction outputs are empty"); } #[test] fn test_absolute_locktime_block_height() { let tx = Transaction { - version: 1, + version: 2, is_segwit: false, inputs: array![ TxIn { @@ -288,7 +293,7 @@ mod tests { let mut utxo_set: UtxoSet = Default::default(); // Transaction should be invalid when current block height is less than locktime - let result = validate_transaction(@tx, 500000, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, 500000, 0, 0, txid, ref utxo_set); assert_eq!( result.unwrap_err().into(), "Transaction locktime 500000 is not lesser than current block height 500000" @@ -298,14 +303,14 @@ mod tests { // Transaction should be valid when current block height is equal to or greater than // locktime - let result = validate_transaction(@tx, 500001, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, 500001, 0, 0, txid, ref utxo_set); assert!(result.is_ok()); } #[test] fn test_absolute_locktime_block_time() { let tx = Transaction { - version: 1, + version: 2, is_segwit: false, inputs: array![ TxIn { @@ -341,7 +346,7 @@ mod tests { let mut utxo_set: UtxoSet = Default::default(); // Transaction should be invalid when current block time is not greater than locktime - let result = validate_transaction(@tx, 0, 1600000000, txid, ref utxo_set); + let result = validate_transaction(@tx, 0, 1600000000, 1600000000, txid, ref utxo_set); assert_eq!( result.unwrap_err().into(), "Transaction locktime 1600000000 is not lesser than current block time 1600000000" @@ -350,7 +355,7 @@ mod tests { utxo_set = Default::default(); // Transaction should be valid when current block time is equal to or greater than locktime - let result = validate_transaction(@tx, 0, 1600000001, txid, ref utxo_set); + let result = validate_transaction(@tx, 0, 1600000001, 1600000001, txid, ref utxo_set); assert!(result.is_ok()); } @@ -393,13 +398,13 @@ mod tests { let mut utxo_set: UtxoSet = Default::default(); // Transaction should still valid when current block time is not greater than locktime - let result = validate_transaction(@tx, 0, 1600000000, txid, ref utxo_set); + let result = validate_transaction(@tx, 0, 1600000000, 1600000000, txid, ref utxo_set); assert!(result.is_ok()); utxo_set = Default::default(); // Transaction should be valid when current block time is greater than locktime - let result = validate_transaction(@tx, 0, 1600000001, txid, ref utxo_set); + let result = validate_transaction(@tx, 0, 1600000001, 1600000001, txid, ref utxo_set); assert!(result.is_ok()); } @@ -442,13 +447,13 @@ mod tests { let mut utxo_set: UtxoSet = Default::default(); // Transaction should still valid when current block time is not greater than locktime - let result = validate_transaction(@tx, 500000, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, 500000, 0, 0, txid, ref utxo_set); assert!(result.is_ok()); utxo_set = Default::default(); // Transaction should be valid when current block time is greater than locktime - let result = validate_transaction(@tx, 500001, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, 500001, 0, 0, txid, ref utxo_set); assert!(result.is_ok()); } @@ -492,7 +497,7 @@ mod tests { let txid = double_sha256_byte_array(tx_bytes_legacy); let mut utxo_set: UtxoSet = Default::default(); - validate_transaction(@tx, block_height, 0, txid, ref utxo_set).unwrap_err(); + validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set).unwrap_err(); } #[test] @@ -535,7 +540,7 @@ mod tests { let txid = double_sha256_byte_array(tx_bytes_legacy); let mut utxo_set: UtxoSet = Default::default(); - validate_transaction(@tx, block_height, 0, txid, ref utxo_set).unwrap(); + validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set).unwrap(); } #[test] @@ -579,7 +584,7 @@ mod tests { let txid = double_sha256_byte_array(tx_bytes_legacy); let mut utxo_set: UtxoSet = Default::default(); - validate_transaction(@tx, block_height, 0, txid, ref utxo_set).unwrap(); + validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set).unwrap(); } #[test] @@ -627,7 +632,7 @@ mod tests { cache.insert(outpoint_hash, TX_OUTPUT_STATUS_UNSPENT); let mut utxo_set = UtxoSet { cache, ..Default::default() }; - validate_transaction(@tx, block_height, 0, txid, ref utxo_set).unwrap(); + validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set).unwrap(); } #[test] @@ -674,7 +679,7 @@ mod tests { cache.insert(outpoint_hash, TX_OUTPUT_STATUS_UNSPENT); let mut utxo_set = UtxoSet { cache, ..Default::default() }; - validate_transaction(@tx, block_height, 0, txid, ref utxo_set).unwrap(); + validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set).unwrap(); } #[test] @@ -729,7 +734,7 @@ mod tests { cache.insert(outpoint_hash, TX_OUTPUT_STATUS_UNSPENT); let mut utxo_set = UtxoSet { cache, ..Default::default() }; - let result = validate_transaction(@tx, block_height, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set); assert_eq!(result.unwrap_err(), "The output has already been added"); } @@ -792,7 +797,7 @@ mod tests { cache.insert(outpoint_hash, TX_OUTPUT_STATUS_UNSPENT); let mut utxo_set = UtxoSet { cache, ..Default::default() }; - let result = validate_transaction(@tx, block_height, 0, txid, ref utxo_set); + let result = validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set); assert_eq!(result.unwrap_err(), "The output has already been spent"); } @@ -850,8 +855,8 @@ mod tests { let tx_bytes_legacy = @tx.encode(); let txid = double_sha256_byte_array(tx_bytes_legacy); let mut utxo_set: UtxoSet = Default::default(); - - let result = validate_transaction(@tx, block_height, 0, txid, ref utxo_set); + + let result = validate_transaction(@tx, block_height, 0, 0, txid, ref utxo_set); assert_eq!(result.unwrap_err(), "The output has already been spent"); } diff --git a/packages/utreexo/src/test.cairo b/packages/utreexo/src/test.cairo index 1017a5f2..c43fe522 100644 --- a/packages/utreexo/src/test.cairo +++ b/packages/utreexo/src/test.cairo @@ -10,12 +10,14 @@ struct Args { leaves_to_del: Array, leaves_to_add: Array, expected_state: UtreexoStumpState, + _unused: felt252, } -fn main(mut arguments: Span, _flags: felt252) { +fn main(args: Array) -> Array { let mut gas_before = get_available_gas(); + let mut arguments = args.span(); - let Args { mut state, proof, leaves_to_del, leaves_to_add, expected_state } = + let Args { mut state, proof, leaves_to_del, leaves_to_add, expected_state, _unused: _ } = Serde::deserialize( ref arguments ) @@ -50,4 +52,5 @@ fn main(mut arguments: Span, _flags: felt252) { } println!("OK: gas_spent={}", gas_before - get_available_gas()); + array![] } diff --git a/scripts/data/format_args.py b/scripts/data/format_args.py index 05ace152..c6c5f7ab 100755 --- a/scripts/data/format_args.py +++ b/scripts/data/format_args.py @@ -65,7 +65,7 @@ def serialize(obj): raise NotImplementedError(obj) -def flatten_tuples(src): +def flatten_tuples(src) -> list: """Recursively flattens tuples. Example: (0, (1, 2), [(3, 4, [5, 6])]) -> [0, 1, 2, [3, 4, [5, 6]]] @@ -117,11 +117,11 @@ def format_args(input_file, execute_script, cairo1_run): """ args = json.loads(Path(input_file).read_text()) res = flatten_tuples(serialize(args)) - flag = 1 if execute_script else 0 + res.append(1 if execute_script else 0) if cairo1_run: - return f"{format_cairo1_run(res)} {flag}" + return format_cairo1_run(res) else: - return [res, flag] + return [res] if __name__ == "__main__": diff --git a/scripts/data/generate_data.py b/scripts/data/generate_data.py index 1335966e..43f6667e 100755 --- a/scripts/data/generate_data.py +++ b/scripts/data/generate_data.py @@ -11,7 +11,7 @@ import requests from generate_timestamp_data import get_timestamp_data -from generate_utreexo_data import UtreexoData +from generate_utreexo_data import get_utreexo_data from generate_utxo_data import get_utxo_set logger = logging.getLogger(__name__) @@ -84,7 +84,7 @@ def fetch_chain_state(block_height: int): prev_timestamps = [int(head["time"])] for _ in range(10): if prev_header["height"] == 0: - prev_timestamps.insert(0, 0) + break else: prev_header = request_rpc( "getblockheader", [prev_header["previousblockhash"]] @@ -323,6 +323,7 @@ def generate_data( "utreexo" — only last block from the batch is included, but it is extended with Utreexo state/proofs :param initial_height: The block height of the initial chain state (0 means the state after genesis) :param num_blocks: The number of blocks to apply on top of it (has to be at least 1) + :param fast: Use data exported from BigQuery rather than Bitcoin node :return: tuple (arguments, expected output) """ @@ -335,12 +336,10 @@ def generate_data( if fast else fetch_chain_state(initial_height) ) - prev_chain_state = None initial_chain_state = chain_state next_block_hash = chain_state["nextblockhash"] blocks = [] - utreexo_data = {} for i in range(num_blocks): logger.debug(f"Fetching block {initial_height + i + 1} {i + 1}/{num_blocks}...") @@ -380,7 +379,6 @@ def generate_data( raise NotImplementedError(mode) blocks.append(block) - prev_chain_state = chain_state chain_state = next_chain_state(chain_state, block) next_block_hash = block["nextblockhash"] @@ -396,13 +394,8 @@ def generate_data( } if mode == "utreexo": - utreexo_data = UtreexoData() - result["utreexo"] = utreexo_data.apply_blocks( - blocks, int(chain_state["mediantime"]) - ) - result["blocks"] = [result["blocks"][-1]] - if num_blocks > 1: - result["chain_state"] = format_chain_state(prev_chain_state) + assert len(blocks) == 1, "Cannot handle more than one block in Utreexo mode" + result["utreexo"] = get_utreexo_data(blocks[0]["height"]) return result diff --git a/scripts/data/generate_utreexo_data.py b/scripts/data/generate_utreexo_data.py index 307752e5..2f923f6e 100644 --- a/scripts/data/generate_utreexo_data.py +++ b/scripts/data/generate_utreexo_data.py @@ -1,227 +1,59 @@ #!/usr/bin/env python3 -import typing as t -from collections import OrderedDict -from utreexo import Utreexo -from poseidon_py.poseidon_hash import poseidon_hash_many +import os +import json +BASE_DIR = f"{os.path.dirname(os.path.realpath(__file__))}/.utreexo_data" +BUCKET_SIZE = 10000 -class TxOut: - def __init__(self, value, pk_script, cached): - self.value = value - self.pk_script = pk_script - self.cached = cached - - def _calculate_sub_data(self) -> t.List[int]: - sub_data = [] - pending_word = 0 - pending_word_len = 0 - - # Convert `pk_script` to bytes - if self.pk_script.startswith("0x") or self.pk_script.startswith("0X"): - hex_str = self.pk_script[2:] - byte_data = bytes.fromhex(hex_str) - else: - byte_data = self.pk_script.encode("utf-8") - - for byte in byte_data: - if not (0 <= byte <= 255): - raise ValueError("Not between 0 and 255.") - - if pending_word_len == 0: - pending_word = byte - pending_word_len = 1 - continue - - new_pending = pending_word * 0x100 + byte - if pending_word_len < 31 - 1: - pending_word = new_pending - pending_word_len += 1 - continue - - # Convert new_pending to 31 bytes (big endian) - new_pending_bytes = new_pending.to_bytes(31, byteorder="big") - word_int = int.from_bytes(new_pending_bytes, "big") - sub_data.append(word_int) - - pending_word = 0 - pending_word_len = 0 - - sub_data.append(pending_word) - sub_data.append(pending_word_len) - - return sub_data - - def serialize(self) -> OrderedDict: - res = OrderedDict() - - sub_data = self._calculate_sub_data() - - res["value"] = self.value - # length of the array containing full words - res["sub_data_len"] = len(sub_data) - 2 - - for idx, word in enumerate(sub_data): - res["sub_data_{}".format(idx)] = word - - return res - - def __repr__(self): - return f"TxOut(\n\ - value={self.value}\n\ - pk_script={self.pk_script}\n\ - cached={self.cached})" - - -class OutPoint: - def __init__( - self, txid, vout, data: TxOut, block_height, median_time_past, is_coinbase - ): - self.txid = txid - self.vout = vout - self.data = data # Instance de TxOut - self.block_height = block_height - self.median_time_past = median_time_past - self.is_coinbase = is_coinbase - - def hash(self): - tab = [] - - # txid (2x u128 in little endian high/low) - txid_bytes = bytes.fromhex(self.txid) - tab.append(int.from_bytes(txid_bytes[:16], "big")) - tab.append(int.from_bytes(txid_bytes[16:], "big")) - - # vout - tab.append(self.vout) - - # prev output - for _, e in self.data.serialize().items(): - tab.append(e) - - tab.append(self.block_height) - tab.append(self.median_time_past) - - tab.append(int(self.is_coinbase)) - hash = poseidon_hash_many(tab) - return hash - - def __repr__(self): - return f"OutPoint(\n\ - txid={self.txid}\n\ - vout={self.vout}\n\ - tx_out={self.data}\n\ - block_height={self.block_height}\n\ - median_time_past={self.median_time_past}\n\ - is_coinbase={self.is_coinbase}\n\ - hash={self.hash()})" +def get_utreexo_data(block_height): + bucket_number = block_height // BUCKET_SIZE + utreexo_data = {} - -class UtreexoData: - def __init__(self) -> None: - self.utreexo = Utreexo() - - def snapshot_state(self) -> dict: - return { - "roots": list(map(format_root_node, self.utreexo.root_nodes)), + if block_height > 1: + with open(f"{BASE_DIR}/{bucket_number}/{block_height - 1}.json") as f: + data = json.loads(f.read()) + utreexo_data["state"] = convert_state(data["utreexo_state"]) + else: + utreexo_data["state"] = { + "roots": [], + "num_leaves": 0, } - def apply_blocks(self, blocks: list, prev_mtp: int) -> dict: - state = {} - proofs = [] - for block_idx, block in enumerate(blocks): - if block_idx == len(blocks) - 1: - state = self.snapshot_state() - - # First we remove all STXOs - for i, (txid, tx) in enumerate(block["data"].items()): - # Skip coinbase tx input - if i != 0: - inc_proofs = self.handle_txin(tx["inputs"]) - # Storing proofs for last block only - if block_idx == len(blocks) - 1: - proofs.extend(inc_proofs) - - # Then we add all new UTXOs - for i, (txid, tx) in enumerate(block["data"].items()): - self.handle_txout( - tx["outputs"], - block["height"], - prev_mtp, - txid, - i == 0, - ) - prev_mtp = block["mediantime"] - - return {"state": state, "proofs": proofs, "expected": self.snapshot_state()} - - def handle_txin(self, inputs: list) -> list: - proofs = [] - for input in inputs: - outpoint = input["previous_output"] - - # Skip if output is cached - if outpoint["data"]["cached"]: - continue - - outpoint = OutPoint( - txid=outpoint["txid"], - vout=outpoint["vout"], - data=TxOut( - value=outpoint["data"]["value"], - pk_script=outpoint["data"]["pk_script"], - cached=outpoint["data"]["cached"], - ), - block_height=outpoint["block_height"], - median_time_past=outpoint["median_time_past"], - is_coinbase=outpoint["is_coinbase"], - ) - - # Remove OutPoint from accumulator and get proof - proof, leaf_index = self.utreexo.delete(outpoint.hash()) - proofs.append( - {"proof": list(map(format_node, proof)), "leaf_index": leaf_index} - ) - - return proofs - - def handle_txout( - self, - outputs: list, - block_height: int, - median_time_past: int, - txid: str, - is_coinbase: bool, - ): - for i, output in enumerate(outputs): - # Skip if output is cached - if output["cached"]: - continue - - new_outpoint = OutPoint( - txid=txid, - vout=i, - data=TxOut( - value=output["value"], - pk_script=output["pk_script"], - cached=output["cached"], - ), - block_height=block_height, - median_time_past=median_time_past, - is_coinbase=is_coinbase, - ) - - # Add OutPoint to accumulator - self.utreexo.add(new_outpoint.hash()) - - -def format_root_node(node) -> str: - if node: - return {"variant_id": 0, "value": format_node(node)} - else: + with open(f"{BASE_DIR}/{bucket_number}/{block_height}.json") as f: + data = json.loads(f.read()) + utreexo_data["proof"] = convert_proof(data["inclusion_proof"]) + utreexo_data["expected_state"] = convert_state(data["utreexo_state"]) + + return utreexo_data + + +def convert_proof(proof): + return { + "proof": list(map(convert_felt, proof["hashes"])), + "targets": proof["targets"], + } + + +def convert_state(state): + return { + "roots": list(map(convert_root, state["roots"])), + "num_leaves": state["num_leaves"], + } + + +def convert_root(root: str) -> int: + if root is None: return None + else: + return {"variant_id": 0, "value": convert_felt(root)} + + +def convert_felt(felt: str) -> int: + return int.from_bytes(bytes.fromhex(felt[2:]), "big") -def format_node(node) -> int: - return int.from_bytes(bytes.fromhex(node.val[2:]), "big") +if __name__ == "__main__": + # TODO: download ~/.bridge/blocks folder bucket by bucket from a remote server + raise NotImplementedError diff --git a/scripts/data/regenerate_tests.sh b/scripts/data/regenerate_tests.sh index 53ff811b..a5ba7fe4 100755 --- a/scripts/data/regenerate_tests.sh +++ b/scripts/data/regenerate_tests.sh @@ -66,7 +66,10 @@ full_test_cases=( ) utreexo_test_cases=( + 0 # To test leaf serialization compliance + 1 169 # Block containing first P2P tx to Hal Finney (170) + 182 ) mkdir $data_dir || true @@ -94,5 +97,5 @@ done for test_case in "${utreexo_test_cases[@]}"; do echo -e "\nGenerating test data: utreexo mode, chain state @ $test_case, single block" - generate_test "utreexo" 0 $(($test_case+1)) + generate_test "utreexo" $test_case done diff --git a/scripts/data/utreexo.py b/scripts/data/utreexo.py deleted file mode 100644 index 19865098..00000000 --- a/scripts/data/utreexo.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 ZeroSync -# -# SPDX-License-Identifier: MIT -from poseidon_py.poseidon_hash import poseidon_hash_many - - -class Node: - def __init__(self, key, left=None, right=None): - self.val = key - self.left = left - self.right = right - self.parent = None - - def __repr__(self) -> str: - return str(self.val) - - -class Utreexo: - def __init__(self): - self.root_nodes = [] - self.leaf_nodes = {} - - def print_state(self): - print("--- Utreexo State ---") - print("Root Nodes:") - for idx, node in enumerate(self.root_nodes): - if node is not None: - print(f" Height {idx}: {node.val}") - print("\nLeaf Nodes:") - for leaf_val, node in self.leaf_nodes.items(): - print(f" Leaf {leaf_val}: {node.val}") - print("---------------------\n") - - def print_tree(self): - print("--- Utreexo Tree ---") - - def _print_subtree(node, prefix=""): - if node is not None: - print(f"{prefix}{node.val}") - if node.left is not None or node.right is not None: - # Display branches of the left subtree - _print_subtree(node.left, prefix + "├── ") - # Display branches of the right subtree - _print_subtree(node.right, prefix + "└── ") - - for idx, root in enumerate(self.root_nodes): - if root is not None: - print(f"Root at height {idx}:") - _print_subtree(root) - print("---------------------\n") - - def parent_node(self, root1, root2): - val1 = int(root1.val, 16) - val2 = int(root2.val, 16) - root = poseidon_hash_many([val1, val2]) - root_hex = f"0x{root:064x}" - root_node = Node(root_hex, root1, root2) - root1.parent = root_node - root2.parent = root_node - return root_node - - def add(self, leaf): - if leaf in self.leaf_nodes: - raise Exception("Leaf exists already") - - n = Node(f"0x{leaf:064x}") - self.leaf_nodes[leaf] = n - h = 0 - - while len(self.root_nodes) <= h: - self.root_nodes.append(None) - - r = self.root_nodes[h] - while r is not None: - n = self.parent_node(r, n) - self.root_nodes[h] = None - h += 1 - r = self.root_nodes[h] - self.root_nodes[h] = n - - if h == len(self.root_nodes) - 1: - self.root_nodes.append(None) - - return self.root_nodes - - def delete(self, leaf): - if leaf not in self.leaf_nodes: - raise Exception(f"Leaf {leaf} does not exist in the Utreexo accumulator") - - leaf_node = self.leaf_nodes[leaf] - del self.leaf_nodes[leaf] - - proof, leaf_index = self.inclusion_proof(leaf_node) - - n = None - h = 0 - while h < len(proof): - p = proof[h] - if n is not None: - n = self.parent_node(p, n) - elif self.root_nodes[h] is None: - p.parent = None - self.root_nodes[h] = p - else: - n = self.parent_node(p, self.root_nodes[h]) - self.root_nodes[h] = None - h += 1 - - if n is not None: - n.parent = None - - self.root_nodes[h] = n - - return proof, leaf_index - - def inclusion_proof(self, node): - if node.parent is None: - return [], 0 - - parent = node.parent - path, leaf_index = self.inclusion_proof(parent) - - if node == parent.left: - path.insert(0, parent.right) - leaf_index *= 2 - else: - path.insert(0, parent.left) - leaf_index = leaf_index * 2 + 1 - - return path, leaf_index - - def reset(self): - self.root_nodes = [] - self.leaf_nodes = {} - - def print_roots(self): - roots = [node.val if node is not None else "" for node in self.root_nodes] - print("Roots:", roots) - - -if __name__ == "__main__": - utreexo = Utreexo() - - # Add some elements - utreexo.add(0x111111111111111111111111) - utreexo.add(0x222222222222222222222222) - utreexo.add(0x333333333333333333333333) - utreexo.add(0x444444444444444444444444) - utreexo.add(0x555555555555555555555555) - utreexo.add(0x666666666666666666666666) - utreexo.add(0x777777777777777777777777) - utreexo.add(0x888888888888888888888888) - utreexo.add(0x999999999999999999999999) - utreexo.add(0xAAAAAAAAAAAAAAAAAAAAAAAA) - utreexo.print_roots() - - # Reset the Utreexo - utreexo.reset() - utreexo.print_roots()