From f76ac13aaf26dcf6f783d34c351b243e9b720340 Mon Sep 17 00:00:00 2001 From: f50033134 Date: Thu, 14 Dec 2023 14:47:25 +0100 Subject: [PATCH 01/24] template for parallel implementation --- .gitignore | 2 +- src/merkle_tree/mod.rs | 31 +++++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 448a8bb6..9d1f1106 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ Cargo.lock params *.swp *.swo - +.vscode \ No newline at end of file diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index d8655bbb..e2cbf091 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -233,12 +233,14 @@ impl MerkleTree

{ Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } - /// Returns a new merkle tree. `leaves.len()` should be power of two. - pub fn new>( + /// Serial computation of merkle tree + #[cfg(not(feature = "parallel"))] + fn new_serial>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoIterator, ) -> Result { + // serial let mut leaves_digests = Vec::new(); // compute and store hash values for each leaf @@ -249,6 +251,31 @@ impl MerkleTree

{ Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digests) } + /// Parallel computation of merkle tree + #[cfg(feature = "parallel")] + fn new_parallel>( + leaf_hash_param: &LeafParam

, + two_to_one_hash_param: &TwoToOneParam

, + leaves: impl IntoIterator, + ) -> Result { + !unimplemented("Parallel Merkle Tree not implemented") + } + + + /// Returns a new merkle tree. `leaves.len()` should be power of two. + pub fn new>( + leaf_hash_param: &LeafParam

, + two_to_one_hash_param: &TwoToOneParam

, + leaves: impl IntoIterator, + ) -> Result { + + if !cfg(feature="parallel"){ + + }else{ + + } + } + pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, From 63dc5fddb511998d85fd33ca879c5b7862812453 Mon Sep 17 00:00:00 2001 From: f50033134 Date: Thu, 14 Dec 2023 17:11:02 +0100 Subject: [PATCH 02/24] added a maybe useful snippet of rayon mut_par_iter as comment in the parallel func --- src/merkle_tree/mod.rs | 44 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index e2cbf091..a7844cf7 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -240,7 +240,6 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoIterator, ) -> Result { - // serial let mut leaves_digests = Vec::new(); // compute and store hash values for each leaf @@ -259,6 +258,37 @@ impl MerkleTree

{ leaves: impl IntoIterator, ) -> Result { !unimplemented("Parallel Merkle Tree not implemented") + /* + perhaps useful snippet + + use rayon::prelude::*; // 1.5.0 + use std::time::*; + use std::thread::sleep; + fn main() { + let mut x :Vec = (0..10).collect(); + println!("{}",x.len()); + let mut now = Instant::now(); + x.par_iter_mut().enumerate().for_each(|(_index,val)| { + *val = 0; + sleep(Duration::from_secs(1)); + }); + let mut elapsed_time = now.elapsed(); + println!("Running parallel took {} seconds.", elapsed_time.as_secs()); + for v in &x{ + assert_eq!(*v,0); + } + now = Instant::now(); + x.iter_mut().enumerate().for_each(|(_index, val)| { + *val = 1; + sleep(Duration::from_secs(1)); + }); + elapsed_time = now.elapsed(); + println!("Running serial took {} seconds.", elapsed_time.as_secs()); + for v in &x{ + assert_eq!(*v,1); + } + } + */ } @@ -270,9 +300,17 @@ impl MerkleTree

{ ) -> Result { if !cfg(feature="parallel"){ - + new_parallel( + leaf_hash_param, + two_to_one_hash_param, + leaves + ) }else{ - + new_serial( + leaf_hash_param, + two_to_one_hash_param, + leaves + ) } } From 6c745c7a36e04a3489ada99744e187d2e2d71fdf Mon Sep 17 00:00:00 2001 From: f50033134 Date: Thu, 14 Dec 2023 20:12:47 +0100 Subject: [PATCH 03/24] to do: try build and run, then test --- src/merkle_tree/mod.rs | 157 ++++++++++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 35 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index a7844cf7..eb5887dd 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -8,6 +8,9 @@ use ark_std::borrow::Borrow; use ark_std::hash::Hash; use ark_std::vec::Vec; +#[cfg(feature = "parallel")] +use rayon::{prelude::*,iter::Zip}; + #[cfg(test)] mod tests; @@ -257,38 +260,15 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoIterator, ) -> Result { - !unimplemented("Parallel Merkle Tree not implemented") - /* - perhaps useful snippet - - use rayon::prelude::*; // 1.5.0 - use std::time::*; - use std::thread::sleep; - fn main() { - let mut x :Vec = (0..10).collect(); - println!("{}",x.len()); - let mut now = Instant::now(); - x.par_iter_mut().enumerate().for_each(|(_index,val)| { - *val = 0; - sleep(Duration::from_secs(1)); - }); - let mut elapsed_time = now.elapsed(); - println!("Running parallel took {} seconds.", elapsed_time.as_secs()); - for v in &x{ - assert_eq!(*v,0); - } - now = Instant::now(); - x.iter_mut().enumerate().for_each(|(_index, val)| { - *val = 1; - sleep(Duration::from_secs(1)); - }); - elapsed_time = now.elapsed(); - println!("Running serial took {} seconds.", elapsed_time.as_secs()); - for v in &x{ - assert_eq!(*v,1); - } - } - */ + // !unimplemented("Parallel Merkle Tree not implemented") + + let mut leaves_digest = vec![P::LeafDigest::default(); leaves.len()]; + + leaves_digest.par_iter_mut().zip(leaves).for_each(|digest,leaf|{ + *digest = P::LeafHash::evaluate(leaf_hash_param, leaf)? + }); + + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digests) } @@ -300,13 +280,13 @@ impl MerkleTree

{ ) -> Result { if !cfg(feature="parallel"){ - new_parallel( + Self::new_parallel( leaf_hash_param, two_to_one_hash_param, leaves ) }else{ - new_serial( + Self::new_serial( leaf_hash_param, two_to_one_hash_param, leaves @@ -314,7 +294,93 @@ impl MerkleTree

{ } } - pub fn new_with_leaf_digest( + + fn new_with_leaf_digest_parallel( + leaf_hash_param: &LeafParam

, + two_to_one_hash_param: &TwoToOneParam

, + leaves_digest: Vec, + ) -> Result { + let leaf_nodes_size = leaves_digest.len(); + assert!( + leaf_nodes_size.is_power_of_two() && leaf_nodes_size > 1, + "`leaves.len() should be power of two and greater than one" + ); + let non_leaf_nodes_size = leaf_nodes_size - 1; + + let tree_height = tree_height(leaf_nodes_size); + + let hash_of_empty: P::InnerDigest = P::InnerDigest::default(); + + // initialize the merkle tree as array of nodes in level order + let mut non_leaf_nodes: Vec = (0..non_leaf_nodes_size) + .par_iter_mut().map(|_| hash_of_empty.clone()) + .collect(); + + // Compute the starting indices for each non-leaf level of the tree + let mut index = 0; + let mut level_indices = Vec::with_capacity(tree_height - 1); + for _ in 0..(tree_height - 1) { + level_indices.push(index); + index = left_child(index); + } + + // compute the hash values for the non-leaf bottom layer + { + let start_index = level_indices.pop().unwrap(); + let upper_bound = left_child(start_index); + let bottom_nodes = &mut non_leaf_nodes[start_index..upper_bound]; + + bottom_nodes.par_iter_mut().enumerate().for_each(|(i,n)|{ + // `left_child(current_index)` and `right_child(current_index) returns the position of + // leaf in the whole tree (represented as a list in level order). We need to shift it + // by `-upper_bound` to get the index in `leaf_nodes` list. + + //similarly, we need to rescale i by start_index + //to get the index outside the slice and in the level-ordered list of nodes + + let current_index = i+start_index; + let left_leaf_index = left_child(current_index) - upper_bound; + let right_leaf_index = right_child(current_index) - upper_bound; + // compute hash + *n = P::TwoToOneHash::evaluate( + two_to_one_hash_param, + P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone())?, + P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone())?, + )? + }); + } + + // compute the hash values for nodes in every other layer in the tree + level_indices.reverse(); + for &start_index in &level_indices { + // The layer beginning `start_index` ends at `upper_bound` (exclusive). + let upper_bound = left_child(start_index); + let nodes_at_level = &mut non_leaf_nodes[start_index..upper_bound]; + let nodes_at_prev_level = &non_leaf_nodes[upper_bound..left_child(upper_bound)]; + + nodes_at_level.par_iter_mut().enumerate().for_each(|(i,n)|{ + let current_index = i+start_index; + let left_leaf_index = left_child(current_index); + let right_leaf_index = right_child(current_index); + + *n = P::TwoToOneHash::compress( + two_to_one_hash_param, + nodes_at_prev_level[left_leaf_index].clone(), + nodes_at_prev_level[right_leaf_index].clone(), + )? + }); + } + + Ok(MerkleTree { + leaf_nodes: leaves_digest, + non_leaf_nodes, + height: tree_height, + leaf_hash_param: leaf_hash_param.clone(), + two_to_one_hash_param: two_to_one_hash_param.clone(), + }) + } + + fn new_with_leaf_digest_serial( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, @@ -387,6 +453,27 @@ impl MerkleTree

{ }) } + pub fn new_with_leaf_digest( + leaf_hash_param: &LeafParam

, + two_to_one_hash_param: &TwoToOneParam

, + leaves_digest: Vec, + ) -> Result { + if !cfg(feature="parallel"){ + Self::new_with_leaf_digest_parallel( + leaf_hash_param, + two_to_one_hash_param, + leaves + ) + }else{ + Self::new_with_leaf_digest_serial( + leaf_hash_param, + two_to_one_hash_param, + leaves + ) + } + } + + /// Returns the root of the Merkle tree. pub fn root(&self) -> P::InnerDigest { self.non_leaf_nodes[0].clone() From fcf8843f66715b1a3d745711db4df789997cf9ff Mon Sep 17 00:00:00 2001 From: f50033134 Date: Mon, 18 Dec 2023 17:06:43 +0100 Subject: [PATCH 04/24] solved build issues --- src/merkle_tree/mod.rs | 130 +++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 69 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index eb5887dd..17e6f6c8 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -9,7 +9,8 @@ use ark_std::hash::Hash; use ark_std::vec::Vec; #[cfg(feature = "parallel")] -use rayon::{prelude::*,iter::Zip}; +use rayon::{prelude::*,iter::IntoParallelIterator}; + #[cfg(test)] mod tests; @@ -63,7 +64,8 @@ pub trait Config { + Hash + Default + CanonicalSerialize - + CanonicalDeserialize; + + CanonicalDeserialize + + Send; // transition between leaf layer to inner layer type LeafInnerDigestConverter: DigestConverter< Self::LeafDigest, @@ -76,7 +78,8 @@ pub trait Config { + Hash + Default + CanonicalSerialize - + CanonicalDeserialize; + + CanonicalDeserialize + + Send; // Tom's Note: in the future, if we want different hash function, we can simply add more // types of digest here and specify a digest converter. Same for constraints. @@ -237,64 +240,55 @@ impl MerkleTree

{ } /// Serial computation of merkle tree - #[cfg(not(feature = "parallel"))] fn new_serial>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoIterator, ) -> Result { - let mut leaves_digests = Vec::new(); + let mut leaves_digest = Vec::new(); // compute and store hash values for each leaf for leaf in leaves.into_iter() { - leaves_digests.push(P::LeafHash::evaluate(leaf_hash_param, leaf)?) + leaves_digest.push(P::LeafHash::evaluate(leaf_hash_param, leaf)?) } - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digests) + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } /// Parallel computation of merkle tree - #[cfg(feature = "parallel")] fn new_parallel>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaves: impl IntoIterator, + leaves: impl IntoParallelIterator, ) -> Result { - // !unimplemented("Parallel Merkle Tree not implemented") - - let mut leaves_digest = vec![P::LeafDigest::default(); leaves.len()]; - leaves_digest.par_iter_mut().zip(leaves).for_each(|digest,leaf|{ - *digest = P::LeafHash::evaluate(leaf_hash_param, leaf)? - }); + //let mut leaves_digest = vec![P::LeafDigest::default(); leaves.len()]; - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digests) + //leaves_digest + // .par_iter_mut() + // .zip(leaves) + // .for_each(|digest, leaf| *digest = P::LeafHash::evaluate(leaf_hash_param, leaf)?); + + let leaves_digest = leaves. + into_par_iter(). + map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) + .collect(); + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } - /// Returns a new merkle tree. `leaves.len()` should be power of two. pub fn new>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaves: impl IntoIterator, + leaves: impl IntoIterator + IntoParallelIterator, ) -> Result { - - if !cfg(feature="parallel"){ - Self::new_parallel( - leaf_hash_param, - two_to_one_hash_param, - leaves - ) - }else{ - Self::new_serial( - leaf_hash_param, - two_to_one_hash_param, - leaves - ) + if cfg!(feature = "parallel") { + Self::new_parallel(leaf_hash_param, two_to_one_hash_param, leaves) + } else { + Self::new_serial(leaf_hash_param, two_to_one_hash_param, leaves) } } - fn new_with_leaf_digest_parallel( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, @@ -313,7 +307,8 @@ impl MerkleTree

{ // initialize the merkle tree as array of nodes in level order let mut non_leaf_nodes: Vec = (0..non_leaf_nodes_size) - .par_iter_mut().map(|_| hash_of_empty.clone()) + .into_par_iter() + .map(|_| hash_of_empty.clone()) .collect(); // Compute the starting indices for each non-leaf level of the tree @@ -329,24 +324,24 @@ impl MerkleTree

{ let start_index = level_indices.pop().unwrap(); let upper_bound = left_child(start_index); let bottom_nodes = &mut non_leaf_nodes[start_index..upper_bound]; - - bottom_nodes.par_iter_mut().enumerate().for_each(|(i,n)|{ + + bottom_nodes.par_iter_mut().enumerate().for_each(|(i, n)| { // `left_child(current_index)` and `right_child(current_index) returns the position of // leaf in the whole tree (represented as a list in level order). We need to shift it // by `-upper_bound` to get the index in `leaf_nodes` list. - + //similarly, we need to rescale i by start_index //to get the index outside the slice and in the level-ordered list of nodes - - let current_index = i+start_index; + + let current_index = i + start_index; let left_leaf_index = left_child(current_index) - upper_bound; let right_leaf_index = right_child(current_index) - upper_bound; // compute hash *n = P::TwoToOneHash::evaluate( two_to_one_hash_param, - P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone())?, - P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone())?, - )? + P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone()).unwrap(), + P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone()).unwrap(), + ).unwrap() }); } @@ -354,22 +349,28 @@ impl MerkleTree

{ level_indices.reverse(); for &start_index in &level_indices { // The layer beginning `start_index` ends at `upper_bound` (exclusive). - let upper_bound = left_child(start_index); - let nodes_at_level = &mut non_leaf_nodes[start_index..upper_bound]; - let nodes_at_prev_level = &non_leaf_nodes[upper_bound..left_child(upper_bound)]; - - nodes_at_level.par_iter_mut().enumerate().for_each(|(i,n)|{ - let current_index = i+start_index; - let left_leaf_index = left_child(current_index); - let right_leaf_index = right_child(current_index); + + + let upper_bound = left_child(start_index); + //let nodes_at_level = &mut non_leaf_nodes[start_index..upper_bound]; + //let nodes_at_prev_level = &mut non_leaf_nodes[upper_bound..left_child(upper_bound)]; - *n = P::TwoToOneHash::compress( - two_to_one_hash_param, - nodes_at_prev_level[left_leaf_index].clone(), - nodes_at_prev_level[right_leaf_index].clone(), - )? - }); - } + let (nodes_at_level, nodes_at_prev_level) = non_leaf_nodes[..].split_at_mut(upper_bound); + nodes_at_level[start_index..] + .par_iter_mut() + .enumerate() + .for_each(|(i, n)| { + let current_index = i + start_index; + let left_leaf_index = left_child(current_index); + let right_leaf_index = right_child(current_index); + + *n = P::TwoToOneHash::compress( + two_to_one_hash_param, + nodes_at_prev_level[left_leaf_index].clone(), + nodes_at_prev_level[right_leaf_index].clone(), + ).unwrap() + }); + } Ok(MerkleTree { leaf_nodes: leaves_digest, @@ -458,22 +459,13 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, ) -> Result { - if !cfg(feature="parallel"){ - Self::new_with_leaf_digest_parallel( - leaf_hash_param, - two_to_one_hash_param, - leaves - ) - }else{ - Self::new_with_leaf_digest_serial( - leaf_hash_param, - two_to_one_hash_param, - leaves - ) + if cfg!(feature = "parallel") { + Self::new_with_leaf_digest_parallel(leaf_hash_param, two_to_one_hash_param, leaves_digest) + } else { + Self::new_with_leaf_digest_serial(leaf_hash_param, two_to_one_hash_param, leaves_digest) } } - /// Returns the root of the Merkle tree. pub fn root(&self) -> P::InnerDigest { self.non_leaf_nodes[0].clone() From 3d114118bf7cb7a5b051649cb084d1a956d61703 Mon Sep 17 00:00:00 2001 From: f50033134 Date: Mon, 18 Dec 2023 17:09:31 +0100 Subject: [PATCH 05/24] solved bug where in new_with_leaf_digest_parallel the index of non leaf nodes at prev layer was not normalized by upper_bound --- src/merkle_tree/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 17e6f6c8..670c4085 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -361,8 +361,8 @@ impl MerkleTree

{ .enumerate() .for_each(|(i, n)| { let current_index = i + start_index; - let left_leaf_index = left_child(current_index); - let right_leaf_index = right_child(current_index); + let left_leaf_index = left_child(current_index) - upper_bound; + let right_leaf_index = right_child(current_index) - upper_bound; *n = P::TwoToOneHash::compress( two_to_one_hash_param, From 4fec1c5c1827730c68517b9199eddbf6d9aa9352 Mon Sep 17 00:00:00 2001 From: f50033134 Date: Tue, 19 Dec 2023 12:18:54 +0100 Subject: [PATCH 06/24] modified mod.rs to build function only if feature is enabled rather than building all and then calling based on feature --- src/merkle_tree/mod.rs | 76 +++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 670c4085..43cc242a 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -239,12 +239,16 @@ impl MerkleTree

{ Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } - /// Serial computation of merkle tree - fn new_serial>( + /// Returns a new merkle tree. `leaves.len()` should be power of two. + #[cfg(not(feature = "parallel"))] + pub fn new>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoIterator, ) -> Result { + + // Serial version of new + let mut leaves_digest = Vec::new(); // compute and store hash values for each leaf @@ -255,13 +259,16 @@ impl MerkleTree

{ Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } - /// Parallel computation of merkle tree - fn new_parallel>( + /// Returns a new merkle tree. `leaves.len()` should be power of two. + #[cfg(feature = "parallel")] + pub fn new>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoParallelIterator, ) -> Result { - + + // Parallel version of new + //let mut leaves_digest = vec![P::LeafDigest::default(); leaves.len()]; //leaves_digest @@ -276,24 +283,27 @@ impl MerkleTree

{ Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } - /// Returns a new merkle tree. `leaves.len()` should be power of two. - pub fn new>( - leaf_hash_param: &LeafParam

, - two_to_one_hash_param: &TwoToOneParam

, - leaves: impl IntoIterator + IntoParallelIterator, - ) -> Result { - if cfg!(feature = "parallel") { - Self::new_parallel(leaf_hash_param, two_to_one_hash_param, leaves) - } else { - Self::new_serial(leaf_hash_param, two_to_one_hash_param, leaves) - } - } - - fn new_with_leaf_digest_parallel( +// pub fn new>( +// leaf_hash_param: &LeafParam

, +// two_to_one_hash_param: &TwoToOneParam

, +// leaves: impl IntoIterator + IntoParallelIterator, +// ) -> Result { +// if cfg!(feature = "parallel") { +// Self::new_parallel(leaf_hash_param, two_to_one_hash_param, leaves) +// } else { +// Self::new_serial(leaf_hash_param, two_to_one_hash_param, leaves) +// } +// } + + #[cfg(feature = "parallel")] + pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, ) -> Result { + + // Parallel implementation of new_with_leaf_digest + let leaf_nodes_size = leaves_digest.len(); assert!( leaf_nodes_size.is_power_of_two() && leaf_nodes_size > 1, @@ -381,11 +391,15 @@ impl MerkleTree

{ }) } - fn new_with_leaf_digest_serial( + #[cfg(not(feature = "parallel"))] + pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, ) -> Result { + + // Serial implementation of new_with_leaf_digest + let leaf_nodes_size = leaves_digest.len(); assert!( leaf_nodes_size.is_power_of_two() && leaf_nodes_size > 1, @@ -454,17 +468,17 @@ impl MerkleTree

{ }) } - pub fn new_with_leaf_digest( - leaf_hash_param: &LeafParam

, - two_to_one_hash_param: &TwoToOneParam

, - leaves_digest: Vec, - ) -> Result { - if cfg!(feature = "parallel") { - Self::new_with_leaf_digest_parallel(leaf_hash_param, two_to_one_hash_param, leaves_digest) - } else { - Self::new_with_leaf_digest_serial(leaf_hash_param, two_to_one_hash_param, leaves_digest) - } - } + //pub fn new_with_leaf_digest( + // leaf_hash_param: &LeafParam

, + // two_to_one_hash_param: &TwoToOneParam

, + // leaves_digest: Vec, + //) -> Result { + // if cfg!(feature = "parallel") { + // Self::new_with_leaf_digest_parallel(leaf_hash_param, two_to_one_hash_param, leaves_digest) + // } else { + // Self::new_with_leaf_digest_serial(leaf_hash_param, two_to_one_hash_param, leaves_digest) + // } + //} /// Returns the root of the Merkle tree. pub fn root(&self) -> P::InnerDigest { From 571b65140e5a40a6c1158637527b902e9816c9f3 Mon Sep 17 00:00:00 2001 From: f50033134 Date: Wed, 20 Dec 2023 10:37:13 +0100 Subject: [PATCH 07/24] adapted unit tests for parallel merkle_tree in merkle_tree/tests/mod.rs --- src/merkle_tree/tests/mod.rs | 233 +++++++++++++++++++++++++---------- 1 file changed, 167 insertions(+), 66 deletions(-) diff --git a/src/merkle_tree/tests/mod.rs b/src/merkle_tree/tests/mod.rs index d328b352..da9060ae 100644 --- a/src/merkle_tree/tests/mod.rs +++ b/src/merkle_tree/tests/mod.rs @@ -12,6 +12,9 @@ mod bytes_mt_tests { use ark_ff::BigInteger256; use ark_std::{test_rng, UniformRand}; + #[cfg(feature="parallel")] + use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + #[derive(Clone)] pub(super) struct Window4x256; impl pedersen::Window for Window4x256 { @@ -47,35 +50,76 @@ mod bytes_mt_tests { let two_to_one_params = ::setup(&mut rng) .unwrap() .clone(); - let mut tree = JubJubMerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); - let mut root = tree.root(); - // test merkle tree functionality without update - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } + + #[cfg(not(feature="parallel"))] + { + // Serial + println!("TEST SERIAL"); + let mut tree = JubJubMerkleTree::new( + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves.iter().map(|x| x.as_slice()) + ).unwrap(); + + let mut root = tree.root(); + // test merkle tree functionality without update + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } - // test merkle tree update functionality - for (i, v) in update_query { - let v = crate::to_uncompressed_bytes!(v).unwrap(); - tree.update(*i, &v).unwrap(); - leaves[*i] = v.clone(); + // test merkle tree update functionality + for (i, v) in update_query { + let v = crate::to_uncompressed_bytes!(v).unwrap(); + tree.update(*i, &v).unwrap(); + leaves[*i] = v.clone(); + } + // update the root + root = tree.root(); + // verify again + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } } - // update the root - root = tree.root(); - // verify again - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); + + #[cfg(feature="parallel")] + { + // Parallel + let mut tree = JubJubMerkleTree::new( + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves.par_iter().map(|x| x.as_slice()) + ).unwrap(); + + let mut root = tree.root(); + // test merkle tree functionality without update + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } + + // test merkle tree update functionality + for (i, v) in update_query { + let v = crate::to_uncompressed_bytes!(v).unwrap(); + tree.update(*i, &v).unwrap(); + leaves[*i] = v.clone(); + } + // update the root + root = tree.root(); + // verify again + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } } } @@ -124,6 +168,9 @@ mod field_mt_tests { use crate::merkle_tree::{Config, IdentityDigestConverter, MerkleTree}; use ark_std::{test_rng, vec::Vec, One, UniformRand}; + #[cfg(feature="parallel")] + use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + type F = ark_ed_on_bls12_381::Fr; type H = poseidon::CRH; type TwoToOneH = poseidon::TwoToOneCRH; @@ -145,52 +192,106 @@ mod field_mt_tests { let leaf_crh_params = poseidon_parameters(); let two_to_one_params = leaf_crh_params.clone(); - let mut tree = FieldMT::new( - &leaf_crh_params, - &two_to_one_params, - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); + #[cfg(not(feature = "parallel"))] + { + let mut tree = FieldMT::new( + &leaf_crh_params, + &two_to_one_params, + leaves.iter().map(|x| x.as_slice()), + ) + .unwrap(); - let mut root = tree.root(); + let mut root = tree.root(); - // test merkle tree functionality without update - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } + // test merkle tree functionality without update + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } - { - // wrong root should lead to error but do not panic - let wrong_root = root + F::one(); - let proof = tree.generate_proof(0).unwrap(); - assert!(!proof - .verify( - &leaf_crh_params, - &two_to_one_params, - &wrong_root, - leaves[0].as_slice() - ) - .unwrap()) - } + { + // wrong root should lead to error but do not panic + let wrong_root = root + F::one(); + let proof = tree.generate_proof(0).unwrap(); + assert!(!proof + .verify( + &leaf_crh_params, + &two_to_one_params, + &wrong_root, + leaves[0].as_slice() + ) + .unwrap()) + } + + // test merkle tree update functionality + for (i, v) in update_query { + tree.update(*i, v).unwrap(); + leaves[*i] = v.to_vec(); + } + + // update the root + root = tree.root(); - // test merkle tree update functionality - for (i, v) in update_query { - tree.update(*i, v).unwrap(); - leaves[*i] = v.to_vec(); + // verify again + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } } - // update the root - root = tree.root(); + #[cfg(feature="parallel")] + { + let mut tree = FieldMT::new( + &leaf_crh_params, + &two_to_one_params, + leaves.par_iter().map(|x| x.as_slice()), + ) + .unwrap(); + + let mut root = tree.root(); + + // test merkle tree functionality without update + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } + + { + // wrong root should lead to error but do not panic + let wrong_root = root + F::one(); + let proof = tree.generate_proof(0).unwrap(); + assert!(!proof + .verify( + &leaf_crh_params, + &two_to_one_params, + &wrong_root, + leaves[0].as_slice() + ) + .unwrap()) + } + + // test merkle tree update functionality + for (i, v) in update_query { + tree.update(*i, v).unwrap(); + leaves[*i] = v.to_vec(); + } + + // update the root + root = tree.root(); - // verify again - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); + // verify again + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } } } From f44806f7f73004adc7db4baa1f08f9f19e47b6f1 Mon Sep 17 00:00:00 2001 From: f50033134 Date: Wed, 20 Dec 2023 14:59:14 +0100 Subject: [PATCH 08/24] implemented benches for merkle_tree --- Cargo.toml | 6 + benches/merkle_tree.rs | 152 ++++++++++ benches/utils.rs | 657 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 815 insertions(+) create mode 100644 benches/merkle_tree.rs create mode 100644 benches/utils.rs diff --git a/Cargo.toml b/Cargo.toml index 909b59a8..aa001515 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,12 @@ path = "benches/signature.rs" harness = false required-features = [ "signature" ] +[[bench]] +name = "merkle_tree" +path = "benches/merkle_tree.rs" +harness = false +required-features = [ "merkle_tree" ] + [patch.crates-io] ark-r1cs-std = { git = "https://github.com/arkworks-rs/r1cs-std/" } ark-ff = { git = "https://github.com/arkworks-rs/algebra/" } diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs new file mode 100644 index 00000000..7c293845 --- /dev/null +++ b/benches/merkle_tree.rs @@ -0,0 +1,152 @@ +#[macro_use] +extern crate criterion; + +mod utils; + +static NUM_LEAVES :i32 = i32::pow(2,8); + +mod bytes_mt_benches { + use std::borrow::Borrow; + use ark_serialize::CanonicalSerialize; + use ark_crypto_primitives::merkle_tree::*; + use ark_crypto_primitives::to_uncompressed_bytes; + use ark_crypto_primitives::crh::*; + use ark_ed_on_bls12_381::EdwardsProjective as JubJub; + use ark_ff::BigInteger256; + use ark_std::{test_rng, UniformRand}; + use criterion::Criterion; + + #[cfg(feature="parallel")] + use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + + use crate::NUM_LEAVES; + + #[derive(Clone)] + pub(super) struct Window4x256; + impl pedersen::Window for Window4x256 { + const WINDOW_SIZE: usize = 4; + const NUM_WINDOWS: usize = 256; + } + + type LeafH = pedersen::CRH; + type CompressH = pedersen::TwoToOneCRH; + + struct JubJubMerkleTreeParams; + + impl Config for JubJubMerkleTreeParams { + type Leaf = [u8]; + + type LeafDigest = ::Output; + type LeafInnerDigestConverter = ByteDigestConverter; + type InnerDigest = ::Output; + + type LeafHash = LeafH; + type TwoToOneHash = CompressH; + } + type JubJubMerkleTree = MerkleTree; + + pub fn merkle_tree_create(c: &mut Criterion) { + let mut rng = test_rng(); + let leaves: Vec<_> = (0..NUM_LEAVES) + .map(|_| { + let rnd = BigInteger256::rand(&mut rng); + to_uncompressed_bytes!(rnd).unwrap() + }).collect(); + let leaf_crh_params = ::setup(&mut rng).unwrap(); + let two_to_one_params = ::setup(&mut rng) + .unwrap() + .clone(); + c.bench_function("Merkle Tree Create (Leaves as [u8])", move |b| { + b.iter(|| { + #[cfg(not(feature="parallel"))] + { + _ = JubJubMerkleTree::new( + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves.iter().map(|x| x.as_slice()) + ).unwrap(); + } + #[cfg(feature="parallel")] + { + _ = JubJubMerkleTree::new( + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves.par_iter().map(|x| x.as_slice()) + ).unwrap(); + } + }) + }); + } + + criterion_group! { + name = mt_create; + config = Criterion::default().sample_size(10); + targets = merkle_tree_create + } +} +mod field_mt_benches { + use ark_crypto_primitives::crh::poseidon; + use ark_crypto_primitives::merkle_tree::*; + use ark_std::{test_rng, vec::Vec, UniformRand}; + use criterion::Criterion; + use crate::utils::merkle_tree_utils; + + #[cfg(feature="parallel")] + use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + + use crate::NUM_LEAVES; + + type F = ark_ed_on_bls12_381::Fr; + type H = poseidon::CRH; + type TwoToOneH = poseidon::TwoToOneCRH; + + struct FieldMTConfig; + impl Config for FieldMTConfig { + type Leaf = [F]; + type LeafDigest = F; + type LeafInnerDigestConverter = IdentityDigestConverter; + type InnerDigest = F; + type LeafHash = H; + type TwoToOneHash = TwoToOneH; + } + + type FieldMerkleTree = MerkleTree; + + pub fn merkle_tree_create(c: &mut Criterion) { + let mut rng = test_rng(); + let mut rand_leaves = || (0..3).map(|_| F::rand(&mut rng)).collect(); + let leaves: Vec> = (0..NUM_LEAVES) + .map(|_| rand_leaves()).collect(); + let leaf_crh_params = merkle_tree_utils::poseidon_parameters(); + let two_to_one_params = leaf_crh_params.clone(); + c.bench_function("Merkle Tree Create (Leaves as [F.E])", move |b| { + b.iter(|| { + #[cfg(not(feature="parallel"))] + { + _ = FieldMerkleTree::new( + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves.iter().map(|x| x.as_slice()) + ).unwrap(); + } + #[cfg(feature="parallel")] + { + _ = FieldMerkleTree::new( + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves.par_iter().map(|x| x.as_slice()) + ).unwrap(); + } + }) + }); + } + + criterion_group! { + name = mt_create; + config = Criterion::default().sample_size(10); + targets = merkle_tree_create + } +} +criterion_main!(crate::bytes_mt_benches::mt_create, crate::field_mt_benches::mt_create); + + diff --git a/benches/utils.rs b/benches/utils.rs new file mode 100644 index 00000000..8b76d824 --- /dev/null +++ b/benches/utils.rs @@ -0,0 +1,657 @@ +pub (crate) mod merkle_tree_utils{ + use ark_crypto_primitives::sponge::poseidon::PoseidonConfig; + use ark_std::str::FromStr; + use ark_std::{One, Zero}; + type F = ark_ed_on_bls12_381::Fr; + + pub(crate) fn poseidon_parameters() -> PoseidonConfig { + let full_rounds = 8; + let partial_rounds = 29; + let alpha = 17; + + let ark = vec![ + vec![ + F::from_str( + "9478896780421655835758496955063136571251874317427585180076394551808670301829", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "1410220424381727336803825453763847584610565307685015130563813219659976870089", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "12324248147325396388933912754817224521085038231095815415485781874375379288849", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "5869197693688547188262203345939784760013629955870738354032535473827837048029", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "7027675418691353855077049716619550622043312043660992344940177187528247727783", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "12525656923125347519081182951439180216858859245949104467678704676398049957654", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "2393593257638453164081539737606611596909105394156134386135868506931280124380", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "21284282509779560826339329447865344953337633312148348516557075030360788076689", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "9426009547297688316907727916185688178981799243406990694957955230529774886223", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "5340930120720868177469579986808462005013697381998009281661327587975132166755", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "13224952063922250960936823741448973692264041750100990569445192064567307041002", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "5263772922985715307758718731861278699232625525745635678504665316187832057553", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "12905140589386545724352113723305099554526316070018892915579084990225436501424", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "3682692866591423277196501877256311982914914533289815428970072263880360882202", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "19681976272543335942352939522328215645129363120562038296975370569202780487598", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "5636115553781577891149626756201577064573186936824720926267940879716772984728", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "9501050736957980494328252533770324735114766672253853282051029963140075785396", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "2809392708032113981798687947163092027958611686144429680366467696224014505992", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "18433043696013996573551852847056868761017170818820490351056924728720017242180", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "1600424531609887868281118752288673305222025191763201214001133841689879221076", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "4077863666335789263839414578443702921867998881654209770993100224779179660280", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "10750183821931976144366649760909312224094512474274826686974526305203678408743", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "5876585841304782856135279046524906005004905983316552629403091395701737015709", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "13484299981373196201166722380389594773562113262309564134825386266765751213853", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "17382139184029132729706972098151128411278461930818849113274328379445169530719", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "20539300163698134245746932972121993866388520784246731402041866252259697791654", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "149101987103211771991327927827692640556911620408176100290586418839323044234", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "3772300053282831651551351000101118094165364582047053942163129913249479587871", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "1859494671365748569037492975272316924127197175139843386363551067183747450207", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "6056775412522970299341516426839343188000696076848171109448990325789072743616", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "13535861576199801040709157556664030757939966797019046516538528720719863222691", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "3166287940256215995277151981354337176516077219230228956292184356796876826882", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "3878105211417696553129343540655091450996375987051865710523878345663272335218", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "3234972450165117119793849127765475568944145932922109597427102281521349833458", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "4245107901241859301876588161430872878162557070919886440605528540426123750702", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "14797507122636944484020484450153618519329103538375805997650508264647579279513", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "3893725073760673244819994221888005992135922325903832357013427303988853516024", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "21641836396029226240087625131527365621781742784615208902930655613239471409203", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "4622082908476410083286670201138165773322781640914243047922441301693321472984", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "14738633807199650048753490173004870343158648561341211428780666160270584694255", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "2635090520059500019661864086615522409798872905401305311748231832709078452746", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "19070766579582338321241892986615538320421651429118757507174186491084617237586", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "12622420533971517050761060317049369208980632120901481436392835424625664738526", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "4395637216713203985567958440367812800809784906642242330796693491855644277207", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "13856237567677889405904897420967317137820909836352033096836527506967315017500", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "2152570847472117965131784005129148028733701170858744625211808968788882229984", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "6585203416839617436007268534508514569040432229287367393560615429950244309612", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "2153122337593625580331500314713439203221416612327349850130027435376816262006", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "7340485916200743279276570085958556798507770452421357119145466906520506506342", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "12717879727828017519339312786933302720905962296193775803009326830415523871745", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "5392903649799167854181087360481925061021040403603926349022734894553054536405", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "7221669722700687417346373353960536661883467014204005276831020252277657076044", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "8259126917996748375739426565773281408349947402369855975457055235880500335093", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "9272385735015968356236075957906198733226196415690072035874639311675477515202", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "10999027991078055598627757097261950281899485771669414759870674222957875237568", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "15453393396765207016379045014101989306173462885430532298601655955681532648226", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "5478929644476681096437469958231489102974161353940993351588559414552523375472", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "6864274099016679903139678736335228538241825704814597078997020342617052506183", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "12133526413093116990739357861671284889661106676453313677855438696597541491864", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "4363234898901124667709814170397096827222883770682185860994495523839008586252", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "16799465577487943696587954846666404704275729737273450161871875150400464433797", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "3466902930973160737502426090330438125630820207992414876720169645462530526357", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "10062441698891350053170325824989022858836994651376301483266809451301259521913", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "5849282602749563270643968237860161465694876981255295041960826011116890638924", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "18460093993858702487671589299005229942046272739124591066186726570539410116617", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "9812100862165422922235757591915383485338044715409891361026651619010947646011", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "3387849124775103843519196664933515074848119722071551419682472701704619249120", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "5283840871671971215904992681385681067319154145921438770232973796570506340281", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "14450974197863079729258614455552607708855872944526185987072755641686663205867", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "12613293459867195704822743599193025685229122593088639435739984309110321350551", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "6228273556621778927381918766322387348845347649737780310185999880647567569148", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "7482296435079443913598332362891173417094991594500715575107878549173583070413", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "18655449861670697203232484600163743308157596453845950955559776266093852537258", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "19948920146235041970991269588233091409704340607794045065548049409652881283328", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "13866078374565054775555309394949653928903776100036987352339975076159400168494", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "19398653685274645718325650121748668221118186023117741800737442235635318532994", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "4234154881267169381851681265196336178292466185695662916289548353755778788440", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "12763628380946395634691260884409562631856128057257959813602172954351304541746", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "7882453112990894293341171586279209575183467873317150236705310601775347127762", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "5669812778237054435250482766817044415794242063465169363632154286378940417646", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "16998738906020038479274018881471127087312245548341958049900081105113388112420", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "3923902724726826782251513956816550869721438812970437824859252798290604500141", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "8649850619802776810849631749100283821801281306919958924112424995025830909252", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "11095642206650177249637693917287763476332497377393343056089442602164577098005", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "6935839211798937659784055008131602708847374430164859822530563797964932598700", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "7009671085960032501857416946339379996865118520008277046653124221544059312084", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "14361753917538892938870644779277430374939140280641641154553910654644462796654", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "6296738827713642491839335218022320853584196754765009910619998033694434027436", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "13849351053619304861036345979638534258290466678610892122310972291285921828452", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "434708832289952835651719825370636597763362139118091644948171210201038442144", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "16633750393567936099837698146248798150044883935695159627422586429892098538881", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "12944939557587269500508410478785174192748264930676627398550886896505925728421", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "13132297714437965464312509267711212830308064898189789451541658159340762509645", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "3197382106307730326149017386920960267079843887376371149099833465681078850285", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "1219439673853113792340300173186247996249367102884530407862469123523013083971", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "3493891993991676033939225547105305872211028239751045376877382816726002847983", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "17474961424148900675164871904345354895260557993970869987490270849177572737815", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "14496326112831768456074139601688618143496262542471380389977686658437504436331", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "2924472580096769678506212811457662807142794313402961128576445038927398235897", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "4628296006426596599826873705217702584581936573072175641058168144816722698331", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "21191637522268746884323101636631937283436518241594045635071026927358145697662", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "16951212238971640283544926666565087199118390400059790490897089817025688673127", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "19613695336435411200907478310503966803576648245805018042761984388590288078910", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "19408817842355340096520725353160494939342325645253279486424056603334799168015", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "21454045045501902703155952158575095010854214688097850310899813261125869452799", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "7770328480231095569114093553841085793308707788942057894109603074902652929530", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "16464571997310094273270381226660568195148193554716113613093103468413654931642", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "17470702407108506528534764015553049093186219898758900659217736458688524875937", + ) + .map_err(|_| ()) + .unwrap(), + ], + vec![ + F::from_str( + "18550730212998825286534234924565339469725380540133305684933015562293032312245", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "2896017217286658654468296502214232988965841950467453595108246966331694256153", + ) + .map_err(|_| ()) + .unwrap(), + F::from_str( + "14675299739240143232464986549869467617250208852063994519435190317578889428919", + ) + .map_err(|_| ()) + .unwrap(), + ], + ]; + + // We use a near MDS matrix of the form: + // [[1, 0, 1], + // [1, 1, 0], + // [0, 1, 1]] + let mds = vec![ + vec![F::one(), F::zero(), F::one()], + vec![F::one(), F::one(), F::zero()], + vec![F::zero(), F::one(), F::one()], + ]; + + PoseidonConfig::::new(full_rounds, partial_rounds, alpha, mds, ark, 2, 1) + } + +} \ No newline at end of file From 4ebc098a3b94b08e93ed532d0011321feff6709e Mon Sep 17 00:00:00 2001 From: f50033134 Date: Wed, 20 Dec 2023 15:37:05 +0100 Subject: [PATCH 09/24] modified field merkle tree bench --- benches/merkle_tree.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 7c293845..d923b788 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -114,7 +114,8 @@ mod field_mt_benches { pub fn merkle_tree_create(c: &mut Criterion) { let mut rng = test_rng(); - let mut rand_leaves = || (0..3).map(|_| F::rand(&mut rng)).collect(); + let field_elems_in_leaf = 3; + let mut rand_leaves = || (0..field_elems_in_leaf).map(|_| F::rand(&mut rng)).collect(); let leaves: Vec> = (0..NUM_LEAVES) .map(|_| rand_leaves()).collect(); let leaf_crh_params = merkle_tree_utils::poseidon_parameters(); From 5eecd5ab0e8d14c6f7aece358ec6f903e17bceaf Mon Sep 17 00:00:00 2001 From: Giacomo Fenzi Date: Thu, 21 Dec 2023 20:29:16 +0100 Subject: [PATCH 10/24] fmt --- benches/merkle_tree.rs | 73 +++++++++++++++------------ benches/utils.rs | 5 +- src/merkle_tree/mod.rs | 95 ++++++++++++++++++------------------ src/merkle_tree/tests/mod.rs | 28 ++++++----- 4 files changed, 105 insertions(+), 96 deletions(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index d923b788..1cad8099 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -3,24 +3,24 @@ extern crate criterion; mod utils; -static NUM_LEAVES :i32 = i32::pow(2,8); +static NUM_LEAVES: i32 = i32::pow(2, 8); mod bytes_mt_benches { - use std::borrow::Borrow; - use ark_serialize::CanonicalSerialize; + use ark_crypto_primitives::crh::*; use ark_crypto_primitives::merkle_tree::*; use ark_crypto_primitives::to_uncompressed_bytes; - use ark_crypto_primitives::crh::*; use ark_ed_on_bls12_381::EdwardsProjective as JubJub; use ark_ff::BigInteger256; + use ark_serialize::CanonicalSerialize; use ark_std::{test_rng, UniformRand}; use criterion::Criterion; + use std::borrow::Borrow; - #[cfg(feature="parallel")] - use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + #[cfg(feature = "parallel")] + use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use crate::NUM_LEAVES; - + #[derive(Clone)] pub(super) struct Window4x256; impl pedersen::Window for Window4x256 { @@ -44,40 +44,43 @@ mod bytes_mt_benches { type TwoToOneHash = CompressH; } type JubJubMerkleTree = MerkleTree; - + pub fn merkle_tree_create(c: &mut Criterion) { let mut rng = test_rng(); let leaves: Vec<_> = (0..NUM_LEAVES) .map(|_| { let rnd = BigInteger256::rand(&mut rng); to_uncompressed_bytes!(rnd).unwrap() - }).collect(); + }) + .collect(); let leaf_crh_params = ::setup(&mut rng).unwrap(); let two_to_one_params = ::setup(&mut rng) .unwrap() .clone(); c.bench_function("Merkle Tree Create (Leaves as [u8])", move |b| { b.iter(|| { - #[cfg(not(feature="parallel"))] + #[cfg(not(feature = "parallel"))] { _ = JubJubMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()) - ).unwrap(); + leaves.iter().map(|x| x.as_slice()), + ) + .unwrap(); } - #[cfg(feature="parallel")] + #[cfg(feature = "parallel")] { _ = JubJubMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.par_iter().map(|x| x.as_slice()) - ).unwrap(); + leaves.par_iter().map(|x| x.as_slice()), + ) + .unwrap(); } }) }); } - + criterion_group! { name = mt_create; config = Criterion::default().sample_size(10); @@ -85,14 +88,14 @@ mod bytes_mt_benches { } } mod field_mt_benches { + use crate::utils::merkle_tree_utils; use ark_crypto_primitives::crh::poseidon; use ark_crypto_primitives::merkle_tree::*; use ark_std::{test_rng, vec::Vec, UniformRand}; use criterion::Criterion; - use crate::utils::merkle_tree_utils; - #[cfg(feature="parallel")] - use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + #[cfg(feature = "parallel")] + use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use crate::NUM_LEAVES; @@ -115,39 +118,45 @@ mod field_mt_benches { pub fn merkle_tree_create(c: &mut Criterion) { let mut rng = test_rng(); let field_elems_in_leaf = 3; - let mut rand_leaves = || (0..field_elems_in_leaf).map(|_| F::rand(&mut rng)).collect(); - let leaves: Vec> = (0..NUM_LEAVES) - .map(|_| rand_leaves()).collect(); + let mut rand_leaves = || { + (0..field_elems_in_leaf) + .map(|_| F::rand(&mut rng)) + .collect() + }; + let leaves: Vec> = (0..NUM_LEAVES).map(|_| rand_leaves()).collect(); let leaf_crh_params = merkle_tree_utils::poseidon_parameters(); let two_to_one_params = leaf_crh_params.clone(); c.bench_function("Merkle Tree Create (Leaves as [F.E])", move |b| { b.iter(|| { - #[cfg(not(feature="parallel"))] + #[cfg(not(feature = "parallel"))] { _ = FieldMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()) - ).unwrap(); + leaves.iter().map(|x| x.as_slice()), + ) + .unwrap(); } - #[cfg(feature="parallel")] + #[cfg(feature = "parallel")] { _ = FieldMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.par_iter().map(|x| x.as_slice()) - ).unwrap(); + leaves.par_iter().map(|x| x.as_slice()), + ) + .unwrap(); } }) }); } - + criterion_group! { name = mt_create; config = Criterion::default().sample_size(10); targets = merkle_tree_create } } -criterion_main!(crate::bytes_mt_benches::mt_create, crate::field_mt_benches::mt_create); - - +criterion_main!( + crate::bytes_mt_benches::mt_create, + crate::field_mt_benches::mt_create +); diff --git a/benches/utils.rs b/benches/utils.rs index 8b76d824..2a968a79 100644 --- a/benches/utils.rs +++ b/benches/utils.rs @@ -1,4 +1,4 @@ -pub (crate) mod merkle_tree_utils{ +pub(crate) mod merkle_tree_utils { use ark_crypto_primitives::sponge::poseidon::PoseidonConfig; use ark_std::str::FromStr; use ark_std::{One, Zero}; @@ -653,5 +653,4 @@ pub (crate) mod merkle_tree_utils{ PoseidonConfig::::new(full_rounds, partial_rounds, alpha, mds, ark, 2, 1) } - -} \ No newline at end of file +} diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 43cc242a..b7f199dd 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -9,8 +9,7 @@ use ark_std::hash::Hash; use ark_std::vec::Vec; #[cfg(feature = "parallel")] -use rayon::{prelude::*,iter::IntoParallelIterator}; - +use rayon::{iter::IntoParallelIterator, prelude::*}; #[cfg(test)] mod tests; @@ -246,9 +245,8 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoIterator, ) -> Result { - // Serial version of new - + let mut leaves_digest = Vec::new(); // compute and store hash values for each leaf @@ -266,34 +264,33 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves: impl IntoParallelIterator, ) -> Result { - // Parallel version of new - + //let mut leaves_digest = vec![P::LeafDigest::default(); leaves.len()]; //leaves_digest // .par_iter_mut() // .zip(leaves) // .for_each(|digest, leaf| *digest = P::LeafHash::evaluate(leaf_hash_param, leaf)?); - - let leaves_digest = leaves. - into_par_iter(). - map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) + + let leaves_digest = leaves + .into_par_iter() + .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) .collect(); Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } -// pub fn new>( -// leaf_hash_param: &LeafParam

, -// two_to_one_hash_param: &TwoToOneParam

, -// leaves: impl IntoIterator + IntoParallelIterator, -// ) -> Result { -// if cfg!(feature = "parallel") { -// Self::new_parallel(leaf_hash_param, two_to_one_hash_param, leaves) -// } else { -// Self::new_serial(leaf_hash_param, two_to_one_hash_param, leaves) -// } -// } + // pub fn new>( + // leaf_hash_param: &LeafParam

, + // two_to_one_hash_param: &TwoToOneParam

, + // leaves: impl IntoIterator + IntoParallelIterator, + // ) -> Result { + // if cfg!(feature = "parallel") { + // Self::new_parallel(leaf_hash_param, two_to_one_hash_param, leaves) + // } else { + // Self::new_serial(leaf_hash_param, two_to_one_hash_param, leaves) + // } + // } #[cfg(feature = "parallel")] pub fn new_with_leaf_digest( @@ -301,7 +298,6 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, ) -> Result { - // Parallel implementation of new_with_leaf_digest let leaf_nodes_size = leaves_digest.len(); @@ -349,9 +345,12 @@ impl MerkleTree

{ // compute hash *n = P::TwoToOneHash::evaluate( two_to_one_hash_param, - P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone()).unwrap(), - P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone()).unwrap(), - ).unwrap() + P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone()) + .unwrap(), + P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone()) + .unwrap(), + ) + .unwrap() }); } @@ -359,28 +358,29 @@ impl MerkleTree

{ level_indices.reverse(); for &start_index in &level_indices { // The layer beginning `start_index` ends at `upper_bound` (exclusive). - - - let upper_bound = left_child(start_index); - //let nodes_at_level = &mut non_leaf_nodes[start_index..upper_bound]; - //let nodes_at_prev_level = &mut non_leaf_nodes[upper_bound..left_child(upper_bound)]; - - let (nodes_at_level, nodes_at_prev_level) = non_leaf_nodes[..].split_at_mut(upper_bound); - nodes_at_level[start_index..] - .par_iter_mut() - .enumerate() - .for_each(|(i, n)| { - let current_index = i + start_index; - let left_leaf_index = left_child(current_index) - upper_bound; - let right_leaf_index = right_child(current_index) - upper_bound; - - *n = P::TwoToOneHash::compress( - two_to_one_hash_param, - nodes_at_prev_level[left_leaf_index].clone(), - nodes_at_prev_level[right_leaf_index].clone(), - ).unwrap() - }); - } + + let upper_bound = left_child(start_index); + //let nodes_at_level = &mut non_leaf_nodes[start_index..upper_bound]; + //let nodes_at_prev_level = &mut non_leaf_nodes[upper_bound..left_child(upper_bound)]; + + let (nodes_at_level, nodes_at_prev_level) = + non_leaf_nodes[..].split_at_mut(upper_bound); + nodes_at_level[start_index..] + .par_iter_mut() + .enumerate() + .for_each(|(i, n)| { + let current_index = i + start_index; + let left_leaf_index = left_child(current_index) - upper_bound; + let right_leaf_index = right_child(current_index) - upper_bound; + + *n = P::TwoToOneHash::compress( + two_to_one_hash_param, + nodes_at_prev_level[left_leaf_index].clone(), + nodes_at_prev_level[right_leaf_index].clone(), + ) + .unwrap() + }); + } Ok(MerkleTree { leaf_nodes: leaves_digest, @@ -397,7 +397,6 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, ) -> Result { - // Serial implementation of new_with_leaf_digest let leaf_nodes_size = leaves_digest.len(); diff --git a/src/merkle_tree/tests/mod.rs b/src/merkle_tree/tests/mod.rs index da9060ae..fa36cc80 100644 --- a/src/merkle_tree/tests/mod.rs +++ b/src/merkle_tree/tests/mod.rs @@ -12,8 +12,8 @@ mod bytes_mt_tests { use ark_ff::BigInteger256; use ark_std::{test_rng, UniformRand}; - #[cfg(feature="parallel")] - use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + #[cfg(feature = "parallel")] + use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; #[derive(Clone)] pub(super) struct Window4x256; @@ -50,16 +50,17 @@ mod bytes_mt_tests { let two_to_one_params = ::setup(&mut rng) .unwrap() .clone(); - - #[cfg(not(feature="parallel"))] + + #[cfg(not(feature = "parallel"))] { // Serial println!("TEST SERIAL"); let mut tree = JubJubMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()) - ).unwrap(); + leaves.iter().map(|x| x.as_slice()), + ) + .unwrap(); let mut root = tree.root(); // test merkle tree functionality without update @@ -87,14 +88,15 @@ mod bytes_mt_tests { } } - #[cfg(feature="parallel")] + #[cfg(feature = "parallel")] { // Parallel let mut tree = JubJubMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.par_iter().map(|x| x.as_slice()) - ).unwrap(); + leaves.par_iter().map(|x| x.as_slice()), + ) + .unwrap(); let mut root = tree.root(); // test merkle tree functionality without update @@ -168,8 +170,8 @@ mod field_mt_tests { use crate::merkle_tree::{Config, IdentityDigestConverter, MerkleTree}; use ark_std::{test_rng, vec::Vec, One, UniformRand}; - #[cfg(feature="parallel")] - use rayon::iter::{IntoParallelRefIterator,ParallelIterator}; + #[cfg(feature = "parallel")] + use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; type F = ark_ed_on_bls12_381::Fr; type H = poseidon::CRH; @@ -243,8 +245,8 @@ mod field_mt_tests { } } - #[cfg(feature="parallel")] - { + #[cfg(feature = "parallel")] + { let mut tree = FieldMT::new( &leaf_crh_params, &two_to_one_params, From 0056600c60ecf23f213d5eb09f8290dd08707b53 Mon Sep 17 00:00:00 2001 From: Giacomo Fenzi Date: Thu, 21 Dec 2023 20:34:48 +0100 Subject: [PATCH 11/24] cleanup --- src/merkle_tree/mod.rs | 47 +++++++----------------------------------- 1 file changed, 7 insertions(+), 40 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index b7f199dd..0f0726b5 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -246,13 +246,13 @@ impl MerkleTree

{ leaves: impl IntoIterator, ) -> Result { // Serial version of new - let mut leaves_digest = Vec::new(); - // compute and store hash values for each leaf - for leaf in leaves.into_iter() { - leaves_digest.push(P::LeafHash::evaluate(leaf_hash_param, leaf)?) - } + // Compute and store hash values for each leaf + leaves + .into_iter() + .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)?) + .collect(); Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } @@ -265,34 +265,15 @@ impl MerkleTree

{ leaves: impl IntoParallelIterator, ) -> Result { // Parallel version of new - - //let mut leaves_digest = vec![P::LeafDigest::default(); leaves.len()]; - - //leaves_digest - // .par_iter_mut() - // .zip(leaves) - // .for_each(|digest, leaf| *digest = P::LeafHash::evaluate(leaf_hash_param, leaf)?); - let leaves_digest = leaves .into_par_iter() - .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) + .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)?) .collect(); Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } - // pub fn new>( - // leaf_hash_param: &LeafParam

, - // two_to_one_hash_param: &TwoToOneParam

, - // leaves: impl IntoIterator + IntoParallelIterator, - // ) -> Result { - // if cfg!(feature = "parallel") { - // Self::new_parallel(leaf_hash_param, two_to_one_hash_param, leaves) - // } else { - // Self::new_serial(leaf_hash_param, two_to_one_hash_param, leaves) - // } - // } - #[cfg(feature = "parallel")] +#[cfg(feature = "parallel")] pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, @@ -360,8 +341,6 @@ impl MerkleTree

{ // The layer beginning `start_index` ends at `upper_bound` (exclusive). let upper_bound = left_child(start_index); - //let nodes_at_level = &mut non_leaf_nodes[start_index..upper_bound]; - //let nodes_at_prev_level = &mut non_leaf_nodes[upper_bound..left_child(upper_bound)]; let (nodes_at_level, nodes_at_prev_level) = non_leaf_nodes[..].split_at_mut(upper_bound); @@ -467,18 +446,6 @@ impl MerkleTree

{ }) } - //pub fn new_with_leaf_digest( - // leaf_hash_param: &LeafParam

, - // two_to_one_hash_param: &TwoToOneParam

, - // leaves_digest: Vec, - //) -> Result { - // if cfg!(feature = "parallel") { - // Self::new_with_leaf_digest_parallel(leaf_hash_param, two_to_one_hash_param, leaves_digest) - // } else { - // Self::new_with_leaf_digest_serial(leaf_hash_param, two_to_one_hash_param, leaves_digest) - // } - //} - /// Returns the root of the Merkle tree. pub fn root(&self) -> P::InnerDigest { self.non_leaf_nodes[0].clone() From f88817196ea77112ad581f74da86794e7992fe08 Mon Sep 17 00:00:00 2001 From: Giacomo Fenzi Date: Thu, 21 Dec 2023 20:51:50 +0100 Subject: [PATCH 12/24] Fix compilation issue --- benches/merkle_tree.rs | 3 +- benches/utils.rs | 656 ----------------------------------------- src/merkle_tree/mod.rs | 17 +- 3 files changed, 8 insertions(+), 668 deletions(-) delete mode 100644 benches/utils.rs diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 1cad8099..7325f956 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -1,8 +1,6 @@ #[macro_use] extern crate criterion; -mod utils; - static NUM_LEAVES: i32 = i32::pow(2, 8); mod bytes_mt_benches { @@ -87,6 +85,7 @@ mod bytes_mt_benches { targets = merkle_tree_create } } + mod field_mt_benches { use crate::utils::merkle_tree_utils; use ark_crypto_primitives::crh::poseidon; diff --git a/benches/utils.rs b/benches/utils.rs deleted file mode 100644 index 2a968a79..00000000 --- a/benches/utils.rs +++ /dev/null @@ -1,656 +0,0 @@ -pub(crate) mod merkle_tree_utils { - use ark_crypto_primitives::sponge::poseidon::PoseidonConfig; - use ark_std::str::FromStr; - use ark_std::{One, Zero}; - type F = ark_ed_on_bls12_381::Fr; - - pub(crate) fn poseidon_parameters() -> PoseidonConfig { - let full_rounds = 8; - let partial_rounds = 29; - let alpha = 17; - - let ark = vec![ - vec![ - F::from_str( - "9478896780421655835758496955063136571251874317427585180076394551808670301829", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "1410220424381727336803825453763847584610565307685015130563813219659976870089", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "12324248147325396388933912754817224521085038231095815415485781874375379288849", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "5869197693688547188262203345939784760013629955870738354032535473827837048029", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "7027675418691353855077049716619550622043312043660992344940177187528247727783", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "12525656923125347519081182951439180216858859245949104467678704676398049957654", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "2393593257638453164081539737606611596909105394156134386135868506931280124380", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "21284282509779560826339329447865344953337633312148348516557075030360788076689", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "9426009547297688316907727916185688178981799243406990694957955230529774886223", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "5340930120720868177469579986808462005013697381998009281661327587975132166755", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "13224952063922250960936823741448973692264041750100990569445192064567307041002", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "5263772922985715307758718731861278699232625525745635678504665316187832057553", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "12905140589386545724352113723305099554526316070018892915579084990225436501424", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "3682692866591423277196501877256311982914914533289815428970072263880360882202", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "19681976272543335942352939522328215645129363120562038296975370569202780487598", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "5636115553781577891149626756201577064573186936824720926267940879716772984728", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "9501050736957980494328252533770324735114766672253853282051029963140075785396", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "2809392708032113981798687947163092027958611686144429680366467696224014505992", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "18433043696013996573551852847056868761017170818820490351056924728720017242180", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "1600424531609887868281118752288673305222025191763201214001133841689879221076", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "4077863666335789263839414578443702921867998881654209770993100224779179660280", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "10750183821931976144366649760909312224094512474274826686974526305203678408743", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "5876585841304782856135279046524906005004905983316552629403091395701737015709", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "13484299981373196201166722380389594773562113262309564134825386266765751213853", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "17382139184029132729706972098151128411278461930818849113274328379445169530719", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "20539300163698134245746932972121993866388520784246731402041866252259697791654", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "149101987103211771991327927827692640556911620408176100290586418839323044234", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "3772300053282831651551351000101118094165364582047053942163129913249479587871", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "1859494671365748569037492975272316924127197175139843386363551067183747450207", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "6056775412522970299341516426839343188000696076848171109448990325789072743616", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "13535861576199801040709157556664030757939966797019046516538528720719863222691", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "3166287940256215995277151981354337176516077219230228956292184356796876826882", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "3878105211417696553129343540655091450996375987051865710523878345663272335218", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "3234972450165117119793849127765475568944145932922109597427102281521349833458", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "4245107901241859301876588161430872878162557070919886440605528540426123750702", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "14797507122636944484020484450153618519329103538375805997650508264647579279513", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "3893725073760673244819994221888005992135922325903832357013427303988853516024", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "21641836396029226240087625131527365621781742784615208902930655613239471409203", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "4622082908476410083286670201138165773322781640914243047922441301693321472984", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "14738633807199650048753490173004870343158648561341211428780666160270584694255", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "2635090520059500019661864086615522409798872905401305311748231832709078452746", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "19070766579582338321241892986615538320421651429118757507174186491084617237586", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "12622420533971517050761060317049369208980632120901481436392835424625664738526", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "4395637216713203985567958440367812800809784906642242330796693491855644277207", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "13856237567677889405904897420967317137820909836352033096836527506967315017500", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "2152570847472117965131784005129148028733701170858744625211808968788882229984", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "6585203416839617436007268534508514569040432229287367393560615429950244309612", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "2153122337593625580331500314713439203221416612327349850130027435376816262006", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "7340485916200743279276570085958556798507770452421357119145466906520506506342", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "12717879727828017519339312786933302720905962296193775803009326830415523871745", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "5392903649799167854181087360481925061021040403603926349022734894553054536405", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "7221669722700687417346373353960536661883467014204005276831020252277657076044", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "8259126917996748375739426565773281408349947402369855975457055235880500335093", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "9272385735015968356236075957906198733226196415690072035874639311675477515202", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "10999027991078055598627757097261950281899485771669414759870674222957875237568", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "15453393396765207016379045014101989306173462885430532298601655955681532648226", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "5478929644476681096437469958231489102974161353940993351588559414552523375472", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "6864274099016679903139678736335228538241825704814597078997020342617052506183", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "12133526413093116990739357861671284889661106676453313677855438696597541491864", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "4363234898901124667709814170397096827222883770682185860994495523839008586252", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "16799465577487943696587954846666404704275729737273450161871875150400464433797", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "3466902930973160737502426090330438125630820207992414876720169645462530526357", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "10062441698891350053170325824989022858836994651376301483266809451301259521913", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "5849282602749563270643968237860161465694876981255295041960826011116890638924", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "18460093993858702487671589299005229942046272739124591066186726570539410116617", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "9812100862165422922235757591915383485338044715409891361026651619010947646011", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "3387849124775103843519196664933515074848119722071551419682472701704619249120", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "5283840871671971215904992681385681067319154145921438770232973796570506340281", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "14450974197863079729258614455552607708855872944526185987072755641686663205867", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "12613293459867195704822743599193025685229122593088639435739984309110321350551", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "6228273556621778927381918766322387348845347649737780310185999880647567569148", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "7482296435079443913598332362891173417094991594500715575107878549173583070413", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "18655449861670697203232484600163743308157596453845950955559776266093852537258", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "19948920146235041970991269588233091409704340607794045065548049409652881283328", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "13866078374565054775555309394949653928903776100036987352339975076159400168494", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "19398653685274645718325650121748668221118186023117741800737442235635318532994", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "4234154881267169381851681265196336178292466185695662916289548353755778788440", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "12763628380946395634691260884409562631856128057257959813602172954351304541746", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "7882453112990894293341171586279209575183467873317150236705310601775347127762", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "5669812778237054435250482766817044415794242063465169363632154286378940417646", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "16998738906020038479274018881471127087312245548341958049900081105113388112420", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "3923902724726826782251513956816550869721438812970437824859252798290604500141", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "8649850619802776810849631749100283821801281306919958924112424995025830909252", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "11095642206650177249637693917287763476332497377393343056089442602164577098005", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "6935839211798937659784055008131602708847374430164859822530563797964932598700", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "7009671085960032501857416946339379996865118520008277046653124221544059312084", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "14361753917538892938870644779277430374939140280641641154553910654644462796654", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "6296738827713642491839335218022320853584196754765009910619998033694434027436", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "13849351053619304861036345979638534258290466678610892122310972291285921828452", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "434708832289952835651719825370636597763362139118091644948171210201038442144", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "16633750393567936099837698146248798150044883935695159627422586429892098538881", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "12944939557587269500508410478785174192748264930676627398550886896505925728421", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "13132297714437965464312509267711212830308064898189789451541658159340762509645", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "3197382106307730326149017386920960267079843887376371149099833465681078850285", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "1219439673853113792340300173186247996249367102884530407862469123523013083971", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "3493891993991676033939225547105305872211028239751045376877382816726002847983", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "17474961424148900675164871904345354895260557993970869987490270849177572737815", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "14496326112831768456074139601688618143496262542471380389977686658437504436331", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "2924472580096769678506212811457662807142794313402961128576445038927398235897", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "4628296006426596599826873705217702584581936573072175641058168144816722698331", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "21191637522268746884323101636631937283436518241594045635071026927358145697662", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "16951212238971640283544926666565087199118390400059790490897089817025688673127", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "19613695336435411200907478310503966803576648245805018042761984388590288078910", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "19408817842355340096520725353160494939342325645253279486424056603334799168015", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "21454045045501902703155952158575095010854214688097850310899813261125869452799", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "7770328480231095569114093553841085793308707788942057894109603074902652929530", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "16464571997310094273270381226660568195148193554716113613093103468413654931642", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "17470702407108506528534764015553049093186219898758900659217736458688524875937", - ) - .map_err(|_| ()) - .unwrap(), - ], - vec![ - F::from_str( - "18550730212998825286534234924565339469725380540133305684933015562293032312245", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "2896017217286658654468296502214232988965841950467453595108246966331694256153", - ) - .map_err(|_| ()) - .unwrap(), - F::from_str( - "14675299739240143232464986549869467617250208852063994519435190317578889428919", - ) - .map_err(|_| ()) - .unwrap(), - ], - ]; - - // We use a near MDS matrix of the form: - // [[1, 0, 1], - // [1, 1, 0], - // [0, 1, 1]] - let mds = vec![ - vec![F::one(), F::zero(), F::one()], - vec![F::one(), F::one(), F::zero()], - vec![F::zero(), F::one(), F::one()], - ]; - - PoseidonConfig::::new(full_rounds, partial_rounds, alpha, mds, ark, 2, 1) - } -} diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 0f0726b5..cf90f10f 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -246,15 +246,13 @@ impl MerkleTree

{ leaves: impl IntoIterator, ) -> Result { // Serial version of new - let mut leaves_digest = Vec::new(); - // Compute and store hash values for each leaf - leaves + let leaves_digest: Result, _> = leaves .into_iter() - .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)?) + .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)) .collect(); - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest?) } /// Returns a new merkle tree. `leaves.len()` should be power of two. @@ -265,15 +263,14 @@ impl MerkleTree

{ leaves: impl IntoParallelIterator, ) -> Result { // Parallel version of new - let leaves_digest = leaves + let leaves_digest: Result, _> = leaves .into_par_iter() - .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)?) + .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)) .collect(); - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest?) } - -#[cfg(feature = "parallel")] + #[cfg(feature = "parallel")] pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, From 4b6be6cfbc9b15b10e7a69b78bb3048d259cec42 Mon Sep 17 00:00:00 2001 From: Giacomo Fenzi Date: Thu, 21 Dec 2023 20:57:39 +0100 Subject: [PATCH 13/24] Move to SHA256 --- benches/merkle_tree.rs | 92 ++++-------------------------------------- 1 file changed, 7 insertions(+), 85 deletions(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 7325f956..1cdf9445 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -7,7 +7,6 @@ mod bytes_mt_benches { use ark_crypto_primitives::crh::*; use ark_crypto_primitives::merkle_tree::*; use ark_crypto_primitives::to_uncompressed_bytes; - use ark_ed_on_bls12_381::EdwardsProjective as JubJub; use ark_ff::BigInteger256; use ark_serialize::CanonicalSerialize; use ark_std::{test_rng, UniformRand}; @@ -19,19 +18,12 @@ mod bytes_mt_benches { use crate::NUM_LEAVES; - #[derive(Clone)] - pub(super) struct Window4x256; - impl pedersen::Window for Window4x256 { - const WINDOW_SIZE: usize = 4; - const NUM_WINDOWS: usize = 256; - } - - type LeafH = pedersen::CRH; - type CompressH = pedersen::TwoToOneCRH; + type LeafH = sha2::Sha256; + type CompressH = sha2::Sha256; - struct JubJubMerkleTreeParams; + struct Sha256MerkleTreeParams; - impl Config for JubJubMerkleTreeParams { + impl Config for Sha256MerkleTreeParams { type Leaf = [u8]; type LeafDigest = ::Output; @@ -41,7 +33,7 @@ mod bytes_mt_benches { type LeafHash = LeafH; type TwoToOneHash = CompressH; } - type JubJubMerkleTree = MerkleTree; + type Sha256MerkleTree = MerkleTree; pub fn merkle_tree_create(c: &mut Criterion) { let mut rng = test_rng(); @@ -59,7 +51,7 @@ mod bytes_mt_benches { b.iter(|| { #[cfg(not(feature = "parallel"))] { - _ = JubJubMerkleTree::new( + _ = Sha256MerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), leaves.iter().map(|x| x.as_slice()), @@ -68,7 +60,7 @@ mod bytes_mt_benches { } #[cfg(feature = "parallel")] { - _ = JubJubMerkleTree::new( + _ = Sha256MerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), leaves.par_iter().map(|x| x.as_slice()), @@ -86,76 +78,6 @@ mod bytes_mt_benches { } } -mod field_mt_benches { - use crate::utils::merkle_tree_utils; - use ark_crypto_primitives::crh::poseidon; - use ark_crypto_primitives::merkle_tree::*; - use ark_std::{test_rng, vec::Vec, UniformRand}; - use criterion::Criterion; - - #[cfg(feature = "parallel")] - use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - - use crate::NUM_LEAVES; - - type F = ark_ed_on_bls12_381::Fr; - type H = poseidon::CRH; - type TwoToOneH = poseidon::TwoToOneCRH; - - struct FieldMTConfig; - impl Config for FieldMTConfig { - type Leaf = [F]; - type LeafDigest = F; - type LeafInnerDigestConverter = IdentityDigestConverter; - type InnerDigest = F; - type LeafHash = H; - type TwoToOneHash = TwoToOneH; - } - - type FieldMerkleTree = MerkleTree; - - pub fn merkle_tree_create(c: &mut Criterion) { - let mut rng = test_rng(); - let field_elems_in_leaf = 3; - let mut rand_leaves = || { - (0..field_elems_in_leaf) - .map(|_| F::rand(&mut rng)) - .collect() - }; - let leaves: Vec> = (0..NUM_LEAVES).map(|_| rand_leaves()).collect(); - let leaf_crh_params = merkle_tree_utils::poseidon_parameters(); - let two_to_one_params = leaf_crh_params.clone(); - c.bench_function("Merkle Tree Create (Leaves as [F.E])", move |b| { - b.iter(|| { - #[cfg(not(feature = "parallel"))] - { - _ = FieldMerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); - } - #[cfg(feature = "parallel")] - { - _ = FieldMerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - leaves.par_iter().map(|x| x.as_slice()), - ) - .unwrap(); - } - }) - }); - } - - criterion_group! { - name = mt_create; - config = Criterion::default().sample_size(10); - targets = merkle_tree_create - } -} criterion_main!( crate::bytes_mt_benches::mt_create, - crate::field_mt_benches::mt_create ); From a37041bb522b08bf5cac902c1e6cbdda475b44fc Mon Sep 17 00:00:00 2001 From: Giacomo Fenzi Date: Thu, 21 Dec 2023 21:05:41 +0100 Subject: [PATCH 14/24] More realistic benches --- benches/merkle_tree.rs | 2 +- src/merkle_tree/mod.rs | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 1cdf9445..13ce3d47 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -1,7 +1,7 @@ #[macro_use] extern crate criterion; -static NUM_LEAVES: i32 = i32::pow(2, 8); +static NUM_LEAVES: i32 = 1 << 20; mod bytes_mt_benches { use ark_crypto_primitives::crh::*; diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index cf90f10f..6e2bd452 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -263,11 +263,13 @@ impl MerkleTree

{ leaves: impl IntoParallelIterator, ) -> Result { // Parallel version of new - let leaves_digest: Result, _> = leaves + // TODO: Box is not Send, I have used .unwrap() but this changes semantics between + // parallel and serial version. We should fix this in the future. + let leaves_digest: Vec<_> = leaves .into_par_iter() - .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)) + .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) .collect(); - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest?) + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } #[cfg(feature = "parallel")] From 278445e215a4fff079bcaec5317e265aac880686 Mon Sep 17 00:00:00 2001 From: intx4 Date: Thu, 21 Dec 2023 21:46:43 +0100 Subject: [PATCH 15/24] spacing in new (parallel) --- src/merkle_tree/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 6e2bd452..1501c5ef 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -269,6 +269,7 @@ impl MerkleTree

{ .into_par_iter() .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) .collect(); + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } From efc0d61a7e8b8fdb48469cd86b50b8d2bf6dc07a Mon Sep 17 00:00:00 2001 From: intx4 Date: Sat, 23 Dec 2023 17:03:22 +0100 Subject: [PATCH 16/24] refactored code inside merkle_tree/mod.rs --- src/merkle_tree/mod.rs | 172 ++++++++++++----------------------------- 1 file changed, 48 insertions(+), 124 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 1501c5ef..7cb925c0 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -8,15 +8,15 @@ use ark_std::borrow::Borrow; use ark_std::hash::Hash; use ark_std::vec::Vec; -#[cfg(feature = "parallel")] -use rayon::{iter::IntoParallelIterator, prelude::*}; - #[cfg(test)] mod tests; #[cfg(feature = "r1cs")] pub mod constraints; +#[cfg(feature = "parallel")] +use rayon::prelude::*; + /// Convert the hash digest in different layers by converting previous layer's output to /// `TargetType`, which is a `Borrow` to next layer's input. pub trait DigestConverter { @@ -64,7 +64,7 @@ pub trait Config { + Default + CanonicalSerialize + CanonicalDeserialize - + Send; + + Send; // needed for parallel creation of MT with rayon // transition between leaf layer to inner layer type LeafInnerDigestConverter: DigestConverter< Self::LeafDigest, @@ -78,7 +78,7 @@ pub trait Config { + Default + CanonicalSerialize + CanonicalDeserialize - + Send; + + Send; // needed for parallel creation of MT with rayon // Tom's Note: in the future, if we want different hash function, we can simply add more // types of digest here and specify a digest converter. Same for constraints. @@ -239,41 +239,23 @@ impl MerkleTree

{ } /// Returns a new merkle tree. `leaves.len()` should be power of two. - #[cfg(not(feature = "parallel"))] pub fn new>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaves: impl IntoIterator, + leaves: impl IntoIterator + IntoParallelIterator, ) -> Result { - // Serial version of new - // Compute and store hash values for each leaf - let leaves_digest: Result, _> = leaves - .into_iter() - .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)) - .collect(); - - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest?) - } + + // TODO: Box is not Send, I have used .unwrap() but this causes panic + // might wish to change in the future - /// Returns a new merkle tree. `leaves.len()` should be power of two. - #[cfg(feature = "parallel")] - pub fn new>( - leaf_hash_param: &LeafParam

, - two_to_one_hash_param: &TwoToOneParam

, - leaves: impl IntoParallelIterator, - ) -> Result { - // Parallel version of new - // TODO: Box is not Send, I have used .unwrap() but this changes semantics between - // parallel and serial version. We should fix this in the future. - let leaves_digest: Vec<_> = leaves - .into_par_iter() + let leaves_iter = cfg_into_iter!(leaves); // par_iter() if cfg!(feature="parallel") else iter() + let leaves_digest: Vec<_> = leaves_iter .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) .collect(); Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } - #[cfg(feature = "parallel")] pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, @@ -293,8 +275,8 @@ impl MerkleTree

{ let hash_of_empty: P::InnerDigest = P::InnerDigest::default(); // initialize the merkle tree as array of nodes in level order - let mut non_leaf_nodes: Vec = (0..non_leaf_nodes_size) - .into_par_iter() + let non_leaf_nodes_iter = cfg_into_iter!((0..non_leaf_nodes_size)); + let mut non_leaf_nodes: Vec = non_leaf_nodes_iter .map(|_| hash_of_empty.clone()) .collect(); @@ -310,133 +292,75 @@ impl MerkleTree

{ { let start_index = level_indices.pop().unwrap(); let upper_bound = left_child(start_index); + let bottom_nodes = &mut non_leaf_nodes[start_index..upper_bound]; - - bottom_nodes.par_iter_mut().enumerate().for_each(|(i, n)| { + let bottom_nodes_iter = cfg_into_iter!(bottom_nodes); + + bottom_nodes_iter.enumerate().for_each(|(i,n)|{ + // `left_child(current_index)` and `right_child(current_index) returns the position of // leaf in the whole tree (represented as a list in level order). We need to shift it // by `-upper_bound` to get the index in `leaf_nodes` list. //similarly, we need to rescale i by start_index //to get the index outside the slice and in the level-ordered list of nodes - + let current_index = i + start_index; let left_leaf_index = left_child(current_index) - upper_bound; let right_leaf_index = right_child(current_index) - upper_bound; - // compute hash + + // need for unwrap as Box does not implement trait Send *n = P::TwoToOneHash::evaluate( two_to_one_hash_param, - P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone()) - .unwrap(), - P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone()) - .unwrap(), - ) - .unwrap() + P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone()).unwrap(), + P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone()).unwrap() + ).unwrap() }); } // compute the hash values for nodes in every other layer in the tree level_indices.reverse(); for &start_index in &level_indices { + // The layer beginning `start_index` ends at `upper_bound` (exclusive). - let upper_bound = left_child(start_index); let (nodes_at_level, nodes_at_prev_level) = non_leaf_nodes[..].split_at_mut(upper_bound); - nodes_at_level[start_index..] - .par_iter_mut() + //let nodes_at_level_iter = cfg_into_iter!(nodes_at_level[start_index..]); + let nodes_at_level_iter; + #[cfg(feature="parallel")] + { + nodes_at_level_iter = nodes_at_level[start_index..].par_iter_mut(); + } + + #[cfg(not(feature="parallel"))] + { + nodes_at_level_iter = nodes_at_level[start_index..].iter_mut(); + } + + nodes_at_level_iter .enumerate() - .for_each(|(i, n)| { + .for_each(|(i,n)|{ + + // `left_child(current_index)` and `right_child(current_index) returns the position of + // leaf in the whole tree (represented as a list in level order). We need to shift it + // by `-upper_bound` to get the index in `leaf_nodes` list. + + //similarly, we need to rescale i by start_index + //to get the index outside the slice and in the level-ordered list of nodes let current_index = i + start_index; let left_leaf_index = left_child(current_index) - upper_bound; let right_leaf_index = right_child(current_index) - upper_bound; + // need for unwrap as Box does not implement trait Send *n = P::TwoToOneHash::compress( two_to_one_hash_param, nodes_at_prev_level[left_leaf_index].clone(), nodes_at_prev_level[right_leaf_index].clone(), - ) - .unwrap() + ).unwrap(); }); } - - Ok(MerkleTree { - leaf_nodes: leaves_digest, - non_leaf_nodes, - height: tree_height, - leaf_hash_param: leaf_hash_param.clone(), - two_to_one_hash_param: two_to_one_hash_param.clone(), - }) - } - - #[cfg(not(feature = "parallel"))] - pub fn new_with_leaf_digest( - leaf_hash_param: &LeafParam

, - two_to_one_hash_param: &TwoToOneParam

, - leaves_digest: Vec, - ) -> Result { - // Serial implementation of new_with_leaf_digest - - let leaf_nodes_size = leaves_digest.len(); - assert!( - leaf_nodes_size.is_power_of_two() && leaf_nodes_size > 1, - "`leaves.len() should be power of two and greater than one" - ); - let non_leaf_nodes_size = leaf_nodes_size - 1; - - let tree_height = tree_height(leaf_nodes_size); - - let hash_of_empty: P::InnerDigest = P::InnerDigest::default(); - - // initialize the merkle tree as array of nodes in level order - let mut non_leaf_nodes: Vec = (0..non_leaf_nodes_size) - .map(|_| hash_of_empty.clone()) - .collect(); - - // Compute the starting indices for each non-leaf level of the tree - let mut index = 0; - let mut level_indices = Vec::with_capacity(tree_height - 1); - for _ in 0..(tree_height - 1) { - level_indices.push(index); - index = left_child(index); - } - - // compute the hash values for the non-leaf bottom layer - { - let start_index = level_indices.pop().unwrap(); - let upper_bound = left_child(start_index); - for current_index in start_index..upper_bound { - // `left_child(current_index)` and `right_child(current_index) returns the position of - // leaf in the whole tree (represented as a list in level order). We need to shift it - // by `-upper_bound` to get the index in `leaf_nodes` list. - let left_leaf_index = left_child(current_index) - upper_bound; - let right_leaf_index = right_child(current_index) - upper_bound; - // compute hash - non_leaf_nodes[current_index] = P::TwoToOneHash::evaluate( - &two_to_one_hash_param, - P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone())?, - P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone())?, - )? - } - } - - // compute the hash values for nodes in every other layer in the tree - level_indices.reverse(); - for &start_index in &level_indices { - // The layer beginning `start_index` ends at `upper_bound` (exclusive). - let upper_bound = left_child(start_index); - for current_index in start_index..upper_bound { - let left_index = left_child(current_index); - let right_index = right_child(current_index); - non_leaf_nodes[current_index] = P::TwoToOneHash::compress( - &two_to_one_hash_param, - non_leaf_nodes[left_index].clone(), - non_leaf_nodes[right_index].clone(), - )? - } - } - Ok(MerkleTree { leaf_nodes: leaves_digest, non_leaf_nodes, From 007aeefb02981aeedbba566c67e2935cc9b8b2ac Mon Sep 17 00:00:00 2001 From: intx4 Date: Sun, 24 Dec 2023 08:00:29 +0100 Subject: [PATCH 17/24] refactored merkle_tree/mod.rs with cfg_into_iter macro. Refactored accordingly unit tests --- src/merkle_tree/mod.rs | 10 +- src/merkle_tree/tests/mod.rs | 270 +++++++++++++---------------------- 2 files changed, 105 insertions(+), 175 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 7cb925c0..4e14e475 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -242,7 +242,13 @@ impl MerkleTree

{ pub fn new>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaves: impl IntoIterator + IntoParallelIterator, + + // leaves is rayon::Iter::IntoParallelIterator if parallel feature is enabled + // else leaves is a std::iter::IntoIterator + #[cfg(feature="parallel")] + leaves: impl IntoParallelIterator, + #[cfg(not(feature="parallel"))] + leaves: impl IntoIterator, ) -> Result { // TODO: Box is not Send, I have used .unwrap() but this causes panic @@ -261,7 +267,6 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, ) -> Result { - // Parallel implementation of new_with_leaf_digest let leaf_nodes_size = leaves_digest.len(); assert!( @@ -342,7 +347,6 @@ impl MerkleTree

{ nodes_at_level_iter .enumerate() .for_each(|(i,n)|{ - // `left_child(current_index)` and `right_child(current_index) returns the position of // leaf in the whole tree (represented as a list in level order). We need to shift it // by `-upper_bound` to get the index in `leaf_nodes` list. diff --git a/src/merkle_tree/tests/mod.rs b/src/merkle_tree/tests/mod.rs index fa36cc80..ad94cafb 100644 --- a/src/merkle_tree/tests/mod.rs +++ b/src/merkle_tree/tests/mod.rs @@ -12,8 +12,8 @@ mod bytes_mt_tests { use ark_ff::BigInteger256; use ark_std::{test_rng, UniformRand}; - #[cfg(feature = "parallel")] - use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + #[cfg(feature="parallel")] + use rayon::prelude::*; #[derive(Clone)] pub(super) struct Window4x256; @@ -42,86 +42,56 @@ mod bytes_mt_tests { /// Pedersen only takes bytes as leaf, so we use `ToBytes` trait. fn merkle_tree_test(leaves: &[L], update_query: &[(usize, L)]) -> () { let mut rng = ark_std::test_rng(); - let mut leaves: Vec<_> = leaves - .iter() + + let mut leaves: Vec<_> = leaves.iter() .map(|leaf| crate::to_uncompressed_bytes!(leaf).unwrap()) .collect(); + let leaf_crh_params = ::setup(&mut rng).unwrap(); let two_to_one_params = ::setup(&mut rng) .unwrap() .clone(); - #[cfg(not(feature = "parallel"))] + let leaves_iter; //serial or parallel + #[cfg(feature="parallel")] { - // Serial - println!("TEST SERIAL"); - let mut tree = JubJubMerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); - - let mut root = tree.root(); - // test merkle tree functionality without update - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } - - // test merkle tree update functionality - for (i, v) in update_query { - let v = crate::to_uncompressed_bytes!(v).unwrap(); - tree.update(*i, &v).unwrap(); - leaves[*i] = v.clone(); - } - // update the root - root = tree.root(); - // verify again - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } + leaves_iter = leaves.par_iter(); } - - #[cfg(feature = "parallel")] + #[cfg(not(feature="parallel"))] { - // Parallel - let mut tree = JubJubMerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - leaves.par_iter().map(|x| x.as_slice()), - ) - .unwrap(); - - let mut root = tree.root(); - // test merkle tree functionality without update - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } - - // test merkle tree update functionality - for (i, v) in update_query { - let v = crate::to_uncompressed_bytes!(v).unwrap(); - tree.update(*i, &v).unwrap(); - leaves[*i] = v.clone(); - } - // update the root - root = tree.root(); - // verify again - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } + leaves_iter = leaves.iter(); + } + + let mut tree = JubJubMerkleTree::new( + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves_iter.map(|x| x.as_slice()), + ) + .unwrap(); + + let mut root = tree.root(); + // test merkle tree functionality without update + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); + } + + // test merkle tree update functionality + for (i, v) in update_query { + let v = crate::to_uncompressed_bytes!(v).unwrap(); + tree.update(*i, &v).unwrap(); + leaves[*i] = v.clone(); + } + // update the root + root = tree.root(); + // verify again + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); } } @@ -170,9 +140,9 @@ mod field_mt_tests { use crate::merkle_tree::{Config, IdentityDigestConverter, MerkleTree}; use ark_std::{test_rng, vec::Vec, One, UniformRand}; - #[cfg(feature = "parallel")] - use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - + #[cfg(feature="parallel")] + use rayon::prelude::*; + type F = ark_ed_on_bls12_381::Fr; type H = poseidon::CRH; type TwoToOneH = poseidon::TwoToOneCRH; @@ -193,107 +163,63 @@ mod field_mt_tests { let mut leaves = leaves.to_vec(); let leaf_crh_params = poseidon_parameters(); let two_to_one_params = leaf_crh_params.clone(); - - #[cfg(not(feature = "parallel"))] + + let leaves_iter; //serial or parallel + #[cfg(feature="parallel")] + { + leaves_iter = leaves.par_iter(); + } + #[cfg(not(feature="parallel"))] { - let mut tree = FieldMT::new( - &leaf_crh_params, - &two_to_one_params, - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); - - let mut root = tree.root(); - - // test merkle tree functionality without update - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } - - { - // wrong root should lead to error but do not panic - let wrong_root = root + F::one(); - let proof = tree.generate_proof(0).unwrap(); - assert!(!proof - .verify( - &leaf_crh_params, - &two_to_one_params, - &wrong_root, - leaves[0].as_slice() - ) - .unwrap()) - } - - // test merkle tree update functionality - for (i, v) in update_query { - tree.update(*i, v).unwrap(); - leaves[*i] = v.to_vec(); - } - - // update the root - root = tree.root(); - - // verify again - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } + leaves_iter = leaves.iter(); + } + + let mut tree = FieldMT::new( + &leaf_crh_params, + &two_to_one_params, + leaves_iter.map(|x| x.as_slice()), + ) + .unwrap(); + + let mut root = tree.root(); + + // test merkle tree functionality without update + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); } - #[cfg(feature = "parallel")] { - let mut tree = FieldMT::new( - &leaf_crh_params, - &two_to_one_params, - leaves.par_iter().map(|x| x.as_slice()), - ) - .unwrap(); - - let mut root = tree.root(); - - // test merkle tree functionality without update - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } - - { - // wrong root should lead to error but do not panic - let wrong_root = root + F::one(); - let proof = tree.generate_proof(0).unwrap(); - assert!(!proof - .verify( - &leaf_crh_params, - &two_to_one_params, - &wrong_root, - leaves[0].as_slice() - ) - .unwrap()) - } - - // test merkle tree update functionality - for (i, v) in update_query { - tree.update(*i, v).unwrap(); - leaves[*i] = v.to_vec(); - } - - // update the root - root = tree.root(); - - // verify again - for (i, leaf) in leaves.iter().enumerate() { - let proof = tree.generate_proof(i).unwrap(); - assert!(proof - .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) - .unwrap()); - } + // wrong root should lead to error but do not panic + let wrong_root = root + F::one(); + let proof = tree.generate_proof(0).unwrap(); + assert!(!proof + .verify( + &leaf_crh_params, + &two_to_one_params, + &wrong_root, + leaves[0].as_slice() + ) + .unwrap()) + } + + // test merkle tree update functionality + for (i, v) in update_query { + tree.update(*i, v).unwrap(); + leaves[*i] = v.to_vec(); + } + + // update the root + root = tree.root(); + + // verify again + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree.generate_proof(i).unwrap(); + assert!(proof + .verify(&leaf_crh_params, &two_to_one_params, &root, leaf.as_slice()) + .unwrap()); } } From f9ae45a5e352477f067c67bf70858c91a5d8ef8c Mon Sep 17 00:00:00 2001 From: intx4 Date: Sun, 24 Dec 2023 09:38:16 +0100 Subject: [PATCH 18/24] refactored merkle_tree benches and tests with cfg_iter macro --- benches/merkle_tree.rs | 35 +++++++++++++++++++++-------------- src/merkle_tree/mod.rs | 5 +---- src/merkle_tree/tests/mod.rs | 31 ++++--------------------------- 3 files changed, 26 insertions(+), 45 deletions(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 13ce3d47..88880621 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -7,6 +7,7 @@ mod bytes_mt_benches { use ark_crypto_primitives::crh::*; use ark_crypto_primitives::merkle_tree::*; use ark_crypto_primitives::to_uncompressed_bytes; + use ark_std::cfg_iter; use ark_ff::BigInteger256; use ark_serialize::CanonicalSerialize; use ark_std::{test_rng, UniformRand}; @@ -49,24 +50,30 @@ mod bytes_mt_benches { .clone(); c.bench_function("Merkle Tree Create (Leaves as [u8])", move |b| { b.iter(|| { - #[cfg(not(feature = "parallel"))] - { - _ = Sha256MerkleTree::new( + //#[cfg(not(feature = "parallel"))] + //{ + // _ = Sha256MerkleTree::new( + // &leaf_crh_params.clone(), + // &two_to_one_params.clone(), + // leaves.iter().map(|x| x.as_slice()), + // ) + // .unwrap(); + //} + //#[cfg(feature = "parallel")] + //{ + // _ = Sha256MerkleTree::new( + // &leaf_crh_params.clone(), + // &two_to_one_params.clone(), + // leaves.par_iter().map(|x| x.as_slice()), + // ) + // .unwrap(); + //} + Sha256MerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()), + cfg_iter!(leaves).map(|x| x.as_slice()), ) .unwrap(); - } - #[cfg(feature = "parallel")] - { - _ = Sha256MerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - leaves.par_iter().map(|x| x.as_slice()), - ) - .unwrap(); - } }) }); } diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 4e14e475..804deb38 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -242,7 +242,6 @@ impl MerkleTree

{ pub fn new>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - // leaves is rayon::Iter::IntoParallelIterator if parallel feature is enabled // else leaves is a std::iter::IntoIterator #[cfg(feature="parallel")] @@ -253,9 +252,7 @@ impl MerkleTree

{ // TODO: Box is not Send, I have used .unwrap() but this causes panic // might wish to change in the future - - let leaves_iter = cfg_into_iter!(leaves); // par_iter() if cfg!(feature="parallel") else iter() - let leaves_digest: Vec<_> = leaves_iter + let leaves_digest: Vec<_> = cfg_into_iter!(leaves) // into_par_iter() if cfg!(feature="parallel") else into_iter() .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) .collect(); diff --git a/src/merkle_tree/tests/mod.rs b/src/merkle_tree/tests/mod.rs index ad94cafb..a1afaaaf 100644 --- a/src/merkle_tree/tests/mod.rs +++ b/src/merkle_tree/tests/mod.rs @@ -10,10 +10,7 @@ mod bytes_mt_tests { }; use ark_ed_on_bls12_381::EdwardsProjective as JubJub; use ark_ff::BigInteger256; - use ark_std::{test_rng, UniformRand}; - - #[cfg(feature="parallel")] - use rayon::prelude::*; + use ark_std::{test_rng, cfg_iter, UniformRand}; #[derive(Clone)] pub(super) struct Window4x256; @@ -52,20 +49,10 @@ mod bytes_mt_tests { .unwrap() .clone(); - let leaves_iter; //serial or parallel - #[cfg(feature="parallel")] - { - leaves_iter = leaves.par_iter(); - } - #[cfg(not(feature="parallel"))] - { - leaves_iter = leaves.iter(); - } - let mut tree = JubJubMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves_iter.map(|x| x.as_slice()), + cfg_iter!(leaves).map(|x| x.as_slice()), ) .unwrap(); @@ -138,7 +125,7 @@ mod field_mt_tests { use crate::crh::poseidon; use crate::merkle_tree::tests::test_utils::poseidon_parameters; use crate::merkle_tree::{Config, IdentityDigestConverter, MerkleTree}; - use ark_std::{test_rng, vec::Vec, One, UniformRand}; + use ark_std::{test_rng, cfg_iter, vec::Vec, One, UniformRand}; #[cfg(feature="parallel")] use rayon::prelude::*; @@ -164,20 +151,10 @@ mod field_mt_tests { let leaf_crh_params = poseidon_parameters(); let two_to_one_params = leaf_crh_params.clone(); - let leaves_iter; //serial or parallel - #[cfg(feature="parallel")] - { - leaves_iter = leaves.par_iter(); - } - #[cfg(not(feature="parallel"))] - { - leaves_iter = leaves.iter(); - } - let mut tree = FieldMT::new( &leaf_crh_params, &two_to_one_params, - leaves_iter.map(|x| x.as_slice()), + cfg_iter!(leaves).map(|x| x.as_slice()), ) .unwrap(); From a7684e92869020cf818d0de702ea38eddc5e770d Mon Sep 17 00:00:00 2001 From: intx4 Date: Sun, 24 Dec 2023 19:24:53 +0100 Subject: [PATCH 19/24] modified merkle_tree/mod.rs new_with_leaf_digest to use macro cfg_iter_mut --- src/merkle_tree/mod.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 804deb38..3df4016b 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -329,17 +329,7 @@ impl MerkleTree

{ let (nodes_at_level, nodes_at_prev_level) = non_leaf_nodes[..].split_at_mut(upper_bound); - //let nodes_at_level_iter = cfg_into_iter!(nodes_at_level[start_index..]); - let nodes_at_level_iter; - #[cfg(feature="parallel")] - { - nodes_at_level_iter = nodes_at_level[start_index..].par_iter_mut(); - } - - #[cfg(not(feature="parallel"))] - { - nodes_at_level_iter = nodes_at_level[start_index..].iter_mut(); - } + let nodes_at_level_iter = cfg_iter_mut!(nodes_at_level[start_index..]); nodes_at_level_iter .enumerate() From 8f0a6d4c158697e57d6e733f5cbc492df8b7b341 Mon Sep 17 00:00:00 2001 From: Pratyush Mishra Date: Thu, 28 Dec 2023 10:18:14 -0500 Subject: [PATCH 20/24] Make error send --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index c2083553..2f828a4a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,7 +43,7 @@ pub mod snark; #[cfg(feature = "sponge")] pub mod sponge; -pub type Error = Box; +pub type Error = Box; #[derive(Debug)] pub enum CryptoError { From a781861a9f29fe82dc75a6d1493cf0200988c0e5 Mon Sep 17 00:00:00 2001 From: Pratyush Mishra Date: Thu, 28 Dec 2023 10:36:46 -0500 Subject: [PATCH 21/24] Parallel iteration fixes --- benches/merkle_tree.rs | 34 +++----------- src/merkle_tree/mod.rs | 90 +++++++++++++++++------------------- src/merkle_tree/tests/mod.rs | 18 ++++---- 3 files changed, 57 insertions(+), 85 deletions(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 88880621..0c10f11d 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -7,9 +7,9 @@ mod bytes_mt_benches { use ark_crypto_primitives::crh::*; use ark_crypto_primitives::merkle_tree::*; use ark_crypto_primitives::to_uncompressed_bytes; - use ark_std::cfg_iter; use ark_ff::BigInteger256; use ark_serialize::CanonicalSerialize; + use ark_std::cfg_iter; use ark_std::{test_rng, UniformRand}; use criterion::Criterion; use std::borrow::Borrow; @@ -50,30 +50,12 @@ mod bytes_mt_benches { .clone(); c.bench_function("Merkle Tree Create (Leaves as [u8])", move |b| { b.iter(|| { - //#[cfg(not(feature = "parallel"))] - //{ - // _ = Sha256MerkleTree::new( - // &leaf_crh_params.clone(), - // &two_to_one_params.clone(), - // leaves.iter().map(|x| x.as_slice()), - // ) - // .unwrap(); - //} - //#[cfg(feature = "parallel")] - //{ - // _ = Sha256MerkleTree::new( - // &leaf_crh_params.clone(), - // &two_to_one_params.clone(), - // leaves.par_iter().map(|x| x.as_slice()), - // ) - // .unwrap(); - //} Sha256MerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - cfg_iter!(leaves).map(|x| x.as_slice()), - ) - .unwrap(); + &leaf_crh_params.clone(), + &two_to_one_params.clone(), + leaves.iter().map(|x| x.as_slice()), + ) + .unwrap(); }) }); } @@ -85,6 +67,4 @@ mod bytes_mt_benches { } } -criterion_main!( - crate::bytes_mt_benches::mt_create, -); +criterion_main!(crate::bytes_mt_benches::mt_create,); diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index 3df4016b..ecd90777 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -64,7 +64,7 @@ pub trait Config { + Default + CanonicalSerialize + CanonicalDeserialize - + Send; // needed for parallel creation of MT with rayon + + Send; // transition between leaf layer to inner layer type LeafInnerDigestConverter: DigestConverter< Self::LeafDigest, @@ -78,7 +78,7 @@ pub trait Config { + Default + CanonicalSerialize + CanonicalDeserialize - + Send; // needed for parallel creation of MT with rayon + + Send; // Tom's Note: in the future, if we want different hash function, we can simply add more // types of digest here and specify a digest converter. Same for constraints. @@ -242,20 +242,15 @@ impl MerkleTree

{ pub fn new>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - // leaves is rayon::Iter::IntoParallelIterator if parallel feature is enabled - // else leaves is a std::iter::IntoIterator - #[cfg(feature="parallel")] - leaves: impl IntoParallelIterator, - #[cfg(not(feature="parallel"))] - leaves: impl IntoIterator, + leaves: impl IntoIterator, ) -> Result { - - // TODO: Box is not Send, I have used .unwrap() but this causes panic - // might wish to change in the future - let leaves_digest: Vec<_> = cfg_into_iter!(leaves) // into_par_iter() if cfg!(feature="parallel") else into_iter() - .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf).unwrap()) - .collect(); - + #[cfg(feature = "parallel")] + let leaves = leaves.into_iter().par_bridge(); + + let leaves_digest: Vec<_> = cfg_into_iter!(leaves) + .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)) + .collect()?; + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) } @@ -264,7 +259,6 @@ impl MerkleTree

{ two_to_one_hash_param: &TwoToOneParam

, leaves_digest: Vec, ) -> Result { - let leaf_nodes_size = leaves_digest.len(); assert!( leaf_nodes_size.is_power_of_two() && leaf_nodes_size > 1, @@ -277,8 +271,7 @@ impl MerkleTree

{ let hash_of_empty: P::InnerDigest = P::InnerDigest::default(); // initialize the merkle tree as array of nodes in level order - let non_leaf_nodes_iter = cfg_into_iter!((0..non_leaf_nodes_size)); - let mut non_leaf_nodes: Vec = non_leaf_nodes_iter + let mut non_leaf_nodes: Vec = cfg_into_iter!(0..non_leaf_nodes_size) .map(|_| hash_of_empty.clone()) .collect(); @@ -294,46 +287,46 @@ impl MerkleTree

{ { let start_index = level_indices.pop().unwrap(); let upper_bound = left_child(start_index); - - let bottom_nodes = &mut non_leaf_nodes[start_index..upper_bound]; - let bottom_nodes_iter = cfg_into_iter!(bottom_nodes); - - bottom_nodes_iter.enumerate().for_each(|(i,n)|{ - - // `left_child(current_index)` and `right_child(current_index) returns the position of - // leaf in the whole tree (represented as a list in level order). We need to shift it - // by `-upper_bound` to get the index in `leaf_nodes` list. - - //similarly, we need to rescale i by start_index - //to get the index outside the slice and in the level-ordered list of nodes - - let current_index = i + start_index; - let left_leaf_index = left_child(current_index) - upper_bound; - let right_leaf_index = right_child(current_index) - upper_bound; - - // need for unwrap as Box does not implement trait Send - *n = P::TwoToOneHash::evaluate( - two_to_one_hash_param, - P::LeafInnerDigestConverter::convert(leaves_digest[left_leaf_index].clone()).unwrap(), - P::LeafInnerDigestConverter::convert(leaves_digest[right_leaf_index].clone()).unwrap() - ).unwrap() - }); + + cfg_iter_mut!(non_leaf_nodes[start_index..upper_bound]) + .enumerate() + .try_for_each(|(i, n)| { + // `left_child(current_index)` and `right_child(current_index) returns the position of + // leaf in the whole tree (represented as a list in level order). We need to shift it + // by `-upper_bound` to get the index in `leaf_nodes` list. + + //similarly, we need to rescale i by start_index + //to get the index outside the slice and in the level-ordered list of nodes + + let current_index = i + start_index; + let left_leaf_index = left_child(current_index) - upper_bound; + let right_leaf_index = right_child(current_index) - upper_bound; + + *n = P::TwoToOneHash::evaluate( + two_to_one_hash_param, + P::LeafInnerDigestConverter::convert( + leaves_digest[left_leaf_index].clone(), + )?, + P::LeafInnerDigestConverter::convert( + leaves_digest[right_leaf_index].clone(), + )?, + )?; + Ok::<(), crate::Error>(()) + }); } // compute the hash values for nodes in every other layer in the tree level_indices.reverse(); for &start_index in &level_indices { - // The layer beginning `start_index` ends at `upper_bound` (exclusive). let upper_bound = left_child(start_index); let (nodes_at_level, nodes_at_prev_level) = non_leaf_nodes[..].split_at_mut(upper_bound); - let nodes_at_level_iter = cfg_iter_mut!(nodes_at_level[start_index..]); - - nodes_at_level_iter + // Iterate over the nodes at the current level, and compute the hash of each node + cfg_iter_mut!(nodes_at_level[start_index..]) .enumerate() - .for_each(|(i,n)|{ + .try_for_each(|(i, n)| { // `left_child(current_index)` and `right_child(current_index) returns the position of // leaf in the whole tree (represented as a list in level order). We need to shift it // by `-upper_bound` to get the index in `leaf_nodes` list. @@ -349,7 +342,8 @@ impl MerkleTree

{ two_to_one_hash_param, nodes_at_prev_level[left_leaf_index].clone(), nodes_at_prev_level[right_leaf_index].clone(), - ).unwrap(); + )?; + Ok::<(), crate::Error>(()) }); } Ok(MerkleTree { diff --git a/src/merkle_tree/tests/mod.rs b/src/merkle_tree/tests/mod.rs index a1afaaaf..91116ea1 100644 --- a/src/merkle_tree/tests/mod.rs +++ b/src/merkle_tree/tests/mod.rs @@ -10,7 +10,7 @@ mod bytes_mt_tests { }; use ark_ed_on_bls12_381::EdwardsProjective as JubJub; use ark_ff::BigInteger256; - use ark_std::{test_rng, cfg_iter, UniformRand}; + use ark_std::{test_rng, UniformRand}; #[derive(Clone)] pub(super) struct Window4x256; @@ -40,10 +40,11 @@ mod bytes_mt_tests { fn merkle_tree_test(leaves: &[L], update_query: &[(usize, L)]) -> () { let mut rng = ark_std::test_rng(); - let mut leaves: Vec<_> = leaves.iter() + let mut leaves: Vec<_> = leaves + .iter() .map(|leaf| crate::to_uncompressed_bytes!(leaf).unwrap()) .collect(); - + let leaf_crh_params = ::setup(&mut rng).unwrap(); let two_to_one_params = ::setup(&mut rng) .unwrap() @@ -52,7 +53,7 @@ mod bytes_mt_tests { let mut tree = JubJubMerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - cfg_iter!(leaves).map(|x| x.as_slice()), + leaves.iter().map(|x| x.as_slice()), ) .unwrap(); @@ -125,11 +126,8 @@ mod field_mt_tests { use crate::crh::poseidon; use crate::merkle_tree::tests::test_utils::poseidon_parameters; use crate::merkle_tree::{Config, IdentityDigestConverter, MerkleTree}; - use ark_std::{test_rng, cfg_iter, vec::Vec, One, UniformRand}; + use ark_std::{test_rng, vec::Vec, One, UniformRand}; - #[cfg(feature="parallel")] - use rayon::prelude::*; - type F = ark_ed_on_bls12_381::Fr; type H = poseidon::CRH; type TwoToOneH = poseidon::TwoToOneCRH; @@ -150,11 +148,11 @@ mod field_mt_tests { let mut leaves = leaves.to_vec(); let leaf_crh_params = poseidon_parameters(); let two_to_one_params = leaf_crh_params.clone(); - + let mut tree = FieldMT::new( &leaf_crh_params, &two_to_one_params, - cfg_iter!(leaves).map(|x| x.as_slice()), + leaves.iter().map(|x| x.as_slice()), ) .unwrap(); From c17337f437d26d4ddb780e1d76a856d38bb62b26 Mon Sep 17 00:00:00 2001 From: Pratyush Mishra Date: Thu, 28 Dec 2023 11:16:00 -0500 Subject: [PATCH 22/24] Fixes --- benches/merkle_tree.rs | 4 ---- src/crh/bowe_hopwood/mod.rs | 4 ++-- src/crh/injective_map/mod.rs | 6 +++--- src/lib.rs | 27 +++++++++++++++--------- src/merkle_tree/mod.rs | 41 ++++++++++++++++++++---------------- src/prf/blake2s/mod.rs | 4 ++-- src/prf/mod.rs | 4 ++-- 7 files changed, 49 insertions(+), 41 deletions(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 0c10f11d..71322879 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -9,14 +9,10 @@ mod bytes_mt_benches { use ark_crypto_primitives::to_uncompressed_bytes; use ark_ff::BigInteger256; use ark_serialize::CanonicalSerialize; - use ark_std::cfg_iter; use ark_std::{test_rng, UniformRand}; use criterion::Criterion; use std::borrow::Borrow; - #[cfg(feature = "parallel")] - use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - use crate::NUM_LEAVES; type LeafH = sha2::Sha256; diff --git a/src/crh/bowe_hopwood/mod.rs b/src/crh/bowe_hopwood/mod.rs index e74386b0..6dd758e5 100644 --- a/src/crh/bowe_hopwood/mod.rs +++ b/src/crh/bowe_hopwood/mod.rs @@ -17,7 +17,7 @@ use ark_ec::{ twisted_edwards::Projective as TEProjective, twisted_edwards::TECurveConfig, AdditiveGroup, CurveGroup, }; -use ark_ff::{biginteger::BigInteger, fields::PrimeField}; +use ark_ff::fields::PrimeField; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use ark_std::borrow::Borrow; use ark_std::cfg_chunks; @@ -82,7 +82,7 @@ impl CRHScheme for CRH { let mut c = 0; let mut range = F::BigInt::from(2_u64); while range < upper_limit { - range.muln(4); + range <<= 4; c += 1; } diff --git a/src/crh/injective_map/mod.rs b/src/crh/injective_map/mod.rs index e138149d..fbd99fd1 100644 --- a/src/crh/injective_map/mod.rs +++ b/src/crh/injective_map/mod.rs @@ -1,4 +1,4 @@ -use crate::{CryptoError, Error}; +use crate::Error; use ark_std::rand::Rng; use ark_std::{fmt::Debug, hash::Hash, marker::PhantomData}; @@ -16,7 +16,7 @@ pub mod constraints; pub trait InjectiveMap { type Output: Clone + Eq + Hash + Default + Debug + CanonicalSerialize + CanonicalDeserialize; - fn injective_map(ge: &C::Affine) -> Result; + fn injective_map(ge: &C::Affine) -> Result; } pub struct TECompressor; @@ -24,7 +24,7 @@ pub struct TECompressor; impl InjectiveMap> for TECompressor { type Output =

::BaseField; - fn injective_map(ge: &TEAffine

) -> Result { + fn injective_map(ge: &TEAffine

) -> Result { debug_assert!(ge.is_in_correct_subgroup_assuming_on_curve()); Ok(ge.x) } diff --git a/src/lib.rs b/src/lib.rs index 2f828a4a..31ae920b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,22 +43,29 @@ pub mod snark; #[cfg(feature = "sponge")] pub mod sponge; -pub type Error = Box; - #[derive(Debug)] -pub enum CryptoError { +pub enum Error { IncorrectInputLength(usize), NotPrimeOrder, + GenericError(Box), + SerializationError(ark_serialize::SerializationError), } -impl core::fmt::Display for CryptoError { +impl core::fmt::Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let msg = match self { - CryptoError::IncorrectInputLength(len) => format!("input length is wrong: {}", len), - CryptoError::NotPrimeOrder => "element is not prime order".to_owned(), - }; - write!(f, "{}", msg) + match self { + Self::IncorrectInputLength(len) => write!(f, "incorrect input length: {len}"), + Self::NotPrimeOrder => write!(f, "element is not prime order"), + Self::GenericError(e) => write!(f, "{e}"), + Self::SerializationError(e) => write!(f, "{e}"), + } } } -impl ark_std::error::Error for CryptoError {} +impl ark_std::error::Error for Error {} + +impl From for Error { + fn from(e: ark_serialize::SerializationError) -> Self { + Self::SerializationError(e) + } +} diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index ecd90777..d6acd254 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -55,8 +55,8 @@ impl DigestConverter for ByteDigestConverter /// * `LeafHash`: Convert leaf to leaf digest /// * `TwoToOneHash`: Compress two inner digests to one inner digest pub trait Config { - type Leaf: ?Sized; // merkle tree does not store the leaf - // leaf layer + type Leaf: ?Sized + Send; // merkle tree does not store the leaf + // leaf layer type LeafDigest: Clone + Eq + core::fmt::Debug @@ -234,32 +234,37 @@ impl MerkleTree

{ height: usize, ) -> Result { // use empty leaf digest - let leaves_digest = vec![P::LeafDigest::default(); 1 << (height - 1)]; - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) + let leaf_digests = vec![P::LeafDigest::default(); 1 << (height - 1)]; + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, &leaf_digests) } /// Returns a new merkle tree. `leaves.len()` should be power of two. - pub fn new>( + pub fn new( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaves: impl IntoIterator, - ) -> Result { + leaves: T, + ) -> Result + where + L: Borrow + Send, + T: IntoIterator, + T::IntoIter: Send, + { #[cfg(feature = "parallel")] let leaves = leaves.into_iter().par_bridge(); - let leaves_digest: Vec<_> = cfg_into_iter!(leaves) + let leaf_digests: Vec<_> = cfg_into_iter!(leaves) .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)) - .collect()?; + .collect::, _>>()?; - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaves_digest) + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, &leaf_digests) } pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaves_digest: Vec, + leaf_digests: &[P::LeafDigest], ) -> Result { - let leaf_nodes_size = leaves_digest.len(); + let leaf_nodes_size = leaf_digests.len(); assert!( leaf_nodes_size.is_power_of_two() && leaf_nodes_size > 1, "`leaves.len() should be power of two and greater than one" @@ -305,14 +310,14 @@ impl MerkleTree

{ *n = P::TwoToOneHash::evaluate( two_to_one_hash_param, P::LeafInnerDigestConverter::convert( - leaves_digest[left_leaf_index].clone(), + leaf_digests[left_leaf_index].clone(), )?, P::LeafInnerDigestConverter::convert( - leaves_digest[right_leaf_index].clone(), + leaf_digests[right_leaf_index].clone(), )?, )?; Ok::<(), crate::Error>(()) - }); + })?; } // compute the hash values for nodes in every other layer in the tree @@ -343,11 +348,11 @@ impl MerkleTree

{ nodes_at_prev_level[left_leaf_index].clone(), nodes_at_prev_level[right_leaf_index].clone(), )?; - Ok::<(), crate::Error>(()) - }); + Ok::<_, crate::Error>(()) + })?; } Ok(MerkleTree { - leaf_nodes: leaves_digest, + leaf_nodes: leaf_digests.to_vec(), non_leaf_nodes, height: tree_height, leaf_hash_param: leaf_hash_param.clone(), diff --git a/src/prf/blake2s/mod.rs b/src/prf/blake2s/mod.rs index 6add07d9..7455e18a 100644 --- a/src/prf/blake2s/mod.rs +++ b/src/prf/blake2s/mod.rs @@ -3,7 +3,7 @@ use blake2::{Blake2s256 as B2s, Blake2sMac}; use digest::Digest; use super::PRF; -use crate::CryptoError; +use crate::Error; #[cfg(feature = "r1cs")] pub mod constraints; @@ -16,7 +16,7 @@ impl PRF for Blake2s { type Output = [u8; 32]; type Seed = [u8; 32]; - fn evaluate(seed: &Self::Seed, input: &Self::Input) -> Result { + fn evaluate(seed: &Self::Seed, input: &Self::Input) -> Result { let eval_time = start_timer!(|| "Blake2s::Eval"); let mut h = B2s::new(); h.update(seed.as_ref()); diff --git a/src/prf/mod.rs b/src/prf/mod.rs index d8f870cf..fa3da3a4 100644 --- a/src/prf/mod.rs +++ b/src/prf/mod.rs @@ -2,7 +2,7 @@ use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use core::{fmt::Debug, hash::Hash}; -use crate::CryptoError; +use crate::Error; #[cfg(feature = "r1cs")] pub mod constraints; @@ -17,5 +17,5 @@ pub trait PRF { type Output: CanonicalSerialize + Eq + Clone + Debug + Default + Hash; type Seed: CanonicalDeserialize + CanonicalSerialize + Clone + Default + Debug; - fn evaluate(seed: &Self::Seed, input: &Self::Input) -> Result; + fn evaluate(seed: &Self::Seed, input: &Self::Input) -> Result; } From 93e0e8d4acca541ac03aa4384b5c1a12819480f2 Mon Sep 17 00:00:00 2001 From: Pratyush Mishra Date: Thu, 28 Dec 2023 11:55:26 -0500 Subject: [PATCH 23/24] Change signature to enable correct iteration --- benches/merkle_tree.rs | 2 +- src/crh/mod.rs | 2 +- src/merkle_tree/mod.rs | 17 +++++------------ src/merkle_tree/tests/constraints.rs | 15 +++------------ src/merkle_tree/tests/mod.rs | 21 +++++---------------- 5 files changed, 15 insertions(+), 42 deletions(-) diff --git a/benches/merkle_tree.rs b/benches/merkle_tree.rs index 71322879..b32168e8 100644 --- a/benches/merkle_tree.rs +++ b/benches/merkle_tree.rs @@ -49,7 +49,7 @@ mod bytes_mt_benches { Sha256MerkleTree::new( &leaf_crh_params.clone(), &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()), + &leaves, ) .unwrap(); }) diff --git a/src/crh/mod.rs b/src/crh/mod.rs index 4a6e5174..e66be16e 100644 --- a/src/crh/mod.rs +++ b/src/crh/mod.rs @@ -21,7 +21,7 @@ pub use constraints::*; /// Interface to CRH. Note that in this release, while all implementations of `CRH` have fixed length, /// variable length CRH may also implement this trait in future. pub trait CRHScheme { - type Input: ?Sized; + type Input: ?Sized + Send; type Output: Clone + Eq + core::fmt::Debug diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index d6acd254..d2739d7a 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -239,21 +239,14 @@ impl MerkleTree

{ } /// Returns a new merkle tree. `leaves.len()` should be power of two. - pub fn new( + pub fn new + Send>( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaves: T, - ) -> Result - where - L: Borrow + Send, - T: IntoIterator, - T::IntoIter: Send, - { - #[cfg(feature = "parallel")] - let leaves = leaves.into_iter().par_bridge(); - + #[cfg(not(feature = "parallel"))] leaves: impl IntoIterator, + #[cfg(feature = "parallel")] leaves: impl IntoParallelIterator, + ) -> Result { let leaf_digests: Vec<_> = cfg_into_iter!(leaves) - .map(|leaf| P::LeafHash::evaluate(leaf_hash_param, leaf)) + .map(|input| P::LeafHash::evaluate(leaf_hash_param, input.as_ref())) .collect::, _>>()?; Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, &leaf_digests) diff --git a/src/merkle_tree/tests/constraints.rs b/src/merkle_tree/tests/constraints.rs index fbe5217c..8f1602d7 100644 --- a/src/merkle_tree/tests/constraints.rs +++ b/src/merkle_tree/tests/constraints.rs @@ -61,12 +61,8 @@ mod byte_mt_tests { let leaf_crh_params = ::setup(&mut rng).unwrap(); let two_to_one_crh_params = ::setup(&mut rng).unwrap(); - let mut tree = JubJubMerkleTree::new( - &leaf_crh_params, - &two_to_one_crh_params, - leaves.iter().map(|v| v.as_slice()), - ) - .unwrap(); + let mut tree = + JubJubMerkleTree::new(&leaf_crh_params, &two_to_one_crh_params, leaves).unwrap(); let root = tree.root(); for (i, leaf) in leaves.iter().enumerate() { let cs = ConstraintSystem::::new_ref(); @@ -288,12 +284,7 @@ mod field_mt_tests { ) { let leaf_crh_params = poseidon_parameters(); let two_to_one_params = leaf_crh_params.clone(); - let mut tree = FieldMT::new( - &leaf_crh_params, - &two_to_one_params, - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); + let mut tree = FieldMT::new(&leaf_crh_params, &two_to_one_params, leaves).unwrap(); let root = tree.root(); for (i, leaf) in leaves.iter().enumerate() { let cs = ConstraintSystem::::new_ref(); diff --git a/src/merkle_tree/tests/mod.rs b/src/merkle_tree/tests/mod.rs index 91116ea1..a4968917 100644 --- a/src/merkle_tree/tests/mod.rs +++ b/src/merkle_tree/tests/mod.rs @@ -46,16 +46,10 @@ mod bytes_mt_tests { .collect(); let leaf_crh_params = ::setup(&mut rng).unwrap(); - let two_to_one_params = ::setup(&mut rng) - .unwrap() - .clone(); - - let mut tree = JubJubMerkleTree::new( - &leaf_crh_params.clone(), - &two_to_one_params.clone(), - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); + let two_to_one_params = ::setup(&mut rng).unwrap(); + + let mut tree = + JubJubMerkleTree::new(&leaf_crh_params, &two_to_one_params, &leaves).unwrap(); let mut root = tree.root(); // test merkle tree functionality without update @@ -149,12 +143,7 @@ mod field_mt_tests { let leaf_crh_params = poseidon_parameters(); let two_to_one_params = leaf_crh_params.clone(); - let mut tree = FieldMT::new( - &leaf_crh_params, - &two_to_one_params, - leaves.iter().map(|x| x.as_slice()), - ) - .unwrap(); + let mut tree = FieldMT::new(&leaf_crh_params, &two_to_one_params, &leaves).unwrap(); let mut root = tree.root(); From bf076db23c0167113c7745ff430567fe9e7e73fd Mon Sep 17 00:00:00 2001 From: Pratyush Mishra Date: Thu, 28 Dec 2023 12:22:51 -0500 Subject: [PATCH 24/24] Change back to taking `Vec` --- src/merkle_tree/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/merkle_tree/mod.rs b/src/merkle_tree/mod.rs index d2739d7a..2b624903 100644 --- a/src/merkle_tree/mod.rs +++ b/src/merkle_tree/mod.rs @@ -235,7 +235,7 @@ impl MerkleTree

{ ) -> Result { // use empty leaf digest let leaf_digests = vec![P::LeafDigest::default(); 1 << (height - 1)]; - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, &leaf_digests) + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaf_digests) } /// Returns a new merkle tree. `leaves.len()` should be power of two. @@ -249,13 +249,13 @@ impl MerkleTree

{ .map(|input| P::LeafHash::evaluate(leaf_hash_param, input.as_ref())) .collect::, _>>()?; - Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, &leaf_digests) + Self::new_with_leaf_digest(leaf_hash_param, two_to_one_hash_param, leaf_digests) } pub fn new_with_leaf_digest( leaf_hash_param: &LeafParam

, two_to_one_hash_param: &TwoToOneParam

, - leaf_digests: &[P::LeafDigest], + leaf_digests: Vec, ) -> Result { let leaf_nodes_size = leaf_digests.len(); assert!( @@ -345,7 +345,7 @@ impl MerkleTree

{ })?; } Ok(MerkleTree { - leaf_nodes: leaf_digests.to_vec(), + leaf_nodes: leaf_digests, non_leaf_nodes, height: tree_height, leaf_hash_param: leaf_hash_param.clone(),