From 7d1aa9c911bdce46ccd86fec285b14dd5fe67767 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Sun, 8 Sep 2024 12:42:19 +0000 Subject: [PATCH 01/84] Initial impl --- src/ds_impl/hp/list.rs | 114 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 90d89b89..931d2f20 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -5,7 +5,7 @@ use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; use hp_pp::{ - decompose_ptr, light_membarrier, tag, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, + decompose_ptr, light_membarrier, retire, tag, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, }; // `#[repr(C)]` is used to ensure the first field @@ -45,6 +45,9 @@ impl Drop for List { pub struct Handle<'domain> { prev_h: HazardPointer<'domain>, curr_h: HazardPointer<'domain>, + // `anchor_h` and `anchor_next_h` are used for `find_harris` + anchor_h: HazardPointer<'domain>, + anchor_next_h: HazardPointer<'domain>, thread: Thread<'domain>, } @@ -53,6 +56,8 @@ impl Default for Handle<'static> { Self { prev_h: HazardPointer::default(), curr_h: HazardPointer::default(), + anchor_h: HazardPointer::default(), + anchor_next_h: HazardPointer::default(), thread: Thread::new(&DEFAULT_DOMAIN), } } @@ -68,7 +73,12 @@ impl<'domain> Handle<'domain> { pub struct Cursor<'domain, 'hp, K, V> { prev: *mut Node, // not &AtomicPtr because we can't construct the cursor out of thin air + // For harris, this keeps the mark bit. Don't mix harris and harris-micheal. curr: *mut Node, + // `anchor` is used for `find_harris` + // anchor and anchor_next are non-null iff exist + anchor: *mut Node, + anchor_next: *mut Node, handle: &'hp mut Handle<'domain>, } @@ -77,6 +87,8 @@ impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> { Self { prev: head as *const _ as *mut _, curr: head.load(Ordering::Acquire), + anchor: ptr::null_mut(), + anchor_next: ptr::null_mut(), handle, } } @@ -86,6 +98,106 @@ impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> where K: Ord, { + fn find_harris(&mut self, key: &K) -> Result { + // Finding phase + // - cursor.curr: first unmarked node w/ key >= search key (4) + // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) + // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) + + let found = loop { + if self.curr.is_null() { + break false; + } + + let prev = unsafe { &(*self.prev).next }; + + let (curr_base, curr_tag) = decompose_ptr(self.curr); + + self.handle.curr_h.protect_raw(curr_base); + light_membarrier(); + + // Validation depending on the state of self.curr. + // + // - If it is marked, validate on anchor. + // - If it is not marked, validate on curr. + + if curr_tag != 0 { + debug_assert!(!self.anchor.is_null()); + debug_assert!(!self.anchor_next.is_null()); + let (an_base, an_tag) = + decompose_ptr(unsafe { &(*self.anchor).next }.load(Ordering::Acquire)); + if an_tag != 0 { + return Err(()); + } else if an_base != self.anchor_next { + // TODO: optimization here, can restart from anchor, but need to setup some protection for prev & curr. + return Err(()); + } + } else { + let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); + // TODO: is this optimization correct? + // - The first one is a bit more dicy. + // - The second one is for sure correct as original HMList does this. + if curr_new_tag != 0 || curr_new_base != self.curr { + // In contrary to what HP04 paper does, it's fine to retry protecting the new node + // without restarting from head as long as prev is not logically deleted. + self.curr = curr_new_base; + continue; + } + } + + let curr_node = unsafe { &*curr_base }; + let (next_base, next_tag) = decompose_ptr(curr_node.next.load(Ordering::Acquire)); + // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. + if next_tag == 0 { + if curr_node.key < *key { + self.prev = self.curr; + self.curr = next_base; + self.anchor = ptr::null_mut(); + HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); + } else { + break curr_node.key == *key; + } + } else { + if self.anchor.is_null() { + self.anchor = self.prev; + self.anchor_next = self.curr; + HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); + } else if self.anchor_next == self.prev { + HazardPointer::swap(&mut self.handle.anchor_next_h, &mut self.handle.prev_h); + } + self.prev = self.curr; + self.curr = next_base; + HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); + } + }; + + if self.anchor.is_null() { + Ok(found) + } else { + // Should have seen a untagged curr + debug_assert_eq!(tag(self.curr), 0); + debug_assert_eq!(tag(self.anchor_next), 0); + // CAS + unsafe { (*self.anchor).next }.compare_exchange( + self.anchor_next, + self.curr, + Ordering::Release, + Ordering::Relaxed, + )?; + + let mut node = self.anchor_next; + while untagged(node) != self.curr { + // SAFETY: the fact that node node is tagged means that it cannot be modified, hence we can safety do an non-atomic load. + let next = unsafe { *{ *node }.next.as_ptr() }; + debug_assert!(tag(next) != 0); + unsafe { retire(untagged(node)) }; + node = next; + } + + Ok(found) + } + } + #[inline] fn find_harris_michael(&mut self, key: &K) -> Result { loop { From 203b7114960dad50a2c3bcfe8caca911e7e41139 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Mon, 9 Sep 2024 03:43:20 +0000 Subject: [PATCH 02/84] Implement HList for HP. --- src/bin/hp.rs | 3 +- src/ds_impl/hp/list.rs | 111 ++++++++++++++++++++++++++++++++++++----- src/ds_impl/hp/mod.rs | 2 +- 3 files changed, 102 insertions(+), 14 deletions(-) diff --git a/src/bin/hp.rs b/src/bin/hp.rs index f2bf09cb..125eef2e 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -11,7 +11,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HMList, HashMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, HList, HMList, HashMap, SkipList, }; fn main() { @@ -28,6 +28,7 @@ fn main() { fn bench(config: &Config, output: BenchWriter) { println!("{}", config); let perf = match config.ds { + DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 931d2f20..a91e8171 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -134,10 +134,12 @@ where } } else { let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); - // TODO: is this optimization correct? - // - The first one is a bit more dicy. - // - The second one is for sure correct as original HMList does this. - if curr_new_tag != 0 || curr_new_base != self.curr { + if curr_new_tag != 0 { + // TODO: this seems correct, but deadlocks? Might not be dealing with stuff correctly. + // self.curr = curr_new_base; + // continue; + return Err(()); + } else if curr_new_base != self.curr { // In contrary to what HP04 paper does, it's fine to retry protecting the new node // without restarting from head as long as prev is not logically deleted. self.curr = curr_new_base; @@ -178,17 +180,19 @@ where debug_assert_eq!(tag(self.curr), 0); debug_assert_eq!(tag(self.anchor_next), 0); // CAS - unsafe { (*self.anchor).next }.compare_exchange( - self.anchor_next, - self.curr, - Ordering::Release, - Ordering::Relaxed, - )?; + unsafe { &(*self.anchor).next } + .compare_exchange( + self.anchor_next, + self.curr, + Ordering::Release, + Ordering::Relaxed, + ) + .map_err(|_| ())?; let mut node = self.anchor_next; while untagged(node) != self.curr { - // SAFETY: the fact that node node is tagged means that it cannot be modified, hence we can safety do an non-atomic load. - let next = unsafe { *{ *node }.next.as_ptr() }; + // SAFETY: the fact that node is tagged means that it cannot be modified, hence we can safety do an non-atomic load. + let next = unsafe { *({ &*node }.next.as_ptr()) }; debug_assert!(tag(next) != 0); unsafe { retire(untagged(node)) }; node = next; @@ -429,6 +433,18 @@ where } } + pub fn harris_get<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option<&'hp V> { + self.get(key, Cursor::find_harris, handle) + } + + pub fn harris_insert(&self, key: K, value: V, handle: &mut Handle<'_>) -> bool { + self.insert(key, value, Cursor::find_harris, handle) + } + + pub fn harris_remove<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option<&'hp V> { + self.remove(key, Cursor::find_harris, handle) + } + pub fn harris_michael_get<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option<&'hp V> { self.get(key, Cursor::find_harris_michael, handle) } @@ -446,6 +462,49 @@ where } } +pub struct HList { + inner: List, +} + +impl HList +where + K: Ord, +{ + /// Pop the first element efficiently. + /// This method is used for only the fine grained benchmark (src/bin/long_running). + pub fn pop<'hp>(&self, handle: &'hp mut Handle<'_>) -> Option<(&'hp K, &'hp V)> { + self.inner.pop(handle) + } +} + +impl ConcurrentMap for HList +where + K: Ord, +{ + type Handle<'domain> = Handle<'domain>; + + fn handle() -> Self::Handle<'static> { + Handle::default() + } + + fn new() -> Self { + HList { inner: List::new() } + } + + #[inline(always)] + fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + self.inner.harris_get(key, handle) + } + #[inline(always)] + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + self.inner.harris_insert(key, value, handle) + } + #[inline(always)] + fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + self.inner.harris_remove(key, handle) + } +} + pub struct HMList { inner: List, } @@ -519,4 +578,32 @@ mod tests { assert_eq(map.pop(handle).unwrap(), (3, "3".to_string())); assert_eq!(map.pop(handle), None); } + + use super::HList; + + #[test] + fn smoke_h_list() { + concurrent_map::tests::smoke::>(); + } + + #[test] + fn litmus_h_pop() { + use concurrent_map::ConcurrentMap; + let map = HList::new(); + + let handle = &mut HList::::handle(); + map.insert(handle, 1, "1".to_string()); + map.insert(handle, 2, "2".to_string()); + map.insert(handle, 3, "3".to_string()); + + fn assert_eq(a: (&i32, &String), b: (i32, String)) { + assert_eq!(*a.0, b.0); + assert_eq!(*a.1, b.1); + } + + assert_eq(map.pop(handle).unwrap(), (1, "1".to_string())); + assert_eq(map.pop(handle).unwrap(), (2, "2".to_string())); + assert_eq(map.pop(handle).unwrap(), (3, "3".to_string())); + assert_eq!(map.pop(handle), None); + } } diff --git a/src/ds_impl/hp/mod.rs b/src/ds_impl/hp/mod.rs index 329e422d..16f1b2ac 100644 --- a/src/ds_impl/hp/mod.rs +++ b/src/ds_impl/hp/mod.rs @@ -13,7 +13,7 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; pub use self::ellen_tree::EFRBTree; -pub use self::list::HMList; +pub use self::list::{HList, HMList}; pub use self::michael_hash_map::HashMap; pub use self::natarajan_mittal_tree::NMTreeMap; pub use self::skip_list::SkipList; From cabf9b3cf1df44bdc16aac227d5fc625b5806905 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Mon, 9 Sep 2024 07:31:08 +0000 Subject: [PATCH 03/84] WIP --- src/ds_impl/hp/list.rs | 106 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index a91e8171..b835b95b 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -98,6 +98,13 @@ impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> where K: Ord, { + // TODO: Deadlocking. + + // Invariants: + // anchor, anchor_next: protected if they are not null. + // prev: always protected with prev_sh + // curr: not protected. + // curr: also has tag value when it is obtained from prev. fn find_harris(&mut self, key: &K) -> Result { // Finding phase // - cursor.curr: first unmarked node w/ key >= search key (4) @@ -122,13 +129,19 @@ where // - If it is not marked, validate on curr. if curr_tag != 0 { + // This means that prev -> curr was marked, hence persistent. debug_assert!(!self.anchor.is_null()); debug_assert!(!self.anchor_next.is_null()); let (an_base, an_tag) = decompose_ptr(unsafe { &(*self.anchor).next }.load(Ordering::Acquire)); + // Validate on anchor, which should still be the same and cleared. if an_tag != 0 { + // Anchor dirty -> someone logically deleted it -> progress. return Err(()); } else if an_base != self.anchor_next { + // Anchor changed -> someone updated it. + // - Someone else cleared the logically deleted chain -> their find must return -> they must attempt CAS. + // TODO: optimization here, can restart from anchor, but need to setup some protection for prev & curr. return Err(()); } @@ -140,10 +153,14 @@ where // continue; return Err(()); } else if curr_new_base != self.curr { + return Err(()); + // In contrary to what HP04 paper does, it's fine to retry protecting the new node // without restarting from head as long as prev is not logically deleted. - self.curr = curr_new_base; - continue; + + // TODO: this should be correct. + // self.curr = curr_new_base; + // continue; } } @@ -174,6 +191,7 @@ where }; if self.anchor.is_null() { + self.curr = untagged(self.curr); Ok(found) } else { // Should have seen a untagged curr @@ -187,7 +205,10 @@ where Ordering::Release, Ordering::Relaxed, ) - .map_err(|_| ())?; + .map_err(|_| { + self.curr = untagged(self.curr); + () + })?; let mut node = self.anchor_next; while untagged(node) != self.curr { @@ -197,7 +218,7 @@ where unsafe { retire(untagged(node)) }; node = next; } - + self.curr = untagged(self.curr); Ok(found) } } @@ -249,6 +270,83 @@ where self.curr = next_base; } } + + // fn find_hhs(&mut self, key: &K) -> Result { + // // Finding phase + // // - cursor.curr: first unmarked node w/ key >= search key (4) + // // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) + // // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) + + // loop { + // if self.curr.is_null() { + // return Ok(false); + // } + + // let prev = unsafe { &(*self.prev).next }; + + // let (curr_base, curr_tag) = decompose_ptr(self.curr); + + // self.handle.curr_h.protect_raw(curr_base); + // light_membarrier(); + + // // Validation depending on the state of self.curr. + // // + // // - If it is marked, validate on anchor. + // // - If it is not marked, validate on curr. + + // if curr_tag != 0 { + // debug_assert!(!self.anchor.is_null()); + // debug_assert!(!self.anchor_next.is_null()); + // let (an_base, an_tag) = + // decompose_ptr(unsafe { &(*self.anchor).next }.load(Ordering::Acquire)); + // if an_tag != 0 { + // return Err(()); + // } else if an_base != self.anchor_next { + // // TODO: optimization here, can restart from anchor, but need to setup some protection for prev & curr. + // return Err(()); + // } + // } else { + // let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); + // if curr_new_tag != 0 { + // // TODO: this seems correct, but deadlocks? Might not be dealing with stuff correctly. + // // self.curr = curr_new_base; + // // continue; + // return Err(()); + // } else if curr_new_base != self.curr { + // // In contrary to what HP04 paper does, it's fine to retry protecting the new node + // // without restarting from head as long as prev is not logically deleted. + // self.curr = curr_new_base; + // continue; + // } + // } + + // let curr_node = unsafe { &*curr_base }; + // let (next_base, next_tag) = decompose_ptr(curr_node.next.load(Ordering::Acquire)); + // // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. + // // TODO: check key first, then traversal. + // if next_tag == 0 { + // if curr_node.key < *key { + // self.prev = self.curr; + // self.curr = next_base; + // self.anchor = ptr::null_mut(); + // HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); + // } else { + // return Ok(curr_node.key == *key); + // } + // } else { + // if self.anchor.is_null() { + // self.anchor = self.prev; + // self.anchor_next = self.curr; + // HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); + // } else if self.anchor_next == self.prev { + // HazardPointer::swap(&mut self.handle.anchor_next_h, &mut self.handle.prev_h); + // } + // self.prev = self.curr; + // self.curr = next_base; + // HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); + // } + // } + // } } impl List From c51edf58a9ed66464572523c0c67893d820e55de Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 10 Sep 2024 12:28:04 +0000 Subject: [PATCH 04/84] Fix a bug in HP HList --- src/ds_impl/hp/list.rs | 51 +++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index b835b95b..beab3a3d 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -112,11 +112,11 @@ where // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) let found = loop { - if self.curr.is_null() { + if untagged(self.curr).is_null() { break false; } - let prev = unsafe { &(*self.prev).next }; + let prev = unsafe { &(*untagged(self.prev)).next }; let (curr_base, curr_tag) = decompose_ptr(self.curr); @@ -130,10 +130,10 @@ where if curr_tag != 0 { // This means that prev -> curr was marked, hence persistent. - debug_assert!(!self.anchor.is_null()); - debug_assert!(!self.anchor_next.is_null()); + debug_assert!(!untagged(self.anchor).is_null()); + debug_assert!(!untagged(self.anchor_next).is_null()); let (an_base, an_tag) = - decompose_ptr(unsafe { &(*self.anchor).next }.load(Ordering::Acquire)); + decompose_ptr(unsafe { &(*untagged(self.anchor)).next }.load(Ordering::Acquire)); // Validate on anchor, which should still be the same and cleared. if an_tag != 0 { // Anchor dirty -> someone logically deleted it -> progress. @@ -146,38 +146,27 @@ where return Err(()); } } else { - let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); - if curr_new_tag != 0 { - // TODO: this seems correct, but deadlocks? Might not be dealing with stuff correctly. - // self.curr = curr_new_base; - // continue; + // TODO: optimize here. + if prev.load(Ordering::Acquire) != self.curr { return Err(()); - } else if curr_new_base != self.curr { - return Err(()); - - // In contrary to what HP04 paper does, it's fine to retry protecting the new node - // without restarting from head as long as prev is not logically deleted. - - // TODO: this should be correct. - // self.curr = curr_new_base; - // continue; } } let curr_node = unsafe { &*curr_base }; - let (next_base, next_tag) = decompose_ptr(curr_node.next.load(Ordering::Acquire)); + let next = curr_node.next.load(Ordering::Acquire); + let (_, next_tag) = decompose_ptr(next); // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. if next_tag == 0 { if curr_node.key < *key { self.prev = self.curr; - self.curr = next_base; + self.curr = next; self.anchor = ptr::null_mut(); HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); } else { break curr_node.key == *key; } } else { - if self.anchor.is_null() { + if untagged(self.anchor).is_null() { self.anchor = self.prev; self.anchor_next = self.curr; HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); @@ -185,24 +174,23 @@ where HazardPointer::swap(&mut self.handle.anchor_next_h, &mut self.handle.prev_h); } self.prev = self.curr; - self.curr = next_base; + self.curr = next; HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); } }; - if self.anchor.is_null() { + if untagged(self.anchor).is_null() { + self.prev = untagged(self.prev); self.curr = untagged(self.curr); Ok(found) } else { - // Should have seen a untagged curr - debug_assert_eq!(tag(self.curr), 0); debug_assert_eq!(tag(self.anchor_next), 0); // CAS - unsafe { &(*self.anchor).next } + unsafe { &(*untagged(self.anchor)).next } .compare_exchange( self.anchor_next, - self.curr, - Ordering::Release, + untagged(self.curr), + Ordering::AcqRel, Ordering::Relaxed, ) .map_err(|_| { @@ -211,13 +199,14 @@ where })?; let mut node = self.anchor_next; - while untagged(node) != self.curr { + while untagged(node) != untagged(self.curr) { // SAFETY: the fact that node is tagged means that it cannot be modified, hence we can safety do an non-atomic load. - let next = unsafe { *({ &*node }.next.as_ptr()) }; + let next = unsafe { *{ &*untagged(node) }.next.as_ptr() }; debug_assert!(tag(next) != 0); unsafe { retire(untagged(node)) }; node = next; } + self.prev = untagged(self.anchor); self.curr = untagged(self.curr); Ok(found) } From a67bb165e174a648bc3ce15cc624686047a2aa4b Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 10 Sep 2024 14:41:01 +0000 Subject: [PATCH 05/84] Apply `Shared` API for HP Lists --- src/ds_impl/hp/list.rs | 190 ++++++++++++----------- src/ds_impl/hp/michael_hash_map.rs | 4 +- src/ds_impl/hp/mod.rs | 1 + src/ds_impl/hp/pointers.rs | 238 +++++++++++++++++++++++++++++ 4 files changed, 340 insertions(+), 93 deletions(-) create mode 100644 src/ds_impl/hp/pointers.rs diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index beab3a3d..660c668e 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -1,31 +1,28 @@ use super::concurrent_map::ConcurrentMap; +use super::pointers::{Atomic, Pointer, Shared}; use std::cmp::Ordering::{Equal, Greater, Less}; -use std::ptr; -use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::atomic::Ordering; -use hp_pp::{ - decompose_ptr, light_membarrier, retire, tag, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, -}; +use hp_pp::{light_membarrier, retire, HazardPointer, Thread, DEFAULT_DOMAIN}; // `#[repr(C)]` is used to ensure the first field // is also the first data in the memory alignment. #[repr(C)] -#[derive(Debug)] pub struct Node { /// Mark: tag(), Tag: not needed - next: AtomicPtr>, + next: Atomic>, key: K, value: V, } pub struct List { - head: AtomicPtr>, + head: Atomic>, } impl Default for List where - K: Ord, + K: Ord + 'static, { fn default() -> Self { Self::new() @@ -34,10 +31,10 @@ where impl Drop for List { fn drop(&mut self) { - let mut curr = untagged(*self.head.get_mut()); + let mut curr = self.head.as_shared(); while !curr.is_null() { - curr = untagged(*unsafe { Box::from_raw(curr) }.next.get_mut()); + curr = unsafe { curr.into_owned() }.next.as_shared(); } } } @@ -72,23 +69,23 @@ impl<'domain> Handle<'domain> { } pub struct Cursor<'domain, 'hp, K, V> { - prev: *mut Node, // not &AtomicPtr because we can't construct the cursor out of thin air + prev: Shared>, // not &Atomic because we can't construct the cursor out of thin air // For harris, this keeps the mark bit. Don't mix harris and harris-micheal. - curr: *mut Node, + curr: Shared>, // `anchor` is used for `find_harris` // anchor and anchor_next are non-null iff exist - anchor: *mut Node, - anchor_next: *mut Node, + anchor: Shared>, + anchor_next: Shared>, handle: &'hp mut Handle<'domain>, } impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> { - pub fn new(head: &AtomicPtr>, handle: &'hp mut Handle<'domain>) -> Self { + pub fn new(head: &Atomic>, handle: &'hp mut Handle<'domain>) -> Self { Self { - prev: head as *const _ as *mut _, + prev: unsafe { Shared::from_raw(head as *const _ as *mut _) }, curr: head.load(Ordering::Acquire), - anchor: ptr::null_mut(), - anchor_next: ptr::null_mut(), + anchor: Shared::null(), + anchor_next: Shared::null(), handle, } } @@ -105,6 +102,7 @@ where // prev: always protected with prev_sh // curr: not protected. // curr: also has tag value when it is obtained from prev. + #[inline] fn find_harris(&mut self, key: &K) -> Result { // Finding phase // - cursor.curr: first unmarked node w/ key >= search key (4) @@ -112,15 +110,14 @@ where // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) let found = loop { - if untagged(self.curr).is_null() { + if self.curr.is_null() { break false; } - let prev = unsafe { &(*untagged(self.prev)).next }; - - let (curr_base, curr_tag) = decompose_ptr(self.curr); - - self.handle.curr_h.protect_raw(curr_base); + let prev = unsafe { &self.prev.deref().next }; + self.handle + .curr_h + .protect_raw(self.curr.with_tag(0).into_raw()); light_membarrier(); // Validation depending on the state of self.curr. @@ -128,17 +125,16 @@ where // - If it is marked, validate on anchor. // - If it is not marked, validate on curr. - if curr_tag != 0 { + if self.curr.tag() != 0 { // This means that prev -> curr was marked, hence persistent. - debug_assert!(!untagged(self.anchor).is_null()); - debug_assert!(!untagged(self.anchor_next).is_null()); - let (an_base, an_tag) = - decompose_ptr(unsafe { &(*untagged(self.anchor)).next }.load(Ordering::Acquire)); + debug_assert!(!self.anchor.is_null()); + debug_assert!(!self.anchor_next.is_null()); + let an_new = unsafe { &self.anchor.deref().next }.load(Ordering::Acquire); // Validate on anchor, which should still be the same and cleared. - if an_tag != 0 { + if an_new.tag() != 0 { // Anchor dirty -> someone logically deleted it -> progress. return Err(()); - } else if an_base != self.anchor_next { + } else if an_new != self.anchor_next { // Anchor changed -> someone updated it. // - Someone else cleared the logically deleted chain -> their find must return -> they must attempt CAS. @@ -152,21 +148,20 @@ where } } - let curr_node = unsafe { &*curr_base }; + let curr_node = unsafe { self.curr.deref() }; let next = curr_node.next.load(Ordering::Acquire); - let (_, next_tag) = decompose_ptr(next); // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. - if next_tag == 0 { + if next.tag() == 0 { if curr_node.key < *key { self.prev = self.curr; self.curr = next; - self.anchor = ptr::null_mut(); + self.anchor = Shared::null(); HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); } else { break curr_node.key == *key; } } else { - if untagged(self.anchor).is_null() { + if self.anchor.is_null() { self.anchor = self.prev; self.anchor_next = self.curr; HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); @@ -179,35 +174,35 @@ where } }; - if untagged(self.anchor).is_null() { - self.prev = untagged(self.prev); - self.curr = untagged(self.curr); + if self.anchor.is_null() { + self.prev = self.prev.with_tag(0); + self.curr = self.curr.with_tag(0); Ok(found) } else { - debug_assert_eq!(tag(self.anchor_next), 0); + debug_assert_eq!(self.anchor_next.tag(), 0); // CAS - unsafe { &(*untagged(self.anchor)).next } + unsafe { &self.anchor.deref().next } .compare_exchange( self.anchor_next, - untagged(self.curr), + self.curr.with_tag(0), Ordering::AcqRel, Ordering::Relaxed, ) .map_err(|_| { - self.curr = untagged(self.curr); + self.curr = self.curr.with_tag(0); () })?; let mut node = self.anchor_next; - while untagged(node) != untagged(self.curr) { + while node.with_tag(0) != self.curr.with_tag(0) { // SAFETY: the fact that node is tagged means that it cannot be modified, hence we can safety do an non-atomic load. - let next = unsafe { *{ &*untagged(node) }.next.as_ptr() }; - debug_assert!(tag(next) != 0); - unsafe { retire(untagged(node)) }; + let next = unsafe { node.deref().next.as_shared() }; + debug_assert!(next.tag() != 0); + unsafe { retire(node.with_tag(0).into_raw()) }; node = next; } - self.prev = untagged(self.anchor); - self.curr = untagged(self.curr); + self.prev = self.anchor.with_tag(0); + self.curr = self.curr.with_tag(0); Ok(found) } } @@ -215,31 +210,32 @@ where #[inline] fn find_harris_michael(&mut self, key: &K) -> Result { loop { - debug_assert_eq!(tag(self.curr), 0); + debug_assert_eq!(self.curr.tag(), 0); if self.curr.is_null() { return Ok(false); } - let prev = unsafe { &(*self.prev).next }; + let prev = unsafe { &self.prev.deref().next }; - self.handle.curr_h.protect_raw(self.curr); + self.handle + .curr_h + .protect_raw(self.curr.with_tag(0).into_raw()); light_membarrier(); - let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); - if curr_new_tag != 0 { + let curr_new = prev.load(Ordering::Acquire); + if curr_new.tag() != 0 { return Err(()); - } else if curr_new_base != self.curr { + } else if curr_new.with_tag(0) != self.curr { // In contrary to what HP04 paper does, it's fine to retry protecting the new node // without restarting from head as long as prev is not logically deleted. - self.curr = curr_new_base; + self.curr = curr_new.with_tag(0); continue; } - let curr_node = unsafe { &*self.curr }; + let curr_node = unsafe { self.curr.deref() }; let next = curr_node.next.load(Ordering::Acquire); - let (next_base, next_tag) = decompose_ptr(next); - if next_tag == 0 { + if next.tag() == 0 { match curr_node.key.cmp(key) { Less => { self.prev = self.curr; @@ -249,14 +245,19 @@ where Greater => return Ok(false), } } else if prev - .compare_exchange(self.curr, next_base, Ordering::Release, Ordering::Relaxed) + .compare_exchange( + self.curr, + next.with_tag(0), + Ordering::Release, + Ordering::Relaxed, + ) .is_ok() { - unsafe { self.handle.thread.retire(self.curr) }; + unsafe { self.handle.thread.retire(self.curr.with_tag(0).into_raw()) }; } else { return Err(()); } - self.curr = next_base; + self.curr = next.with_tag(0); } } @@ -340,12 +341,12 @@ where impl List where - K: Ord, + K: Ord + 'static, { /// Creates a new list. pub fn new() -> Self { List { - head: AtomicPtr::new(ptr::null_mut()), + head: Atomic::null(), } } @@ -362,7 +363,7 @@ where loop { let mut cursor = Cursor::new(&self.head, handle.launder()); match find(&mut cursor, key) { - Ok(true) => return unsafe { Some(&((*cursor.curr).value)) }, + Ok(true) => return unsafe { Some(&(cursor.curr.deref().value)) }, Ok(false) => return None, Err(_) => continue, } @@ -371,7 +372,7 @@ where fn insert_inner<'domain, 'hp, F>( &self, - node: *mut Node, + node: Shared>, find: &F, handle: &'hp mut Handle<'domain>, ) -> Result @@ -380,14 +381,16 @@ where { loop { let mut cursor = Cursor::new(&self.head, handle.launder()); - let found = find(&mut cursor, unsafe { &(*node).key })?; + let found = find(&mut cursor, unsafe { &node.deref().key })?; if found { - drop(unsafe { Box::from_raw(node) }); + drop(unsafe { node.into_owned() }); return Ok(false); } - unsafe { &*node }.next.store(cursor.curr, Ordering::Relaxed); - if unsafe { &*cursor.prev } + unsafe { node.deref() } + .next + .store(cursor.curr, Ordering::Relaxed); + if unsafe { cursor.prev.deref() } .next .compare_exchange(cursor.curr, node, Ordering::Release, Ordering::Relaxed) .is_ok() @@ -408,11 +411,11 @@ where where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { - let node = Box::into_raw(Box::new(Node { + let node = Shared::from_owned(Node { key, value, - next: AtomicPtr::new(ptr::null_mut()), - })); + next: Atomic::null(), + }); loop { match self.insert_inner(node, &find, handle.launder()) { @@ -438,20 +441,24 @@ where return Ok(None); } - let curr_node = unsafe { &*cursor.curr }; + let curr_node = unsafe { cursor.curr.deref() }; let next = curr_node.next.fetch_or(1, Ordering::AcqRel); - let next_tag = tag(next); - if next_tag == 1 { + if next.tag() == 1 { continue; } - let prev = unsafe { &(*cursor.prev).next }; + let prev = unsafe { &cursor.prev.deref().next }; if prev .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) .is_ok() { - unsafe { cursor.handle.thread.retire(cursor.curr) }; + unsafe { + cursor + .handle + .thread + .retire(cursor.curr.with_tag(0).into_raw()) + }; } return Ok(Some(&curr_node.value)); @@ -479,12 +486,14 @@ where #[inline] fn pop_inner<'hp>(&self, handle: &'hp mut Handle<'_>) -> Result, ()> { let cursor = Cursor::new(&self.head, handle.launder()); - let prev = unsafe { &(*cursor.prev).next }; + let prev = unsafe { &cursor.prev.deref().next }; - handle.curr_h.protect_raw(cursor.curr); + handle + .curr_h + .protect_raw(cursor.curr.with_tag(0).into_raw()); light_membarrier(); - let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); - if curr_new_tag != 0 || curr_new_base != cursor.curr { + let curr_new = prev.load(Ordering::Acquire); + if curr_new.tag() != 0 || curr_new.with_tag(0) != cursor.curr { return Err(()); } @@ -492,11 +501,10 @@ where return Ok(None); } - let curr_node = unsafe { &*cursor.curr }; + let curr_node = unsafe { cursor.curr.deref() }; let next = curr_node.next.fetch_or(1, Ordering::AcqRel); - let next_tag = tag(next); - if next_tag == 1 { + if next.tag() == 1 { return Err(()); } @@ -504,7 +512,7 @@ where .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) .is_ok() { - unsafe { handle.thread.retire(cursor.curr) }; + unsafe { handle.thread.retire(cursor.curr.with_tag(0).into_raw()) }; } Ok(Some((&curr_node.key, &curr_node.value))) @@ -555,7 +563,7 @@ pub struct HList { impl HList where - K: Ord, + K: Ord + 'static, { /// Pop the first element efficiently. /// This method is used for only the fine grained benchmark (src/bin/long_running). @@ -566,7 +574,7 @@ where impl ConcurrentMap for HList where - K: Ord, + K: Ord + 'static, { type Handle<'domain> = Handle<'domain>; @@ -598,7 +606,7 @@ pub struct HMList { impl HMList where - K: Ord, + K: Ord + 'static, { /// Pop the first element efficiently. /// This method is used for only the fine grained benchmark (src/bin/long_running). @@ -609,7 +617,7 @@ where impl ConcurrentMap for HMList where - K: Ord, + K: Ord + 'static, { type Handle<'domain> = Handle<'domain>; diff --git a/src/ds_impl/hp/michael_hash_map.rs b/src/ds_impl/hp/michael_hash_map.rs index a0bb9557..f42336b0 100644 --- a/src/ds_impl/hp/michael_hash_map.rs +++ b/src/ds_impl/hp/michael_hash_map.rs @@ -11,7 +11,7 @@ pub struct HashMap { impl HashMap where - K: Ord + Hash, + K: Ord + Hash + 'static, { pub fn with_capacity(n: usize) -> Self { let mut buckets = Vec::with_capacity(n); @@ -52,7 +52,7 @@ where impl ConcurrentMap for HashMap where - K: Ord + Hash + Send, + K: Ord + Hash + Send + 'static, V: Send, { type Handle<'domain> = Handle<'domain>; diff --git a/src/ds_impl/hp/mod.rs b/src/ds_impl/hp/mod.rs index 16f1b2ac..1a4d88a6 100644 --- a/src/ds_impl/hp/mod.rs +++ b/src/ds_impl/hp/mod.rs @@ -1,4 +1,5 @@ pub mod concurrent_map; +pub mod pointers; pub mod bonsai_tree; pub mod double_link; diff --git a/src/ds_impl/hp/pointers.rs b/src/ds_impl/hp/pointers.rs new file mode 100644 index 00000000..ff334667 --- /dev/null +++ b/src/ds_impl/hp/pointers.rs @@ -0,0 +1,238 @@ +use core::mem; +use std::{ + ptr::null_mut, + sync::atomic::{AtomicPtr, Ordering}, +}; + +pub struct CompareExchangeError> { + pub new: P, + pub current: Shared, +} + +pub struct Atomic { + link: AtomicPtr, +} + +unsafe impl Sync for Atomic {} +unsafe impl Send for Atomic {} + +impl Atomic { + #[inline] + pub fn new(init: T) -> Self { + let link = AtomicPtr::new(Box::into_raw(Box::new(init))); + Self { link } + } + + #[inline] + pub fn null() -> Self { + let link = AtomicPtr::new(null_mut()); + Self { link } + } + + #[inline] + pub fn load(&self, order: Ordering) -> Shared { + let ptr = self.link.load(order); + Shared { ptr } + } + + #[inline] + pub fn store(&self, ptr: Shared, order: Ordering) { + self.link.store(ptr.into_raw(), order) + } + + #[inline] + pub fn fetch_or(&self, val: usize, order: Ordering) -> Shared { + let ptr = self.link.fetch_or(val, order); + Shared { ptr } + } + + #[inline] + pub fn compare_exchange>( + &self, + current: Shared, + new: P, + success: Ordering, + failure: Ordering, + ) -> Result, CompareExchangeError> { + let current = current.into_raw(); + let new = new.into_raw(); + + match self.link.compare_exchange(current, new, success, failure) { + Ok(current) => Ok(Shared { ptr: current }), + Err(current) => { + let new = unsafe { P::from_raw(new) }; + Err(CompareExchangeError { + new, + current: Shared { ptr: current }, + }) + } + } + } + + #[inline] + pub unsafe fn into_owned(self) -> Box { + Box::from_raw(self.link.into_inner()) + } + + #[inline] + pub fn as_shared(&self) -> Shared { + Shared { + ptr: unsafe { *self.link.as_ptr() }, + } + } +} + +impl Default for Atomic { + #[inline] + fn default() -> Self { + Self { + link: AtomicPtr::default(), + } + } +} + +impl From> for Atomic { + #[inline] + fn from(value: Shared) -> Self { + let link = AtomicPtr::new(value.into_raw()); + Self { link } + } +} + +pub struct Shared { + ptr: *mut T, +} + +impl Shared { + #[inline] + pub fn from_owned(init: T) -> Shared { + let ptr = Box::into_raw(Box::new(init)); + Self { ptr } + } + + #[inline] + pub unsafe fn into_owned(self) -> T { + *Box::from_raw(base_ptr(self.ptr)) + } + + #[inline] + pub fn null() -> Self { + Self { ptr: null_mut() } + } + + #[inline] + pub fn tag(&self) -> usize { + tag(self.ptr) + } + + #[inline] + pub fn is_null(&self) -> bool { + base_ptr(self.ptr).is_null() + } + + #[inline] + pub fn with_tag(&self, tag: usize) -> Self { + let ptr = compose_tag(self.ptr, tag); + Self { ptr } + } + + #[inline] + pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { + base_ptr(self.ptr).as_ref() + } + + #[inline] + pub unsafe fn as_mut<'g>(&self) -> Option<&'g mut T> { + base_ptr(self.ptr).as_mut() + } + + #[inline] + pub unsafe fn deref<'g>(&self) -> &'g T { + &*base_ptr(self.ptr) + } + + #[inline] + pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { + &mut *base_ptr(self.ptr) + } +} + +impl From for Shared { + #[inline] + fn from(val: usize) -> Self { + Self { + ptr: val as *const T as *mut T, + } + } +} + +impl Clone for Shared { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Shared {} + +impl PartialEq for Shared { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl Eq for Shared {} + +pub trait Pointer { + fn into_raw(self) -> *mut T; + unsafe fn from_raw(val: *mut T) -> Self; +} + +impl Pointer for Shared { + #[inline] + fn into_raw(self) -> *mut T { + self.ptr + } + + #[inline] + unsafe fn from_raw(val: *mut T) -> Self { + Shared::from(val as usize) + } +} + +impl Pointer for Box { + #[inline] + fn into_raw(self) -> *mut T { + Box::into_raw(self) + } + + #[inline] + unsafe fn from_raw(val: *mut T) -> Self { + Box::from_raw(val) + } +} + +/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. +#[inline] +fn low_bits() -> usize { + (1 << mem::align_of::().trailing_zeros()) - 1 +} + +/// Given a tagged pointer `data`, returns the same pointer, but tagged with `tag`. +/// +/// `tag` is truncated to fit into the unused bits of the pointer to `T`. +#[inline] +pub(crate) fn compose_tag(ptr: *mut T, tag: usize) -> *mut T { + ((ptr as usize & !low_bits::()) | (tag & low_bits::())) as _ +} + +#[inline] +pub(crate) fn base_ptr(ptr: *mut T) -> *mut T { + (ptr as usize & !low_bits::()) as _ +} + +#[inline] +pub(crate) fn tag(ptr: *mut T) -> usize { + ptr as usize & low_bits::() +} From 46d5366fee8220043fa3cd456ab52723f6619792 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Wed, 11 Sep 2024 05:10:25 +0000 Subject: [PATCH 06/84] Cleanup impl --- src/ds_impl/hp/list.rs | 36 +++++++++++++++++------------------- src/ds_impl/hp/pointers.rs | 17 ++++++++++++++--- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 660c668e..9634c60f 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -1,6 +1,7 @@ use super::concurrent_map::ConcurrentMap; use super::pointers::{Atomic, Pointer, Shared}; +use core::mem; use std::cmp::Ordering::{Equal, Greater, Less}; use std::sync::atomic::Ordering; @@ -31,10 +32,10 @@ where impl Drop for List { fn drop(&mut self) { - let mut curr = self.head.as_shared(); + let mut o_curr = mem::take(&mut self.head); - while !curr.is_null() { - curr = unsafe { curr.into_owned() }.next.as_shared(); + while let Some(curr) = unsafe { o_curr.try_into_owned() } { + o_curr = curr.next; } } } @@ -95,8 +96,6 @@ impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> where K: Ord, { - // TODO: Deadlocking. - // Invariants: // anchor, anchor_next: protected if they are not null. // prev: always protected with prev_sh @@ -363,7 +362,7 @@ where loop { let mut cursor = Cursor::new(&self.head, handle.launder()); match find(&mut cursor, key) { - Ok(true) => return unsafe { Some(&(cursor.curr.deref().value)) }, + Ok(true) => return Some(&unsafe { cursor.curr.deref() }.value), Ok(false) => return None, Err(_) => continue, } @@ -372,7 +371,7 @@ where fn insert_inner<'domain, 'hp, F>( &self, - node: Shared>, + mut node: Box>, find: &F, handle: &'hp mut Handle<'domain>, ) -> Result @@ -381,21 +380,20 @@ where { loop { let mut cursor = Cursor::new(&self.head, handle.launder()); - let found = find(&mut cursor, unsafe { &node.deref().key })?; + let found = find(&mut cursor, &node.key)?; if found { - drop(unsafe { node.into_owned() }); return Ok(false); } - unsafe { node.deref() } - .next - .store(cursor.curr, Ordering::Relaxed); - if unsafe { cursor.prev.deref() } - .next - .compare_exchange(cursor.curr, node, Ordering::Release, Ordering::Relaxed) - .is_ok() - { - return Ok(true); + node.next = cursor.curr.into(); + match unsafe { cursor.prev.deref() }.next.compare_exchange( + cursor.curr, + node, + Ordering::Release, + Ordering::Relaxed, + ) { + Ok(_) => return Ok(true), + Err(e) => node = e.new, } } } @@ -411,7 +409,7 @@ where where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { - let node = Shared::from_owned(Node { + let node = Box::new(Node { key, value, next: Atomic::null(), diff --git a/src/ds_impl/hp/pointers.rs b/src/ds_impl/hp/pointers.rs index ff334667..68ff9308 100644 --- a/src/ds_impl/hp/pointers.rs +++ b/src/ds_impl/hp/pointers.rs @@ -69,13 +69,24 @@ impl Atomic { } } + #[inline] + pub unsafe fn try_into_owned(self) -> Option> { + let ptr = base_ptr(self.link.into_inner()); + if ptr.is_null() { + return None; + } else { + Some(unsafe { Box::from_raw(ptr) }) + } + } + #[inline] pub unsafe fn into_owned(self) -> Box { - Box::from_raw(self.link.into_inner()) + unsafe { Box::from_raw(self.link.into_inner()) } } #[inline] - pub fn as_shared(&self) -> Shared { + // TODO: best API? might be better to just wrap as_ptr, without the deref. + pub unsafe fn as_shared(&self) -> Shared { Shared { ptr: unsafe { *self.link.as_ptr() }, } @@ -112,7 +123,7 @@ impl Shared { #[inline] pub unsafe fn into_owned(self) -> T { - *Box::from_raw(base_ptr(self.ptr)) + unsafe { *Box::from_raw(base_ptr(self.ptr)) } } #[inline] From ffcbcb2ad440806c79a4b136012ca983f70f02e8 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Wed, 11 Sep 2024 05:23:15 +0000 Subject: [PATCH 07/84] Cleanup --- src/ds_impl/hp/list.rs | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 9634c60f..94454216 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -374,15 +374,17 @@ where mut node: Box>, find: &F, handle: &'hp mut Handle<'domain>, - ) -> Result + ) -> bool where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { loop { let mut cursor = Cursor::new(&self.head, handle.launder()); - let found = find(&mut cursor, &node.key)?; + let Ok(found) = find(&mut cursor, &node.key) else { + continue; + }; if found { - return Ok(false); + return false; } node.next = cursor.curr.into(); @@ -392,7 +394,7 @@ where Ordering::Release, Ordering::Relaxed, ) { - Ok(_) => return Ok(true), + Ok(_) => return true, Err(e) => node = e.new, } } @@ -415,12 +417,7 @@ where next: Atomic::null(), }); - loop { - match self.insert_inner(node, &find, handle.launder()) { - Ok(r) => return r, - Err(()) => continue, - } - } + self.insert_inner(node, &find, handle.launder()) } fn remove_inner<'domain, 'hp, F>( @@ -428,15 +425,17 @@ where key: &K, find: &F, handle: &'hp mut Handle<'domain>, - ) -> Result, ()> + ) -> Option<&'hp V> where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { loop { let mut cursor = Cursor::new(&self.head, handle.launder()); - let found = find(&mut cursor, key)?; + let Ok(found) = find(&mut cursor, key) else { + continue; + }; if !found { - return Ok(None); + return None; } let curr_node = unsafe { cursor.curr.deref() }; @@ -459,7 +458,7 @@ where }; } - return Ok(Some(&curr_node.value)); + return Some(&curr_node.value); } } @@ -473,12 +472,7 @@ where where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { - loop { - match self.remove_inner(key, &find, handle.launder()) { - Ok(r) => return r, - Err(_) => continue, - } - } + self.remove_inner(key, &find, handle.launder()) } #[inline] From 716c85d2d70df5342d9789f663b2ea43c4d06f37 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Wed, 11 Sep 2024 07:01:52 +0000 Subject: [PATCH 08/84] Use optimized retire --- src/ds_impl/hp/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 94454216..debf0d45 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -197,7 +197,7 @@ where // SAFETY: the fact that node is tagged means that it cannot be modified, hence we can safety do an non-atomic load. let next = unsafe { node.deref().next.as_shared() }; debug_assert!(next.tag() != 0); - unsafe { retire(node.with_tag(0).into_raw()) }; + unsafe { self.handle.thread.retire(node.with_tag(0).into_raw()) }; node = next; } self.prev = self.anchor.with_tag(0); From f34c09396be54fc9256267b8dd6d6da314250f60 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 11 Sep 2024 07:10:01 +0000 Subject: [PATCH 09/84] Remove an unused import (`retire`) --- src/ds_impl/hp/list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index debf0d45..2ee27a22 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -5,7 +5,7 @@ use core::mem; use std::cmp::Ordering::{Equal, Greater, Less}; use std::sync::atomic::Ordering; -use hp_pp::{light_membarrier, retire, HazardPointer, Thread, DEFAULT_DOMAIN}; +use hp_pp::{light_membarrier, HazardPointer, Thread, DEFAULT_DOMAIN}; // `#[repr(C)]` is used to ensure the first field // is also the first data in the memory alignment. From c011a6e50e77bd317def7ee46292d35d1903d3a9 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 11 Sep 2024 07:11:29 +0000 Subject: [PATCH 10/84] Add a sanitizing script for HP --- test-scripts/sanitize-hp.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 test-scripts/sanitize-hp.sh diff --git a/test-scripts/sanitize-hp.sh b/test-scripts/sanitize-hp.sh new file mode 100755 index 00000000..ebe40215 --- /dev/null +++ b/test-scripts/sanitize-hp.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' + +hps="cargo run --bin hp --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " + +set -e +for i in {1..5000}; do + $hps -dh-list -i3 -t256 -r10 -g1 +done From e19c5624db171522cfc8112c50829388dcfcbd17 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 12 Sep 2024 07:00:27 +0000 Subject: [PATCH 11/84] Implement HP HHSList --- src/bin/hp.rs | 3 +- src/ds_impl/hp/list.rs | 242 +++++++++++++++++++++-------------------- src/ds_impl/hp/mod.rs | 2 +- 3 files changed, 130 insertions(+), 117 deletions(-) diff --git a/src/bin/hp.rs b/src/bin/hp.rs index 125eef2e..fb9f36e8 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -11,7 +11,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HList, HMList, HashMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, SkipList, }; fn main() { @@ -29,6 +29,7 @@ fn bench(config: &Config, output: BenchWriter) { println!("{}", config); let perf = match config.ds { DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 2ee27a22..d66ea9d4 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -260,82 +260,83 @@ where } } - // fn find_hhs(&mut self, key: &K) -> Result { - // // Finding phase - // // - cursor.curr: first unmarked node w/ key >= search key (4) - // // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) - // // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) - - // loop { - // if self.curr.is_null() { - // return Ok(false); - // } - - // let prev = unsafe { &(*self.prev).next }; - - // let (curr_base, curr_tag) = decompose_ptr(self.curr); - - // self.handle.curr_h.protect_raw(curr_base); - // light_membarrier(); - - // // Validation depending on the state of self.curr. - // // - // // - If it is marked, validate on anchor. - // // - If it is not marked, validate on curr. - - // if curr_tag != 0 { - // debug_assert!(!self.anchor.is_null()); - // debug_assert!(!self.anchor_next.is_null()); - // let (an_base, an_tag) = - // decompose_ptr(unsafe { &(*self.anchor).next }.load(Ordering::Acquire)); - // if an_tag != 0 { - // return Err(()); - // } else if an_base != self.anchor_next { - // // TODO: optimization here, can restart from anchor, but need to setup some protection for prev & curr. - // return Err(()); - // } - // } else { - // let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); - // if curr_new_tag != 0 { - // // TODO: this seems correct, but deadlocks? Might not be dealing with stuff correctly. - // // self.curr = curr_new_base; - // // continue; - // return Err(()); - // } else if curr_new_base != self.curr { - // // In contrary to what HP04 paper does, it's fine to retry protecting the new node - // // without restarting from head as long as prev is not logically deleted. - // self.curr = curr_new_base; - // continue; - // } - // } - - // let curr_node = unsafe { &*curr_base }; - // let (next_base, next_tag) = decompose_ptr(curr_node.next.load(Ordering::Acquire)); - // // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. - // // TODO: check key first, then traversal. - // if next_tag == 0 { - // if curr_node.key < *key { - // self.prev = self.curr; - // self.curr = next_base; - // self.anchor = ptr::null_mut(); - // HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); - // } else { - // return Ok(curr_node.key == *key); - // } - // } else { - // if self.anchor.is_null() { - // self.anchor = self.prev; - // self.anchor_next = self.curr; - // HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); - // } else if self.anchor_next == self.prev { - // HazardPointer::swap(&mut self.handle.anchor_next_h, &mut self.handle.prev_h); - // } - // self.prev = self.curr; - // self.curr = next_base; - // HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); - // } - // } - // } + fn find_harris_herlihy_shavit(&mut self, key: &K) -> Result { + // Finding phase + // - cursor.curr: first unmarked node w/ key >= search key (4) + // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) + // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) + + let found = loop { + if self.curr.is_null() { + break false; + } + + let prev = unsafe { &self.prev.deref().next }; + self.handle + .curr_h + .protect_raw(self.curr.with_tag(0).into_raw()); + light_membarrier(); + + // Validation depending on the state of self.curr. + // + // - If it is marked, validate on anchor. + // - If it is not marked, validate on curr. + + if self.curr.tag() != 0 { + // This means that prev -> curr was marked, hence persistent. + debug_assert!(!self.anchor.is_null()); + debug_assert!(!self.anchor_next.is_null()); + let an_new = unsafe { &self.anchor.deref().next }.load(Ordering::Acquire); + // Validate on anchor, which should still be the same and cleared. + if an_new.tag() != 0 { + // Anchor dirty -> someone logically deleted it -> progress. + return Err(()); + } else if an_new != self.anchor_next { + // Anchor changed -> someone updated it. + // - Someone else cleared the logically deleted chain -> their find must return -> they must attempt CAS. + + // TODO: optimization here, can restart from anchor, but need to setup some protection for prev & curr. + return Err(()); + } + } else { + // TODO: optimize here. + if prev.load(Ordering::Acquire) != self.curr { + return Err(()); + } + } + + let curr_node = unsafe { self.curr.deref() }; + let next = curr_node.next.load(Ordering::Acquire); + // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. + if next.tag() == 0 { + if curr_node.key < *key { + self.prev = self.curr; + self.curr = next; + self.anchor = Shared::null(); + HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); + } else { + break curr_node.key == *key; + } + } else { + if self.anchor.is_null() { + self.anchor = self.prev; + self.anchor_next = self.curr; + HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); + } else if self.anchor_next == self.prev { + HazardPointer::swap(&mut self.handle.anchor_next_h, &mut self.handle.prev_h); + } + self.prev = self.curr; + self.curr = next; + HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); + } + }; + + // Return only the found `curr` node. + // Others are not necessary because we are not going to do insertion or deletion + // with this Harris-Herlihy-Shavit traversal. + self.curr = self.curr.with_tag(0); + Ok(found) + } } impl List @@ -547,23 +548,20 @@ where ) -> Option<&'hp V> { self.remove(key, Cursor::find_harris_michael, handle) } + + pub fn harris_herlihy_shavit_get<'hp>( + &self, + key: &K, + handle: &'hp mut Handle<'_>, + ) -> Option<&'hp V> { + self.get(key, Cursor::find_harris_herlihy_shavit, handle) + } } pub struct HList { inner: List, } -impl HList -where - K: Ord + 'static, -{ - /// Pop the first element efficiently. - /// This method is used for only the fine grained benchmark (src/bin/long_running). - pub fn pop<'hp>(&self, handle: &'hp mut Handle<'_>) -> Option<(&'hp K, &'hp V)> { - self.inner.pop(handle) - } -} - impl ConcurrentMap for HList where K: Ord + 'static, @@ -635,50 +633,64 @@ where } } +pub struct HHSList { + inner: List, +} + +impl ConcurrentMap for HHSList +where + K: Ord + 'static, +{ + type Handle<'domain> = Handle<'domain>; + + fn handle() -> Self::Handle<'static> { + Handle::default() + } + + fn new() -> Self { + HHSList { inner: List::new() } + } + + #[inline(always)] + fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + self.inner.harris_michael_get(key, handle) + } + #[inline(always)] + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + self.inner.harris_michael_insert(key, value, handle) + } + #[inline(always)] + fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + self.inner.harris_michael_remove(key, handle) + } +} + #[cfg(test)] mod tests { - use super::HMList; + use super::{HHSList, HList, HMList}; use crate::ds_impl::hp::concurrent_map; #[test] - fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + fn smoke_h_list() { + concurrent_map::tests::smoke::>(); } #[test] - fn litmus_hm_pop() { - use concurrent_map::ConcurrentMap; - let map = HMList::new(); - - let handle = &mut HMList::::handle(); - map.insert(handle, 1, "1".to_string()); - map.insert(handle, 2, "2".to_string()); - map.insert(handle, 3, "3".to_string()); - - fn assert_eq(a: (&i32, &String), b: (i32, String)) { - assert_eq!(*a.0, b.0); - assert_eq!(*a.1, b.1); - } - - assert_eq(map.pop(handle).unwrap(), (1, "1".to_string())); - assert_eq(map.pop(handle).unwrap(), (2, "2".to_string())); - assert_eq(map.pop(handle).unwrap(), (3, "3".to_string())); - assert_eq!(map.pop(handle), None); + fn smoke_hm_list() { + concurrent_map::tests::smoke::>(); } - use super::HList; - #[test] - fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + fn smoke_hhs_list() { + concurrent_map::tests::smoke::>(); } #[test] - fn litmus_h_pop() { + fn litmus_hm_pop() { use concurrent_map::ConcurrentMap; - let map = HList::new(); + let map = HMList::new(); - let handle = &mut HList::::handle(); + let handle = &mut HMList::::handle(); map.insert(handle, 1, "1".to_string()); map.insert(handle, 2, "2".to_string()); map.insert(handle, 3, "3".to_string()); diff --git a/src/ds_impl/hp/mod.rs b/src/ds_impl/hp/mod.rs index 1a4d88a6..bd66b816 100644 --- a/src/ds_impl/hp/mod.rs +++ b/src/ds_impl/hp/mod.rs @@ -14,7 +14,7 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; pub use self::ellen_tree::EFRBTree; -pub use self::list::{HList, HMList}; +pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; pub use self::natarajan_mittal_tree::NMTreeMap; pub use self::skip_list::SkipList; From 35a36ad33312de8a5a943f49667f4def177cabf8 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 12 Sep 2024 18:32:29 +0000 Subject: [PATCH 12/84] Implement HHS-based Hashmap for HP --- src/ds_impl/hp/michael_hash_map.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ds_impl/hp/michael_hash_map.rs b/src/ds_impl/hp/michael_hash_map.rs index f42336b0..beaef9df 100644 --- a/src/ds_impl/hp/michael_hash_map.rs +++ b/src/ds_impl/hp/michael_hash_map.rs @@ -2,11 +2,11 @@ use super::concurrent_map::ConcurrentMap; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -use super::list::HMList; +use super::list::HHSList; pub use super::list::{Cursor, Handle}; pub struct HashMap { - buckets: Vec>, + buckets: Vec>, } impl HashMap @@ -16,14 +16,14 @@ where pub fn with_capacity(n: usize) -> Self { let mut buckets = Vec::with_capacity(n); for _ in 0..n { - buckets.push(HMList::new()); + buckets.push(HHSList::new()); } HashMap { buckets } } #[inline] - pub fn get_bucket(&self, index: usize) -> &HMList { + pub fn get_bucket(&self, index: usize) -> &HHSList { unsafe { self.buckets.get_unchecked(index % self.buckets.len()) } } From cd0e7a5fa2b64d8a616e4de4062f418cbd67b4ed Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 13 Sep 2024 09:42:35 +0000 Subject: [PATCH 13/84] Implement HP NMTree --- src/bin/hp.rs | 4 +- src/ds_impl/hp/natarajan_mittal_tree.rs | 146 ++++++++++++++++++------ test-scripts/sanitize-hp.sh | 1 + 3 files changed, 111 insertions(+), 40 deletions(-) diff --git a/src/bin/hp.rs b/src/bin/hp.rs index fb9f36e8..ce24724d 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -11,7 +11,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -35,7 +35,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), - _ => panic!("Unsupported(or unimplemented) data structure for HP"), + DS::NMTree => bench_map::>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); diff --git a/src/ds_impl/hp/natarajan_mittal_tree.rs b/src/ds_impl/hp/natarajan_mittal_tree.rs index 43861a2d..976dddf3 100644 --- a/src/ds_impl/hp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp/natarajan_mittal_tree.rs @@ -1,9 +1,8 @@ -use hp_pp::Thread; +use hp_pp::{light_membarrier, Thread}; use hp_pp::{tag, tagged, untagged, HazardPointer, DEFAULT_DOMAIN}; use super::concurrent_map::ConcurrentMap; use std::cmp; -use std::mem; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; @@ -30,6 +29,10 @@ impl Marks { fn tag(self) -> bool { !(self & Marks::TAG).is_empty() } + + fn marked(self) -> bool { + !(self & (Marks::TAG | Marks::FLAG)).is_empty() + } } #[derive(Clone, PartialEq, Eq, Debug)] @@ -274,12 +277,11 @@ where NMTreeMap { r } } - // All `Shared<_>` fields are unmarked. fn seek(&self, key: &K, record: &mut SeekRecord<'_, '_, K, V>) -> Result<(), ()> { let s = untagged(self.r.left.load(Ordering::Relaxed)); let s_node = unsafe { &*s }; - // We doesn't have to defend with hazard pointers here + // The root node is always alive; we do not have to protect it. record.ancestor = &self.r as *const _ as *mut _; record.successor = s; // TODO: should preserve tag? @@ -287,11 +289,12 @@ where let leaf = tagged(s_node.left.load(Ordering::Relaxed), Marks::empty().bits()); - // We doesn't have to defend with hazard pointers here + // The `s` node is always alive; we do not have to protect it. record.parent = s; record.handle.leaf_h.protect_raw(leaf); - if leaf != tagged(s_node.left.load(Ordering::Relaxed), Marks::empty().bits()) { + light_membarrier(); + if leaf != tagged(s_node.left.load(Ordering::Acquire), Marks::empty().bits()) { return Err(()); } record.leaf = leaf; @@ -301,47 +304,114 @@ where let mut curr_dir = Direction::L; let mut curr = unsafe { &*record.leaf }.left.load(Ordering::Relaxed); + // `ancestor` always points untagged node. while !untagged(curr).is_null() { if !prev_tag { // untagged edge: advance ancestor and successor pointers - record - .handle - .ancestor_h - .protect_raw(untagged(record.parent)); record.ancestor = record.parent; - record.handle.successor_h.protect_raw(untagged(record.leaf)); record.successor = record.leaf; record.successor_dir = record.leaf_dir; + // `ancestor` and `successor` are already protected by + // hazard pointers of `parent` and `leaf`. + + // Advance the parent and leaf pointers when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (parent), ancestor -> O (ancestor) -> O + // / \ / \ + // (leaf), successor -> O O => (parent), successor -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardPointer::swap(&mut record.handle.ancestor_h, &mut record.handle.parent_h); + HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); + } else if record.successor == record.parent { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O (ancestor) -> O + // / \ / \ + // (parent), successor -> O O (successor) -> O O + // / \ => / \ + // (leaf) -> O O (parent) -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardPointer::swap(&mut record.handle.successor_h, &mut record.handle.parent_h); + HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); + } else { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O + // / \ + // (successor) -> O O + // ... ... + // (parent) -> O + // / \ + // (leaf) -> O O + record.parent = record.leaf; + HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); } - - // advance parent and leaf pointers - mem::swap(&mut record.parent, &mut record.leaf); - HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); - let mut curr_base = untagged(curr); - loop { - record.handle.leaf_h.protect_raw(curr_base); - let curr_base_new = untagged(match curr_dir { - Direction::L => unsafe { &*record.parent }.left.load(Ordering::Acquire), - Direction::R => unsafe { &*record.parent }.right.load(Ordering::Acquire), - }); - if curr_base_new == curr_base { - break; + debug_assert_eq!(tag(record.successor), 0); + + let curr_base = untagged(curr); + record.handle.leaf_h.protect_raw(curr_base); + light_membarrier(); + + if Marks::from_bits_truncate(tag(curr)).marked() { + // `curr` is marked. Validate by `ancestor`. + let succ_new = record.successor_addr().load(Ordering::Acquire); + if Marks::from_bits_truncate(tag(succ_new)).marked() || record.successor != succ_new + { + // Validation is failed. Let's restart from the root. + // TODO: Maybe it can be optimized (by restarting from the anchor), but + // it would require a serious reasoning (including shield swapping, etc). + return Err(()); + } + } else { + // `curr` is unmarked. Validate by `parent`. + let curr_new = match curr_dir { + Direction::L => unsafe { &*untagged(record.parent) } + .left + .load(Ordering::Acquire), + Direction::R => unsafe { &*untagged(record.parent) } + .right + .load(Ordering::Acquire), + }; + if curr_new != curr { + // Validation is failed. Let's restart from the root. + // TODO: Maybe it can be optimized (by restarting from the parent), but + // it would require a serious reasoning (including shield swapping, etc). + return Err(()); } - curr_base = curr_base_new; } - record.leaf = curr_base; + record.leaf = curr; record.leaf_dir = curr_dir; // update other variables prev_tag = Marks::from_bits_truncate(tag(curr)).tag(); - let curr_node = unsafe { &*curr_base }; - if curr_node.key.cmp(key) == cmp::Ordering::Greater { - curr_dir = Direction::L; - curr = curr_node.left.load(Ordering::Acquire); + if Marks::from_bits_truncate(tag(curr)).flag() { + std::hint::black_box({ + let curr_node = unsafe { &*curr_base }; + if curr_node.key.cmp(key) == cmp::Ordering::Greater { + curr_dir = Direction::L; + curr = curr_node.left.load(Ordering::Acquire); + } else { + curr_dir = Direction::R; + curr = curr_node.right.load(Ordering::Acquire); + } + }) } else { - curr_dir = Direction::R; - curr = curr_node.right.load(Ordering::Acquire); + let curr_node = unsafe { &*curr_base }; + if curr_node.key.cmp(key) == cmp::Ordering::Greater { + curr_dir = Direction::L; + curr = curr_node.left.load(Ordering::Acquire); + } else { + curr_dir = Direction::R; + curr = curr_node.right.load(Ordering::Acquire); + } } } Ok(()) @@ -373,7 +443,7 @@ where let is_unlinked = record .successor_addr() .compare_exchange( - record.successor, + untagged(record.successor), tagged(target_sibling, Marks::new(flag, false).bits()), Ordering::AcqRel, Ordering::Acquire, @@ -446,9 +516,9 @@ where drop(Box::from_raw(new_internal)); Err(value) })?; - let leaf = record.leaf; + let leaf = untagged(record.leaf); - let (new_left, new_right) = match unsafe { &*untagged(leaf) }.key.cmp(key) { + let (new_left, new_right) = match unsafe { &*leaf }.key.cmp(key) { cmp::Ordering::Equal => { // Newly created nodes that failed to be inserted are free'd here. let value = unsafe { &mut *new_leaf }.value.take().unwrap(); @@ -510,8 +580,8 @@ where self.seek(key, &mut record)?; // candidates - let leaf = record.leaf; - let leaf_node = unsafe { &*untagged(record.leaf) }; + let leaf = untagged(record.leaf); + let leaf_node = unsafe { &*record.leaf }; if leaf_node.key.cmp(key) != cmp::Ordering::Equal { return Ok(None); @@ -551,7 +621,7 @@ where // cleanup phase loop { self.seek(key, &mut record)?; - if record.leaf != leaf { + if untagged(record.leaf) != leaf { // The edge to leaf flagged for deletion was removed by a helping thread return Ok(Some(value)); } diff --git a/test-scripts/sanitize-hp.sh b/test-scripts/sanitize-hp.sh index ebe40215..6f70fc75 100755 --- a/test-scripts/sanitize-hp.sh +++ b/test-scripts/sanitize-hp.sh @@ -7,4 +7,5 @@ hps="cargo run --bin hp --profile=release-simple --target x86_64-unknown-linux-g set -e for i in {1..5000}; do $hps -dh-list -i3 -t256 -r10 -g1 + $hps -dnm-tree -i3 -t256 -r10 -g1 done From cecc5ccad30e64d19d8f4011213c774556e19240 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Tue, 17 Sep 2024 10:30:54 +0000 Subject: [PATCH 14/84] Optimize hlist impl. --- src/ds_impl/hp/list.rs | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index d66ea9d4..4eaefd76 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -119,31 +119,45 @@ where .protect_raw(self.curr.with_tag(0).into_raw()); light_membarrier(); - // Validation depending on the state of self.curr. + // Validation depending on the state of `self.curr`. // // - If it is marked, validate on anchor. - // - If it is not marked, validate on curr. + // - If it is not marked, validate on prev. if self.curr.tag() != 0 { - // This means that prev -> curr was marked, hence persistent. + // Validate on anchor. + debug_assert!(!self.anchor.is_null()); debug_assert!(!self.anchor_next.is_null()); let an_new = unsafe { &self.anchor.deref().next }.load(Ordering::Acquire); + // Validate on anchor, which should still be the same and cleared. if an_new.tag() != 0 { - // Anchor dirty -> someone logically deleted it -> progress. return Err(()); } else if an_new != self.anchor_next { - // Anchor changed -> someone updated it. - // - Someone else cleared the logically deleted chain -> their find must return -> they must attempt CAS. + // Anchor is updated but clear, so can restart from anchor. - // TODO: optimization here, can restart from anchor, but need to setup some protection for prev & curr. - return Err(()); + self.prev = self.anchor; + self.curr = an_new; + + // Set prev HP as anchor HP, since prev should always be protected. + HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.anchor_h); } } else { - // TODO: optimize here. - if prev.load(Ordering::Acquire) != self.curr { + // Validate on prev. + + let curr_new = prev.load(Ordering::Acquire); + + if curr_new.tag() != 0 { + // If prev is marked, then restart from head. return Err(()); + } else if curr_new != self.curr { + // self.curr's tag was 0, so the above comparison ignores tags. + + // In contrary to what HP04 paper does, it's fine to retry protecting the new node + // without restarting from head as long as prev is not logically deleted. + self.curr = curr_new; + continue; } } From bc9da72c0e3fb5dd0c5d4d5a1cb1b37a0fe062ea Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Thu, 19 Sep 2024 05:16:37 +0000 Subject: [PATCH 15/84] Add some comments. --- src/ds_impl/hp/list.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 4eaefd76..0be85645 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -131,7 +131,6 @@ where debug_assert!(!self.anchor_next.is_null()); let an_new = unsafe { &self.anchor.deref().next }.load(Ordering::Acquire); - // Validate on anchor, which should still be the same and cleared. if an_new.tag() != 0 { return Err(()); } else if an_new != self.anchor_next { @@ -163,7 +162,6 @@ where let curr_node = unsafe { self.curr.deref() }; let next = curr_node.next.load(Ordering::Acquire); - // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. if next.tag() == 0 { if curr_node.key < *key { self.prev = self.curr; @@ -193,7 +191,7 @@ where Ok(found) } else { debug_assert_eq!(self.anchor_next.tag(), 0); - // CAS + // TODO: on CAS failure, if anchor is not tagged, we can restart from anchor. unsafe { &self.anchor.deref().next } .compare_exchange( self.anchor_next, From 8469da7065fcb1ad714b877d475e36b87667e715 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 20 Sep 2024 02:56:59 +0000 Subject: [PATCH 16/84] Fix a null reference bug in HP HList --- src/ds_impl/hp/list.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 0be85645..819dea83 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -138,12 +138,15 @@ where self.prev = self.anchor; self.curr = an_new; + self.anchor = Shared::null(); // Set prev HP as anchor HP, since prev should always be protected. HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.anchor_h); + continue; } } else { // Validate on prev. + debug_assert!(self.anchor.is_null()); let curr_new = prev.load(Ordering::Acquire); From c39c4fa489a86d5f91bd508825dfd6bcd9fdd9a3 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 20 Sep 2024 03:04:15 +0000 Subject: [PATCH 17/84] Temporarily enlarge small bag sizes --- src/bin/ebr.rs | 2 +- src/bin/hp.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/ebr.rs b/src/bin/ebr.rs index d67c8a7e..3acadb6e 100644 --- a/src/bin/ebr.rs +++ b/src/bin/ebr.rs @@ -102,7 +102,7 @@ fn bench_map + Send + Sync, N: Unsigned>( strategy: PrefillStrategy, ) -> Perf { match config.bag_size { - BagSize::Small => crossbeam_ebr::set_bag_capacity(64), + BagSize::Small => crossbeam_ebr::set_bag_capacity(512), BagSize::Large => crossbeam_ebr::set_bag_capacity(4096), } let map = &M::new(); diff --git a/src/bin/hp.rs b/src/bin/hp.rs index ce24724d..ebed6c36 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -95,7 +95,7 @@ fn bench_map + Send + Sync>( strategy: PrefillStrategy, ) -> Perf { match config.bag_size { - BagSize::Small => set_counts_between_flush(64), + BagSize::Small => set_counts_between_flush(512), BagSize::Large => set_counts_between_flush(4096), } let map = &M::new(); From 5f6d6939547410a3f3ac5ea9c02e7da991830114 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 20 Sep 2024 03:25:19 +0000 Subject: [PATCH 18/84] Refactor HP H(HS)List implementation --- src/ds_impl/hp/list.rs | 104 +++++++++-------------------------------- 1 file changed, 21 insertions(+), 83 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 819dea83..b2b4ac56 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -96,19 +96,16 @@ impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> where K: Ord, { - // Invariants: - // anchor, anchor_next: protected if they are not null. - // prev: always protected with prev_sh - // curr: not protected. - // curr: also has tag value when it is obtained from prev. + /// Optimistically traverses while maintaining `anchor` and `anchor_next`. + /// It is used for both Harris and Harris-Herlihy-Shavit traversals. #[inline] - fn find_harris(&mut self, key: &K) -> Result { - // Finding phase - // - cursor.curr: first unmarked node w/ key >= search key (4) - // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) - // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) - - let found = loop { + fn traverse_with_anchor(&mut self, key: &K) -> Result { + // Invariants: + // anchor, anchor_next: protected if they are not null. + // prev: always protected with prev_sh + // curr: not protected. + // curr: also has tag value when it is obtained from prev. + Ok(loop { if self.curr.is_null() { break false; } @@ -186,7 +183,17 @@ where self.curr = next; HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); } - }; + }) + } + + #[inline] + fn find_harris(&mut self, key: &K) -> Result { + // Finding phase + // - cursor.curr: first unmarked node w/ key >= search key (4) + // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) + // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) + + let found = self.traverse_with_anchor(key)?; if self.anchor.is_null() { self.prev = self.prev.with_tag(0); @@ -276,76 +283,7 @@ where } fn find_harris_herlihy_shavit(&mut self, key: &K) -> Result { - // Finding phase - // - cursor.curr: first unmarked node w/ key >= search key (4) - // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) - // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) - - let found = loop { - if self.curr.is_null() { - break false; - } - - let prev = unsafe { &self.prev.deref().next }; - self.handle - .curr_h - .protect_raw(self.curr.with_tag(0).into_raw()); - light_membarrier(); - - // Validation depending on the state of self.curr. - // - // - If it is marked, validate on anchor. - // - If it is not marked, validate on curr. - - if self.curr.tag() != 0 { - // This means that prev -> curr was marked, hence persistent. - debug_assert!(!self.anchor.is_null()); - debug_assert!(!self.anchor_next.is_null()); - let an_new = unsafe { &self.anchor.deref().next }.load(Ordering::Acquire); - // Validate on anchor, which should still be the same and cleared. - if an_new.tag() != 0 { - // Anchor dirty -> someone logically deleted it -> progress. - return Err(()); - } else if an_new != self.anchor_next { - // Anchor changed -> someone updated it. - // - Someone else cleared the logically deleted chain -> their find must return -> they must attempt CAS. - - // TODO: optimization here, can restart from anchor, but need to setup some protection for prev & curr. - return Err(()); - } - } else { - // TODO: optimize here. - if prev.load(Ordering::Acquire) != self.curr { - return Err(()); - } - } - - let curr_node = unsafe { self.curr.deref() }; - let next = curr_node.next.load(Ordering::Acquire); - // TODO: REALLY THINK HARD ABOUT THIS SHIELD STUFF. - if next.tag() == 0 { - if curr_node.key < *key { - self.prev = self.curr; - self.curr = next; - self.anchor = Shared::null(); - HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); - } else { - break curr_node.key == *key; - } - } else { - if self.anchor.is_null() { - self.anchor = self.prev; - self.anchor_next = self.curr; - HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); - } else if self.anchor_next == self.prev { - HazardPointer::swap(&mut self.handle.anchor_next_h, &mut self.handle.prev_h); - } - self.prev = self.curr; - self.curr = next; - HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); - } - }; - + let found = self.traverse_with_anchor(key)?; // Return only the found `curr` node. // Others are not necessary because we are not going to do insertion or deletion // with this Harris-Herlihy-Shavit traversal. From 8f9e176c8de089cda5e5c16bcfea63248300ac33 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 20 Sep 2024 03:27:41 +0000 Subject: [PATCH 19/84] Temporarily enlarge small bag sizes for HP++ --- src/bin/hp-pp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/hp-pp.rs b/src/bin/hp-pp.rs index c526cc12..55aadb4d 100644 --- a/src/bin/hp-pp.rs +++ b/src/bin/hp-pp.rs @@ -96,7 +96,7 @@ fn bench_map + Send + Sync>( strategy: PrefillStrategy, ) -> Perf { match config.bag_size { - BagSize::Small => set_counts_between_flush(64), + BagSize::Small => set_counts_between_flush(512), BagSize::Large => set_counts_between_flush(4096), } let map = &M::new(); From 317d8535f2ed96a4f9f387221396a101a95fb23c Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 20 Sep 2024 04:01:20 +0000 Subject: [PATCH 20/84] Refactor HP NMTree implementation --- src/ds_impl/hp/natarajan_mittal_tree.rs | 29 +++++++------------------ 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/src/ds_impl/hp/natarajan_mittal_tree.rs b/src/ds_impl/hp/natarajan_mittal_tree.rs index 976dddf3..30882e13 100644 --- a/src/ds_impl/hp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp/natarajan_mittal_tree.rs @@ -320,8 +320,8 @@ where // (parent), ancestor -> O (ancestor) -> O // / \ / \ // (leaf), successor -> O O => (parent), successor -> O O - // / \ / \ - // O O (leaf) -> O O + // / \ / \ + // O O (leaf) -> O O record.parent = record.leaf; HazardPointer::swap(&mut record.handle.ancestor_h, &mut record.handle.parent_h); HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); @@ -392,26 +392,13 @@ where // update other variables prev_tag = Marks::from_bits_truncate(tag(curr)).tag(); - if Marks::from_bits_truncate(tag(curr)).flag() { - std::hint::black_box({ - let curr_node = unsafe { &*curr_base }; - if curr_node.key.cmp(key) == cmp::Ordering::Greater { - curr_dir = Direction::L; - curr = curr_node.left.load(Ordering::Acquire); - } else { - curr_dir = Direction::R; - curr = curr_node.right.load(Ordering::Acquire); - } - }) + let curr_node = unsafe { &*curr_base }; + if curr_node.key.cmp(key) == cmp::Ordering::Greater { + curr_dir = Direction::L; + curr = curr_node.left.load(Ordering::Acquire); } else { - let curr_node = unsafe { &*curr_base }; - if curr_node.key.cmp(key) == cmp::Ordering::Greater { - curr_dir = Direction::L; - curr = curr_node.left.load(Ordering::Acquire); - } else { - curr_dir = Direction::R; - curr = curr_node.right.load(Ordering::Acquire); - } + curr_dir = Direction::R; + curr = curr_node.right.load(Ordering::Acquire); } } Ok(()) From 7056a1ccaee0c65ecb36235a83e2b5b9d5fa6aa4 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 23 Sep 2024 07:49:01 +0000 Subject: [PATCH 21/84] Fix mistakes in HP(++) implementations that `handle.thread` and hazard pointers are independent --- src/ds_impl/hp/bonsai_tree.rs | 11 ++-- src/ds_impl/hp/double_link.rs | 9 +-- src/ds_impl/hp/ellen_tree.rs | 23 +++---- src/ds_impl/hp/list.rs | 13 ++-- src/ds_impl/hp/natarajan_mittal_tree.rs | 13 ++-- src/ds_impl/hp/skip_list.rs | 14 ++--- src/ds_impl/hp_pp/bonsai_tree.rs | 11 ++-- src/ds_impl/hp_pp/ellen_tree.rs | 23 +++---- src/ds_impl/hp_pp/list.rs | 73 ++++++++++++++-------- src/ds_impl/hp_pp/natarajan_mittal_tree.rs | 56 +++++++++-------- src/ds_impl/hp_pp/skip_list.rs | 11 ++-- 11 files changed, 144 insertions(+), 113 deletions(-) diff --git a/src/ds_impl/hp/bonsai_tree.rs b/src/ds_impl/hp/bonsai_tree.rs index b351a417..7abd11cb 100644 --- a/src/ds_impl/hp/bonsai_tree.rs +++ b/src/ds_impl/hp/bonsai_tree.rs @@ -113,20 +113,21 @@ pub struct State<'domain, K, V> { retired_nodes: Vec<*mut Node>, /// Nodes newly constructed by the op. Should be destroyed if CAS fails. (`destroy`) new_nodes: Vec<*mut Node>, - thread: Thread<'domain>, + thread: Box>, } impl Default for State<'static, K, V> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { root_link: ptr::null(), curr_root: ptr::null_mut(), - root_h: Default::default(), - succ_h: Default::default(), - removed_h: Default::default(), + root_h: HazardPointer::new(&mut thread), + succ_h: HazardPointer::new(&mut thread), + removed_h: HazardPointer::new(&mut thread), retired_nodes: vec![], new_nodes: vec![], - thread: Thread::new(&DEFAULT_DOMAIN), + thread, } } } diff --git a/src/ds_impl/hp/double_link.rs b/src/ds_impl/hp/double_link.rs index dba1ef35..0dd04958 100644 --- a/src/ds_impl/hp/double_link.rs +++ b/src/ds_impl/hp/double_link.rs @@ -41,15 +41,16 @@ pub struct DoubleLink { pub struct Handle<'domain> { pri: HazardPointer<'domain>, sub: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - pri: HazardPointer::default(), - sub: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + pri: HazardPointer::new(&mut thread), + sub: HazardPointer::new(&mut thread), + thread, } } } diff --git a/src/ds_impl/hp/ellen_tree.rs b/src/ds_impl/hp/ellen_tree.rs index fd8ce7ab..72eacb0d 100644 --- a/src/ds_impl/hp/ellen_tree.rs +++ b/src/ds_impl/hp/ellen_tree.rs @@ -230,22 +230,23 @@ pub struct Handle<'domain> { new_internal_h: HazardPointer<'domain>, // Protect an owner of update which is currently being helped. help_src_h: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - gp_h: HazardPointer::default(), - p_h: HazardPointer::default(), - l_h: HazardPointer::default(), - l_other_h: HazardPointer::default(), - pupdate_h: HazardPointer::default(), - gpupdate_h: HazardPointer::default(), - aux_update_h: HazardPointer::default(), - new_internal_h: HazardPointer::default(), - help_src_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + gp_h: HazardPointer::new(&mut thread), + p_h: HazardPointer::new(&mut thread), + l_h: HazardPointer::new(&mut thread), + l_other_h: HazardPointer::new(&mut thread), + pupdate_h: HazardPointer::new(&mut thread), + gpupdate_h: HazardPointer::new(&mut thread), + aux_update_h: HazardPointer::new(&mut thread), + new_internal_h: HazardPointer::new(&mut thread), + help_src_h: HazardPointer::new(&mut thread), + thread, } } } diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index b2b4ac56..9cba3dc7 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -46,17 +46,18 @@ pub struct Handle<'domain> { // `anchor_h` and `anchor_next_h` are used for `find_harris` anchor_h: HazardPointer<'domain>, anchor_next_h: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - prev_h: HazardPointer::default(), - curr_h: HazardPointer::default(), - anchor_h: HazardPointer::default(), - anchor_next_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + prev_h: HazardPointer::new(&mut thread), + curr_h: HazardPointer::new(&mut thread), + anchor_h: HazardPointer::new(&mut thread), + anchor_next_h: HazardPointer::new(&mut thread), + thread, } } } diff --git a/src/ds_impl/hp/natarajan_mittal_tree.rs b/src/ds_impl/hp/natarajan_mittal_tree.rs index 30882e13..714eedd9 100644 --- a/src/ds_impl/hp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp/natarajan_mittal_tree.rs @@ -139,17 +139,18 @@ pub struct Handle<'domain> { successor_h: HazardPointer<'domain>, parent_h: HazardPointer<'domain>, leaf_h: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - ancestor_h: HazardPointer::default(), - successor_h: HazardPointer::default(), - parent_h: HazardPointer::default(), - leaf_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + ancestor_h: HazardPointer::new(&mut thread), + successor_h: HazardPointer::new(&mut thread), + parent_h: HazardPointer::new(&mut thread), + leaf_h: HazardPointer::new(&mut thread), + thread, } } } diff --git a/src/ds_impl/hp/skip_list.rs b/src/ds_impl/hp/skip_list.rs index 0f43fe93..29f716ec 100644 --- a/src/ds_impl/hp/skip_list.rs +++ b/src/ds_impl/hp/skip_list.rs @@ -2,8 +2,7 @@ use std::mem::transmute; use std::ptr; use std::sync::atomic::{fence, AtomicPtr, AtomicUsize, Ordering}; -use hp_pp::{light_membarrier, tagged, Thread}; -use hp_pp::{tag, untagged, HazardPointer, DEFAULT_DOMAIN}; +use hp_pp::{light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN}; use super::concurrent_map::ConcurrentMap; @@ -89,16 +88,17 @@ pub struct Handle<'g> { preds_h: [HazardPointer<'g>; MAX_HEIGHT], succs_h: [HazardPointer<'g>; MAX_HEIGHT], removed_h: HazardPointer<'g>, - thread: Thread<'g>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - preds_h: Default::default(), - succs_h: Default::default(), - removed_h: Default::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + preds_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + succs_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + removed_h: HazardPointer::new(&mut thread), + thread, } } } diff --git a/src/ds_impl/hp_pp/bonsai_tree.rs b/src/ds_impl/hp_pp/bonsai_tree.rs index 252a1784..220bd7c5 100644 --- a/src/ds_impl/hp_pp/bonsai_tree.rs +++ b/src/ds_impl/hp_pp/bonsai_tree.rs @@ -119,20 +119,21 @@ pub struct State<'domain, K, V> { retired_nodes: Vec<*mut Node>, /// Nodes newly constructed by the op. Should be destroyed if CAS fails. (`destroy`) new_nodes: Vec<*mut Node>, - thread: Thread<'domain>, + thread: Box>, } impl Default for State<'static, K, V> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { root_link: ptr::null(), curr_root: ptr::null_mut(), - root_h: Default::default(), - succ_h: Default::default(), - removed_h: Default::default(), + root_h: HazardPointer::new(&mut thread), + succ_h: HazardPointer::new(&mut thread), + removed_h: HazardPointer::new(&mut thread), retired_nodes: vec![], new_nodes: vec![], - thread: Thread::new(&DEFAULT_DOMAIN), + thread, } } } diff --git a/src/ds_impl/hp_pp/ellen_tree.rs b/src/ds_impl/hp_pp/ellen_tree.rs index cdfd5306..df622fe0 100644 --- a/src/ds_impl/hp_pp/ellen_tree.rs +++ b/src/ds_impl/hp_pp/ellen_tree.rs @@ -240,22 +240,23 @@ pub struct Handle<'domain> { new_internal_h: HazardPointer<'domain>, // Protect an owner of update which is currently being helped. help_src_h: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - gp_h: HazardPointer::default(), - p_h: HazardPointer::default(), - l_h: HazardPointer::default(), - l_other_h: HazardPointer::default(), - pupdate_h: HazardPointer::default(), - gpupdate_h: HazardPointer::default(), - aux_update_h: HazardPointer::default(), - new_internal_h: HazardPointer::default(), - help_src_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + gp_h: HazardPointer::new(&mut thread), + p_h: HazardPointer::new(&mut thread), + l_h: HazardPointer::new(&mut thread), + l_other_h: HazardPointer::new(&mut thread), + pupdate_h: HazardPointer::new(&mut thread), + gpupdate_h: HazardPointer::new(&mut thread), + aux_update_h: HazardPointer::new(&mut thread), + new_internal_h: HazardPointer::new(&mut thread), + help_src_h: HazardPointer::new(&mut thread), + thread, } } } diff --git a/src/ds_impl/hp_pp/list.rs b/src/ds_impl/hp_pp/list.rs index 8c8c21c5..30b329d6 100644 --- a/src/ds_impl/hp_pp/list.rs +++ b/src/ds_impl/hp_pp/list.rs @@ -4,7 +4,9 @@ use std::cmp::Ordering::{Equal, Greater, Less}; use std::sync::atomic::{AtomicPtr, Ordering}; use std::{mem, ptr, slice}; -use hp_pp::{decompose_ptr, light_membarrier, tag, tagged, try_unlink, untagged, HazardPointer}; +use hp_pp::{ + decompose_ptr, light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, +}; // `#[repr(C)]` is used to ensure the first field // is also the first data in the memory alignment. @@ -46,15 +48,18 @@ pub struct Handle<'domain> { // `anchor_h` and `anchor_next_h` are used for `find_harris` anchor_h: HazardPointer<'domain>, anchor_next_h: HazardPointer<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - prev_h: HazardPointer::default(), - curr_h: HazardPointer::default(), - anchor_h: HazardPointer::default(), - anchor_next_h: HazardPointer::default(), + prev_h: HazardPointer::new(&mut thread), + curr_h: HazardPointer::new(&mut thread), + anchor_h: HazardPointer::new(&mut thread), + anchor_next_h: HazardPointer::new(&mut thread), + thread, } } } @@ -96,26 +101,28 @@ impl hp_pp::Invalidate for Node { } } -struct HarrisUnlink<'c, 'domain, 'hp, K, V> { - cursor: &'c Cursor<'domain, 'hp, K, V>, +struct HarrisUnlink { + anchor: *mut Node, + anchor_next: *mut Node, + curr: *mut Node, } -impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for HarrisUnlink<'r, 'domain, 'hp, K, V> { +impl hp_pp::Unlink> for HarrisUnlink { fn do_unlink(&self) -> Result>, ()> { - if unsafe { &*self.cursor.anchor } + if unsafe { &*self.anchor } .next .compare_exchange( - self.cursor.anchor_next, - self.cursor.curr, + self.anchor_next, + self.curr, Ordering::AcqRel, Ordering::Relaxed, ) .is_ok() { let mut collected = Vec::with_capacity(16); - let mut node = self.cursor.anchor_next; + let mut node = self.anchor_next; loop { - if untagged(node) == self.cursor.curr { + if untagged(node) == self.curr { break; } let node_ref = unsafe { node.as_ref() }.unwrap(); @@ -130,24 +137,25 @@ impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for HarrisUnlink<'r, 'dom } } -struct Unlink<'c, 'domain, 'hp, K, V> { - cursor: &'c Cursor<'domain, 'hp, K, V>, +struct Unlink { + prev: *mut Node, + curr: *mut Node, next_base: *mut Node, } -impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for Unlink<'r, 'domain, 'hp, K, V> { +impl hp_pp::Unlink> for Unlink { fn do_unlink(&self) -> Result>, ()> { - let prev = unsafe { &(*self.cursor.prev).next }; + let prev = unsafe { &(*self.prev).next }; if prev .compare_exchange( - self.cursor.curr, + self.curr, self.next_base, Ordering::Release, Ordering::Relaxed, ) .is_ok() { - Ok(vec![self.cursor.curr]) + Ok(vec![self.curr]) } else { Err(()) } @@ -212,8 +220,16 @@ where if self.anchor.is_null() { Ok(found) } else { - let unlink = HarrisUnlink { cursor: &self }; - if unsafe { try_unlink(unlink, slice::from_ref(&self.curr)) } { + let unlink = HarrisUnlink { + anchor: self.anchor, + anchor_next: self.anchor_next, + curr: self.curr, + }; + if unsafe { + self.handle + .thread + .try_unlink(unlink, slice::from_ref(&self.curr)) + } { self.prev = self.anchor; Ok(found) } else { @@ -262,10 +278,11 @@ where } else { let links = slice::from_ref(&next_base); let unlink = Unlink { - cursor: self, + prev: self.prev, + curr: self.curr, next_base, }; - if unsafe { !try_unlink(unlink, links) } { + if unsafe { !self.handle.thread.try_unlink(unlink, links) } { return Err(()); } } @@ -421,10 +438,11 @@ where let links = slice::from_ref(&next); let unlink = Unlink { - cursor: &cursor, + prev: cursor.prev, + curr: cursor.curr, next_base: next, }; - unsafe { try_unlink(unlink, links) }; + unsafe { handle.thread.try_unlink(unlink, links) }; return Ok(Some(&curr_node.value)); } @@ -476,10 +494,11 @@ where let links = slice::from_ref(&next); let unlink = Unlink { - cursor: &cursor, + prev: cursor.prev, + curr: cursor.curr, next_base: next, }; - unsafe { try_unlink(unlink, links) }; + unsafe { handle.thread.try_unlink(unlink, links) }; return Some((&curr_node.key, &curr_node.value)); } diff --git a/src/ds_impl/hp_pp/natarajan_mittal_tree.rs b/src/ds_impl/hp_pp/natarajan_mittal_tree.rs index 8ed1faa5..556425c5 100644 --- a/src/ds_impl/hp_pp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp_pp/natarajan_mittal_tree.rs @@ -1,5 +1,6 @@ -use hp_pp::{light_membarrier, tag, tagged, untagged, HazardPointer}; -use hp_pp::{try_unlink, ProtectError}; +use hp_pp::{ + light_membarrier, tag, tagged, untagged, HazardPointer, ProtectError, Thread, DEFAULT_DOMAIN, +}; use crate::ds_impl::hp::concurrent_map::ConcurrentMap; use std::mem; @@ -144,15 +145,18 @@ pub struct Handle<'domain> { successor_h: HazardPointer<'domain>, parent_h: HazardPointer<'domain>, leaf_h: HazardPointer<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - ancestor_h: HazardPointer::default(), - successor_h: HazardPointer::default(), - parent_h: HazardPointer::default(), - leaf_h: HazardPointer::default(), + ancestor_h: HazardPointer::new(&mut thread), + successor_h: HazardPointer::new(&mut thread), + parent_h: HazardPointer::new(&mut thread), + leaf_h: HazardPointer::new(&mut thread), + thread, } } } @@ -276,31 +280,25 @@ impl hp_pp::Invalidate for Node { } } -struct Unlink<'r, 'domain, 'hp, K, V> { - record: &'r SeekRecord<'domain, 'hp, K, V>, +struct Unlink { + successor_addr: *mut AtomicPtr>, + successor: *mut Node, target_sibling: *mut Node, flag: bool, } -impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for Unlink<'r, 'domain, 'hp, K, V> { +impl hp_pp::Unlink> for Unlink { fn do_unlink(&self) -> Result>, ()> { let link = tagged( self.target_sibling, Marks::new(false, self.flag, false).bits(), ); - if self - .record - .successor_addr() - .compare_exchange( - self.record.successor, - link, - Ordering::AcqRel, - Ordering::Acquire, - ) + if unsafe { &*self.successor_addr } + .compare_exchange(self.successor, link, Ordering::AcqRel, Ordering::Acquire) .is_ok() { // destroy the subtree of successor except target_sibling - let mut stack = vec![self.record.successor]; + let mut stack = vec![self.successor]; let mut collected = Vec::with_capacity(32); while let Some(node) = stack.pop() { @@ -518,7 +516,7 @@ where /// Physically removes node. /// /// Returns true if it successfully unlinks the flagged node in `record`. - fn cleanup(&self, record: &SeekRecord) -> bool { + fn cleanup(&self, record: &mut SeekRecord) -> bool { // Identify the node(subtree) that will replace `successor`. let leaf_marked = record.leaf_addr().load(Ordering::Acquire); let leaf_flag = Marks::from_bits_truncate(tag(leaf_marked)).flag(); @@ -540,13 +538,19 @@ where let flag = Marks::from_bits_truncate(tag(target_sibling)).flag(); let link = tagged(target_sibling, Marks::new(false, flag, false).bits()); let unlink = Unlink { - record, + successor_addr: record.successor_addr() as *const _ as *mut _, + successor: record.successor, target_sibling, flag, }; let link = untagged(link); - unsafe { try_unlink(unlink, slice::from_ref(&link)) } + unsafe { + record + .handle + .thread + .try_unlink(unlink, slice::from_ref(&link)) + } } fn get_inner<'domain, 'hp>( @@ -629,7 +633,7 @@ where // Insertion failed. Help the conflicting remove operation if needed. // NOTE: The paper version checks if any of the mark is set, which is redundant. if untagged(current) == leaf { - self.cleanup(&record); + self.cleanup(record); } } } @@ -683,7 +687,7 @@ where ) { Ok(_) => { // Finalize the node to be removed - if self.cleanup(&record) { + if self.cleanup(&mut record) { return Ok(Some(value)); } // In-place cleanup failed. Enter the cleanup phase. @@ -695,7 +699,7 @@ where // case 2. Another thread flagged/tagged the edge to leaf: help and restart // NOTE: The paper version checks if any of the mark is set, which is redundant. if leaf == tagged(current, Marks::empty().bits()) { - self.cleanup(&record); + self.cleanup(&mut record); } } } @@ -712,7 +716,7 @@ where } // leaf is still present in the tree. - if self.cleanup(&record) { + if self.cleanup(&mut record) { return Ok(Some(value)); } } diff --git a/src/ds_impl/hp_pp/skip_list.rs b/src/ds_impl/hp_pp/skip_list.rs index cc71abe3..bbe54215 100644 --- a/src/ds_impl/hp_pp/skip_list.rs +++ b/src/ds_impl/hp_pp/skip_list.rs @@ -100,16 +100,17 @@ pub struct Handle<'g> { preds_h: [HazardPointer<'g>; MAX_HEIGHT], succs_h: [HazardPointer<'g>; MAX_HEIGHT], removed_h: HazardPointer<'g>, - thread: Thread<'g>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - preds_h: Default::default(), - succs_h: Default::default(), - removed_h: Default::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + preds_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + succs_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + removed_h: HazardPointer::new(&mut thread), + thread, } } } From 78e7cfbb7c867958fc7bf09a025115fbd00a64dd Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 23 Sep 2024 08:53:18 +0000 Subject: [PATCH 22/84] Fix a linearizability bug in HP++ SkipList --- src/ds_impl/hp_pp/skip_list.rs | 92 +++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/src/ds_impl/hp_pp/skip_list.rs b/src/ds_impl/hp_pp/skip_list.rs index bbe54215..00c61eec 100644 --- a/src/ds_impl/hp_pp/skip_list.rs +++ b/src/ds_impl/hp_pp/skip_list.rs @@ -165,9 +165,23 @@ where &self, key: &K, handle: &'hp mut Handle<'domain>, - ) -> Option> { + ) -> Option<*mut Node> { 'search: loop { - let mut cursor = Cursor::new(&self.head); + // This optimistic traversal doesn't have to use all shields in the `handle`. + let (anchor_h, anchor_next_h) = unsafe { + let splited = handle.preds_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; + let (pred_h, curr_h) = unsafe { + let splited = handle.succs_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; let mut level = MAX_HEIGHT; while level >= 1 && self.head[level - 1].load(Ordering::Relaxed).is_null() { @@ -175,55 +189,64 @@ where } let mut pred = &self.head as *const _ as *mut Node; + let mut curr = ptr::null_mut(); + let mut anchor = pred; + let mut anchor_next = ptr::null_mut(); + while level >= 1 { level -= 1; // untagged - let mut curr = untagged(unsafe { &*pred }.next[level].load(Ordering::Acquire)); + curr = untagged(unsafe { &*anchor }.next[level].load(Ordering::Acquire)); loop { if curr.is_null() { break; } - - let pred_ref = unsafe { &*pred }; - - // Inlined version of hp++ protection, without duplicate load - handle.succs_h[level].protect_raw(curr); - light_membarrier(); - let (curr_new_base, curr_new_tag) = - decompose_ptr(pred_ref.next[level].load(Ordering::Acquire)); - if curr_new_tag == 3 { - // Invalidated. Restart from head. + if curr_h + .try_protect_pp( + curr, + unsafe { &*pred }, + unsafe { &(*pred).next[level] }, + &|node| node.next[level].load(Ordering::Acquire) as usize & 3 == 3, + ) + .is_err() + { continue 'search; - } else if curr_new_base != curr { - // If link changed but not invalidated, retry protecting the new node. - curr = curr_new_base; - continue; } let curr_node = unsafe { &*curr }; - - match curr_node.key.cmp(key) { - std::cmp::Ordering::Less => { + let (next_base, next_tag) = + decompose_ptr(curr_node.next[level].load(Ordering::Acquire)); + if next_tag == 0 { + if curr_node.key < *key { pred = curr; - curr = untagged(curr_node.next[level].load(Ordering::Acquire)); - HazardPointer::swap( - &mut handle.preds_h[level], - &mut handle.succs_h[level], - ); + curr = next_base; + anchor = pred; + HazardPointer::swap(curr_h, pred_h); + } else { + break; } - std::cmp::Ordering::Equal => { - if curr_new_tag == 0 { - cursor.found = Some(curr); - return Some(cursor); - } else { - return None; - } + } else { + if anchor == pred { + anchor_next = curr; + HazardPointer::swap(anchor_h, pred_h); + } else if anchor_next == pred { + HazardPointer::swap(anchor_next_h, pred_h); } - std::cmp::Ordering::Greater => break, + pred = curr; + curr = next_base; + HazardPointer::swap(pred_h, curr_h); } } } + + if let Some(curr_node) = unsafe { curr.as_ref() } { + if curr_node.key == *key + && (curr_node.next[0].load(Ordering::Acquire) as usize & 1) == 0 + { + return Some(curr); + } + } return None; } } @@ -504,8 +527,7 @@ where #[inline(always)] fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { - let cursor = self.find_optimistic(key, handle)?; - let node = unsafe { &*cursor.found? }; + let node = unsafe { &*self.find_optimistic(key, handle)? }; if node.key.eq(&key) { Some(&node.value) } else { From 12a7b47eecde9b1c1106f8208194bf40fa1b4a69 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 23 Sep 2024 14:21:47 +0000 Subject: [PATCH 23/84] Implement HP SkipList with optimistic traversals --- src/ds_impl/hp/list.rs | 4 +- src/ds_impl/hp/skip_list.rs | 127 ++++++++++++++++++++++++++++++++++-- 2 files changed, 125 insertions(+), 6 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 9cba3dc7..33f1ee9b 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -111,7 +111,7 @@ where break false; } - let prev = unsafe { &self.prev.deref().next }; + let prev_next = unsafe { &self.prev.deref().next }; self.handle .curr_h .protect_raw(self.curr.with_tag(0).into_raw()); @@ -146,7 +146,7 @@ where // Validate on prev. debug_assert!(self.anchor.is_null()); - let curr_new = prev.load(Ordering::Acquire); + let curr_new = prev_next.load(Ordering::Acquire); if curr_new.tag() != 0 { // If prev is marked, then restart from head. diff --git a/src/ds_impl/hp/skip_list.rs b/src/ds_impl/hp/skip_list.rs index 29f716ec..babf32fb 100644 --- a/src/ds_impl/hp/skip_list.rs +++ b/src/ds_impl/hp/skip_list.rs @@ -159,6 +159,126 @@ where } } + fn find_optimistic(&self, key: &K, handle: &mut Handle<'_>) -> Option<*mut Node> { + 'search: loop { + // This optimistic traversal doesn't have to use all shields in the `handle`. + let (anchor_h, anchor_next_h) = unsafe { + let splited = handle.preds_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; + let (pred_h, curr_h) = unsafe { + let splited = handle.succs_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; + + let mut level = MAX_HEIGHT; + while level >= 1 && self.head[level - 1].load(Ordering::Relaxed).is_null() { + level -= 1; + } + + let mut pred = &self.head as *const _ as *mut Node; + let mut curr = ptr::null_mut::>(); + let mut anchor = pred; + let mut anchor_next = ptr::null_mut::>(); + + while level >= 1 { + level -= 1; + curr = unsafe { &*untagged(anchor) }.next[level].load(Ordering::Acquire); + + loop { + if untagged(curr).is_null() { + break; + } + + curr_h.protect_raw(untagged(curr)); + light_membarrier(); + + // Validation depending on the state of `curr`. + // + // - If it is marked, validate on anchor. + // - If it is not marked, validate on pred. + + if tag(curr) != 0 { + // Validate on anchor. + debug_assert_ne!(untagged(anchor), untagged(pred)); + + let an_new = + unsafe { &*untagged(anchor) }.next[level].load(Ordering::Acquire); + + if tag(an_new) != 0 { + continue 'search; + } else if an_new != anchor_next { + // Anchor is updated but clear, so can restart from anchor. + + pred = anchor; + curr = an_new; + + // Set prev HP as anchor HP, since prev should always be protected. + HazardPointer::swap(pred_h, anchor_h); + continue; + } + } else { + // Validate on prev. + debug_assert_eq!(untagged(anchor), untagged(pred)); + + let curr_new = + unsafe { &*untagged(pred) }.next[level].load(Ordering::Acquire); + + if tag(curr_new) != 0 { + // If prev is marked, then restart from head. + continue 'search; + } else if curr_new != curr { + // curr's tag was 0, so the above comparison ignores tags. + + // In contrary to what HP04 paper does, it's fine to retry protecting the new node + // without restarting from head as long as prev is not logically deleted. + curr = curr_new; + continue; + } + } + + let curr_node = unsafe { &*untagged(curr) }; + let next = curr_node.next[level].load(Ordering::Acquire); + if tag(next) == 0 { + if curr_node.key < *key { + pred = curr; + curr = next; + anchor = pred; + HazardPointer::swap(curr_h, pred_h); + } else { + break; + } + } else { + if untagged(anchor) == untagged(pred) { + anchor_next = curr; + HazardPointer::swap(anchor_h, pred_h); + } else if untagged(anchor_next) == untagged(pred) { + HazardPointer::swap(anchor_next_h, pred_h); + } + pred = curr; + curr = next; + HazardPointer::swap(pred_h, curr_h); + } + } + } + + if let Some(curr_node) = unsafe { untagged(curr).as_ref() } { + if curr_node.key == *key + && (curr_node.next[0].load(Ordering::Acquire) as usize & 1) == 0 + { + return Some(untagged(curr)); + } + } + return None; + } + } + fn find(&self, key: &K, handle: &mut Handle<'_>) -> Cursor { 'search: loop { let mut cursor = Cursor::new(&self.head); @@ -387,10 +507,9 @@ where #[inline(always)] fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - let cursor = self.find(key, handle); - let node = unsafe { cursor.found?.as_ref()? }; - if node.key.eq(key) { - Some(unsafe { transmute(&node.value) }) + let node = unsafe { &*self.find_optimistic(key, handle)? }; + if node.key.eq(&key) { + Some(&node.value) } else { None } From f804f05f706904516ef4ebf4dd3aff763a5cd101 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 23 Sep 2024 16:07:58 +0000 Subject: [PATCH 24/84] Add bag size configurations for major SMRs --- Cargo.lock | 13 ++++++------- smrs/hp-brcu/src/deferred.rs | 28 +++++++++++++++++++++------- smrs/hp-brcu/src/handle.rs | 4 ++-- smrs/hp-brcu/src/lib.rs | 1 + smrs/vbr/Cargo.toml | 1 - smrs/vbr/src/lib.rs | 36 +++++++++++++++++++++++++++--------- src/bin/hp-brcu.rs | 7 ++++--- src/bin/hp-rcu.rs | 7 ++++--- src/bin/nbr.rs | 2 +- src/bin/pebr.rs | 5 +++-- src/bin/vbr.rs | 5 +++-- 11 files changed, 72 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7eac0b38..7877a97c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -259,14 +259,14 @@ dependencies = [ [[package]] name = "crossbeam-pebr-epoch" version = "0.7.1" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#8c4b969c866a7d1b1c698f6b374e781615efb8c8" dependencies = [ "arrayvec 0.4.12", "bitflags 1.3.2", "cfg-if 0.1.10", "crossbeam-pebr-utils", "lazy_static", - "membarrier 0.2.3 (git+https://github.com/kaist-cp/crossbeam?branch=pebr)", + "membarrier 0.2.3 (git+https://github.com/jeehoonkang/membarrier-rs.git?rev=99254da47b6dab382c9860a024cab3ab9bac6504)", "memoffset", "murmur3", "scopeguard", @@ -276,7 +276,7 @@ dependencies = [ [[package]] name = "crossbeam-pebr-utils" version = "0.6.5" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#8c4b969c866a7d1b1c698f6b374e781615efb8c8" dependencies = [ "cfg-if 0.1.10", "lazy_static", @@ -454,7 +454,7 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "membarrier" version = "0.2.3" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/jeehoonkang/membarrier-rs.git?branch=smr-benchmark#bc3cf47144212655ea3daf38b10bf646ba94fbe9" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -465,7 +465,7 @@ dependencies = [ [[package]] name = "membarrier" version = "0.2.3" -source = "git+https://github.com/jeehoonkang/membarrier-rs.git?branch=smr-benchmark#bc3cf47144212655ea3daf38b10bf646ba94fbe9" +source = "git+https://github.com/jeehoonkang/membarrier-rs.git?rev=99254da47b6dab382c9860a024cab3ab9bac6504#99254da47b6dab382c9860a024cab3ab9bac6504" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -497,7 +497,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "murmur3" version = "0.4.1" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/kaist-cp/murmur3.git?branch=upgrade#0843ec5d49329d24e7c646d681a55701dbf7cf52" dependencies = [ "byteorder", "static_assertions 0.3.4", @@ -909,7 +909,6 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" name = "vbr" version = "0.1.0" dependencies = [ - "arrayvec 0.7.4", "atomic", "crossbeam-utils 0.8.20", "portable-atomic", diff --git a/smrs/hp-brcu/src/deferred.rs b/smrs/hp-brcu/src/deferred.rs index c42fe70b..23dfcb11 100644 --- a/smrs/hp-brcu/src/deferred.rs +++ b/smrs/hp-brcu/src/deferred.rs @@ -2,11 +2,25 @@ use std::mem::forget; use crate::epoch::Epoch; -/// Maximum number of objects a bag can contain. -#[cfg(not(sanitize = "address"))] -pub(crate) const MAX_OBJECTS: usize = 128; -#[cfg(sanitize = "address")] -pub(crate) const MAX_OBJECTS: usize = 4; +#[cfg(not(feature = "sanitize"))] +static mut MAX_OBJECTS: usize = 64; +#[cfg(feature = "sanitize")] +static mut MAX_OBJECTS: usize = 4; + +/// Sets the capacity of thread-local garbage bag. +/// +/// This value applies to all threads. +#[inline] +pub fn set_bag_capacity(cap: usize) { + assert!(cap > 1, "capacity must be greater than 1."); + unsafe { MAX_OBJECTS = cap }; +} + +/// Returns the current capacity of thread-local garbage bag. +#[inline] +pub fn bag_capacity() -> usize { + unsafe { MAX_OBJECTS } +} /// A deferred task consisted of data and a callable function. /// @@ -67,7 +81,7 @@ impl Bag { #[inline] pub fn new() -> Self { Self { - defs: Vec::with_capacity(MAX_OBJECTS), + defs: Vec::with_capacity(bag_capacity()), } } @@ -77,7 +91,7 @@ impl Bag { /// full. #[inline] pub fn try_push(&mut self, def: Deferred) -> Result<(), Deferred> { - if self.len() == MAX_OBJECTS { + if self.len() == bag_capacity() { return Err(def); } self.defs.push(def); diff --git a/smrs/hp-brcu/src/handle.rs b/smrs/hp-brcu/src/handle.rs index 70b8ce58..2b67296b 100644 --- a/smrs/hp-brcu/src/handle.rs +++ b/smrs/hp-brcu/src/handle.rs @@ -1,7 +1,7 @@ use std::mem::{transmute, ManuallyDrop}; use std::sync::atomic::{compiler_fence, fence, AtomicUsize, Ordering}; -use crate::deferred::{Deferred, MAX_OBJECTS}; +use crate::deferred::{bag_capacity, Deferred}; use crate::internal::{free, Local}; use crate::pointers::Shared; use crate::rollback::Rollbacker; @@ -259,7 +259,7 @@ impl Thread { pub(crate) unsafe fn do_reclamation(&mut self, mut deferred: Vec) -> Vec { let deferred_len = deferred.len(); - if deferred_len < MAX_OBJECTS * 2 { + if deferred_len < bag_capacity() * 2 { return deferred; } diff --git a/smrs/hp-brcu/src/lib.rs b/smrs/hp-brcu/src/lib.rs index e27fe539..2370d2fa 100644 --- a/smrs/hp-brcu/src/lib.rs +++ b/smrs/hp-brcu/src/lib.rs @@ -11,6 +11,7 @@ mod pointers; mod queue; mod rollback; +pub use deferred::{bag_capacity, set_bag_capacity}; pub use handle::*; pub use internal::*; pub use pointers::*; diff --git a/smrs/vbr/Cargo.toml b/smrs/vbr/Cargo.toml index 2261a5ee..be3a5bc2 100644 --- a/smrs/vbr/Cargo.toml +++ b/smrs/vbr/Cargo.toml @@ -9,4 +9,3 @@ edition = "2021" crossbeam-utils = "0.8" atomic = "0.5" portable-atomic = "1" -arrayvec = "0.7.4" diff --git a/smrs/vbr/src/lib.rs b/smrs/vbr/src/lib.rs index 3fa9d796..0a1f575e 100644 --- a/smrs/vbr/src/lib.rs +++ b/smrs/vbr/src/lib.rs @@ -8,15 +8,29 @@ use std::{ sync::atomic::AtomicU64, }; -use arrayvec::ArrayVec; use atomic::{Atomic, Ordering}; use crossbeam_utils::CachePadded; use portable_atomic::{compiler_fence, AtomicU128}; -pub const ENTRIES_PER_BAG: usize = 128; +static mut ENTRIES_PER_BAG: usize = 128; pub const INIT_BAGS_PER_LOCAL: usize = 32; pub const NOT_RETIRED: u64 = u64::MAX; +/// Sets the capacity of thread-local garbage bag. +/// +/// This value applies to all threads. +#[inline] +pub fn set_bag_capacity(cap: usize) { + assert!(cap > 1, "capacity must be greater than 1."); + unsafe { ENTRIES_PER_BAG = cap }; +} + +/// Returns the current capacity of thread-local garbage bag. +#[inline] +pub fn bag_capacity() -> usize { + unsafe { ENTRIES_PER_BAG } +} + pub struct Inner { birth: AtomicU64, retire: AtomicU64, @@ -34,7 +48,7 @@ unsafe impl Send for Global {} impl Global { pub fn new(capacity: usize) -> Self { let avail = BagStack::new(); - let count = capacity / ENTRIES_PER_BAG + if capacity % ENTRIES_PER_BAG > 0 { 1 } else { 0 }; + let count = capacity / bag_capacity() + if capacity % bag_capacity() > 0 { 1 } else { 0 }; for _ in 0..count { avail.push(Box::into_raw(Box::new(Bag::new_with_alloc()))); } @@ -153,30 +167,34 @@ impl Drop for BagStack { pub struct Bag { /// NOTE: A timestamp is necessary to prevent ABA problems. next: AtomicU128, - entries: ArrayVec<*mut T, ENTRIES_PER_BAG>, + entries: Vec<*mut T>, } impl Bag { fn new() -> Self { Self { next: AtomicU128::new(0), - entries: ArrayVec::new(), + entries: Vec::with_capacity(bag_capacity()), } } fn new_with_alloc() -> Self { - let mut alloc = [null_mut(); ENTRIES_PER_BAG]; - for ptr in &mut alloc { + let mut entries = vec![null_mut(); bag_capacity()]; + for ptr in &mut entries { *ptr = unsafe { Box::into_raw(Box::new(zeroed())) }; } Self { next: AtomicU128::new(0), - entries: ArrayVec::from(alloc), + entries, } } fn push(&mut self, obj: *mut T) -> bool { - self.entries.try_push(obj).is_ok() + if self.entries.len() < bag_capacity() { + self.entries.push(obj); + return true; + } + false } fn pop(&mut self) -> Option<*mut T> { diff --git a/src/bin/hp-brcu.rs b/src/bin/hp-brcu.rs index 862d2db9..4f4884c3 100644 --- a/src/bin/hp-brcu.rs +++ b/src/bin/hp-brcu.rs @@ -1,5 +1,5 @@ use crossbeam_utils::thread::scope; -use hp_brcu::{global, THREAD}; +use hp_brcu::{global, set_bag_capacity, THREAD}; use rand::prelude::*; use std::cmp::max; use std::io::{stdout, Write}; @@ -98,8 +98,9 @@ fn bench_map + Send + Sync>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for HP-BRCU."); + match config.bag_size { + BagSize::Small => set_bag_capacity(512), + BagSize::Large => set_bag_capacity(4096), } let map = &M::new(); strategy.prefill(config, map); diff --git a/src/bin/hp-rcu.rs b/src/bin/hp-rcu.rs index cbdc94ff..6711e3c5 100644 --- a/src/bin/hp-rcu.rs +++ b/src/bin/hp-rcu.rs @@ -1,5 +1,5 @@ use crossbeam_utils::thread::scope; -use hp_brcu::{global, THREAD}; +use hp_brcu::{global, set_bag_capacity, THREAD}; use rand::prelude::*; use std::cmp::max; use std::io::{stdout, Write}; @@ -99,8 +99,9 @@ fn bench_map + Send + Sync>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for HP-BRCU."); + match config.bag_size { + BagSize::Small => set_bag_capacity(512), + BagSize::Large => set_bag_capacity(4096), } let map = &M::new(); strategy.prefill(config, map); diff --git a/src/bin/nbr.rs b/src/bin/nbr.rs index 9f0bfafb..4f7e09d0 100644 --- a/src/bin/nbr.rs +++ b/src/bin/nbr.rs @@ -36,7 +36,7 @@ fn bench(config: &Config, output: BenchWriter) { fn extract_nbr_params(config: &Config) -> (usize, usize) { match config.bag_size { - BagSize::Small => (256, 32), + BagSize::Small => (512, 64), BagSize::Large => (8192, 1024), } } diff --git a/src/bin/pebr.rs b/src/bin/pebr.rs index 4d04c9df..adb1a28b 100644 --- a/src/bin/pebr.rs +++ b/src/bin/pebr.rs @@ -102,8 +102,9 @@ fn bench_map + Send + Sync, N: Unsigned>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for PEBR."); + match config.bag_size { + BagSize::Small => crossbeam_pebr::set_bag_capacity(512), + BagSize::Large => crossbeam_pebr::set_bag_capacity(4096), } let map = &M::new(); strategy.prefill(config, map); diff --git a/src/bin/vbr.rs b/src/bin/vbr.rs index 420f71e8..2ee8f78e 100644 --- a/src/bin/vbr.rs +++ b/src/bin/vbr.rs @@ -96,8 +96,9 @@ fn bench_map + Send + Sync>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for VBR."); + match config.bag_size { + BagSize::Small => vbr::set_bag_capacity(512), + BagSize::Large => vbr::set_bag_capacity(4096), } let global = &M::global(config.prefill); let local = &M::local(global); From a6dd3eef93faafb495fb072a16ec7d37e42b68fc Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 25 Sep 2024 03:23:44 +0000 Subject: [PATCH 25/84] Ignore `__pycache__` --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d670581d..3e9a8c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ tags *.pdf *.csv .python-version +__pycache__ From 1f7dac12eaefeb3eb41fb8c09b8d16416ac93648 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 25 Sep 2024 03:24:02 +0000 Subject: [PATCH 26/84] Use atomic operations to access the bag capacity --- .../src/internal/smr/ebr_impl/internal.rs | 25 +++++++++++-------- smrs/cdrc/src/internal/smr/hp_impl/thread.rs | 10 ++++---- smrs/circ/src/smr/ebr_impl/internal.rs | 19 ++++++++------ smrs/circ/src/smr/hp_impl/thread.rs | 10 ++++---- smrs/hp-brcu/src/deferred.rs | 9 ++++--- smrs/hp-pp/src/thread.rs | 12 ++++----- smrs/vbr/src/lib.rs | 8 +++--- 7 files changed, 50 insertions(+), 43 deletions(-) diff --git a/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs b/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs index 726ebe8d..afc6b1c6 100644 --- a/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs +++ b/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs @@ -58,19 +58,19 @@ pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); /// Maximum number of objects a bag can contain. #[cfg(not(any(crossbeam_sanitize, miri)))] -static mut MAX_OBJECTS: usize = 64; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(64); // Makes it more likely to trigger any potential data races. #[cfg(any(crossbeam_sanitize, miri))] -static mut MAX_OBJECTS: usize = 4; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(4); -static mut MANUAL_EVENTS_BETWEEN_COLLECT: usize = 128; +static MANUAL_EVENTS_BETWEEN_COLLECT: AtomicUsize = AtomicUsize::new(128); /// Sets the capacity of thread-local deferred bag. /// /// Note that an update on this capacity may not be reflected immediately to concurrent threads, /// because there is no synchronization between reads and writes on the capacity variable. pub fn set_bag_capacity(max_objects: usize) { - unsafe { MAX_OBJECTS = max_objects }; + MAX_OBJECTS.store(max_objects, Ordering::Relaxed); } /// Sets the manual collection interval. @@ -78,7 +78,7 @@ pub fn set_bag_capacity(max_objects: usize) { /// Note that an update on this interval may not be reflected immediately to concurrent threads, /// because there is no synchronization between reads and writes on the interval variable. pub fn set_manual_collection_interval(interval: usize) { - unsafe { MANUAL_EVENTS_BETWEEN_COLLECT = interval }; + MANUAL_EVENTS_BETWEEN_COLLECT.store(interval, Ordering::Relaxed); } /// A bag of deferred functions. @@ -123,7 +123,7 @@ impl Bag { impl Default for Bag { fn default() -> Self { - Bag(Vec::with_capacity(unsafe { MAX_OBJECTS })) + Bag(Vec::with_capacity(MAX_OBJECTS.load(Ordering::Relaxed))) } } @@ -342,12 +342,12 @@ fn local_size() { impl Local { #[inline] fn counts_between_collect() -> usize { - unsafe { MAX_OBJECTS } + MAX_OBJECTS.load(Ordering::Relaxed) } #[inline] fn counts_between_try_advance() -> usize { - unsafe { MAX_OBJECTS * 2 } + MAX_OBJECTS.load(Ordering::Relaxed) * 2 } /// Registers a new `Local` in the provided `Global`. @@ -606,7 +606,7 @@ impl Local { let manual_count = self.manual_count.get().wrapping_add(1); self.manual_count.set(manual_count); - if manual_count % unsafe { MANUAL_EVENTS_BETWEEN_COLLECT } == 0 { + if manual_count % MANUAL_EVENTS_BETWEEN_COLLECT.load(Ordering::Relaxed) == 0 { self.flush(guard); } } @@ -663,7 +663,7 @@ mod tests { let mut bag = Bag::new(); assert!(bag.is_empty()); - for _ in 0..unsafe { MAX_OBJECTS } { + for _ in 0..MAX_OBJECTS.load(Ordering::Relaxed) { assert!(unsafe { bag.try_push(Deferred::new(incr)).is_ok() }); assert!(!bag.is_empty()); assert_eq!(FLAG.load(Ordering::Relaxed), 0); @@ -675,6 +675,9 @@ mod tests { assert_eq!(FLAG.load(Ordering::Relaxed), 0); drop(bag); - assert_eq!(FLAG.load(Ordering::Relaxed), unsafe { MAX_OBJECTS }); + assert_eq!( + FLAG.load(Ordering::Relaxed), + MAX_OBJECTS.load(Ordering::Relaxed) + ); } } diff --git a/smrs/cdrc/src/internal/smr/hp_impl/thread.rs b/smrs/cdrc/src/internal/smr/hp_impl/thread.rs index abab3c85..7922f2b7 100644 --- a/smrs/cdrc/src/internal/smr/hp_impl/thread.rs +++ b/smrs/cdrc/src/internal/smr/hp_impl/thread.rs @@ -1,26 +1,26 @@ use core::ptr; -use core::sync::atomic::{AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use std::cell::{Cell, RefCell}; use super::domain::Domain; use super::hazard::ThreadRecord; use super::retire::Retired; -pub static mut COUNTS_BETWEEN_FLUSH: usize = 64; +pub static COUNTS_BETWEEN_FLUSH: AtomicUsize = AtomicUsize::new(64); #[inline] pub fn set_counts_between_flush(counts: usize) { - unsafe { COUNTS_BETWEEN_FLUSH = counts }; + COUNTS_BETWEEN_FLUSH.store(counts, Ordering::Relaxed); } #[inline] pub fn counts_between_flush() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) } #[inline] pub fn counts_between_collect() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH * 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) * 2 } pub struct Thread { diff --git a/smrs/circ/src/smr/ebr_impl/internal.rs b/smrs/circ/src/smr/ebr_impl/internal.rs index 3737f615..e69b2510 100644 --- a/smrs/circ/src/smr/ebr_impl/internal.rs +++ b/smrs/circ/src/smr/ebr_impl/internal.rs @@ -58,19 +58,19 @@ pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); /// Maximum number of objects a bag can contain. #[cfg(not(any(crossbeam_sanitize, miri)))] -static mut MAX_OBJECTS: usize = 64; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(64); // Makes it more likely to trigger any potential data races. #[cfg(any(crossbeam_sanitize, miri))] -static mut MAX_OBJECTS: usize = 4; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(4); -static mut MANUAL_EVENTS_BETWEEN_COLLECT: usize = 64; +static MANUAL_EVENTS_BETWEEN_COLLECT: AtomicUsize = AtomicUsize::new(64); /// Sets the capacity of thread-local deferred bag. /// /// Note that an update on this capacity may not be reflected immediately to concurrent threads, /// because there is no synchronization between reads and writes on the capacity variable. pub fn set_bag_capacity(max_objects: usize) { - unsafe { MAX_OBJECTS = max_objects }; + MAX_OBJECTS.store(max_objects, Ordering::Relaxed); } /// A bag of deferred functions. @@ -115,7 +115,7 @@ impl Bag { impl Default for Bag { fn default() -> Self { - Bag(Vec::with_capacity(unsafe { MAX_OBJECTS })) + Bag(Vec::with_capacity(MAX_OBJECTS.load(Ordering::Relaxed))) } } @@ -602,7 +602,7 @@ impl Local { let manual_count = self.manual_count.get().wrapping_add(1); self.manual_count.set(manual_count); - if manual_count % unsafe { MANUAL_EVENTS_BETWEEN_COLLECT } == 0 { + if manual_count % MANUAL_EVENTS_BETWEEN_COLLECT.load(Ordering::Relaxed) == 0 { self.flush(guard); } } @@ -659,7 +659,7 @@ mod tests { let mut bag = Bag::new(); assert!(bag.is_empty()); - for _ in 0..unsafe { MAX_OBJECTS } { + for _ in 0..MAX_OBJECTS.load(Ordering::Relaxed) { assert!(unsafe { bag.try_push(Deferred::new(incr)).is_ok() }); assert!(!bag.is_empty()); assert_eq!(FLAG.load(Ordering::Relaxed), 0); @@ -671,6 +671,9 @@ mod tests { assert_eq!(FLAG.load(Ordering::Relaxed), 0); drop(bag); - assert_eq!(FLAG.load(Ordering::Relaxed), unsafe { MAX_OBJECTS }); + assert_eq!( + FLAG.load(Ordering::Relaxed), + MAX_OBJECTS.load(Ordering::Relaxed) + ); } } diff --git a/smrs/circ/src/smr/hp_impl/thread.rs b/smrs/circ/src/smr/hp_impl/thread.rs index 04a6e8c1..c16a243a 100644 --- a/smrs/circ/src/smr/hp_impl/thread.rs +++ b/smrs/circ/src/smr/hp_impl/thread.rs @@ -1,26 +1,26 @@ use core::ptr::null_mut; -use core::sync::atomic::{AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use std::cell::{Cell, RefCell}; use super::domain::Domain; use super::hazard::ThreadRecord; use super::retire::{Pile, Retired}; -pub static mut COUNTS_BETWEEN_FLUSH: usize = 64; +pub static COUNTS_BETWEEN_FLUSH: AtomicUsize = AtomicUsize::new(64); #[inline] pub fn set_counts_between_flush(counts: usize) { - unsafe { COUNTS_BETWEEN_FLUSH = counts }; + COUNTS_BETWEEN_FLUSH.store(counts, Ordering::Relaxed); } #[inline] pub fn counts_between_flush() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) } #[inline] pub fn counts_between_collect() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH * 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) * 2 } pub struct Thread { diff --git a/smrs/hp-brcu/src/deferred.rs b/smrs/hp-brcu/src/deferred.rs index 23dfcb11..ee43ae96 100644 --- a/smrs/hp-brcu/src/deferred.rs +++ b/smrs/hp-brcu/src/deferred.rs @@ -1,11 +1,12 @@ use std::mem::forget; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::epoch::Epoch; #[cfg(not(feature = "sanitize"))] -static mut MAX_OBJECTS: usize = 64; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(64); #[cfg(feature = "sanitize")] -static mut MAX_OBJECTS: usize = 4; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(4); /// Sets the capacity of thread-local garbage bag. /// @@ -13,13 +14,13 @@ static mut MAX_OBJECTS: usize = 4; #[inline] pub fn set_bag_capacity(cap: usize) { assert!(cap > 1, "capacity must be greater than 1."); - unsafe { MAX_OBJECTS = cap }; + MAX_OBJECTS.store(cap, Ordering::Relaxed); } /// Returns the current capacity of thread-local garbage bag. #[inline] pub fn bag_capacity() -> usize { - unsafe { MAX_OBJECTS } + MAX_OBJECTS.load(Ordering::Relaxed) } /// A deferred task consisted of data and a callable function. diff --git a/smrs/hp-pp/src/thread.rs b/smrs/hp-pp/src/thread.rs index f7c0d24b..fca788b8 100644 --- a/smrs/hp-pp/src/thread.rs +++ b/smrs/hp-pp/src/thread.rs @@ -1,4 +1,4 @@ -use core::sync::atomic::{AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use core::{mem, ptr}; use std::collections::VecDeque; @@ -9,7 +9,7 @@ use crate::retire::{Retired, Unlinked}; use crate::HazardPointer; use crate::{Invalidate, Unlink}; -pub static mut COUNTS_BETWEEN_FLUSH: usize = 64; +pub static COUNTS_BETWEEN_FLUSH: AtomicUsize = AtomicUsize::new(64); #[inline] pub fn set_counts_between_flush(counts: usize) { @@ -17,22 +17,22 @@ pub fn set_counts_between_flush(counts: usize) { counts >= 2 && counts % 2 == 0, "counts must be even and greater than 1." ); - unsafe { COUNTS_BETWEEN_FLUSH = counts }; + COUNTS_BETWEEN_FLUSH.store(counts, Ordering::Relaxed); } #[inline] pub fn counts_between_invalidation() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH / 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) / 2 } #[inline] pub fn counts_between_flush() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) } #[inline] pub fn counts_between_collect() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH * 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) * 2 } pub struct Thread<'domain> { diff --git a/smrs/vbr/src/lib.rs b/smrs/vbr/src/lib.rs index 0a1f575e..5fad7de9 100644 --- a/smrs/vbr/src/lib.rs +++ b/smrs/vbr/src/lib.rs @@ -10,9 +10,9 @@ use std::{ use atomic::{Atomic, Ordering}; use crossbeam_utils::CachePadded; -use portable_atomic::{compiler_fence, AtomicU128}; +use portable_atomic::{compiler_fence, AtomicU128, AtomicUsize}; -static mut ENTRIES_PER_BAG: usize = 128; +static ENTRIES_PER_BAG: AtomicUsize = AtomicUsize::new(128); pub const INIT_BAGS_PER_LOCAL: usize = 32; pub const NOT_RETIRED: u64 = u64::MAX; @@ -22,13 +22,13 @@ pub const NOT_RETIRED: u64 = u64::MAX; #[inline] pub fn set_bag_capacity(cap: usize) { assert!(cap > 1, "capacity must be greater than 1."); - unsafe { ENTRIES_PER_BAG = cap }; + ENTRIES_PER_BAG.store(cap, Ordering::Relaxed); } /// Returns the current capacity of thread-local garbage bag. #[inline] pub fn bag_capacity() -> usize { - unsafe { ENTRIES_PER_BAG } + ENTRIES_PER_BAG.load(Ordering::Relaxed) } pub struct Inner { From 55bb6412e07a022aa617ab0afb19774ae4c61714 Mon Sep 17 00:00:00 2001 From: Janggun Lee Date: Thu, 3 Oct 2024 07:06:52 +0000 Subject: [PATCH 27/84] Fix race --- src/ds_impl/hp/list.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 33f1ee9b..38dc7e71 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -217,8 +217,9 @@ where let mut node = self.anchor_next; while node.with_tag(0) != self.curr.with_tag(0) { - // SAFETY: the fact that node is tagged means that it cannot be modified, hence we can safety do an non-atomic load. - let next = unsafe { node.deref().next.as_shared() }; + // NOTE: It may seem like this can be done with a NA load, but we do a `fetch_or` in remove, which always does an write. + // This can be a NA load if the `fetch_or` in delete is changed to a CAS, but it is not clear if it is worth it. + let next = unsafe { node.deref().next.load(Ordering::Relaxed) }; debug_assert!(next.tag() != 0); unsafe { self.handle.thread.retire(node.with_tag(0).into_raw()) }; node = next; From 4bb594bc8919d0d5d35f60c314cf4127915e45f2 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 7 Oct 2024 06:19:05 +0000 Subject: [PATCH 28/84] Add bench scripts --- bench-scripts/hp-revisited/bench.py | 102 ++++++++++++++ bench-scripts/hp-revisited/legends.py | 88 ++++++++++++ bench-scripts/hp-revisited/plot.py | 185 ++++++++++++++++++++++++++ 3 files changed, 375 insertions(+) create mode 100644 bench-scripts/hp-revisited/bench.py create mode 100644 bench-scripts/hp-revisited/legends.py create mode 100644 bench-scripts/hp-revisited/plot.py diff --git a/bench-scripts/hp-revisited/bench.py b/bench-scripts/hp-revisited/bench.py new file mode 100644 index 00000000..26f078b1 --- /dev/null +++ b/bench-scripts/hp-revisited/bench.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +import subprocess +import os + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") +BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") + +dss = ['h-list', 'hm-list', 'hhs-list', 'hash-map', 'nm-tree', 'skip-list'] +# "-large" suffix if it uses a large garbage bag. +mms = ['nr', 'ebr', 'pebr', 'hp', 'hp-pp', 'nbr', 'hp-brcu', 'vbr'] +i = 10 +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = list(map(str, [1] + list(range(4, 33, 4)))) +elif cpu_count <= 64: + ts = list(map(str, [1] + list(range(8, 129, 8)))) +else: + ts = list(map(str, [1] + list(range(12, 193, 12)))) +runs = 2 +gs = [0, 1, 2] + +subprocess.run(['cargo', 'build', '--release']) + +def key_ranges(ds): + if ds in ["h-list", "hm-list", "hhs-list"]: + # 1K and 10K + return ["1000", "10000"] + else: + # 100K and 100M + return ["100000", "100000000"] + +def is_suffix(orig, suf): + return len(suf) <= len(orig) and mm[-len(suf):] == suf + +def make_cmd(mm, i, ds, g, t, kr): + bag = "small" + if is_suffix(mm, "-large"): + mm = mm[:len(mm)-len("-large")] + bag = "large" + + return [os.path.join(BIN_PATH, mm), + '-i', str(i), + '-d', str(ds), + '-g', str(g), + '-t', str(t), + '-r', str(kr), + '-b', bag, + '-o', os.path.join(RESULTS_PATH, f'{ds}.csv')] + +def invalid(mm, ds, g): + is_invalid = False + if ds == 'hhs-list': + is_invalid |= g == 0 # HHSList is just HList with faster get() + if mm == 'nbr': + is_invalid |= ds in ["hm-list", "skip-list"] + return is_invalid + +cmds = [] + +for ds in dss: + for kr in key_ranges(ds): + for mm in mms: + for g in gs: + if invalid(mm, ds, g): + continue + for t in ts: + cmds.append(make_cmd(mm, i, ds, g, t, kr)) + +print('number of configurations: ', len(cmds)) +print('estimated time: ', (len(cmds) * i * 1.1) // 60, ' min *', runs, 'times\n') + +for i, cmd in enumerate(cmds): + try: + print(f"\rdry-running commands... ({i+1}/{len(cmds)})", end="") + subprocess.run(cmd + ['--dry-run']) + except: + print(f"A dry-run for the following command is failed:\n{' '.join(cmd)}") + exit(1) +print("\nAll dry-runs passed!\n") + +os.makedirs(RESULTS_PATH, exist_ok=True) +failed = [] +for run in range(runs): + for i, cmd in enumerate(cmds): + print("run {}/{}, bench {}/{}: '{}'".format(run + 1, runs, i + 1, len(cmds), ' '.join(cmd))) + try: + subprocess.run(cmd, timeout=i+30) + except subprocess.TimeoutExpired: + print("timeout") + failed.append(' '.join(cmd)) + except KeyboardInterrupt: + if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) + exit(0) + except: + failed.append(' '.join(cmd)) + +if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) diff --git a/bench-scripts/hp-revisited/legends.py b/bench-scripts/hp-revisited/legends.py new file mode 100644 index 00000000..f038e4a7 --- /dev/null +++ b/bench-scripts/hp-revisited/legends.py @@ -0,0 +1,88 @@ +import os +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from matplotlib.path import Path +from matplotlib.transforms import Affine2D + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +EBR = "ebr" +PEBR = "pebr" +NR = "nr" +HP = "hp" +HP_PP = "hp-pp" +HP_BRCU = "hp-brcu" +HP_RCU = "hp-rcu" +VBR = "vbr" + +SMRs = [NR, EBR, HP_PP, HP, PEBR, HP_BRCU, HP_RCU, VBR] + +FACE_ALPHA = 0.85 + +# https://matplotlib.org/stable/gallery/lines_bars_and_markers/marker_reference.html +line_shapes = { + NR: { + "marker": ".", + "color": "k", + "linestyle": "-", + }, + EBR: { + "marker": "o", + "color": "c", + "linestyle": "-", + }, + HP: { + "marker": "v", + "color": "hotpink", + "linestyle": "dashed", + }, + HP_PP: { + "marker": "^", + "color": "purple", + "linestyle": "dashdot", + }, + PEBR: { + # Diamond("D") shape, but smaller. + "marker": Path.unit_rectangle() + .transformed(Affine2D().translate(-0.5, -0.5) + .rotate_deg(45)), + "color": "y", + "linestyle": (5, (10, 3)), + }, + HP_BRCU: { + "marker": "X", + "color": "r", + "linestyle": (5, (10, 3)), + }, + HP_RCU: { + "marker": "P", + "color": "green", + "linestyle": (5, (10, 3)), + }, + VBR: { + "marker": "d", + "color": "orange", + "linestyle": (0, (2, 1)), + }, +} + +# Add some common or derivable properties. +line_shapes = dict(map( + lambda kv: kv if kv[0] == NR else + (kv[0], { **kv[1], + "markerfacecolor": (*colors.to_rgb(kv[1]["color"]), FACE_ALPHA), + "markeredgecolor": "k", + "markeredgewidth": 0.75 }), + line_shapes.items() +)) + +if __name__ == "__main__": + os.makedirs(f'{RESULTS_PATH}/legends', exist_ok=True) + for smr in SMRs: + fig, ax = plt.subplots(ncols=1) + ax.plot([0] * 3, linewidth=3, markersize=18, **line_shapes[smr]) + ax.set_xlim(2/3, 4/3) + ax.set_axis_off() + fig.set_size_inches(1.25, 0.75) + fig.tight_layout() + fig.savefig(f"{RESULTS_PATH}/legends/{smr}.pdf", bbox_inches="tight") diff --git a/bench-scripts/hp-revisited/plot.py b/bench-scripts/hp-revisited/plot.py new file mode 100644 index 00000000..9b36deb0 --- /dev/null +++ b/bench-scripts/hp-revisited/plot.py @@ -0,0 +1,185 @@ +import pandas as pd +import warnings +import os, math +import matplotlib +import matplotlib.pyplot as plt +from legends import * + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +warnings.filterwarnings("ignore") +pd.set_option('display.max_rows', None) + +# avoid Type 3 fonts +matplotlib.rcParams['pdf.fonttype'] = 42 +matplotlib.rcParams['ps.fonttype'] = 42 + +# raw column names +THREADS = "threads" +THROUGHPUT = "throughput" +PEAK_MEM = "peak_mem" +AVG_GARB = "avg_garb" +PEAK_GARB = "peak_garb" + +# legend +SMR_ONLY = "SMR\n" + +HLIST = "h-list" +HHSLIST = "hhs-list" +HASHMAP = "hash-map" +NMTREE = "nm-tree" +SKIPLIST = "skip-list" + +FORMAL_NAMES = { + HLIST: "HList", + HHSLIST: "HHSList", + HASHMAP: "HashMap", + NMTREE: "NMTree", + SKIPLIST: "SkipList", +} + +# DS with read-dominated bench & write-only bench +dss_all = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST] +dss_read = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST] +dss_write = [HLIST, HASHMAP, NMTREE, SKIPLIST] + +WRITE, HALF, READ = "write", "half", "read" + +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = [1] + list(range(4, 33, 4)) +elif cpu_count <= 64: + ts = [1] + list(range(8, 129, 8)) +else: + ts = [1] + list(range(12, 193, 12)) +n_map = {0: ''} + +(label_size, xtick_size, ytick_size, marker_size) = (24, 20, 18, 20) + +mm_order = [ + HP_BRCU, + EBR, + HP, + HP_PP, + PEBR, + VBR, + NR, +] + +def plot_title(ds, bench): + if bench == WRITE and ds == HLIST: + return 'HList/HHSList' + return FORMAL_NAMES[ds] + +def key_ranges(ds): + if ds in [HLIST, HHSLIST]: + return [1000, 10000] + else: + return [100000, 100000000] + +def range_to_str(kr: int): + UNITS = ["K", "M", "B", "T"] + for i in range(len(UNITS)): + unit = pow(10, 3 * (i+1)) + div = kr // unit + if div < 1000 or i == len(UNITS) - 1: + return f"{div}{UNITS[i]}" + +def draw(title, name, data, y_value, y_label=None, y_max=None, y_min=None): + print(name) + plt.figure(figsize=(10, 7)) + plt.title(title, fontsize=36, pad=15) + + for mm in sorted(list(set(data.mm)), key=lambda mm: len(mm_order) - mm_order.index(mm)): + d = data[data.mm == mm].sort_values(by=[THREADS], axis=0) + plt.plot(d[THREADS], d[y_value], + linewidth=3, markersize=marker_size, **line_shapes[mm], zorder=30) + + plt.xlabel("Threads", fontsize=label_size) + plt.ylabel(y_label, fontsize=label_size) + plt.yticks(fontsize=ytick_size) + plt.xticks(ts, fontsize=xtick_size, rotation=90) + plt.grid(alpha=0.5) + + if data.threads.max() >= cpu_count: + left, right = plt.xlim() + plt.axvspan(cpu_count, right, facecolor="#FF00000A") + plt.xlim(left, right) + + y_max = min(y_max, data[y_value].max()) if y_max else data[y_value].max() + y_min = max(y_min, data[y_value].min()) if y_min else data[y_value].min() + y_margin = (y_max - y_min) * 0.05 + plt.ylim(y_min-y_margin, y_max+y_margin) + + plt.savefig(name, bbox_inches='tight') + + +def draw_throughput(data, ds, bench, key_range): + data = data[ds].copy() + data = data[data.key_range == key_range] + data = data[data.non_coop == 0] + y_label = 'Throughput (M op/s)' + y_max = data.throughput.max() * 1.05 + draw(plot_title(ds, bench), f'{RESULTS_PATH}/{ds}_{bench}_{range_to_str(key_range)}_throughput.pdf', + data, THROUGHPUT, y_label, y_max) + + +def draw_peak_garb(data, ds, bench, key_range): + data = data[ds].copy() + data = data[data.key_range == key_range] + data = data[data.mm != NR] + data = data[data.mm != VBR] + y_label = 'Peak unreclaimed blocks (×10⁴)' + y_max = 0 + for cand in [HP_PP, HP_BRCU]: + max_garb = data[data.mm == cand].peak_garb.max() + if not math.isnan(max_garb): + y_max = max(y_max, max_garb * 2) + draw(plot_title(ds, bench), f'{RESULTS_PATH}/{ds}_{bench}_{range_to_str(key_range)}_peak_garb.pdf', + data, PEAK_GARB, y_label, y_max) + + +raw_data = {} +# averaged data for write:read = 100:0, 50:50, 10:90 +avg_data = { WRITE: {}, HALF: {}, READ: {} } + +# preprocess +for ds in dss_all: + data = pd.read_csv(f'{RESULTS_PATH}/' + ds + '.csv') + + data.throughput = data.throughput.map(lambda x: x / 1000_000) + data.peak_mem = data.peak_mem.map(lambda x: x / (2 ** 20)) + data.avg_mem = data.avg_mem.map(lambda x: x / (2 ** 20)) + data.peak_garb = data.peak_garb.map(lambda x: x / 10000) + data.avg_garb = data.avg_garb.map(lambda x: x / 10000) + data.mm = list(map(lambda tup: tup[0] if tup[1] == "small" else tup[0] + "-large", zip(data.mm, data.bag_size))) + data = data.drop("bag_size", axis=1) + data = data[data.mm.isin(SMRs)] + + raw_data[ds] = data.copy() + + # take average of each runs + avg = data.groupby(['ds', 'mm', 'threads', 'non_coop', 'get_rate', 'key_range']).mean().reset_index() + + avg[SMR_ONLY] = pd.Categorical(avg.mm.map(str), SMRs) + avg.sort_values(by=SMR_ONLY, inplace=True) + for i, bench in enumerate([WRITE, HALF, READ]): + avg_data[bench][ds] = avg[avg.get_rate == i] + +# 1. throughput graphs, 3 lines (SMR_ONLY) each. +for ds in dss_write: + for kr in key_ranges(ds): + draw_throughput(avg_data[WRITE], ds, WRITE, kr) +for ds in dss_read: + for kr in key_ranges(ds): + draw_throughput(avg_data[HALF], ds, HALF, kr) + draw_throughput(avg_data[READ], ds, READ, kr) + +# 2. peak garbage graph +for ds in dss_write: + for kr in key_ranges(ds): + draw_peak_garb(avg_data[WRITE], ds, WRITE, kr) +for ds in dss_read: + for kr in key_ranges(ds): + draw_peak_garb(avg_data[HALF], ds, HALF, kr) + draw_peak_garb(avg_data[READ], ds, READ, kr) From e75062031a27e5a34440002b01322898c1237cf9 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 10 Oct 2024 07:01:15 +0000 Subject: [PATCH 29/84] [WIP] implementing ElimABTree --- src/ds_impl/ebr/elim_ab_tree.rs | 208 ++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 src/ds_impl/ebr/elim_ab_tree.rs diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs new file mode 100644 index 00000000..414c63cc --- /dev/null +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -0,0 +1,208 @@ +use super::concurrent_map::ConcurrentMap; +use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared}; + +use std::cmp::Ordering::{Equal, Greater, Less}; +use std::marker::PhantomData; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +struct MCSLock { + _marker: PhantomData<(K, V)>, +} + +impl MCSLock { + fn new() -> Self { + todo!() + } + + fn acquire(&self) -> MCSLockGuard { + todo!() + } +} + +struct MCSLockGuard { + _marker: PhantomData<(K, V)>, +} + +impl Drop for MCSLockGuard { + fn drop(&mut self) { + todo!("release lock") + } +} + +struct Node { + keys: [K; DEGREE], + search_key: K, + lock: MCSLock, + size: usize, + weight: bool, + marked: AtomicBool, + kind: NodeSpecific, +} + +// Leaf or Internal node specific data. +enum NodeSpecific { + Leaf { + values: [V; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: MCSLock::new(), + size, + weight, + marked: AtomicBool::new(false), + kind: NodeSpecific::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: MCSLock::new(), + size, + weight, + marked: AtomicBool::new(false), + kind: NodeSpecific::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeSpecific::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeSpecific::Leaf { .. } => self.size, + NodeSpecific::Internal { .. } => self.size - 1, + } + } + + fn child_index(&self, key: &K) -> usize { + let mut index = 0; + while index < self.key_count() && !(key < &self.keys[index]) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_value_version(&self, key: &K) -> (usize, Option, usize) { + if let NodeSpecific::Leaf { values, write_version } = &self.kind { + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index] != *key { + key_index += 1; + } + let value = if key_index < DEGREE { Some(values[key_index]) } else { None }; + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value, version); + } + } + } + panic!("Attempted to read value from an internal node.") + } +} + +enum Operation { + Insert, + Delete, + Balance, +} + +struct Cursor<'g, K, V> { + l: Shared<'g, Node>, + p: Shared<'g, Node>, + gp: Shared<'g, Node>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: V, + node_version: usize, +} + +struct ElimABTree { + entry: Node, +} + +impl ElimABTree +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + /// `a` in the original code... TODO: remove later. + fn a() -> usize { + (DEGREE / 4).max(2) + } + + /// `b` in the original code... TODO: remove later. + fn b() -> usize { + DEGREE + } + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let entry = Node::internal(true, 1, K::default()); + entry.next()[0].store(Owned::new(left), Ordering::Relaxed); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, guard: &Guard) -> Option { + let mut node = unsafe { self.entry.next()[0].load(Ordering::Acquire, guard).deref() }; + while let NodeSpecific::Internal { next } = &node.kind { + let next = next[node.child_index(key)].load(Ordering::Acquire, guard); + node = unsafe { next.deref() }; + } + node.read_value_version(key).1 + } + + pub fn search<'g>(&self, key: &K, target: Option>>, guard: &'g Guard) -> Cursor<'g, K, V> { + // let mut cursor = Cursor:: + // let mut gp = Shared::null(); + // let mut p = unsafe { Shared::>::from_usize(&self.entry as *const _ as usize) }; + // let mut p_l_idx = 0; + todo!() + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + todo!() + } +} From ba9872f7848af41fda841267b621404d5a0adef5 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 22 Oct 2024 05:51:27 +0000 Subject: [PATCH 30/84] Fix bugs in prefilling of Bonsai Tree --- src/bin/cdrc-ebr-flush.rs | 11 ++++++++++- src/bin/cdrc-ebr.rs | 11 ++++++++++- src/bin/cdrc-hp.rs | 11 ++++++++++- src/bin/circ-ebr.rs | 13 ++++++++++++- src/bin/circ-hp.rs | 13 ++++++++++++- src/bin/ebr.rs | 11 ++++++++++- src/bin/hp-brcu.rs | 2 ++ src/bin/hp-pp.rs | 6 +++++- src/bin/hp-rcu.rs | 2 ++ src/bin/hp.rs | 6 +++++- src/bin/nbr.rs | 2 ++ src/bin/nr.rs | 6 +++++- src/bin/pebr.rs | 11 ++++++++++- src/bin/vbr.rs | 2 ++ 14 files changed, 97 insertions(+), 10 deletions(-) diff --git a/src/bin/cdrc-ebr-flush.rs b/src/bin/cdrc-ebr-flush.rs index b2cb15c0..0aa4ff7e 100644 --- a/src/bin/cdrc-ebr-flush.rs +++ b/src/bin/cdrc-ebr-flush.rs @@ -41,7 +41,11 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::BonsaiTree => { - bench_map::>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) } _ => panic!("Unsupported(or unimplemented) data structure for CDRC"), }; @@ -51,7 +55,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -69,6 +75,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/cdrc-ebr.rs b/src/bin/cdrc-ebr.rs index 62d60f0c..5337a4d7 100644 --- a/src/bin/cdrc-ebr.rs +++ b/src/bin/cdrc-ebr.rs @@ -41,7 +41,11 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::BonsaiTree => { - bench_map::>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) } _ => panic!("Unsupported(or unimplemented) data structure for CDRC"), }; @@ -51,7 +55,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -69,6 +75,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/cdrc-hp.rs b/src/bin/cdrc-hp.rs index 1f96e803..fe7109d4 100644 --- a/src/bin/cdrc-hp.rs +++ b/src/bin/cdrc-hp.rs @@ -40,7 +40,11 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::BonsaiTree => { - bench_map::>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) } _ => panic!("Unsupported(or unimplemented) data structure for CDRC"), }; @@ -50,7 +54,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -64,6 +70,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/circ-ebr.rs b/src/bin/circ-ebr.rs index fd47e152..33aba25e 100644 --- a/src/bin/circ-ebr.rs +++ b/src/bin/circ-ebr.rs @@ -34,7 +34,13 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) + } _ => panic!("Unsupported(or unimplemented) data structure for CIRC"), }; output.write_record(config, &perf); @@ -43,7 +49,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -57,6 +65,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let rng = &mut rand::thread_rng(); let count = config.prefill / threads diff --git a/src/bin/circ-hp.rs b/src/bin/circ-hp.rs index cbbbb446..8ffb15a1 100644 --- a/src/bin/circ-hp.rs +++ b/src/bin/circ-hp.rs @@ -33,7 +33,13 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) + } _ => panic!("Unsupported(or unimplemented) data structure for CIRC"), }; output.write_record(config, &perf); @@ -42,7 +48,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -56,6 +64,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/ebr.rs b/src/bin/ebr.rs index 3acadb6e..d09a696a 100644 --- a/src/bin/ebr.rs +++ b/src/bin/ebr.rs @@ -38,7 +38,11 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::, N>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::, N>(config, PrefillStrategy::Random), DS::BonsaiTree => { - bench_map::, N>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::, N>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::, N>(config, PrefillStrategy::Random), DS::SkipList => bench_map::, N>(config, PrefillStrategy::Decreasing), @@ -49,7 +53,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -63,6 +69,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let guard = unsafe { crossbeam_ebr::unprotected() }; let rng = &mut rand::thread_rng(); let count = config.prefill / threads diff --git a/src/bin/hp-brcu.rs b/src/bin/hp-brcu.rs index 4f4884c3..36f1040b 100644 --- a/src/bin/hp-brcu.rs +++ b/src/bin/hp-brcu.rs @@ -41,7 +41,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/hp-pp.rs b/src/bin/hp-pp.rs index 55aadb4d..3cbfcd87 100644 --- a/src/bin/hp-pp.rs +++ b/src/bin/hp-pp.rs @@ -36,7 +36,9 @@ fn bench(config: &Config, output: BenchWriter) { DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } }; output.write_record(config, &perf); println!("{}", perf); @@ -44,7 +46,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/hp-rcu.rs b/src/bin/hp-rcu.rs index 6711e3c5..c8fea315 100644 --- a/src/bin/hp-rcu.rs +++ b/src/bin/hp-rcu.rs @@ -42,7 +42,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/hp.rs b/src/bin/hp.rs index ebed6c36..a3e97221 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -34,7 +34,9 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } DS::NMTree => bench_map::>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); @@ -43,7 +45,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/nbr.rs b/src/bin/nbr.rs index 4f7e09d0..afa79743 100644 --- a/src/bin/nbr.rs +++ b/src/bin/nbr.rs @@ -43,7 +43,9 @@ fn extract_nbr_params(config: &Config) -> (usize, usize) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/nr.rs b/src/bin/nr.rs index 2458f436..bd23820a 100644 --- a/src/bin/nr.rs +++ b/src/bin/nr.rs @@ -32,7 +32,9 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); @@ -41,7 +43,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/pebr.rs b/src/bin/pebr.rs index adb1a28b..344e004d 100644 --- a/src/bin/pebr.rs +++ b/src/bin/pebr.rs @@ -38,7 +38,11 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::, N>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::, N>(config, PrefillStrategy::Random), DS::BonsaiTree => { - bench_map::, N>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::, N>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::, N>(config, PrefillStrategy::Random), DS::SkipList => bench_map::, N>(config, PrefillStrategy::Decreasing), @@ -49,7 +53,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -63,6 +69,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let guard = unsafe { crossbeam_pebr::unprotected() }; let mut handle = M::handle(guard); let rng = &mut rand::thread_rng(); diff --git a/src/bin/vbr.rs b/src/bin/vbr.rs index 2ee8f78e..6b2a1b1a 100644 --- a/src/bin/vbr.rs +++ b/src/bin/vbr.rs @@ -40,7 +40,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } From 954c2c66cafe2a4153d10f96a17470293a0cba0e Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 24 Oct 2024 11:30:31 +0000 Subject: [PATCH 31/84] [WIP] Implement a draft version of EBR ElimABTree --- src/ds_impl/ebr/elim_ab_tree.rs | 1272 +++++++++++++++++++++++++++++-- 1 file changed, 1192 insertions(+), 80 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 414c63cc..11734d8b 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -1,44 +1,110 @@ -use super::concurrent_map::ConcurrentMap; +// use super::concurrent_map::ConcurrentMap; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared}; -use std::cmp::Ordering::{Equal, Greater, Less}; -use std::marker::PhantomData; -use std::sync::atomic::{compiler_fence, AtomicBool, AtomicUsize, Ordering}; +use std::cell::UnsafeCell; +use std::hint::spin_loop; +use std::mem::{transmute, MaybeUninit}; +use std::ptr::{null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; // Copied from the original author's code: // https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 const DEGREE: usize = 11; -struct MCSLock { - _marker: PhantomData<(K, V)>, +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: UnsafeCell>, } -impl MCSLock { +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ fn new() -> Self { - todo!() + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: UnsafeCell::new(None), + } } - fn acquire(&self) -> MCSLockGuard { - todo!() + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; } } -struct MCSLockGuard { - _marker: PhantomData<(K, V)>, +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, } -impl Drop for MCSLockGuard { +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { fn drop(&mut self) { - todo!("release lock") + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + let next = if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next + } else { + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + break next; + } + spin_loop(); + } + }; + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); } } +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + struct Node { - keys: [K; DEGREE], + keys: [UnsafeCell>; DEGREE], search_key: K, - lock: MCSLock, - size: usize, - weight: bool, + lock: AtomicPtr>, + size: AtomicUsize, + weight: AtomicBool, marked: AtomicBool, kind: NodeSpecific, } @@ -46,7 +112,7 @@ struct Node { // Leaf or Internal node specific data. enum NodeSpecific { Leaf { - values: [V; DEGREE], + values: [UnsafeCell; DEGREE], write_version: AtomicUsize, }, Internal { @@ -54,6 +120,57 @@ enum NodeSpecific { }, } +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeSpecific::Leaf { .. } => true, + NodeSpecific::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeSpecific::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeSpecific::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn values(&self) -> &[UnsafeCell; DEGREE] { + match &self.kind { + NodeSpecific::Leaf { values, .. } => values, + _ => panic!("No values for an internal node."), + } + } + + fn values_mut(&mut self) -> &mut [UnsafeCell; DEGREE] { + match &mut self.kind { + NodeSpecific::Leaf { values, .. } => values, + _ => panic!("No values for an internal node."), + } + } + + fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeSpecific::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeSpecific::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeSpecific::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } +} + impl Node where K: PartialOrd + Eq + Default + Copy, @@ -63,9 +180,9 @@ where Self { keys: Default::default(), search_key, - lock: MCSLock::new(), - size, - weight, + lock: Default::default(), + size: AtomicUsize::new(size), + weight: AtomicBool::new(weight), marked: AtomicBool::new(false), kind: NodeSpecific::Internal { next: Default::default(), @@ -77,9 +194,9 @@ where Self { keys: Default::default(), search_key, - lock: MCSLock::new(), - size, - weight, + lock: Default::default(), + size: AtomicUsize::new(size), + weight: AtomicBool::new(weight), marked: AtomicBool::new(false), kind: NodeSpecific::Leaf { values: Default::default(), @@ -88,23 +205,11 @@ where } } - fn next(&self) -> &[Atomic; DEGREE] { - match &self.kind { - NodeSpecific::Internal { next } => next, - _ => panic!("No next pointers for a leaf node."), - } - } - - fn key_count(&self) -> usize { - match &self.kind { - NodeSpecific::Leaf { .. } => self.size, - NodeSpecific::Internal { .. } => self.size - 1, - } - } - fn child_index(&self, key: &K) -> usize { let mut index = 0; - while index < self.key_count() && !(key < &self.keys[index]) { + while index < self.key_count() + && !(key < unsafe { &*self.keys[index].get() }.as_ref().unwrap()) + { index += 1; } index @@ -112,32 +217,109 @@ where // Search a node for a key repeatedly until we successfully read a consistent version. fn read_value_version(&self, key: &K) -> (usize, Option, usize) { - if let NodeSpecific::Leaf { values, write_version } = &self.kind { - loop { - let mut version = write_version.load(Ordering::Acquire); - while version & 1 > 0 { - version = write_version.load(Ordering::Acquire); - } - let mut key_index = 0; - while key_index < DEGREE && self.keys[key_index] != *key { - key_index += 1; - } - let value = if key_index < DEGREE { Some(values[key_index]) } else { None }; - compiler_fence(Ordering::SeqCst); - - if version == write_version.load(Ordering::Acquire) { - return (key_index, value, version); - } + let NodeSpecific::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && unsafe { *self.keys[key_index].get() } != Some(*key) { + key_index += 1; + } + let value = if key_index < DEGREE { + Some(unsafe { *values[key_index].get() }) + } else { + None + }; + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value, version); } } - panic!("Attempted to read value from an internal node.") + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::Relaxed); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Relaxed); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(unsafe { *curr.ret.get() }.unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>(&'l self, value: V, old_version: usize, guard: &MCSLockGuard<'l, K, V>) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Relaxed); + self.write_version() + .store(old_version + 2, Ordering::Release); + + if std::ptr::eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + unsafe { (*curr_node.ret.get()) = Some(value) }; + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); } } -enum Operation { - Insert, - Delete, - Balance, +enum InsertResult { + Ok, + Retry, + Exists(V), } struct Cursor<'g, K, V> { @@ -150,28 +332,24 @@ struct Cursor<'g, K, V> { p_l_idx: usize, /// Index of the key in `l`. l_key_idx: usize, - val: V, - node_version: usize, + val: Option, + l_version: usize, } struct ElimABTree { entry: Node, } +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + impl ElimABTree where - K: PartialOrd + Eq + Default + Copy, + K: Ord + Eq + Default + Copy, V: Default + Copy, { - /// `a` in the original code... TODO: remove later. - fn a() -> usize { - (DEGREE / 4).max(2) - } - - /// `b` in the original code... TODO: remove later. - fn b() -> usize { - DEGREE - } + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; pub fn new() -> Self { let left = Node::leaf(true, 0, K::default()); @@ -192,17 +370,951 @@ where node.read_value_version(key).1 } - pub fn search<'g>(&self, key: &K, target: Option>>, guard: &'g Guard) -> Cursor<'g, K, V> { - // let mut cursor = Cursor:: - // let mut gp = Shared::null(); - // let mut p = unsafe { Shared::>::from_usize(&self.entry as *const _ as usize) }; - // let mut p_l_idx = 0; - todo!() + pub fn search<'g>( + &self, + key: &K, + target: Option>>, + guard: &'g Guard, + ) -> (bool, Cursor<'g, K, V>) { + let mut cursor = Cursor { + l: self.entry.next()[0].load(Ordering::Relaxed, guard), + p: unsafe { Shared::from_usize(&self.entry as *const _ as usize) }, + gp: Shared::null(), + gp_p_idx: 0, + p_l_idx: 0, + l_key_idx: 0, + val: None, + l_version: 0, + }; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { cursor.l.deref() }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.l = l_node.next()[cursor.p_l_idx].load(Ordering::Acquire, guard); + } + + if let Some(target) = target { + (cursor.l == target, cursor) + } else { + let (index, value, version) = unsafe { cursor.l.deref() }.read_value_version(key); + cursor.val = value; + cursor.l_key_idx = index; + cursor.l_version = version; + (value.is_some(), cursor) + } + } + + pub fn insert(&self, key: &K, value: &V, guard: &Guard) -> Result<(), V> { + loop { + let (_, cursor) = self.search(key, None, guard); + if let Some(value) = cursor.val { + return Err(value); + } + match self.insert_inner(key, value, &cursor, guard) { + InsertResult::Ok => return Ok(()), + InsertResult::Exists(value) => return Err(value), + InsertResult::Retry => continue, + } + } + } + + pub fn insert_inner<'g>( + &self, + key: &K, + value: &V, + cursor: &Cursor<'g, K, V>, + guard: &'g Guard, + ) -> InsertResult { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return InsertResult::Exists(value), + }; + if node.marked.load(Ordering::SeqCst) { + return InsertResult::Retry; + } + for i in 0..DEGREE { + if unsafe { *node.keys[i].get() } == Some(*key) { + return InsertResult::Exists(unsafe { *node.values()[i].get() }); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if unsafe { *node.keys[i].get() }.is_some() { + continue; + } + let old_version = node.write_version().load(Ordering::Relaxed); + node.write_version() + .store(old_version + 1, Ordering::Relaxed); + debug_assert!(old_version % 2 == 0); + compiler_fence(Ordering::SeqCst); + unsafe { + *node.keys[i].get() = Some(*key); + *node.values()[i].get() = *value; + } + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, old_version, &node_lock); + + // TODO: do smarter (lifetime) + drop(node_lock); + return InsertResult::Ok; + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + let parent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let parent_lock = match ( + parent.acquire(Operation::Insert, None, &parent_lock_slot), + parent.marked.load(Ordering::SeqCst), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => return InsertResult::Retry, + }; + + let mut kv_pairs: [MaybeUninit<(K, V)>; DEGREE + 1] = + [MaybeUninit::uninit(); DEGREE + 1]; + let mut count = 0; + for i in 0..DEGREE { + if let Some(key) = unsafe { *node.keys[i].get() } { + let value = unsafe { *node.values()[i].get() }; + kv_pairs[count].write((key, value)); + count += 1; + } + } + kv_pairs[count].write((*key, *value)); + count += 1; + let kv_pairs = unsafe { transmute::<_, &mut [(K, V)]>(&mut kv_pairs[0..count]) }; + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = count / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + *left.keys[i].get_mut() = Some(kv_pairs[i].0); + *left.values_mut()[i].get_mut() = kv_pairs[i].1; + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + *right.keys[i].get_mut() = Some(kv_pairs[i + left_size].0); + *right.values_mut()[i].get_mut() = kv_pairs[i + left_size].1; + } + + // TODO: understand this comment... + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = + Node::internal(std::ptr::eq(parent, &self.entry), 2, kv_pairs[left_size].0); + *internal.keys[0].get_mut() = Some(kv_pairs[left_size].0); + internal.next()[0].store(Owned::new(left), Ordering::Relaxed); + internal.next()[1].store(Owned::new(right), Ordering::Relaxed); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Owned::new(internal).into_shared(guard); + parent.next()[cursor.p_l_idx].store(new_internal, Ordering::Relaxed); + node.marked.store(true, Ordering::Release); + drop(node_lock); + + // Manually unlock and fix the tag. + drop(parent_lock); + unsafe { guard.defer_destroy(cursor.l) }; + self.fix_tag_violation(new_internal, guard); + + InsertResult::Ok + } + } + + fn fix_tag_violation<'g>(&self, viol: Shared<'g, Node>, guard: &'g Guard) { + loop { + let viol_node = unsafe { viol.deref() }; + if viol_node.weight.load(Ordering::Relaxed) { + return; + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!( + !std::ptr::eq(viol_node, &self.entry) + && self.entry.next()[0].load(Ordering::Relaxed, guard) != viol + ); + + let (found, cursor) = self.search(&viol_node.search_key, Some(viol), guard); + if !found { + return; + } + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + if !std::ptr::eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight.load(Ordering::Relaxed) { + self.fix_tag_violation(cursor.p, guard); + continue; + } + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match ( + node.acquire(Operation::Balance, None, &node_lock_slot), + node.marked.load(Ordering::Relaxed), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => continue, + }; + let parent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let parent_lock = match ( + parent.acquire(Operation::Balance, None, &parent_lock_slot), + parent.marked.load(Ordering::Relaxed), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => continue, + }; + let gparent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let gparent_lock = match ( + gparent.acquire(Operation::Balance, None, &gparent_lock_slot), + gparent.marked.load(Ordering::Relaxed), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => continue, + }; + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = + Node::internal(true, size, unsafe { &*parent.keys[0].get() }.unwrap()); + + ptrs_clone( + &parent.next()[0..], + &mut absorber.next_mut()[0..], + cursor.p_l_idx, + ); + ptrs_clone( + &node.next()[0..], + &mut absorber.next_mut()[cursor.p_l_idx..], + nsize, + ); + ptrs_clone( + &parent.next()[cursor.p_l_idx + 1..], + &mut absorber.next_mut()[cursor.p_l_idx + nsize..], + psize - (cursor.p_l_idx + 1), + ); + + ufcells_clone(&parent.keys[0..], &mut absorber.keys[0..], cursor.p_l_idx); + ufcells_clone( + &node.keys[0..], + &mut absorber.keys[cursor.p_l_idx..], + node.key_count(), + ); + ufcells_clone( + &parent.keys[cursor.p_l_idx..], + &mut absorber.keys[cursor.p_l_idx + node.key_count()..], + parent.key_count() - cursor.p_l_idx, + ); + + gparent.next()[cursor.gp_p_idx].store(Owned::new(absorber), Ordering::Relaxed); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l) }; + unsafe { guard.defer_destroy(cursor.p) }; + return; + } else { + // Split case. + + // Merge keys of p and l into one big array (and similarly for children). + // We essentially replace the pointer to l with the contents of l. + let mut next: [Atomic>; 2 * DEGREE] = Default::default(); + let mut keys: [UnsafeCell>; 2 * DEGREE] = Default::default(); + + ptrs_clone(&parent.next()[0..], &mut next[0..], cursor.p_l_idx); + ptrs_clone(&node.next()[0..], &mut next[cursor.p_l_idx..], nsize); + ptrs_clone( + &parent.next()[cursor.p_l_idx + 1..], + &mut next[cursor.p_l_idx + nsize..], + psize - (cursor.p_l_idx + 1), + ); + + ufcells_clone(&parent.keys[0..], &mut keys[0..], cursor.p_l_idx); + ufcells_clone( + &node.keys[0..], + &mut keys[cursor.p_l_idx..], + node.key_count(), + ); + ufcells_clone( + &parent.keys[cursor.p_l_idx..], + &mut keys[cursor.p_l_idx + node.key_count()..], + parent.key_count() - cursor.p_l_idx, + ); + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, unsafe { *keys[0].get() }.unwrap()); + ufcells_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + ptrs_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = + Node::internal(true, right_size, unsafe { *keys[left_size].get() }.unwrap()); + ufcells_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + ptrs_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + std::ptr::eq(gparent, &self.entry), + 2, + unsafe { *keys[left_size - 1].get() }.unwrap(), + ); + *new_internal.keys[0].get_mut() = unsafe { *keys[left_size - 1].get() }; + new_internal.next()[0].store(Owned::new(left), Ordering::Relaxed); + new_internal.next()[1].store(Owned::new(right), Ordering::Relaxed); + + // TODO: understand this comment... + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Owned::new(new_internal).into_shared(guard); + gparent.next()[cursor.gp_p_idx].store(new_internal, Ordering::Relaxed); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l) }; + unsafe { guard.defer_destroy(cursor.p) }; + + drop(node_lock); + drop(parent_lock); + drop(gparent_lock); + self.fix_tag_violation(new_internal, guard); + return; + } + } + } + + pub fn remove(&self, key: &K, guard: &Guard) -> Option { + loop { + let (_, cursor) = self.search(key, None, guard); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor, guard) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner<'g>( + &self, + key: &K, + cursor: &Cursor<'g, K, V>, + guard: &'g Guard, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Delete, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(_) => return Err(()), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if unsafe { *node.keys[i].get() } == Some(*key) { + let val = unsafe { *node.values()[i].get() }; + let old_version = node.write_version().load(Ordering::Relaxed); + node.write_version() + .store(old_version + 1, Ordering::Relaxed); + compiler_fence(Ordering::SeqCst); + unsafe { *node.keys[i].get() = None }; + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, old_version, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation(cursor.l, guard); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation<'g>(&self, viol: Shared<'g, Node>, guard: &'g Guard) { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + let viol_node = unsafe { viol.deref() }; + loop { + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || std::ptr::eq(viol_node, &self.entry) + || viol == self.entry.next()[0].load(Ordering::Relaxed, guard) + { + // No degree violation at `viol`. + return; + } + + // Search for `viol`. + let (_, cursor) = self.search(&viol_node.search_key, Some(viol), guard); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !std::ptr::eq(parent, &self.entry) + && cursor.p != self.entry.next()[0].load(Ordering::Relaxed, guard) + { + self.fix_underfull_violation(cursor.p, guard); + continue; + } + + if !std::ptr::eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + let sibling_idx = if cursor.p_l_idx > 0 { + cursor.p_l_idx - 1 + } else { + 1 + }; + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling_sh = parent.next()[sibling_idx].load(Ordering::Relaxed, guard); + let sibling = unsafe { sibling_sh.deref() }; + + // Prevent deadlocks by acquiring left node first. + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let sibling_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let ((left, left_idx, left_slot), (right, right_idx, right_slot)) = + if sibling_idx < cursor.p_l_idx { + ( + (sibling, sibling_idx, sibling_lock_slot), + (node, cursor.p_l_idx, node_lock_slot), + ) + } else { + ( + (node, cursor.p_l_idx, node_lock_slot), + (sibling, sibling_idx, sibling_lock_slot), + ) + }; + + // TODO: maybe we should give a reference to a Pin? + let left_lock = match left.acquire(Operation::Balance, None, &left_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(_) => continue, + }; + if left.marked.load(Ordering::Relaxed) { + continue; + } + let right_lock = match right.acquire(Operation::Balance, None, &right_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(_) => continue, + }; + if right.marked.load(Ordering::Relaxed) { + continue; + } + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return; + } + + let parent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let parent_lock = match parent.acquire(Operation::Balance, None, &parent_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(_) => continue, + }; + if parent.marked.load(Ordering::Relaxed) { + continue; + } + + let gparent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let gparent_lock = match gparent.acquire(Operation::Balance, None, &gparent_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(_) => continue, + }; + if gparent.marked.load(Ordering::Relaxed) { + continue; + } + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight.load(Ordering::Relaxed) + || !node.weight.load(Ordering::Relaxed) + || !node.weight.load(Ordering::Relaxed) + { + drop(left_lock); + drop(right_lock); + drop(parent_lock); + drop(gparent_lock); + self.fix_tag_violation(cursor.p, guard); + self.fix_tag_violation(cursor.l, guard); + self.fix_tag_violation(sibling_sh, guard); + continue; + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!( + parent.weight.load(Ordering::Relaxed) + && node.weight.load(Ordering::Relaxed) + && sibling.weight.load(Ordering::Relaxed) + ); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let (mut key_count, mut next_count) = (0, 0); + let new_node = if left.is_leaf() { + let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); + for i in 0..DEGREE { + let key = some_or!(unsafe { *left.keys[i].get() }, continue); + let value = unsafe { *left.values()[i].get() }; + *new_leaf.keys[key_count].get_mut() = Some(key); + *new_leaf.values_mut()[next_count].get_mut() = value; + key_count += 1; + next_count += 1; + } + debug_assert!(right.is_leaf()); + for i in 0..DEGREE { + let key = some_or!(unsafe { *right.keys[i].get() }, continue); + let value = unsafe { *right.values()[i].get() }; + *new_leaf.keys[key_count].get_mut() = Some(key); + *new_leaf.values_mut()[next_count].get_mut() = value; + key_count += 1; + next_count += 1; + } + new_leaf + } else { + let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); + for i in 0..left.key_count() { + *new_internal.keys[key_count].get_mut() = unsafe { *left.keys[i].get() }; + key_count += 1; + } + *new_internal.keys[key_count].get_mut() = + unsafe { *parent.keys[left_idx].get() }; + key_count += 1; + for i in 0..lsize { + new_internal.next_mut()[next_count] = + Atomic::from(left.next()[i].load(Ordering::Relaxed, guard)); + next_count += 1; + } + debug_assert!(!right.is_leaf()); + for i in 0..right.key_count() { + *new_internal.keys[key_count].get_mut() = unsafe { *right.keys[i].get() }; + key_count += 1; + } + for i in 0..rsize { + new_internal.next_mut()[next_count] = + Atomic::from(right.next()[i].load(Ordering::Relaxed, guard)); + next_count += 1; + } + new_internal + } + .into_shared(guard); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if std::ptr::eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.next()[cursor.gp_p_idx].store(new_node, Ordering::Relaxed); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l); + guard.defer_destroy(cursor.p); + guard.defer_destroy(sibling_sh); + } + + drop(left_lock); + drop(right_lock); + drop(parent_lock); + drop(gparent_lock); + self.fix_underfull_violation(new_node, guard); + return; + } else { + debug_assert!(!std::ptr::eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + *new_parent.keys[i].get_mut() = unsafe { *parent.keys[i].get() }; + } + for i in 0..sibling_idx { + new_parent.next_mut()[i] = + Atomic::from(parent.next()[i].load(Ordering::Relaxed, guard)); + } + for i in left_idx + 1..parent.key_count() { + *new_parent.keys[i - 1].get_mut() = unsafe { *parent.keys[i].get() }; + } + for i in cursor.p_l_idx + 1..psize { + new_parent.next_mut()[i - 1] = + Atomic::from(parent.next()[i].load(Ordering::Relaxed, guard)); + } + + new_parent.next_mut() + [cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 })] = + Atomic::from(new_node); + let new_parent = Owned::new(new_parent).into_shared(guard); + + gparent.next()[cursor.gp_p_idx].store(new_parent, Ordering::Relaxed); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l); + guard.defer_destroy(cursor.p); + guard.defer_destroy(sibling_sh); + } + + drop(left_lock); + drop(right_lock); + drop(parent_lock); + drop(gparent_lock); + + self.fix_underfull_violation(new_node, guard); + self.fix_underfull_violation(new_parent, guard); + return; + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + let mut kv_pairs: [(MaybeUninit, MaybeUninit); 2 * DEGREE] = + [(MaybeUninit::uninit(), MaybeUninit::uninit()); 2 * DEGREE]; + + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let (mut key_count, mut val_count) = (0, 0); + if left.is_leaf() { + debug_assert!(right.is_leaf()); + for i in 0..DEGREE { + let key = some_or!(unsafe { *left.keys[i].get() }, continue); + let val = unsafe { *left.values()[i].get() }; + kv_pairs[key_count].0.write(key); + kv_pairs[val_count].1.write(val); + key_count += 1; + val_count += 1; + } + } else { + for i in 0..left.key_count() { + kv_pairs[key_count] + .0 + .write(unsafe { *left.keys[i].get() }.unwrap()); + key_count += 1; + } + for i in 0..lsize { + kv_pairs[val_count] + .1 + .write(unsafe { *left.values()[i].get() }); + val_count += 1; + } + } + + if !left.is_leaf() { + kv_pairs[key_count] + .0 + .write(unsafe { *parent.keys[left_idx].get() }.unwrap()); + key_count += 1; + } + + if right.is_leaf() { + debug_assert!(left.is_leaf()); + for i in 0..DEGREE { + let key = some_or!(unsafe { *right.keys[i].get() }, continue); + let val = unsafe { *right.values()[i].get() }; + kv_pairs[key_count].0.write(key); + kv_pairs[val_count].1.write(val); + key_count += 1; + val_count += 1; + } + } else { + for i in 0..right.key_count() { + kv_pairs[key_count] + .0 + .write(unsafe { *right.keys[i].get() }.unwrap()); + key_count += 1; + } + for i in 0..rsize { + kv_pairs[val_count] + .1 + .write(unsafe { *right.values()[i].get() }); + val_count += 1; + } + } + + let kv_pairs = + unsafe { transmute::<_, &mut [(K, V)]>(&mut kv_pairs[0..key_count]) }; + if left.is_leaf() { + kv_pairs.sort_by_key(|(k, _)| *k); + } + + (key_count, val_count) = (0, 0); + + let (new_left, pivot) = if left.is_leaf() { + let mut new_leaf = + Owned::new(Node::leaf(true, left_size, kv_pairs[key_count].0)); + for i in 0..left_size { + *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); + *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; + key_count += 1; + val_count += 1; + } + (new_leaf, kv_pairs[key_count].0) + } else { + let mut new_internal = + Owned::new(Node::internal(true, left_size, kv_pairs[key_count].0)); + for i in 0..left_size - 1 { + *new_internal.keys[i].get_mut() = Some(kv_pairs[key_count].0); + key_count += 1; + } + for i in 0..left_size { + *new_internal.values_mut()[i].get_mut() = kv_pairs[val_count].1; + val_count += 1; + } + let pivot = kv_pairs[key_count].0; + key_count += 1; + (new_internal, pivot) + }; + + // Reserve one key for the parent (to go between `new_left` and `new_right`). + + let new_right = if right.is_leaf() { + debug_assert!(left.is_leaf()); + let mut new_leaf = + Owned::new(Node::leaf(true, right_size, kv_pairs[key_count].0)); + for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { + *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); + key_count += 1; + } + for i in 0..right_size { + *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; + val_count += 1; + } + new_leaf + } else { + let mut new_internal = + Owned::new(Node::internal(true, right_size, kv_pairs[key_count].0)); + for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { + *new_internal.keys[i].get_mut() = Some(kv_pairs[key_count].0); + key_count += 1; + } + for i in 0..right_size { + *new_internal.values_mut()[i].get_mut() = kv_pairs[val_count].1; + val_count += 1; + } + new_internal + }; + + let mut new_parent = Owned::new(Node::internal( + parent.weight.load(Ordering::Relaxed), + psize, + parent.search_key, + )); + ufcells_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + ptrs_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.next_mut()[left_idx] = Atomic::from(new_left); + new_parent.next_mut()[right_idx] = Atomic::from(new_right); + *new_parent.keys[left_idx].get_mut() = Some(pivot); + + gparent.next()[cursor.gp_p_idx].store(new_parent, Ordering::SeqCst); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l); + guard.defer_destroy(cursor.p); + guard.defer_destroy(sibling_sh); + } + + return; + } + } } } impl Drop for ElimABTree { fn drop(&mut self) { - todo!() + let mut stack = vec![]; + let guard = unsafe { unprotected() }; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +#[inline] +fn ptrs_clone(src: &[Atomic], dst: &mut [Atomic], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +/// TODO: unsafe? +#[inline] +fn ufcells_clone(src: &[UnsafeCell], dst: &mut [UnsafeCell], len: usize) { + for i in 0..len { + unsafe { *dst[i].get_mut() = *src[i].get() }; + } +} + +#[cfg(test)] +pub mod tests { + extern crate rand; + use super::ElimABTree; + use crossbeam_ebr::pin; + use crossbeam_utils::thread; + use rand::prelude::*; + + const THREADS: i32 = 30; + const ELEMENTS_PER_THREADS: i32 = 1000; + + #[test] + pub fn smoke() { + let map = &ElimABTree::new(); + + thread::scope(|s| { + for t in 0..THREADS { + s.spawn(move |_| { + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert!(map.insert(&i, &i, &pin()).is_ok()); + } + }); + } + }) + .unwrap(); + + thread::scope(|s| { + for t in 0..(THREADS / 2) { + s.spawn(move |_| { + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert_eq!(i, map.remove(&i, &pin()).unwrap()); + } + }); + } + }) + .unwrap(); + + thread::scope(|s| { + for t in (THREADS / 2)..THREADS { + s.spawn(move |_| { + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert_eq!(i, map.search_basic(&i, &pin()).unwrap()); + } + }); + } + }) + .unwrap(); } } From cbba802bdd40dcc71ca7954725f8e2e0a3a48697 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 24 Oct 2024 11:38:56 +0000 Subject: [PATCH 32/84] Add `mask_light` and add some useful functionalities --- smrs/hp-brcu/src/handle.rs | 31 +++++++++++++++++++++++++++++++ smrs/hp-brcu/src/pointers.rs | 13 +++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/smrs/hp-brcu/src/handle.rs b/smrs/hp-brcu/src/handle.rs index 2b67296b..d64f9d0d 100644 --- a/smrs/hp-brcu/src/handle.rs +++ b/smrs/hp-brcu/src/handle.rs @@ -333,6 +333,10 @@ impl CsGuard { } /// Starts a non-crashable section where we can conduct operations with global side-effects. + /// + /// It issues a `SeqCst` fence at the beginning, so that a protected hazard pointers can be + /// dereferenced within this section. If you are not going to use such local pointers, + /// it would be better to use `mask_light` instead, which issues a compiler fence only. /// /// In this section, we do not restart immediately when we receive signals from reclaimers. /// The whole critical section restarts after this `mask` section ends, if a reclaimer sent @@ -354,6 +358,33 @@ impl CsGuard { compiler_fence(Ordering::SeqCst); result } + + /// Starts a non-crashable section where we can conduct operations with global side-effects. + /// + /// It issues a compiler fence at the beginning to prevent unexpected instruction reordering + /// across this region boundary. If you are going to use local pointers that are protected + /// with hazard pointers, see `mask` instead. + /// + /// In this section, we do not restart immediately when we receive signals from reclaimers. + /// The whole critical section restarts after this `mask` section ends, if a reclaimer sent + /// a signal, or we advanced our epoch to reclaim a full local garbage bag. + /// + /// The body may return an arbitrary value, and it will be returned without any modifications. + /// However, it is required to return a *rollback-safe* variable from the body. For example, + /// [`String`] or [`Box`] is dangerous to return as it will be leaked on a crash! On the other + /// hand, [`Copy`] types is likely to be safe as they are totally defined by their bit-wise + /// representations, and have no possibilities to be leaked after an unexpected crash. + #[inline(always)] + pub fn mask_light(&self, body: F) -> R + where + F: FnOnce(&mut RaGuard) -> R, + R: Copy, + { + compiler_fence(Ordering::SeqCst); + let result = self.rb.atomic(|_| body(&mut RaGuard { local: self.local })); + compiler_fence(Ordering::SeqCst); + result + } } impl Handle for CsGuard {} diff --git a/smrs/hp-brcu/src/pointers.rs b/smrs/hp-brcu/src/pointers.rs index 6504ef90..183cffc2 100644 --- a/smrs/hp-brcu/src/pointers.rs +++ b/smrs/hp-brcu/src/pointers.rs @@ -125,6 +125,15 @@ impl Atomic { } } +impl<'g, T> From> for Atomic { + fn from(value: Shared<'g, T>) -> Self { + Self { + link: AtomicUsize::new(value.inner), + _marker: PhantomData, + } + } +} + impl Default for Atomic { fn default() -> Self { Self::null() @@ -231,7 +240,7 @@ impl<'r, T> Shared<'r, T> { /// /// The `self` must be a valid memory location. #[inline] - pub unsafe fn deref(&self) -> &T { + pub unsafe fn deref(&self) -> &'r T { &*decompose_data::(self.inner).0 } @@ -241,7 +250,7 @@ impl<'r, T> Shared<'r, T> { /// /// The `self` must be a valid memory location. #[inline] - pub unsafe fn deref_mut(&mut self) -> &mut T { + pub unsafe fn deref_mut(&mut self) -> &'r mut T { &mut *decompose_data::(self.inner).0 } From 789e3d392d3def5b177dcf561e5a4ae484aee13e Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 24 Oct 2024 11:41:00 +0000 Subject: [PATCH 33/84] Minor code formatting --- smrs/hp-brcu/src/handle.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smrs/hp-brcu/src/handle.rs b/smrs/hp-brcu/src/handle.rs index d64f9d0d..dd8c6bbf 100644 --- a/smrs/hp-brcu/src/handle.rs +++ b/smrs/hp-brcu/src/handle.rs @@ -333,7 +333,7 @@ impl CsGuard { } /// Starts a non-crashable section where we can conduct operations with global side-effects. - /// + /// /// It issues a `SeqCst` fence at the beginning, so that a protected hazard pointers can be /// dereferenced within this section. If you are not going to use such local pointers, /// it would be better to use `mask_light` instead, which issues a compiler fence only. @@ -360,7 +360,7 @@ impl CsGuard { } /// Starts a non-crashable section where we can conduct operations with global side-effects. - /// + /// /// It issues a compiler fence at the beginning to prevent unexpected instruction reordering /// across this region boundary. If you are going to use local pointers that are protected /// with hazard pointers, see `mask` instead. From d79a18bb0a391ea2cf4bc080ef4696d63597ef7d Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 24 Oct 2024 11:42:27 +0000 Subject: [PATCH 34/84] Implement HP-(B)RCU Bonsai Trees --- src/bin/hp-brcu.rs | 5 +- src/bin/hp-rcu.rs | 5 +- src/ds_impl/hp_brcu/bonsai_tree.rs | 812 +++++++++++++++++++++++++++++ src/ds_impl/hp_brcu/mod.rs | 2 + 4 files changed, 822 insertions(+), 2 deletions(-) create mode 100644 src/ds_impl/hp_brcu/bonsai_tree.rs diff --git a/src/bin/hp-brcu.rs b/src/bin/hp-brcu.rs index 36f1040b..511fd25c 100644 --- a/src/bin/hp-brcu.rs +++ b/src/bin/hp-brcu.rs @@ -10,7 +10,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp_brcu::{ - ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -33,6 +33,9 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } _ => panic!("Unsupported(or unimplemented) data structure for HP-BRCU"), }; output.write_record(config, &perf); diff --git a/src/bin/hp-rcu.rs b/src/bin/hp-rcu.rs index c8fea315..a5661bd8 100644 --- a/src/bin/hp-rcu.rs +++ b/src/bin/hp-rcu.rs @@ -10,7 +10,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp_brcu::{ - ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -34,6 +34,9 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } _ => panic!("Unsupported(or unimplemented) data structure for HP-BRCU"), }; output.write_record(config, &perf); diff --git a/src/ds_impl/hp_brcu/bonsai_tree.rs b/src/ds_impl/hp_brcu/bonsai_tree.rs new file mode 100644 index 00000000..2d6fbec1 --- /dev/null +++ b/src/ds_impl/hp_brcu/bonsai_tree.rs @@ -0,0 +1,812 @@ +use std::{cmp, sync::atomic::Ordering}; + +use hp_brcu::{ + Atomic, CsGuard, Owned, RaGuard, RollbackProof, Shared, Shield, Thread, Unprotected, +}; + +use super::concurrent_map::{ConcurrentMap, OutputHolder}; + +static WEIGHT: usize = 2; + +// TODO: optimization from the paper? IBR paper doesn't do that + +bitflags! { + /// TODO + struct Retired: usize { + const RETIRED = 1usize; + } +} + +impl Retired { + fn new(retired: bool) -> Self { + if retired { + Retired::RETIRED + } else { + Retired::empty() + } + } + + fn retired(self) -> bool { + !(self & Retired::RETIRED).is_empty() + } +} + +/// a real node in tree or a wrapper of State node +/// Retired node if Shared ptr of Node has RETIRED tag. +struct Node { + key: K, + value: V, + size: usize, + left: Atomic>, + right: Atomic>, +} + +impl Node +where + K: Ord + Clone, + V: Clone, +{ + fn retired_node<'g>() -> Shared<'g, Self> { + Shared::null().with_tag(Retired::new(true).bits()) + } + + fn is_retired<'g>(node: Shared<'g, Self>) -> bool { + Retired::from_bits_truncate(node.tag()).retired() + } + + fn is_retired_spot<'g>(node: Shared<'g, Self>, guard: &'g CsGuard) -> bool { + if Self::is_retired(node) { + return true; + } + + if let Some(node_ref) = unsafe { node.as_ref() } { + Self::is_retired(node_ref.left.load(Ordering::Acquire, guard)) + || Self::is_retired(node_ref.right.load(Ordering::Acquire, guard)) + } else { + false + } + } + + fn node_size<'g>(node: Shared<'g, Self>, _: &'g CsGuard) -> usize { + debug_assert!(!Self::is_retired(node)); + if let Some(node_ref) = unsafe { node.as_ref() } { + node_ref.size + } else { + 0 + } + } +} + +/// Rollback-proof buffer to memorize retired nodes and newly allocated nodes. +struct RBProofBuf { + /// Nodes that current op wants to remove from the tree. Should be retired if CAS succeeds. + /// (`retire`). If not, ignore. + retired_nodes: Vec>>, + /// Nodes newly constructed by the op. Should be destroyed if CAS fails. (`destroy`) + new_nodes: Vec>>, +} + +impl RBProofBuf +where + K: Ord + Clone, + V: Clone, +{ + fn new() -> Self { + Self { + retired_nodes: Vec::new(), + new_nodes: Vec::new(), + } + } + + /// Destroy the newly created state (self) that lost the race (reclaim_state) + fn abort(&mut self, _: &mut G) { + self.retired_nodes.clear(); + + for node in self.new_nodes.drain(..) { + drop(unsafe { node.into_owned() }); + } + } + + /// Retire the old state replaced by the new_state and the new_state.retired_nodes + fn commit(&mut self, guard: &mut G) { + self.new_nodes.clear(); + + for node in self.retired_nodes.drain(..) { + let node = node.load(Ordering::Relaxed, guard); + unsafe { + node.deref() + .left + .store(Node::retired_node(), Ordering::Release, guard); + node.deref() + .right + .store(Node::retired_node(), Ordering::Release, guard); + guard.retire(node); + } + } + } + + fn retire_node(&mut self, node: Shared>, _: &mut RaGuard) { + self.retired_nodes.push(Atomic::from(node)); + } + + fn add_new_node(&mut self, node: Shared>, _: &mut RaGuard) { + self.new_nodes.push(Atomic::from(node)); + } +} + +pub struct Protectors { + old_root: Shield>, + new_root: Shield>, + found_node: Shield>, +} + +impl OutputHolder for Protectors { + fn default(thread: &mut Thread) -> Self { + Self { + old_root: Shield::null(thread), + new_root: Shield::null(thread), + found_node: Shield::null(thread), + } + } + + fn output(&self) -> &V { + &self.found_node.as_ref().unwrap().value + } +} + +/// Each op creates a new local state and tries to update (CAS) the tree with it. +struct State<'g, K, V> { + root_link: &'g Atomic>, + curr_root: Shared<'g, Node>, + buf: &'g mut RBProofBuf, +} + +impl<'g, K, V> State<'g, K, V> +where + K: Ord + Clone, + V: Clone, +{ + fn new( + root_link: &'g Atomic>, + buf: &'g mut RBProofBuf, + guard: &'g CsGuard, + ) -> Self { + Self { + root_link, + curr_root: root_link.load(Ordering::Acquire, guard), + buf, + } + } + + // TODO get ref of K, V and clone here + fn mk_node( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + if Node::is_retired_spot(left, guard) || Node::is_retired_spot(right, guard) { + return Node::retired_node(); + } + + let left_size = Node::node_size(left, guard); + let right_size = Node::node_size(right, guard); + guard.mask_light(|guard| { + let new_node = Owned::new(Node { + key: key.clone(), + value: value.clone(), + size: left_size + right_size + 1, + left: Atomic::from(left), + right: Atomic::from(right), + }) + .into_shared(); + self.buf.add_new_node(new_node, guard); + new_node + }) + } + + /// Make a new balanced tree from cur (the root of a subtree) and newly constructed left and right subtree + fn mk_balanced( + &mut self, + cur: Shared<'g, Node>, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + if Node::is_retired_spot(cur, guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return Node::retired_node(); + } + + let cur_ref = unsafe { cur.deref() }; + let key = &cur_ref.key; + let value = &cur_ref.value; + + let l_size = Node::node_size(left, guard); + let r_size = Node::node_size(right, guard); + let res = if r_size > 0 + && ((l_size > 0 && r_size > WEIGHT * l_size) || (l_size == 0 && r_size > WEIGHT)) + { + self.mk_balanced_left(left, right, key, value, guard) + } else if l_size > 0 + && ((r_size > 0 && l_size > WEIGHT * r_size) || (r_size == 0 && l_size > WEIGHT)) + { + self.mk_balanced_right(left, right, key, value, guard) + } else { + self.mk_node(left, right, key, value, guard) + }; + guard.mask_light(|guard| self.buf.retire_node(cur, guard)); + res + } + + #[inline] + fn mk_balanced_left( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let right_ref = unsafe { right.deref() }; + let right_left = right_ref.left.load(Ordering::Acquire, guard); + let right_right = right_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(right_left, guard) + || Node::is_retired_spot(right_right, guard) + { + return Node::retired_node(); + } + + if Node::node_size(right_left, guard) < Node::node_size(right_right, guard) { + // single left rotation + return self.single_left(left, right, right_left, right_right, key, value, guard); + } + + // double left rotation + return self.double_left(left, right, right_left, right_right, key, value, guard); + } + + #[inline] + fn single_left( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + right_left: Shared<'g, Node>, + right_right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let right_ref = unsafe { right.deref() }; + let new_left = self.mk_node(left, right_left, key, value, guard); + let res = self.mk_node( + new_left, + right_right, + &right_ref.key, + &right_ref.value, + guard, + ); + guard.mask_light(|guard| self.buf.retire_node(right, guard)); + res + } + + #[inline] + fn double_left( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + right_left: Shared<'g, Node>, + right_right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let right_ref = unsafe { right.deref() }; + let right_left_ref = unsafe { right_left.deref() }; + let right_left_left = right_left_ref.left.load(Ordering::Acquire, guard); + let right_left_right = right_left_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(right_left_left, guard) + || Node::is_retired_spot(right_left_right, guard) + { + return Node::retired_node(); + } + + let new_left = self.mk_node(left, right_left_left, key, value, guard); + let new_right = self.mk_node( + right_left_right, + right_right, + &right_ref.key, + &right_ref.value, + guard, + ); + let res = self.mk_node( + new_left, + new_right, + &right_left_ref.key, + &right_left_ref.value, + guard, + ); + guard.mask_light(|guard| { + self.buf.retire_node(right_left, guard); + self.buf.retire_node(right, guard); + }); + res + } + + #[inline] + fn mk_balanced_right( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let left_ref = unsafe { left.deref() }; + let left_right = left_ref.right.load(Ordering::Acquire, guard); + let left_left = left_ref.left.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left_right, guard) + || Node::is_retired_spot(left_left, guard) + { + return Node::retired_node(); + } + + if Node::node_size(left_right, guard) < Node::node_size(left_left, guard) { + // single right rotation (fig 3) + return self.single_right(left, right, left_right, left_left, key, value, guard); + } + // double right rotation + return self.double_right(left, right, left_right, left_left, key, value, guard); + } + + #[inline] + fn single_right( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + left_right: Shared<'g, Node>, + left_left: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let left_ref = unsafe { left.deref() }; + let new_right = self.mk_node(left_right, right, key, value, guard); + let res = self.mk_node(left_left, new_right, &left_ref.key, &left_ref.value, guard); + guard.mask_light(|guard| self.buf.retire_node(left, guard)); + res + } + + #[inline] + fn double_right( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + left_right: Shared<'g, Node>, + left_left: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let left_ref = unsafe { left.deref() }; + let left_right_ref = unsafe { left_right.deref() }; + let left_right_left = left_right_ref.left.load(Ordering::Acquire, guard); + let left_right_right = left_right_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left_right_left, guard) + || Node::is_retired_spot(left_right_right, guard) + { + return Node::retired_node(); + } + + let new_left = self.mk_node( + left_left, + left_right_left, + &left_ref.key, + &left_ref.value, + guard, + ); + let new_right = self.mk_node(left_right_right, right, key, value, guard); + let res = self.mk_node( + new_left, + new_right, + &left_right_ref.key, + &left_right_ref.value, + guard, + ); + guard.mask_light(|guard| { + self.buf.retire_node(left_right, guard); + self.buf.retire_node(left, guard); + }); + res + } + + #[inline] + fn do_insert( + &mut self, + node: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, bool) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), false); + } + + if node.is_null() { + return ( + self.mk_node(Shared::null(), Shared::null(), key, value, guard), + true, + ); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), false); + } + + match node_ref.key.cmp(key) { + cmp::Ordering::Equal => (node, false), + cmp::Ordering::Less => { + let (new_right, inserted) = self.do_insert(right, key, value, guard); + (self.mk_balanced(node, left, new_right, guard), inserted) + } + cmp::Ordering::Greater => { + let (new_left, inserted) = self.do_insert(left, key, value, guard); + (self.mk_balanced(node, new_left, right, guard), inserted) + } + } + } + + /// Hint: Returns a tuple of (new node, removed node if exists). + #[inline] + fn do_remove( + &mut self, + node: Shared<'g, Node>, + key: &K, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, Option>>) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), None); + } + + if node.is_null() { + return (Shared::null(), None); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), None); + } + + match node_ref.key.cmp(key) { + cmp::Ordering::Equal => { + guard.mask_light(|guard| self.buf.retire_node(node, guard)); + if node_ref.size == 1 { + return (Shared::null(), Some(node)); + } + + if !left.is_null() { + let (new_left, succ) = self.pull_rightmost(left, guard); + return (self.mk_balanced(succ, new_left, right, guard), Some(node)); + } + let (new_right, succ) = self.pull_leftmost(right, guard); + (self.mk_balanced(succ, left, new_right, guard), Some(node)) + } + cmp::Ordering::Less => { + let (new_right, removed) = self.do_remove(right, key, guard); + (self.mk_balanced(node, left, new_right, guard), removed) + } + cmp::Ordering::Greater => { + let (new_left, removed) = self.do_remove(left, key, guard); + (self.mk_balanced(node, new_left, right, guard), removed) + } + } + } + + fn pull_leftmost( + &mut self, + node: Shared<'g, Node>, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, Shared<'g, Node>) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), Node::retired_node()); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), Node::retired_node()); + } + + if !left.is_null() { + let (new_left, succ) = self.pull_leftmost(left, guard); + return (self.mk_balanced(node, new_left, right, guard), succ); + } + // node is the leftmost + let succ = self.mk_node( + Shared::null(), + Shared::null(), + &node_ref.key, + &node_ref.value, + guard, + ); + guard.mask_light(|guard| self.buf.retire_node(node, guard)); + (right, succ) + } + + fn pull_rightmost( + &mut self, + node: Shared<'g, Node>, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, Shared<'g, Node>) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), Node::retired_node()); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), Node::retired_node()); + } + + if !right.is_null() { + let (new_right, succ) = self.pull_rightmost(right, guard); + return (self.mk_balanced(node, left, new_right, guard), succ); + } + // node is the rightmost + let succ = self.mk_node( + Shared::null(), + Shared::null(), + &node_ref.key, + &node_ref.value, + guard, + ); + guard.mask_light(|guard| self.buf.retire_node(node, guard)); + (left, succ) + } + + pub fn check_root(&self, guard: &CsGuard) -> bool { + self.curr_root == self.root_link.load(Ordering::Acquire, guard) + } +} + +pub struct BonsaiTreeMap { + root: Atomic>, +} + +impl Default for BonsaiTreeMap +where + K: Ord + Clone, + V: Clone, +{ + fn default() -> Self { + Self::new() + } +} + +impl BonsaiTreeMap +where + K: Ord + Clone, + V: Clone, +{ + pub fn new() -> Self { + Self { + root: Atomic::null(), + } + } + + pub fn get_inner( + &self, + key: &K, + output: &mut Protectors, + guard: &CsGuard, + ) -> Result { + let mut node = self.root.load(Ordering::Acquire, guard); + while !node.is_null() && !Node::is_retired(node) { + let node_ref = unsafe { node.deref() }; + match key.cmp(&node_ref.key) { + cmp::Ordering::Equal => break, + cmp::Ordering::Less => node = node_ref.left.load(Ordering::Acquire, guard), + cmp::Ordering::Greater => node = node_ref.right.load(Ordering::Acquire, guard), + } + } + if Node::is_retired_spot(node, guard) { + return Err(()); + } else if node.is_null() { + return Ok(false); + } + output.found_node.protect(node); + Ok(true) + } + + pub fn get(&self, key: &K, output: &mut Protectors, handle: &mut Thread) -> bool { + loop { + let result = + unsafe { handle.critical_section(|guard| self.get_inner(key, output, guard)) }; + if let Ok(found) = result { + return found; + } + } + } + + pub fn insert( + &self, + key: K, + value: V, + output: &mut Protectors, + handle: &mut Thread, + ) -> bool { + let mut buf = RBProofBuf::new(); + loop { + let inserted = unsafe { + handle.critical_section(|guard| { + output.found_node.release(); + guard.mask_light(|guard| buf.abort(guard)); + let mut state = State::new(&self.root, &mut buf, guard); + let old_root = state.curr_root; + let (new_node, inserted) = state.do_insert(old_root, &key, &value, guard); + output.old_root.protect(old_root); + output.new_root.protect(new_node); + inserted + }) + }; + + if Node::is_retired(output.new_root.shared()) { + buf.abort(handle); + continue; + } + + if self + .root + .compare_exchange( + output.old_root.shared(), + output.new_root.shared(), + Ordering::AcqRel, + Ordering::Acquire, + handle, + ) + .is_ok() + { + buf.commit(handle); + return inserted; + } + buf.abort(handle); + } + } + + pub fn remove(&self, key: &K, output: &mut Protectors, handle: &mut Thread) -> bool { + let mut buf = RBProofBuf::new(); + loop { + unsafe { + handle.critical_section(|guard| { + output.found_node.release(); + guard.mask_light(|guard| buf.abort(guard)); + let mut state = State::new(&self.root, &mut buf, guard); + let old_root = state.curr_root; + let (new_root, removed) = state.do_remove(old_root, key, guard); + if let Some(removed) = removed { + output.found_node.protect(removed); + } + output.old_root.protect(old_root); + output.new_root.protect(new_root); + }) + } + + if Node::is_retired(output.new_root.shared()) { + buf.abort(handle); + continue; + } + + if self + .root + .compare_exchange( + output.old_root.shared(), + output.new_root.shared(), + Ordering::AcqRel, + Ordering::Acquire, + handle, + ) + .is_ok() + { + buf.commit(handle); + return !output.found_node.is_null(); + } + buf.abort(handle); + } + } +} + +impl Drop for BonsaiTreeMap { + fn drop(&mut self) { + unsafe { + let guard = &Unprotected::new(); + let mut stack = vec![self.root.load(Ordering::Relaxed, guard)]; + + while let Some(mut node) = stack.pop() { + if node.is_null() { + continue; + } + + let node_ref = node.deref_mut(); + + stack.push(node_ref.left.load(Ordering::Relaxed, guard)); + stack.push(node_ref.right.load(Ordering::Relaxed, guard)); + drop(node.into_owned()); + } + } + } +} + +impl ConcurrentMap for BonsaiTreeMap +where + K: Ord + Clone, + V: Clone, +{ + type Output = Protectors; + + fn new() -> Self { + Self::new() + } + + fn get(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.get(key, output, thread) + } + + fn insert(&self, key: K, value: V, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.insert(key, value, output, thread) + } + + fn remove<'domain, 'hp>( + &self, + key: &K, + output: &mut Self::Output, + thread: &mut Thread, + ) -> bool { + self.remove(key, output, thread) + } +} + +#[cfg(test)] +mod tests { + use super::BonsaiTreeMap; + use crate::ds_impl::hp_brcu::concurrent_map; + + #[test] + fn smoke_bonsai_tree() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/hp_brcu/mod.rs b/src/ds_impl/hp_brcu/mod.rs index a3668daa..fb1937d7 100644 --- a/src/ds_impl/hp_brcu/mod.rs +++ b/src/ds_impl/hp_brcu/mod.rs @@ -1,5 +1,6 @@ pub mod concurrent_map; +mod bonsai_tree; mod list; pub mod list_alter; mod michael_hash_map; @@ -7,6 +8,7 @@ mod natarajan_mittal_tree; mod skip_list; pub use self::concurrent_map::ConcurrentMap; +pub use bonsai_tree::BonsaiTreeMap; pub use list::{HHSList, HList, HMList}; pub use michael_hash_map::HashMap; pub use natarajan_mittal_tree::NMTreeMap; From e61654929482c5808a556f5a40da35f75a6533c9 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 24 Oct 2024 13:08:19 +0000 Subject: [PATCH 35/84] Support returning an owned `V` in EBR-based DSs --- src/ds_impl/ebr/bonsai_tree.rs | 8 +-- src/ds_impl/ebr/concurrent_map.rs | 36 ++++++++-- src/ds_impl/ebr/elim_ab_tree.rs | 86 +++++++++--------------- src/ds_impl/ebr/ellen_tree.rs | 8 +-- src/ds_impl/ebr/list.rs | 20 +++--- src/ds_impl/ebr/michael_hash_map.rs | 12 ++-- src/ds_impl/ebr/mod.rs | 1 + src/ds_impl/ebr/natarajan_mittal_tree.rs | 8 +-- src/ds_impl/ebr/skip_list.rs | 8 +-- 9 files changed, 92 insertions(+), 95 deletions(-) diff --git a/src/ds_impl/ebr/bonsai_tree.rs b/src/ds_impl/ebr/bonsai_tree.rs index 7b44c8f0..cf3ac70f 100644 --- a/src/ds_impl/ebr/bonsai_tree.rs +++ b/src/ds_impl/ebr/bonsai_tree.rs @@ -1,6 +1,6 @@ use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::sync::atomic::Ordering; @@ -720,7 +720,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.get(key, guard) } #[inline(always)] @@ -728,7 +728,7 @@ where self.insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -740,6 +740,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/concurrent_map.rs b/src/ds_impl/ebr/concurrent_map.rs index aaefdd96..6ef687a3 100644 --- a/src/ds_impl/ebr/concurrent_map.rs +++ b/src/ds_impl/ebr/concurrent_map.rs @@ -1,24 +1,46 @@ use crossbeam_ebr::Guard; +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { fn new() -> Self; - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V>; + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option>; fn insert(&self, key: K, value: V, guard: &Guard) -> bool; - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V>; + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::pin; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -29,7 +51,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(i, i.to_string(), &pin())); + assert!(map.insert(i, to_value(&i), &pin())); } }); } @@ -44,7 +66,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.remove(&i, &pin()).unwrap()); + assert_eq!(to_value(&i), *map.remove(&i, &pin()).unwrap().output()); } }); } @@ -59,7 +81,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.get(&i, &pin()).unwrap()); + assert_eq!(to_value(&i), *map.get(&i, &pin()).unwrap().output()); } }); } diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 11734d8b..e4804325 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -1,4 +1,4 @@ -// use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared}; use std::cell::UnsafeCell; @@ -1257,64 +1257,38 @@ fn ufcells_clone(src: &[UnsafeCell], dst: &mut [UnsafeCell], len: } } +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self::new() + } + + #[inline(always)] + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { + self.search_basic(key, guard) + } + + #[inline(always)] + fn insert(&self, key: K, value: V, guard: &Guard) -> bool { + self.insert(&key, &value, guard).is_ok() + } + + #[inline(always)] + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { + self.remove(key, guard) + } +} + #[cfg(test)] -pub mod tests { - extern crate rand; +mod tests { use super::ElimABTree; - use crossbeam_ebr::pin; - use crossbeam_utils::thread; - use rand::prelude::*; - - const THREADS: i32 = 30; - const ELEMENTS_PER_THREADS: i32 = 1000; + use crate::ds_impl::ebr::concurrent_map; #[test] - pub fn smoke() { - let map = &ElimABTree::new(); - - thread::scope(|s| { - for t in 0..THREADS { - s.spawn(move |_| { - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert!(map.insert(&i, &i, &pin()).is_ok()); - } - }); - } - }) - .unwrap(); - - thread::scope(|s| { - for t in 0..(THREADS / 2) { - s.spawn(move |_| { - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert_eq!(i, map.remove(&i, &pin()).unwrap()); - } - }); - } - }) - .unwrap(); - - thread::scope(|s| { - for t in (THREADS / 2)..THREADS { - s.spawn(move |_| { - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert_eq!(i, map.search_basic(&i, &pin()).unwrap()); - } - }); - } - }) - .unwrap(); + fn smoke_nm_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); } } diff --git a/src/ds_impl/ebr/ellen_tree.rs b/src/ds_impl/ebr/ellen_tree.rs index b08f97b7..dab607a5 100644 --- a/src/ds_impl/ebr/ellen_tree.rs +++ b/src/ds_impl/ebr/ellen_tree.rs @@ -2,7 +2,7 @@ use std::sync::atomic::Ordering; use crossbeam_ebr::{unprotected, Atomic, CompareExchangeError, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -535,7 +535,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { match self.find(key, guard) { Some(node) => Some(node.value.as_ref().unwrap()), None => None, @@ -548,7 +548,7 @@ where } #[inline(always)] - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.delete(key, guard) } } @@ -560,6 +560,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/list.rs b/src/ds_impl/ebr/list.rs index f93d4bd8..0cae89dd 100644 --- a/src/ds_impl/ebr/list.rs +++ b/src/ds_impl/ebr/list.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; use std::cmp::Ordering::{Equal, Greater, Less}; @@ -398,7 +398,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_get(key, guard) } #[inline(always)] @@ -406,7 +406,7 @@ where self.inner.harris_insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_remove(key, guard) } } @@ -425,7 +425,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_michael_get(key, guard) } #[inline(always)] @@ -433,7 +433,7 @@ where self.inner.harris_michael_insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_michael_remove(key, guard) } } @@ -464,7 +464,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_herlihy_shavit_get(key, guard) } #[inline(always)] @@ -472,7 +472,7 @@ where self.inner.harris_insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_remove(key, guard) } } @@ -484,17 +484,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/ebr/michael_hash_map.rs b/src/ds_impl/ebr/michael_hash_map.rs index 2063c5ac..85e31e97 100644 --- a/src/ds_impl/ebr/michael_hash_map.rs +++ b/src/ds_impl/ebr/michael_hash_map.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::Guard; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -35,7 +35,7 @@ where s.finish() as usize } - pub fn get<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option<&'g V> { + pub fn get<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option + 'g> { let i = Self::hash(k); self.get_bucket(i).get(k, guard) } @@ -45,7 +45,7 @@ where self.get_bucket(i).insert(k, v, guard) } - pub fn remove<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option<&'g V> { + pub fn remove<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option + 'g> { let i = Self::hash(k); self.get_bucket(i).remove(k, guard) } @@ -61,7 +61,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.get(key, guard) } #[inline(always)] @@ -69,7 +69,7 @@ where self.insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -81,6 +81,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/mod.rs b/src/ds_impl/ebr/mod.rs index 78d97a1e..b0794d98 100644 --- a/src/ds_impl/ebr/mod.rs +++ b/src/ds_impl/ebr/mod.rs @@ -2,6 +2,7 @@ pub mod concurrent_map; pub mod bonsai_tree; pub mod double_link; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; diff --git a/src/ds_impl/ebr/natarajan_mittal_tree.rs b/src/ds_impl/ebr/natarajan_mittal_tree.rs index 14f62bd1..b433c5d8 100644 --- a/src/ds_impl/ebr/natarajan_mittal_tree.rs +++ b/src/ds_impl/ebr/natarajan_mittal_tree.rs @@ -1,6 +1,6 @@ use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::sync::atomic::Ordering; @@ -506,7 +506,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.get(key, guard) } #[inline(always)] @@ -514,7 +514,7 @@ where self.insert(key, value, guard).is_ok() } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -526,6 +526,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/skip_list.rs b/src/ds_impl/ebr/skip_list.rs index 790b4440..c5db6193 100644 --- a/src/ds_impl/ebr/skip_list.rs +++ b/src/ds_impl/ebr/skip_list.rs @@ -2,7 +2,7 @@ use std::sync::atomic::{fence, AtomicUsize, Ordering}; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -401,7 +401,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { let cursor = self.find_optimistic(key, guard); cursor.found.map(|node| &node.value) } @@ -412,7 +412,7 @@ where } #[inline(always)] - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -424,6 +424,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } From cc2203e961aa11c64c1a64c2e3ae5234f984fa49 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 24 Oct 2024 13:29:48 +0000 Subject: [PATCH 36/84] Register `ElimAbTree` as a runnable data structure --- src/bin/ebr.rs | 23 +++++++++++------------ src/bin/hp-pp.rs | 1 + src/bin/hp.rs | 1 + src/bin/nr.rs | 1 + src/bin/pebr.rs | 1 + src/config/map.rs | 1 + src/ds_impl/ebr/elim_ab_tree.rs | 6 +++--- src/ds_impl/ebr/mod.rs | 1 + 8 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/bin/ebr.rs b/src/bin/ebr.rs index d09a696a..bd278083 100644 --- a/src/bin/ebr.rs +++ b/src/bin/ebr.rs @@ -12,7 +12,8 @@ use typenum::{Unsigned, U1, U4}; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, OpsPerCs, Perf, DS}; use smr_benchmark::ds_impl::ebr::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -38,14 +39,12 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::, N>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::, N>(config, PrefillStrategy::Random), DS::BonsaiTree => { - // Note: Using the `Random` strategy with the Bonsai tree is unsafe - // because it involves multiple threads with unprotected guards. - // It is safe for many other data structures that don't retire elements - // during insertion, but this is not the case for the Bonsai tree. + // For Bonsai Tree, it would be faster to use a single thread to prefill. bench_map::, N>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::, N>(config, PrefillStrategy::Random), DS::SkipList => bench_map::, N>(config, PrefillStrategy::Decreasing), + DS::ElimAbTree => bench_map::, N>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); @@ -61,6 +60,9 @@ pub enum PrefillStrategy { impl PrefillStrategy { fn prefill + Send + Sync>(self, config: &Config, map: &M) { + // Some data structures (e.g., Bonsai tree, Elim AB-Tree) need SMR's retirement + // functionality even during insertions. + let collector = &crossbeam_ebr::Collector::new(); match self { PrefillStrategy::Random => { let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); @@ -69,17 +71,14 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { - // Safety: We assume that the insert operation does not retire - // any elements. Note that this assumption may not hold for all - // data structures (e.g., Bonsai tree). - let guard = unsafe { crossbeam_ebr::unprotected() }; + let handle = collector.register(); let rng = &mut rand::thread_rng(); let count = config.prefill / threads + if t < config.prefill % threads { 1 } else { 0 }; for _ in 0..count { let key = config.key_dist.sample(rng); let value = key; - map.insert(key, value, guard); + map.insert(key, value, &handle.pin()); } }); } @@ -88,7 +87,7 @@ impl PrefillStrategy { for _ in 0..config.prefill {} } PrefillStrategy::Decreasing => { - let guard = unsafe { crossbeam_ebr::unprotected() }; + let handle = collector.register(); let rng = &mut rand::thread_rng(); let mut keys = Vec::with_capacity(config.prefill); for _ in 0..config.prefill { @@ -97,7 +96,7 @@ impl PrefillStrategy { keys.sort_by(|a, b| b.cmp(a)); for key in keys.drain(..) { let value = key; - map.insert(key, value, guard); + map.insert(key, value, &handle.pin()); } } } diff --git a/src/bin/hp-pp.rs b/src/bin/hp-pp.rs index 3cbfcd87..e80608dd 100644 --- a/src/bin/hp-pp.rs +++ b/src/bin/hp-pp.rs @@ -39,6 +39,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::BonsaiTree => { bench_map::>(config, PrefillStrategy::Decreasing) } + _ => panic!("Unsupported(or unimplemented) data structure for HP++"), }; output.write_record(config, &perf); println!("{}", perf); diff --git a/src/bin/hp.rs b/src/bin/hp.rs index a3e97221..1eb79d58 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -38,6 +38,7 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::NMTree => bench_map::>(config, PrefillStrategy::Random), + _ => panic!("Unsupported(or unimplemented) data structure for HP"), }; output.write_record(config, &perf); println!("{}", perf); diff --git a/src/bin/nr.rs b/src/bin/nr.rs index bd23820a..b132fb37 100644 --- a/src/bin/nr.rs +++ b/src/bin/nr.rs @@ -36,6 +36,7 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), + DS::ElimAbTree => todo!(), }; output.write_record(config, &perf); println!("{}", perf); diff --git a/src/bin/pebr.rs b/src/bin/pebr.rs index 344e004d..4097699e 100644 --- a/src/bin/pebr.rs +++ b/src/bin/pebr.rs @@ -46,6 +46,7 @@ fn bench(config: &Config, output: BenchWriter) { } DS::EFRBTree => bench_map::, N>(config, PrefillStrategy::Random), DS::SkipList => bench_map::, N>(config, PrefillStrategy::Decreasing), + _ => panic!("Unsupported(or unimplemented) data structure for PEBR"), }; output.write_record(config, &perf); println!("{}", perf); diff --git a/src/config/map.rs b/src/config/map.rs index 34351cb3..2a9fc670 100644 --- a/src/config/map.rs +++ b/src/config/map.rs @@ -17,6 +17,7 @@ pub enum DS { BonsaiTree, EFRBTree, SkipList, + ElimAbTree, } pub enum OpsPerCs { diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index e4804325..656640b9 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -336,7 +336,7 @@ struct Cursor<'g, K, V> { l_version: usize, } -struct ElimABTree { +pub struct ElimABTree { entry: Node, } @@ -370,7 +370,7 @@ where node.read_value_version(key).1 } - pub fn search<'g>( + fn search<'g>( &self, key: &K, target: Option>>, @@ -423,7 +423,7 @@ where } } - pub fn insert_inner<'g>( + fn insert_inner<'g>( &self, key: &K, value: &V, diff --git a/src/ds_impl/ebr/mod.rs b/src/ds_impl/ebr/mod.rs index b0794d98..9497097f 100644 --- a/src/ds_impl/ebr/mod.rs +++ b/src/ds_impl/ebr/mod.rs @@ -13,6 +13,7 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; From 634f6131902c508e31eeb66c1f16f7db297c110f Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 24 Oct 2024 13:47:02 +0000 Subject: [PATCH 37/84] Fix a typo in EBR ElimABTree --- src/ds_impl/ebr/elim_ab_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 656640b9..09a14af6 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -917,7 +917,7 @@ where // So, we first check for any weight violations and fix any that we see. if !parent.weight.load(Ordering::Relaxed) || !node.weight.load(Ordering::Relaxed) - || !node.weight.load(Ordering::Relaxed) + || !sibling.weight.load(Ordering::Relaxed) { drop(left_lock); drop(right_lock); From af26a7d8265fd5d4181dcb595f0753984160d104 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 25 Oct 2024 06:05:34 +0000 Subject: [PATCH 38/84] Fix a bug in `fix_underfull_violation` --- src/ds_impl/ebr/elim_ab_tree.rs | 202 +++++++++++++++++--------------- 1 file changed, 109 insertions(+), 93 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 09a14af6..837ee7c8 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -1066,14 +1066,16 @@ where let left_size = size / 2; let right_size = size - left_size; - let mut kv_pairs: [(MaybeUninit, MaybeUninit); 2 * DEGREE] = - [(MaybeUninit::uninit(), MaybeUninit::uninit()); 2 * DEGREE]; + assert!(left.is_leaf() == right.is_leaf()); + + let (new_left, new_right, pivot) = if left.is_leaf() { + let mut kv_pairs: [(MaybeUninit, MaybeUninit); 2 * DEGREE] = + [(MaybeUninit::uninit(), MaybeUninit::uninit()); 2 * DEGREE]; + + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let (mut key_count, mut val_count) = (0, 0); - // Combine the contents of `l` and `s` - // (and one key from `p` if `l` and `s` are internal). - let (mut key_count, mut val_count) = (0, 0); - if left.is_leaf() { - debug_assert!(right.is_leaf()); for i in 0..DEGREE { let key = some_or!(unsafe { *left.keys[i].get() }, continue); let val = unsafe { *left.values()[i].get() }; @@ -1082,30 +1084,7 @@ where key_count += 1; val_count += 1; } - } else { - for i in 0..left.key_count() { - kv_pairs[key_count] - .0 - .write(unsafe { *left.keys[i].get() }.unwrap()); - key_count += 1; - } - for i in 0..lsize { - kv_pairs[val_count] - .1 - .write(unsafe { *left.values()[i].get() }); - val_count += 1; - } - } - if !left.is_leaf() { - kv_pairs[key_count] - .0 - .write(unsafe { *parent.keys[left_idx].get() }.unwrap()); - key_count += 1; - } - - if right.is_leaf() { - debug_assert!(left.is_leaf()); for i in 0..DEGREE { let key = some_or!(unsafe { *right.keys[i].get() }, continue); let val = unsafe { *right.values()[i].get() }; @@ -1114,82 +1093,119 @@ where key_count += 1; val_count += 1; } - } else { - for i in 0..right.key_count() { - kv_pairs[key_count] - .0 - .write(unsafe { *right.keys[i].get() }.unwrap()); - key_count += 1; - } - for i in 0..rsize { - kv_pairs[val_count] - .1 - .write(unsafe { *right.values()[i].get() }); - val_count += 1; - } - } - let kv_pairs = - unsafe { transmute::<_, &mut [(K, V)]>(&mut kv_pairs[0..key_count]) }; - if left.is_leaf() { - kv_pairs.sort_by_key(|(k, _)| *k); - } + let kv_pairs = + unsafe { transmute::<_, &mut [(K, V)]>(&mut kv_pairs[0..key_count]) }; - (key_count, val_count) = (0, 0); + kv_pairs.sort_by_key(|(k, _)| *k); - let (new_left, pivot) = if left.is_leaf() { - let mut new_leaf = - Owned::new(Node::leaf(true, left_size, kv_pairs[key_count].0)); - for i in 0..left_size { - *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); - *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; - key_count += 1; - val_count += 1; - } - (new_leaf, kv_pairs[key_count].0) + (key_count, val_count) = (0, 0); + + let (new_left, pivot) = { + let mut new_leaf = + Owned::new(Node::leaf(true, left_size, kv_pairs[key_count].0)); + for i in 0..left_size { + *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); + *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; + key_count += 1; + val_count += 1; + } + (new_leaf, kv_pairs[key_count].0) + }; + + // Reserve one key for the parent (to go between `new_left` and `new_right`). + + let new_right = { + debug_assert!(left.is_leaf()); + let mut new_leaf = + Owned::new(Node::leaf(true, right_size, kv_pairs[key_count].0)); + for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { + *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); + key_count += 1; + } + for i in 0..right_size { + *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; + val_count += 1; + } + new_leaf + }; + + (new_left, new_right, pivot) } else { - let mut new_internal = - Owned::new(Node::internal(true, left_size, kv_pairs[key_count].0)); - for i in 0..left_size - 1 { - *new_internal.keys[i].get_mut() = Some(kv_pairs[key_count].0); + let mut kn_pairs: [(MaybeUninit, Shared<'g, Node>); 2 * DEGREE] = + [(MaybeUninit::uninit(), Shared::null()); 2 * DEGREE]; + + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let (mut key_count, mut nxt_count) = (0, 0); + + for i in 0..left.key_count() { + kn_pairs[key_count] + .0 + .write(unsafe { *left.keys[i].get() }.unwrap()); key_count += 1; } - for i in 0..left_size { - *new_internal.values_mut()[i].get_mut() = kv_pairs[val_count].1; - val_count += 1; + for i in 0..lsize { + kn_pairs[nxt_count].1 = left.next()[i].load(Ordering::Relaxed, guard); + nxt_count += 1; } - let pivot = kv_pairs[key_count].0; - key_count += 1; - (new_internal, pivot) - }; - // Reserve one key for the parent (to go between `new_left` and `new_right`). + kn_pairs[key_count] + .0 + .write(unsafe { *parent.keys[left_idx].get() }.unwrap()); + key_count += 1; - let new_right = if right.is_leaf() { - debug_assert!(left.is_leaf()); - let mut new_leaf = - Owned::new(Node::leaf(true, right_size, kv_pairs[key_count].0)); - for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { - *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); + for i in 0..right.key_count() { + kn_pairs[key_count] + .0 + .write(unsafe { *right.keys[i].get() }.unwrap()); key_count += 1; } - for i in 0..right_size { - *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; - val_count += 1; + for i in 0..rsize { + kn_pairs[nxt_count].1 = right.next()[i].load(Ordering::Relaxed, guard); + nxt_count += 1; } - new_leaf - } else { - let mut new_internal = - Owned::new(Node::internal(true, right_size, kv_pairs[key_count].0)); - for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { - *new_internal.keys[i].get_mut() = Some(kv_pairs[key_count].0); + let kn_pairs = unsafe { + transmute::<_, &mut [(K, Shared<'g, Node>)]>( + &mut kn_pairs[0..key_count], + ) + }; + + (key_count, nxt_count) = (0, 0); + + let (new_left, pivot) = { + let mut new_internal = + Owned::new(Node::internal(true, left_size, kn_pairs[key_count].0)); + for i in 0..left_size - 1 { + *new_internal.keys[i].get_mut() = Some(kn_pairs[key_count].0); + key_count += 1; + } + for i in 0..left_size { + new_internal.next_mut()[i] = Atomic::from(kn_pairs[nxt_count].1); + nxt_count += 1; + } + let pivot = kn_pairs[key_count].0; key_count += 1; - } - for i in 0..right_size { - *new_internal.values_mut()[i].get_mut() = kv_pairs[val_count].1; - val_count += 1; - } - new_internal + (new_internal, pivot) + }; + + // Reserve one key for the parent (to go between `new_left` and `new_right`). + + let new_right = { + let mut new_internal = + Owned::new(Node::internal(true, right_size, kn_pairs[key_count].0)); + for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { + *new_internal.keys[i].get_mut() = Some(kn_pairs[key_count].0); + key_count += 1; + } + for i in 0..right_size { + new_internal.next_mut()[i] = Atomic::from(kn_pairs[nxt_count].1); + nxt_count += 1; + } + new_internal + }; + + (new_left, new_right, pivot) }; let mut new_parent = Owned::new(Node::internal( From f62c260f85cf5af40538874b6220d8c6b69af605 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 25 Oct 2024 06:38:11 +0000 Subject: [PATCH 39/84] Remove redundant `InsertResult` --- src/ds_impl/ebr/elim_ab_tree.rs | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 837ee7c8..59e96a2f 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -316,12 +316,6 @@ where } } -enum InsertResult { - Ok, - Retry, - Exists(V), -} - struct Cursor<'g, K, V> { l: Shared<'g, Node>, p: Shared<'g, Node>, @@ -409,16 +403,15 @@ where } } - pub fn insert(&self, key: &K, value: &V, guard: &Guard) -> Result<(), V> { + pub fn insert(&self, key: &K, value: &V, guard: &Guard) -> Option { loop { let (_, cursor) = self.search(key, None, guard); if let Some(value) = cursor.val { - return Err(value); + return Some(value); } match self.insert_inner(key, value, &cursor, guard) { - InsertResult::Ok => return Ok(()), - InsertResult::Exists(value) => return Err(value), - InsertResult::Retry => continue, + Ok(result) => return result, + Err(_) => continue, } } } @@ -429,7 +422,7 @@ where value: &V, cursor: &Cursor<'g, K, V>, guard: &'g Guard, - ) -> InsertResult { + ) -> Result, ()> { let node = unsafe { cursor.l.deref() }; let parent = unsafe { cursor.p.deref() }; @@ -439,14 +432,14 @@ where let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { AcqResult::Acquired(lock) => lock, - AcqResult::Eliminated(value) => return InsertResult::Exists(value), + AcqResult::Eliminated(value) => return Ok(Some(value)), }; if node.marked.load(Ordering::SeqCst) { - return InsertResult::Retry; + return Err(()); } for i in 0..DEGREE { if unsafe { *node.keys[i].get() } == Some(*key) { - return InsertResult::Exists(unsafe { *node.values()[i].get() }); + return Ok(Some(unsafe { *node.values()[i].get() })); } } // At this point, we are guaranteed key is not in the node. @@ -473,7 +466,7 @@ where // TODO: do smarter (lifetime) drop(node_lock); - return InsertResult::Ok; + return Ok(None); } unreachable!("Should never happen"); } else { @@ -484,7 +477,7 @@ where parent.marked.load(Ordering::SeqCst), ) { (AcqResult::Acquired(lock), false) => lock, - _ => return InsertResult::Retry, + _ => return Err(()), }; let mut kv_pairs: [MaybeUninit<(K, V)>; DEGREE + 1] = @@ -545,7 +538,7 @@ where unsafe { guard.defer_destroy(cursor.l) }; self.fix_tag_violation(new_internal, guard); - InsertResult::Ok + Ok(None) } } @@ -1289,7 +1282,7 @@ where #[inline(always)] fn insert(&self, key: K, value: V, guard: &Guard) -> bool { - self.insert(&key, &value, guard).is_ok() + self.insert(&key, &value, guard).is_none() } #[inline(always)] From b12cc7d7ed6f85033fe3a7d0d767558d67cdc782 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 25 Oct 2024 07:12:13 +0000 Subject: [PATCH 40/84] Make `weight` immutable --- src/ds_impl/ebr/elim_ab_tree.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 59e96a2f..00e8e516 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -104,7 +104,7 @@ struct Node { search_key: K, lock: AtomicPtr>, size: AtomicUsize, - weight: AtomicBool, + weight: bool, marked: AtomicBool, kind: NodeSpecific, } @@ -182,7 +182,7 @@ where search_key, lock: Default::default(), size: AtomicUsize::new(size), - weight: AtomicBool::new(weight), + weight, marked: AtomicBool::new(false), kind: NodeSpecific::Internal { next: Default::default(), @@ -196,7 +196,7 @@ where search_key, lock: Default::default(), size: AtomicUsize::new(size), - weight: AtomicBool::new(weight), + weight, marked: AtomicBool::new(false), kind: NodeSpecific::Leaf { values: Default::default(), @@ -545,7 +545,7 @@ where fn fix_tag_violation<'g>(&self, viol: Shared<'g, Node>, guard: &'g Guard) { loop { let viol_node = unsafe { viol.deref() }; - if viol_node.weight.load(Ordering::Relaxed) { + if viol_node.weight { return; } @@ -578,7 +578,7 @@ where // We cannot apply this update if p has a weight violation. // So, we check if this is the case, and, if so, try to fix it. - if !parent.weight.load(Ordering::Relaxed) { + if !parent.weight { self.fix_tag_violation(cursor.p, guard); continue; } @@ -908,9 +908,9 @@ where // We can only apply AbsorbSibling or Distribute if there are no // weight violations at `parent`, `node`, or `sibling`. // So, we first check for any weight violations and fix any that we see. - if !parent.weight.load(Ordering::Relaxed) - || !node.weight.load(Ordering::Relaxed) - || !sibling.weight.load(Ordering::Relaxed) + if !parent.weight + || !node.weight + || !sibling.weight { drop(left_lock); drop(right_lock); @@ -924,9 +924,9 @@ where // There are no weight violations at `parent`, `node` or `sibling`. debug_assert!( - parent.weight.load(Ordering::Relaxed) - && node.weight.load(Ordering::Relaxed) - && sibling.weight.load(Ordering::Relaxed) + parent.weight + && node.weight + && sibling.weight ); // l and s are either both leaves or both internal nodes, // because there are no weight violations at these nodes. @@ -1202,7 +1202,7 @@ where }; let mut new_parent = Owned::new(Node::internal( - parent.weight.load(Ordering::Relaxed), + parent.weight, psize, parent.search_key, )); From eed971042c8be01b972d0e12a8614fbc054d4c12 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 25 Oct 2024 09:01:57 +0000 Subject: [PATCH 41/84] Clean up the implementation of ElimABTree --- src/ds_impl/ebr/elim_ab_tree.rs | 223 +++++++++++++------------------- 1 file changed, 92 insertions(+), 131 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 00e8e516..12b5926f 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -1,9 +1,8 @@ use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared}; -use std::cell::UnsafeCell; +use std::cell::{Cell, UnsafeCell}; use std::hint::spin_loop; -use std::mem::{transmute, MaybeUninit}; use std::ptr::{null, null_mut}; use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; @@ -18,7 +17,7 @@ struct MCSLockSlot { next: AtomicPtr, owned: AtomicBool, short_circuit: AtomicBool, - ret: UnsafeCell>, + ret: Cell>, } impl MCSLockSlot @@ -34,7 +33,7 @@ where next: Default::default(), owned: AtomicBool::new(false), short_circuit: AtomicBool::new(false), - ret: UnsafeCell::new(None), + ret: Cell::new(None), } } @@ -59,7 +58,8 @@ impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { fn drop(&mut self) { let slot = unsafe { &*self.slot.get() }; let node = unsafe { &*slot.node }; - let next = if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + debug_assert!(slot.owned.load(Ordering::Acquire)); + let next = if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { next } else { if node @@ -67,8 +67,8 @@ impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { .compare_exchange( self.slot.get(), null_mut(), - Ordering::AcqRel, - Ordering::Acquire, + Ordering::SeqCst, + Ordering::SeqCst, ) .is_ok() { @@ -100,7 +100,7 @@ enum Operation { } struct Node { - keys: [UnsafeCell>; DEGREE], + keys: [Cell>; DEGREE], search_key: K, lock: AtomicPtr>, size: AtomicUsize, @@ -112,7 +112,7 @@ struct Node { // Leaf or Internal node specific data. enum NodeSpecific { Leaf { - values: [UnsafeCell; DEGREE], + values: [Cell>; DEGREE], write_version: AtomicUsize, }, Internal { @@ -142,14 +142,14 @@ impl Node { } } - fn values(&self) -> &[UnsafeCell; DEGREE] { + fn values(&self) -> &[Cell>; DEGREE] { match &self.kind { NodeSpecific::Leaf { values, .. } => values, _ => panic!("No values for an internal node."), } } - fn values_mut(&mut self) -> &mut [UnsafeCell; DEGREE] { + fn values_mut(&mut self) -> &mut [Cell>; DEGREE] { match &mut self.kind { NodeSpecific::Leaf { values, .. } => values, _ => panic!("No values for an internal node."), @@ -207,16 +207,14 @@ where fn child_index(&self, key: &K) -> usize { let mut index = 0; - while index < self.key_count() - && !(key < unsafe { &*self.keys[index].get() }.as_ref().unwrap()) - { + while index < self.key_count() && !(key < &self.keys[index].get().unwrap()) { index += 1; } index } // Search a node for a key repeatedly until we successfully read a consistent version. - fn read_value_version(&self, key: &K) -> (usize, Option, usize) { + fn read_consistent(&self, key: &K) -> (usize, Option) { let NodeSpecific::Leaf { values, write_version, @@ -230,18 +228,14 @@ where version = write_version.load(Ordering::Acquire); } let mut key_index = 0; - while key_index < DEGREE && unsafe { *self.keys[key_index].get() } != Some(*key) { + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { key_index += 1; } - let value = if key_index < DEGREE { - Some(unsafe { *values[key_index].get() }) - } else { - None - }; + let value = values.get(key_index).and_then(|value| value.get()); compiler_fence(Ordering::SeqCst); if version == write_version.load(Ordering::Acquire) { - return (key_index, value, version); + return (key_index, value); } } } @@ -253,17 +247,20 @@ where slot: &'l UnsafeCell>, ) -> AcqResult<'l, K, V> { unsafe { &mut *slot.get() }.init(self, op, key); - let old_tail = self.lock.swap(slot.get(), Ordering::Relaxed); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); let curr = unsafe { &*slot.get() }; if let Some(old_tail) = unsafe { old_tail.as_ref() } { - old_tail.next.store(slot.get(), Ordering::Relaxed); + old_tail.next.store(slot.get(), Ordering::Release); while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) { spin_loop(); } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); if curr.short_circuit.load(Ordering::Relaxed) { - return AcqResult::Eliminated(unsafe { *curr.ret.get() }.unwrap()); + return AcqResult::Eliminated(curr.ret.get().unwrap()); } debug_assert!(curr.owned.load(Ordering::Relaxed)); } else { @@ -278,7 +275,7 @@ where debug_assert!(self.is_leaf()); debug_assert!(slot.op != Operation::Balance); - let stop_node = self.lock.load(Ordering::Relaxed); + let stop_node = self.lock.load(Ordering::Acquire); self.write_version() .store(old_version + 2, Ordering::Release); @@ -304,7 +301,7 @@ where prev_alive = curr; } else { // Shortcircuit curr. - unsafe { (*curr_node.ret.get()) = Some(value) }; + curr_node.ret.set(Some(value)); curr_node.short_circuit.store(true, Ordering::Release); } curr = next; @@ -327,7 +324,6 @@ struct Cursor<'g, K, V> { /// Index of the key in `l`. l_key_idx: usize, val: Option, - l_version: usize, } pub struct ElimABTree { @@ -361,7 +357,7 @@ where let next = next[node.child_index(key)].load(Ordering::Acquire, guard); node = unsafe { next.deref() }; } - node.read_value_version(key).1 + node.read_consistent(key).1 } fn search<'g>( @@ -378,7 +374,6 @@ where p_l_idx: 0, l_key_idx: 0, val: None, - l_version: 0, }; while !unsafe { cursor.l.deref() }.is_leaf() @@ -395,10 +390,9 @@ where if let Some(target) = target { (cursor.l == target, cursor) } else { - let (index, value, version) = unsafe { cursor.l.deref() }.read_value_version(key); + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); cursor.val = value; cursor.l_key_idx = index; - cursor.l_version = version; (value.is_some(), cursor) } } @@ -438,8 +432,8 @@ where return Err(()); } for i in 0..DEGREE { - if unsafe { *node.keys[i].get() } == Some(*key) { - return Ok(Some(unsafe { *node.values()[i].get() })); + if node.keys[i].get() == Some(*key) { + return Ok(Some(node.values()[i].get().unwrap())); } } // At this point, we are guaranteed key is not in the node. @@ -447,7 +441,7 @@ where if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { // We have the capacity to fit this new key. So let's just find an empty slot. for i in 0..DEGREE { - if unsafe { *node.keys[i].get() }.is_some() { + if node.keys[i].get().is_some() { continue; } let old_version = node.write_version().load(Ordering::Relaxed); @@ -455,16 +449,13 @@ where .store(old_version + 1, Ordering::Relaxed); debug_assert!(old_version % 2 == 0); compiler_fence(Ordering::SeqCst); - unsafe { - *node.keys[i].get() = Some(*key); - *node.values()[i].get() = *value; - } + node.keys[i].set(Some(*key)); + node.values()[i].set(Some(*value)); node.size .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); node.elim_key_ops(*value, old_version, &node_lock); - // TODO: do smarter (lifetime) drop(node_lock); return Ok(None); } @@ -480,19 +471,18 @@ where _ => return Err(()), }; - let mut kv_pairs: [MaybeUninit<(K, V)>; DEGREE + 1] = - [MaybeUninit::uninit(); DEGREE + 1]; + let mut kv_pairs: [(K, V); DEGREE + 1] = [Default::default(); DEGREE + 1]; let mut count = 0; for i in 0..DEGREE { - if let Some(key) = unsafe { *node.keys[i].get() } { - let value = unsafe { *node.values()[i].get() }; - kv_pairs[count].write((key, value)); + if let Some(key) = node.keys[i].get() { + let value = node.values()[i].get().unwrap(); + kv_pairs[count] = (key, value); count += 1; } } - kv_pairs[count].write((*key, *value)); + kv_pairs[count] = (*key, *value); count += 1; - let kv_pairs = unsafe { transmute::<_, &mut [(K, V)]>(&mut kv_pairs[0..count]) }; + let kv_pairs = &mut kv_pairs[0..count]; kv_pairs.sort_by_key(|(k, _)| *k); // Create new node(s). @@ -506,16 +496,15 @@ where let mut left = Node::leaf(true, left_size, kv_pairs[0].0); for i in 0..left_size { *left.keys[i].get_mut() = Some(kv_pairs[i].0); - *left.values_mut()[i].get_mut() = kv_pairs[i].1; + *left.values_mut()[i].get_mut() = Some(kv_pairs[i].1); } let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); for i in 0..right_size { *right.keys[i].get_mut() = Some(kv_pairs[i + left_size].0); - *right.values_mut()[i].get_mut() = kv_pairs[i + left_size].1; + *right.values_mut()[i].get_mut() = Some(kv_pairs[i + left_size].1); } - // TODO: understand this comment... // The weight of new internal node `n` will be zero, unless it is the root. // This is because we test `p == entry`, above; in doing this, we are actually // performing Root-Zero at the same time as this Overflow if `n` will become the root. @@ -621,8 +610,7 @@ where // Create new node(s). // The new arrays are small enough to fit in a single node, // so we replace p by a new internal node. - let mut absorber = - Node::internal(true, size, unsafe { &*parent.keys[0].get() }.unwrap()); + let mut absorber = Node::internal(true, size, parent.keys[0].get().unwrap()); ptrs_clone( &parent.next()[0..], @@ -640,13 +628,13 @@ where psize - (cursor.p_l_idx + 1), ); - ufcells_clone(&parent.keys[0..], &mut absorber.keys[0..], cursor.p_l_idx); - ufcells_clone( + cells_clone(&parent.keys[0..], &mut absorber.keys[0..], cursor.p_l_idx); + cells_clone( &node.keys[0..], &mut absorber.keys[cursor.p_l_idx..], node.key_count(), ); - ufcells_clone( + cells_clone( &parent.keys[cursor.p_l_idx..], &mut absorber.keys[cursor.p_l_idx + node.key_count()..], parent.key_count() - cursor.p_l_idx, @@ -665,7 +653,7 @@ where // Merge keys of p and l into one big array (and similarly for children). // We essentially replace the pointer to l with the contents of l. let mut next: [Atomic>; 2 * DEGREE] = Default::default(); - let mut keys: [UnsafeCell>; 2 * DEGREE] = Default::default(); + let mut keys: [Cell>; 2 * DEGREE] = Default::default(); ptrs_clone(&parent.next()[0..], &mut next[0..], cursor.p_l_idx); ptrs_clone(&node.next()[0..], &mut next[cursor.p_l_idx..], nsize); @@ -675,13 +663,13 @@ where psize - (cursor.p_l_idx + 1), ); - ufcells_clone(&parent.keys[0..], &mut keys[0..], cursor.p_l_idx); - ufcells_clone( + cells_clone(&parent.keys[0..], &mut keys[0..], cursor.p_l_idx); + cells_clone( &node.keys[0..], &mut keys[cursor.p_l_idx..], node.key_count(), ); - ufcells_clone( + cells_clone( &parent.keys[cursor.p_l_idx..], &mut keys[cursor.p_l_idx + node.key_count()..], parent.key_count() - cursor.p_l_idx, @@ -697,27 +685,25 @@ where // Create new node(s). let left_size = size / 2; - let mut left = Node::internal(true, left_size, unsafe { *keys[0].get() }.unwrap()); - ufcells_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + cells_clone(&keys[0..], &mut left.keys[0..], left_size - 1); ptrs_clone(&next[0..], &mut left.next_mut()[0..], left_size); let right_size = size - left_size; - let mut right = - Node::internal(true, right_size, unsafe { *keys[left_size].get() }.unwrap()); - ufcells_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + cells_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); ptrs_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. let mut new_internal = Node::internal( std::ptr::eq(gparent, &self.entry), 2, - unsafe { *keys[left_size - 1].get() }.unwrap(), + keys[left_size - 1].get().unwrap(), ); - *new_internal.keys[0].get_mut() = unsafe { *keys[left_size - 1].get() }; + *new_internal.keys[0].get_mut() = keys[left_size - 1].get(); new_internal.next()[0].store(Owned::new(left), Ordering::Relaxed); new_internal.next()[1].store(Owned::new(right), Ordering::Relaxed); - // TODO: understand this comment... // The weight of new internal node `n` will be zero, unless it is the root. // This is because we test `p == entry`, above; in doing this, we are actually // performing Root-Zero at the same time @@ -772,19 +758,19 @@ where AcqResult::Acquired(lock) => lock, AcqResult::Eliminated(_) => return Err(()), }; - if node.marked.load(Ordering::SeqCst) { + if node.marked.load(Ordering::Acquire) { return Err(()); } let new_size = node.size.load(Ordering::Relaxed) - 1; for i in 0..DEGREE { - if unsafe { *node.keys[i].get() } == Some(*key) { - let val = unsafe { *node.values()[i].get() }; + if node.keys[i].get() == Some(*key) { + let val = node.values()[i].get().unwrap(); let old_version = node.write_version().load(Ordering::Relaxed); node.write_version() .store(old_version + 1, Ordering::Relaxed); compiler_fence(Ordering::SeqCst); - unsafe { *node.keys[i].get() = None }; + node.keys[i].set(None); node.size.store(new_size, Ordering::Relaxed); node.elim_key_ops(val, old_version, &node_lock); @@ -865,7 +851,6 @@ where ) }; - // TODO: maybe we should give a reference to a Pin? let left_lock = match left.acquire(Operation::Balance, None, &left_slot) { AcqResult::Acquired(lock) => lock, AcqResult::Eliminated(_) => continue, @@ -908,10 +893,7 @@ where // We can only apply AbsorbSibling or Distribute if there are no // weight violations at `parent`, `node`, or `sibling`. // So, we first check for any weight violations and fix any that we see. - if !parent.weight - || !node.weight - || !sibling.weight - { + if !parent.weight || !node.weight || !sibling.weight { drop(left_lock); drop(right_lock); drop(parent_lock); @@ -923,11 +905,7 @@ where } // There are no weight violations at `parent`, `node` or `sibling`. - debug_assert!( - parent.weight - && node.weight - && sibling.weight - ); + debug_assert!(parent.weight && node.weight && sibling.weight); // l and s are either both leaves or both internal nodes, // because there are no weight violations at these nodes. debug_assert!( @@ -945,8 +923,8 @@ where let new_node = if left.is_leaf() { let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); for i in 0..DEGREE { - let key = some_or!(unsafe { *left.keys[i].get() }, continue); - let value = unsafe { *left.values()[i].get() }; + let key = some_or!(left.keys[i].get(), continue); + let value = left.values()[i].get(); *new_leaf.keys[key_count].get_mut() = Some(key); *new_leaf.values_mut()[next_count].get_mut() = value; key_count += 1; @@ -954,8 +932,8 @@ where } debug_assert!(right.is_leaf()); for i in 0..DEGREE { - let key = some_or!(unsafe { *right.keys[i].get() }, continue); - let value = unsafe { *right.values()[i].get() }; + let key = some_or!(right.keys[i].get(), continue); + let value = right.values()[i].get(); *new_leaf.keys[key_count].get_mut() = Some(key); *new_leaf.values_mut()[next_count].get_mut() = value; key_count += 1; @@ -965,11 +943,10 @@ where } else { let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); for i in 0..left.key_count() { - *new_internal.keys[key_count].get_mut() = unsafe { *left.keys[i].get() }; + *new_internal.keys[key_count].get_mut() = left.keys[i].get(); key_count += 1; } - *new_internal.keys[key_count].get_mut() = - unsafe { *parent.keys[left_idx].get() }; + *new_internal.keys[key_count].get_mut() = parent.keys[left_idx].get(); key_count += 1; for i in 0..lsize { new_internal.next_mut()[next_count] = @@ -978,7 +955,7 @@ where } debug_assert!(!right.is_leaf()); for i in 0..right.key_count() { - *new_internal.keys[key_count].get_mut() = unsafe { *right.keys[i].get() }; + *new_internal.keys[key_count].get_mut() = right.keys[i].get(); key_count += 1; } for i in 0..rsize { @@ -1015,14 +992,14 @@ where debug_assert!(!std::ptr::eq(gparent, &self.entry) || psize > 2); let mut new_parent = Node::internal(true, psize - 1, parent.search_key); for i in 0..left_idx { - *new_parent.keys[i].get_mut() = unsafe { *parent.keys[i].get() }; + *new_parent.keys[i].get_mut() = parent.keys[i].get(); } for i in 0..sibling_idx { new_parent.next_mut()[i] = Atomic::from(parent.next()[i].load(Ordering::Relaxed, guard)); } for i in left_idx + 1..parent.key_count() { - *new_parent.keys[i - 1].get_mut() = unsafe { *parent.keys[i].get() }; + *new_parent.keys[i - 1].get_mut() = parent.keys[i].get(); } for i in cursor.p_l_idx + 1..psize { new_parent.next_mut()[i - 1] = @@ -1062,33 +1039,31 @@ where assert!(left.is_leaf() == right.is_leaf()); let (new_left, new_right, pivot) = if left.is_leaf() { - let mut kv_pairs: [(MaybeUninit, MaybeUninit); 2 * DEGREE] = - [(MaybeUninit::uninit(), MaybeUninit::uninit()); 2 * DEGREE]; + let mut kv_pairs: [(K, V); 2 * DEGREE] = [Default::default(); 2 * DEGREE]; // Combine the contents of `l` and `s` // (and one key from `p` if `l` and `s` are internal). let (mut key_count, mut val_count) = (0, 0); for i in 0..DEGREE { - let key = some_or!(unsafe { *left.keys[i].get() }, continue); - let val = unsafe { *left.values()[i].get() }; - kv_pairs[key_count].0.write(key); - kv_pairs[val_count].1.write(val); + let key = some_or!(left.keys[i].get(), continue); + let val = left.values()[i].get().unwrap(); + kv_pairs[key_count].0 = key; + kv_pairs[val_count].1 = val; key_count += 1; val_count += 1; } for i in 0..DEGREE { - let key = some_or!(unsafe { *right.keys[i].get() }, continue); - let val = unsafe { *right.values()[i].get() }; - kv_pairs[key_count].0.write(key); - kv_pairs[val_count].1.write(val); + let key = some_or!(right.keys[i].get(), continue); + let val = right.values()[i].get().unwrap(); + kv_pairs[key_count].0 = key; + kv_pairs[val_count].1 = val; key_count += 1; val_count += 1; } - let kv_pairs = - unsafe { transmute::<_, &mut [(K, V)]>(&mut kv_pairs[0..key_count]) }; + let kv_pairs = &mut kv_pairs[0..key_count]; kv_pairs.sort_by_key(|(k, _)| *k); @@ -1099,7 +1074,7 @@ where Owned::new(Node::leaf(true, left_size, kv_pairs[key_count].0)); for i in 0..left_size { *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); - *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; + *new_leaf.values_mut()[i].get_mut() = Some(kv_pairs[val_count].1); key_count += 1; val_count += 1; } @@ -1117,7 +1092,7 @@ where key_count += 1; } for i in 0..right_size { - *new_leaf.values_mut()[i].get_mut() = kv_pairs[val_count].1; + *new_leaf.values_mut()[i].get_mut() = Some(kv_pairs[val_count].1); val_count += 1; } new_leaf @@ -1125,17 +1100,15 @@ where (new_left, new_right, pivot) } else { - let mut kn_pairs: [(MaybeUninit, Shared<'g, Node>); 2 * DEGREE] = - [(MaybeUninit::uninit(), Shared::null()); 2 * DEGREE]; + let mut kn_pairs: [(K, Shared<'g, Node>); 2 * DEGREE] = + [Default::default(); 2 * DEGREE]; // Combine the contents of `l` and `s` // (and one key from `p` if `l` and `s` are internal). let (mut key_count, mut nxt_count) = (0, 0); for i in 0..left.key_count() { - kn_pairs[key_count] - .0 - .write(unsafe { *left.keys[i].get() }.unwrap()); + kn_pairs[key_count].0 = left.keys[i].get().unwrap(); key_count += 1; } for i in 0..lsize { @@ -1143,26 +1116,18 @@ where nxt_count += 1; } - kn_pairs[key_count] - .0 - .write(unsafe { *parent.keys[left_idx].get() }.unwrap()); + kn_pairs[key_count].0 = parent.keys[left_idx].get().unwrap(); key_count += 1; for i in 0..right.key_count() { - kn_pairs[key_count] - .0 - .write(unsafe { *right.keys[i].get() }.unwrap()); + kn_pairs[key_count].0 = right.keys[i].get().unwrap(); key_count += 1; } for i in 0..rsize { kn_pairs[nxt_count].1 = right.next()[i].load(Ordering::Relaxed, guard); nxt_count += 1; } - let kn_pairs = unsafe { - transmute::<_, &mut [(K, Shared<'g, Node>)]>( - &mut kn_pairs[0..key_count], - ) - }; + let kn_pairs = &mut kn_pairs[0..key_count]; (key_count, nxt_count) = (0, 0); @@ -1201,12 +1166,9 @@ where (new_left, new_right, pivot) }; - let mut new_parent = Owned::new(Node::internal( - parent.weight, - psize, - parent.search_key, - )); - ufcells_clone( + let mut new_parent = + Owned::new(Node::internal(parent.weight, psize, parent.search_key)); + cells_clone( &parent.keys[0..], &mut new_parent.keys[0..], parent.key_count(), @@ -1258,11 +1220,10 @@ fn ptrs_clone(src: &[Atomic], dst: &mut [Atomic], len: usize) { dst[0..len].clone_from_slice(&src[0..len]); } -/// TODO: unsafe? #[inline] -fn ufcells_clone(src: &[UnsafeCell], dst: &mut [UnsafeCell], len: usize) { +fn cells_clone(src: &[Cell], dst: &mut [Cell], len: usize) { for i in 0..len { - unsafe { *dst[i].get_mut() = *src[i].get() }; + *dst[i].get_mut() = src[i].get(); } } @@ -1297,7 +1258,7 @@ mod tests { use crate::ds_impl::ebr::concurrent_map; #[test] - fn smoke_nm_tree() { + fn smoke_elim_ab_tree() { concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); } } From fc2dcaee536bce8da5ca611326ebbbf7b141e061 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 25 Oct 2024 09:53:01 +0000 Subject: [PATCH 42/84] Revailidate the size of the node --- src/ds_impl/ebr/elim_ab_tree.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 12b5926f..24efc980 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -206,8 +206,9 @@ where } fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); let mut index = 0; - while index < self.key_count() && !(key < &self.keys[index].get().unwrap()) { + while index < key_count && !(key < &self.keys[index].get().unwrap()) { index += 1; } index @@ -758,7 +759,10 @@ where AcqResult::Acquired(lock) => lock, AcqResult::Eliminated(_) => return Err(()), }; - if node.marked.load(Ordering::Acquire) { + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.marked.load(Ordering::Acquire) && node.size.load(Ordering::Acquire) > 0 { return Err(()); } From 209c7ab2e1726fab4b1d7251088811733adbd2be Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 28 Oct 2024 07:45:44 +0000 Subject: [PATCH 43/84] Add `try_acq_val_or` to reduce locking boilerplates --- src/ds_impl/ebr/elim_ab_tree.rs | 117 ++++++++++---------------------- 1 file changed, 34 insertions(+), 83 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 24efc980..c59a2445 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -10,6 +10,19 @@ use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Orde // https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 const DEGREE: usize = 11; +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + struct MCSLockSlot { node: *const Node, op: Operation, @@ -463,14 +476,7 @@ where unreachable!("Should never happen"); } else { // We do not have a room for this key. We need to make new nodes. - let parent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let parent_lock = match ( - parent.acquire(Operation::Insert, None, &parent_lock_slot), - parent.marked.load(Ordering::SeqCst), - ) { - (AcqResult::Acquired(lock), false) => lock, - _ => return Err(()), - }; + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); let mut kv_pairs: [(K, V); DEGREE + 1] = [Default::default(); DEGREE + 1]; let mut count = 0; @@ -573,30 +579,9 @@ where continue; } - let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let node_lock = match ( - node.acquire(Operation::Balance, None, &node_lock_slot), - node.marked.load(Ordering::Relaxed), - ) { - (AcqResult::Acquired(lock), false) => lock, - _ => continue, - }; - let parent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let parent_lock = match ( - parent.acquire(Operation::Balance, None, &parent_lock_slot), - parent.marked.load(Ordering::Relaxed), - ) { - (AcqResult::Acquired(lock), false) => lock, - _ => continue, - }; - let gparent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let gparent_lock = match ( - gparent.acquire(Operation::Balance, None, &gparent_lock_slot), - gparent.marked.load(Ordering::Relaxed), - ) { - (AcqResult::Acquired(lock), false) => lock, - _ => continue, - }; + try_acq_val_or!(node, node_lock, Operation::Balance, None, continue); + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); let psize = parent.size.load(Ordering::Relaxed); let nsize = viol_node.size.load(Ordering::Relaxed); @@ -754,15 +739,17 @@ where debug_assert!(!parent.is_leaf()); debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); - let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let node_lock = match node.acquire(Operation::Delete, Some(*key), &node_lock_slot) { - AcqResult::Acquired(lock) => lock, - AcqResult::Eliminated(_) => return Err(()), - }; + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); // Bug Fix: Added a check to ensure the node size is greater than 0. // This prevents underflows caused by decrementing the size value. // This check is not present in the original code. - if node.marked.load(Ordering::Acquire) && node.size.load(Ordering::Acquire) > 0 { + if node.size.load(Ordering::Acquire) == 0 { return Err(()); } @@ -840,35 +827,14 @@ where let sibling = unsafe { sibling_sh.deref() }; // Prevent deadlocks by acquiring left node first. - let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let sibling_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let ((left, left_idx, left_slot), (right, right_idx, right_slot)) = - if sibling_idx < cursor.p_l_idx { - ( - (sibling, sibling_idx, sibling_lock_slot), - (node, cursor.p_l_idx, node_lock_slot), - ) - } else { - ( - (node, cursor.p_l_idx, node_lock_slot), - (sibling, sibling_idx, sibling_lock_slot), - ) - }; - - let left_lock = match left.acquire(Operation::Balance, None, &left_slot) { - AcqResult::Acquired(lock) => lock, - AcqResult::Eliminated(_) => continue, - }; - if left.marked.load(Ordering::Relaxed) { - continue; - } - let right_lock = match right.acquire(Operation::Balance, None, &right_slot) { - AcqResult::Acquired(lock) => lock, - AcqResult::Eliminated(_) => continue, + let ((left, left_idx), (right, right_idx)) = if sibling_idx < cursor.p_l_idx { + ((sibling, sibling_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, sibling_idx)) }; - if right.marked.load(Ordering::Relaxed) { - continue; - } + + try_acq_val_or!(left, left_lock, Operation::Balance, None, continue); + try_acq_val_or!(right, right_lock, Operation::Balance, None, continue); // Repeat this check, this might have changed while we locked `viol`. if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { @@ -876,23 +842,8 @@ where return; } - let parent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let parent_lock = match parent.acquire(Operation::Balance, None, &parent_lock_slot) { - AcqResult::Acquired(lock) => lock, - AcqResult::Eliminated(_) => continue, - }; - if parent.marked.load(Ordering::Relaxed) { - continue; - } - - let gparent_lock_slot = UnsafeCell::new(MCSLockSlot::new()); - let gparent_lock = match gparent.acquire(Operation::Balance, None, &gparent_lock_slot) { - AcqResult::Acquired(lock) => lock, - AcqResult::Eliminated(_) => continue, - }; - if gparent.marked.load(Ordering::Relaxed) { - continue; - } + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); // We can only apply AbsorbSibling or Distribute if there are no // weight violations at `parent`, `node`, or `sibling`. From b714f9417b1747e12501fe8f9e6cf3639fa7e09a Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 28 Oct 2024 10:46:28 +0000 Subject: [PATCH 44/84] Make ElimAbTree more Rusty --- src/ds_impl/ebr/elim_ab_tree.rs | 578 ++++++++++++++++---------------- 1 file changed, 287 insertions(+), 291 deletions(-) diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index c59a2445..1bb4dffd 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -3,7 +3,7 @@ use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared}; use std::cell::{Cell, UnsafeCell}; use std::hint::spin_loop; -use std::ptr::{null, null_mut}; +use std::ptr::{eq, null, null_mut}; use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; // Copied from the original author's code: @@ -65,6 +65,10 @@ impl<'l, K, V> MCSLockGuard<'l, K, V> { fn new(slot: &'l UnsafeCell>) -> Self { Self { slot } } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } } impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { @@ -72,31 +76,34 @@ impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { let slot = unsafe { &*self.slot.get() }; let node = unsafe { &*slot.node }; debug_assert!(slot.owned.load(Ordering::Acquire)); - let next = if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { - next - } else { - if node - .lock - .compare_exchange( - self.slot.get(), - null_mut(), - Ordering::SeqCst, - Ordering::SeqCst, - ) - .is_ok() - { + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); slot.owned.store(false, Ordering::Release); return; } - loop { - if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { - break next; - } - spin_loop(); - } - }; - next.owned.store(true, Ordering::Release); - slot.owned.store(false, Ordering::Release); + spin_loop(); + } } } @@ -119,11 +126,11 @@ struct Node { size: AtomicUsize, weight: bool, marked: AtomicBool, - kind: NodeSpecific, + kind: NodeKind, } // Leaf or Internal node specific data. -enum NodeSpecific { +enum NodeKind { Leaf { values: [Cell>; DEGREE], write_version: AtomicUsize, @@ -136,50 +143,65 @@ enum NodeSpecific { impl Node { fn is_leaf(&self) -> bool { match &self.kind { - NodeSpecific::Leaf { .. } => true, - NodeSpecific::Internal { .. } => false, + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, } } fn next(&self) -> &[Atomic; DEGREE] { match &self.kind { - NodeSpecific::Internal { next } => next, + NodeKind::Internal { next } => next, _ => panic!("No next pointers for a leaf node."), } } fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { match &mut self.kind { - NodeSpecific::Internal { next } => next, + NodeKind::Internal { next } => next, _ => panic!("No next pointers for a leaf node."), } } - fn values(&self) -> &[Cell>; DEGREE] { - match &self.kind { - NodeSpecific::Leaf { values, .. } => values, - _ => panic!("No values for an internal node."), - } + fn load_next<'g>(&self, index: usize, guard: &'g Guard) -> Shared<'g, Self> { + self.next()[index].load(Ordering::Acquire, guard) } - fn values_mut(&mut self) -> &mut [Cell>; DEGREE] { - match &mut self.kind { - NodeSpecific::Leaf { values, .. } => values, - _ => panic!("No values for an internal node."), - } + fn store_next<'g>(&'g self, index: usize, ptr: impl Pointer, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: impl Pointer) { + self.next_mut()[index] = Atomic::from(ptr.into_usize() as *const Self); } - fn write_version(&self) -> &AtomicUsize { + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { match &self.kind { - NodeSpecific::Leaf { write_version, .. } => write_version, + NodeKind::Leaf { write_version, .. } => write_version, _ => panic!("No write version for an internal node."), } } + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + fn key_count(&self) -> usize { match &self.kind { - NodeSpecific::Leaf { .. } => self.size.load(Ordering::Acquire), - NodeSpecific::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, } } } @@ -189,6 +211,39 @@ where K: PartialOrd + Eq + Default + Copy, V: Default + Copy, { + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + fn internal(weight: bool, size: usize, search_key: K) -> Self { Self { keys: Default::default(), @@ -197,7 +252,7 @@ where size: AtomicUsize::new(size), weight, marked: AtomicBool::new(false), - kind: NodeSpecific::Internal { + kind: NodeKind::Internal { next: Default::default(), }, } @@ -211,7 +266,7 @@ where size: AtomicUsize::new(size), weight, marked: AtomicBool::new(false), - kind: NodeSpecific::Leaf { + kind: NodeKind::Leaf { values: Default::default(), write_version: AtomicUsize::new(0), }, @@ -229,7 +284,7 @@ where // Search a node for a key repeatedly until we successfully read a consistent version. fn read_consistent(&self, key: &K) -> (usize, Option) { - let NodeSpecific::Leaf { + let NodeKind::Leaf { values, write_version, } = &self.kind @@ -283,17 +338,21 @@ where return AcqResult::Acquired(MCSLockGuard::new(slot)); } - fn elim_key_ops<'l>(&'l self, value: V, old_version: usize, guard: &MCSLockGuard<'l, K, V>) { + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { let slot = unsafe { &*guard.slot.get() }; debug_assert!(slot.owned.load(Ordering::Relaxed)); debug_assert!(self.is_leaf()); debug_assert!(slot.op != Operation::Balance); let stop_node = self.lock.load(Ordering::Acquire); - self.write_version() - .store(old_version + 2, Ordering::Release); + drop(wguard); - if std::ptr::eq(stop_node.cast(), slot) { + if eq(stop_node.cast(), slot) { return; } @@ -325,6 +384,51 @@ where .next .store(stop_node, Ordering::Release); } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } } struct Cursor<'g, K, V> { @@ -357,8 +461,8 @@ where pub fn new() -> Self { let left = Node::leaf(true, 0, K::default()); - let entry = Node::internal(true, 1, K::default()); - entry.next()[0].store(Owned::new(left), Ordering::Relaxed); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Owned::new(left)); Self { entry } } @@ -366,8 +470,8 @@ where /// or `None` if nothing is found. Unlike other search methods, it does not return /// any path information, making it slightly faster. pub fn search_basic(&self, key: &K, guard: &Guard) -> Option { - let mut node = unsafe { self.entry.next()[0].load(Ordering::Acquire, guard).deref() }; - while let NodeSpecific::Internal { next } = &node.kind { + let mut node = unsafe { self.entry.load_next(0, guard).deref() }; + while let NodeKind::Internal { next } = &node.kind { let next = next[node.child_index(key)].load(Ordering::Acquire, guard); node = unsafe { next.deref() }; } @@ -381,7 +485,7 @@ where guard: &'g Guard, ) -> (bool, Cursor<'g, K, V>) { let mut cursor = Cursor { - l: self.entry.next()[0].load(Ordering::Relaxed, guard), + l: self.entry.load_next(0, guard), p: unsafe { Shared::from_usize(&self.entry as *const _ as usize) }, gp: Shared::null(), gp_p_idx: 0, @@ -398,7 +502,7 @@ where cursor.p = cursor.l; cursor.gp_p_idx = cursor.p_l_idx; cursor.p_l_idx = l_node.child_index(key); - cursor.l = l_node.next()[cursor.p_l_idx].load(Ordering::Acquire, guard); + cursor.l = l_node.load_next(cursor.p_l_idx, guard); } if let Some(target) = target { @@ -446,8 +550,8 @@ where return Err(()); } for i in 0..DEGREE { - if node.keys[i].get() == Some(*key) { - return Ok(Some(node.values()[i].get().unwrap())); + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); } } // At this point, we are guaranteed key is not in the node. @@ -455,20 +559,16 @@ where if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { // We have the capacity to fit this new key. So let's just find an empty slot. for i in 0..DEGREE { - if node.keys[i].get().is_some() { + if node.get_key(i).is_some() { continue; } - let old_version = node.write_version().load(Ordering::Relaxed); - node.write_version() - .store(old_version + 1, Ordering::Relaxed); - debug_assert!(old_version % 2 == 0); - compiler_fence(Ordering::SeqCst); - node.keys[i].set(Some(*key)); - node.values()[i].set(Some(*value)); + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); node.size .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); - node.elim_key_ops(*value, old_version, &node_lock); + node.elim_key_ops(*value, wguard, &node_lock); drop(node_lock); return Ok(None); @@ -479,17 +579,14 @@ where try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); let mut kv_pairs: [(K, V); DEGREE + 1] = [Default::default(); DEGREE + 1]; - let mut count = 0; + let count = &mut 0; for i in 0..DEGREE { - if let Some(key) = node.keys[i].get() { - let value = node.values()[i].get().unwrap(); - kv_pairs[count] = (key, value); - count += 1; - } + let key = some_or!(node.get_key(i), continue); + let value = node.get_value(i).unwrap(); + kv_pairs[tick(count)] = (key, value); } - kv_pairs[count] = (*key, *value); - count += 1; - let kv_pairs = &mut kv_pairs[0..count]; + kv_pairs[tick(count)] = (*key, *value); + let kv_pairs = &mut kv_pairs[0..*count]; kv_pairs.sort_by_key(|(k, _)| *k); // Create new node(s). @@ -497,40 +594,38 @@ where // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. // The array contents are then split between the two new leaves. - let left_size = count / 2; + let left_size = *count / 2; let right_size = DEGREE + 1 - left_size; let mut left = Node::leaf(true, left_size, kv_pairs[0].0); for i in 0..left_size { - *left.keys[i].get_mut() = Some(kv_pairs[i].0); - *left.values_mut()[i].get_mut() = Some(kv_pairs[i].1); + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); } let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); for i in 0..right_size { - *right.keys[i].get_mut() = Some(kv_pairs[i + left_size].0); - *right.values_mut()[i].get_mut() = Some(kv_pairs[i + left_size].1); + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); } // The weight of new internal node `n` will be zero, unless it is the root. // This is because we test `p == entry`, above; in doing this, we are actually // performing Root-Zero at the same time as this Overflow if `n` will become the root. - let mut internal = - Node::internal(std::ptr::eq(parent, &self.entry), 2, kv_pairs[left_size].0); - *internal.keys[0].get_mut() = Some(kv_pairs[left_size].0); - internal.next()[0].store(Owned::new(left), Ordering::Relaxed); - internal.next()[1].store(Owned::new(right), Ordering::Relaxed); + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Owned::new(left)); + internal.init_next(1, Owned::new(right)); // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain // a node since any update to parent would have deleted node (and hence we would have // returned at the `node.marked` check). let new_internal = Owned::new(internal).into_shared(guard); - parent.next()[cursor.p_l_idx].store(new_internal, Ordering::Relaxed); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); node.marked.store(true, Ordering::Release); - drop(node_lock); // Manually unlock and fix the tag. - drop(parent_lock); + drop((parent_lock, node_lock)); unsafe { guard.defer_destroy(cursor.l) }; self.fix_tag_violation(new_internal, guard); @@ -548,10 +643,7 @@ where // `viol` should be internal because leaves always have weight = 1. debug_assert!(!viol_node.is_leaf()); // `viol` is not the entry or root node because both should always have weight = 1. - debug_assert!( - !std::ptr::eq(viol_node, &self.entry) - && self.entry.next()[0].load(Ordering::Relaxed, guard) != viol - ); + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0, guard) != viol); let (found, cursor) = self.search(&viol_node.search_key, Some(viol), guard); if !found { @@ -566,7 +658,7 @@ where debug_assert!(!parent.is_leaf()); debug_assert!(!gparent.is_leaf()); - if !std::ptr::eq(node, viol_node) { + if !eq(node, viol_node) { // `viol` was replaced by another update. // We hand over responsibility for `viol` to that update. return; @@ -589,6 +681,7 @@ where debug_assert_eq!(nsize, 2); let c = psize + nsize; let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); if size <= Self::ABSORB_THRESHOLD { // Absorb case. @@ -596,37 +689,11 @@ where // Create new node(s). // The new arrays are small enough to fit in a single node, // so we replace p by a new internal node. - let mut absorber = Node::internal(true, size, parent.keys[0].get().unwrap()); - - ptrs_clone( - &parent.next()[0..], - &mut absorber.next_mut()[0..], - cursor.p_l_idx, - ); - ptrs_clone( - &node.next()[0..], - &mut absorber.next_mut()[cursor.p_l_idx..], - nsize, - ); - ptrs_clone( - &parent.next()[cursor.p_l_idx + 1..], - &mut absorber.next_mut()[cursor.p_l_idx + nsize..], - psize - (cursor.p_l_idx + 1), - ); + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); - cells_clone(&parent.keys[0..], &mut absorber.keys[0..], cursor.p_l_idx); - cells_clone( - &node.keys[0..], - &mut absorber.keys[cursor.p_l_idx..], - node.key_count(), - ); - cells_clone( - &parent.keys[cursor.p_l_idx..], - &mut absorber.keys[cursor.p_l_idx + node.key_count()..], - parent.key_count() - cursor.p_l_idx, - ); - - gparent.next()[cursor.gp_p_idx].store(Owned::new(absorber), Ordering::Relaxed); + gparent.store_next(cursor.gp_p_idx, Owned::new(absorber), &gparent_lock); node.marked.store(true, Ordering::Relaxed); parent.marked.store(true, Ordering::Relaxed); @@ -636,31 +703,6 @@ where } else { // Split case. - // Merge keys of p and l into one big array (and similarly for children). - // We essentially replace the pointer to l with the contents of l. - let mut next: [Atomic>; 2 * DEGREE] = Default::default(); - let mut keys: [Cell>; 2 * DEGREE] = Default::default(); - - ptrs_clone(&parent.next()[0..], &mut next[0..], cursor.p_l_idx); - ptrs_clone(&node.next()[0..], &mut next[cursor.p_l_idx..], nsize); - ptrs_clone( - &parent.next()[cursor.p_l_idx + 1..], - &mut next[cursor.p_l_idx + nsize..], - psize - (cursor.p_l_idx + 1), - ); - - cells_clone(&parent.keys[0..], &mut keys[0..], cursor.p_l_idx); - cells_clone( - &node.keys[0..], - &mut keys[cursor.p_l_idx..], - node.key_count(), - ); - cells_clone( - &parent.keys[cursor.p_l_idx..], - &mut keys[cursor.p_l_idx + node.key_count()..], - parent.key_count() - cursor.p_l_idx, - ); - // The new arrays are too big to fit in a single node, // so we replace p by a new internal node and two new children. // @@ -672,23 +714,23 @@ where // Create new node(s). let left_size = size / 2; let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); - cells_clone(&keys[0..], &mut left.keys[0..], left_size - 1); - ptrs_clone(&next[0..], &mut left.next_mut()[0..], left_size); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); let right_size = size - left_size; let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); - cells_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); - ptrs_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. let mut new_internal = Node::internal( - std::ptr::eq(gparent, &self.entry), + eq(gparent, &self.entry), 2, keys[left_size - 1].get().unwrap(), ); - *new_internal.keys[0].get_mut() = keys[left_size - 1].get(); - new_internal.next()[0].store(Owned::new(left), Ordering::Relaxed); - new_internal.next()[1].store(Owned::new(right), Ordering::Relaxed); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Owned::new(left)); + new_internal.init_next(1, Owned::new(right)); // The weight of new internal node `n` will be zero, unless it is the root. // This is because we test `p == entry`, above; in doing this, we are actually @@ -696,16 +738,14 @@ where // as this Overflow if `n` will become the root. let new_internal = Owned::new(new_internal).into_shared(guard); - gparent.next()[cursor.gp_p_idx].store(new_internal, Ordering::Relaxed); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); node.marked.store(true, Ordering::Relaxed); parent.marked.store(true, Ordering::Relaxed); unsafe { guard.defer_destroy(cursor.l) }; unsafe { guard.defer_destroy(cursor.p) }; - drop(node_lock); - drop(parent_lock); - drop(gparent_lock); + drop((node_lock, parent_lock, gparent_lock)); self.fix_tag_violation(new_internal, guard); return; } @@ -755,16 +795,13 @@ where let new_size = node.size.load(Ordering::Relaxed) - 1; for i in 0..DEGREE { - if node.keys[i].get() == Some(*key) { - let val = node.values()[i].get().unwrap(); - let old_version = node.write_version().load(Ordering::Relaxed); - node.write_version() - .store(old_version + 1, Ordering::Relaxed); - compiler_fence(Ordering::SeqCst); - node.keys[i].set(None); + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); node.size.store(new_size, Ordering::Relaxed); - node.elim_key_ops(val, old_version, &node_lock); + node.elim_key_ops(val, wguard, &node_lock); if new_size == Self::UNDERFULL_THRESHOLD - 1 { drop(node_lock); @@ -785,8 +822,8 @@ where // "be turned into" the root. The root is only created by the root absorb // operation below, so a node that is not the root will never become the root. if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD - || std::ptr::eq(viol_node, &self.entry) - || viol == self.entry.next()[0].load(Ordering::Relaxed, guard) + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0, guard) { // No degree violation at `viol`. return; @@ -802,14 +839,14 @@ where let gparent = unsafe { cursor.gp.deref() }; if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD - && !std::ptr::eq(parent, &self.entry) - && cursor.p != self.entry.next()[0].load(Ordering::Relaxed, guard) + && !eq(parent, &self.entry) + && cursor.p != self.entry.load_next(0, guard) { self.fix_underfull_violation(cursor.p, guard); continue; } - if !std::ptr::eq(node, viol_node) { + if !eq(node, viol_node) { // `viol` was replaced by another update. // We hand over responsibility for `viol` to that update. return; @@ -823,7 +860,7 @@ where // Don't need a lock on parent here because if the pointer to sibling changes // to a different node after this, sibling will be marked // (Invariant: when a pointer switches away from a node, the node is marked) - let sibling_sh = parent.next()[sibling_idx].load(Ordering::Relaxed, guard); + let sibling_sh = parent.load_next(sibling_idx, guard); let sibling = unsafe { sibling_sh.deref() }; // Prevent deadlocks by acquiring left node first. @@ -849,10 +886,7 @@ where // weight violations at `parent`, `node`, or `sibling`. // So, we first check for any weight violations and fix any that we see. if !parent.weight || !node.weight || !sibling.weight { - drop(left_lock); - drop(right_lock); - drop(parent_lock); - drop(gparent_lock); + drop((left_lock, right_lock, parent_lock, gparent_lock)); self.fix_tag_violation(cursor.p, guard); self.fix_tag_violation(cursor.l, guard); self.fix_tag_violation(sibling_sh, guard); @@ -874,49 +908,38 @@ where if size < 2 * Self::UNDERFULL_THRESHOLD { // AbsorbSibling - let (mut key_count, mut next_count) = (0, 0); + let (key_count, next_count) = (&mut 0, &mut 0); let new_node = if left.is_leaf() { let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); for i in 0..DEGREE { - let key = some_or!(left.keys[i].get(), continue); - let value = left.values()[i].get(); - *new_leaf.keys[key_count].get_mut() = Some(key); - *new_leaf.values_mut()[next_count].get_mut() = value; - key_count += 1; - next_count += 1; + let key = some_or!(left.get_key(i), continue); + let value = left.get_value(i).unwrap(); + new_leaf.init_key(tick(key_count), Some(key)); + new_leaf.init_value(tick(next_count), value); } debug_assert!(right.is_leaf()); for i in 0..DEGREE { - let key = some_or!(right.keys[i].get(), continue); - let value = right.values()[i].get(); - *new_leaf.keys[key_count].get_mut() = Some(key); - *new_leaf.values_mut()[next_count].get_mut() = value; - key_count += 1; - next_count += 1; + let key = some_or!(right.get_key(i), continue); + let value = right.get_value(i).unwrap(); + new_leaf.init_key(tick(key_count), Some(key)); + new_leaf.init_value(tick(next_count), value); } new_leaf } else { let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); for i in 0..left.key_count() { - *new_internal.keys[key_count].get_mut() = left.keys[i].get(); - key_count += 1; + new_internal.init_key(tick(key_count), left.get_key(i)); } - *new_internal.keys[key_count].get_mut() = parent.keys[left_idx].get(); - key_count += 1; + new_internal.init_key(tick(key_count), parent.get_key(left_idx)); for i in 0..lsize { - new_internal.next_mut()[next_count] = - Atomic::from(left.next()[i].load(Ordering::Relaxed, guard)); - next_count += 1; + new_internal.init_next(tick(next_count), left.load_next(i, guard)); } debug_assert!(!right.is_leaf()); for i in 0..right.key_count() { - *new_internal.keys[key_count].get_mut() = right.keys[i].get(); - key_count += 1; + new_internal.init_key(tick(key_count), right.get_key(i)); } for i in 0..rsize { - new_internal.next_mut()[next_count] = - Atomic::from(right.next()[i].load(Ordering::Relaxed, guard)); - next_count += 1; + new_internal.init_next(tick(next_count), right.load_next(i, guard)); } new_internal } @@ -924,9 +947,9 @@ where // Now, we atomically replace `p` and its children with the new nodes. // If appropriate, we perform RootAbsorb at the same time. - if std::ptr::eq(gparent, &self.entry) && psize == 2 { + if eq(gparent, &self.entry) && psize == 2 { debug_assert!(cursor.gp_p_idx == 0); - gparent.next()[cursor.gp_p_idx].store(new_node, Ordering::Relaxed); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); node.marked.store(true, Ordering::Relaxed); parent.marked.store(true, Ordering::Relaxed); sibling.marked.store(true, Ordering::Relaxed); @@ -937,36 +960,32 @@ where guard.defer_destroy(sibling_sh); } - drop(left_lock); - drop(right_lock); - drop(parent_lock); - drop(gparent_lock); + drop((left_lock, right_lock, parent_lock, gparent_lock)); self.fix_underfull_violation(new_node, guard); return; } else { - debug_assert!(!std::ptr::eq(gparent, &self.entry) || psize > 2); + debug_assert!(!eq(gparent, &self.entry) || psize > 2); let mut new_parent = Node::internal(true, psize - 1, parent.search_key); for i in 0..left_idx { - *new_parent.keys[i].get_mut() = parent.keys[i].get(); + new_parent.init_key(i, parent.get_key(i)); } for i in 0..sibling_idx { - new_parent.next_mut()[i] = - Atomic::from(parent.next()[i].load(Ordering::Relaxed, guard)); + new_parent.init_next(i, parent.load_next(i, guard)); } for i in left_idx + 1..parent.key_count() { - *new_parent.keys[i - 1].get_mut() = parent.keys[i].get(); + new_parent.init_key(i - 1, parent.get_key(i)); } for i in cursor.p_l_idx + 1..psize { - new_parent.next_mut()[i - 1] = - Atomic::from(parent.next()[i].load(Ordering::Relaxed, guard)); + new_parent.init_next(i - 1, parent.load_next(i, guard)); } - new_parent.next_mut() - [cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 })] = - Atomic::from(new_node); + new_parent.init_next( + cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 }), + new_node, + ); let new_parent = Owned::new(new_parent).into_shared(guard); - gparent.next()[cursor.gp_p_idx].store(new_parent, Ordering::Relaxed); + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); node.marked.store(true, Ordering::Relaxed); parent.marked.store(true, Ordering::Relaxed); sibling.marked.store(true, Ordering::Relaxed); @@ -977,11 +996,7 @@ where guard.defer_destroy(sibling_sh); } - drop(left_lock); - drop(right_lock); - drop(parent_lock); - drop(gparent_lock); - + drop((left_lock, right_lock, parent_lock, gparent_lock)); self.fix_underfull_violation(new_node, guard); self.fix_underfull_violation(new_parent, guard); return; @@ -998,42 +1013,34 @@ where // Combine the contents of `l` and `s` // (and one key from `p` if `l` and `s` are internal). - let (mut key_count, mut val_count) = (0, 0); + let (key_count, val_count) = (&mut 0, &mut 0); for i in 0..DEGREE { - let key = some_or!(left.keys[i].get(), continue); - let val = left.values()[i].get().unwrap(); - kv_pairs[key_count].0 = key; - kv_pairs[val_count].1 = val; - key_count += 1; - val_count += 1; + let key = some_or!(left.get_key(i), continue); + let val = left.get_value(i).unwrap(); + kv_pairs[tick(key_count)].0 = key; + kv_pairs[tick(val_count)].1 = val; } for i in 0..DEGREE { - let key = some_or!(right.keys[i].get(), continue); - let val = right.values()[i].get().unwrap(); - kv_pairs[key_count].0 = key; - kv_pairs[val_count].1 = val; - key_count += 1; - val_count += 1; + let key = some_or!(right.get_key(i), continue); + let val = right.get_value(i).unwrap(); + kv_pairs[tick(key_count)].0 = key; + kv_pairs[tick(val_count)].1 = val; } - let kv_pairs = &mut kv_pairs[0..key_count]; - + let kv_pairs = &mut kv_pairs[0..*key_count]; kv_pairs.sort_by_key(|(k, _)| *k); - - (key_count, val_count) = (0, 0); + (*key_count, *val_count) = (0, 0); let (new_left, pivot) = { let mut new_leaf = - Owned::new(Node::leaf(true, left_size, kv_pairs[key_count].0)); + Owned::new(Node::leaf(true, left_size, kv_pairs[*key_count].0)); for i in 0..left_size { - *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); - *new_leaf.values_mut()[i].get_mut() = Some(kv_pairs[val_count].1); - key_count += 1; - val_count += 1; + new_leaf.init_key(i, Some(kv_pairs[tick(key_count)].0)); + new_leaf.init_value(i, kv_pairs[tick(val_count)].1); } - (new_leaf, kv_pairs[key_count].0) + (new_leaf, kv_pairs[*key_count].0) }; // Reserve one key for the parent (to go between `new_left` and `new_right`). @@ -1041,14 +1048,12 @@ where let new_right = { debug_assert!(left.is_leaf()); let mut new_leaf = - Owned::new(Node::leaf(true, right_size, kv_pairs[key_count].0)); + Owned::new(Node::leaf(true, right_size, kv_pairs[*key_count].0)); for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { - *new_leaf.keys[i].get_mut() = Some(kv_pairs[key_count].0); - key_count += 1; + new_leaf.init_key(i, Some(kv_pairs[tick(key_count)].0)); } for i in 0..right_size { - *new_leaf.values_mut()[i].get_mut() = Some(kv_pairs[val_count].1); - val_count += 1; + new_leaf.init_value(i, kv_pairs[tick(val_count)].1); } new_leaf }; @@ -1060,45 +1065,37 @@ where // Combine the contents of `l` and `s` // (and one key from `p` if `l` and `s` are internal). - let (mut key_count, mut nxt_count) = (0, 0); + let (key_count, nxt_count) = (&mut 0, &mut 0); for i in 0..left.key_count() { - kn_pairs[key_count].0 = left.keys[i].get().unwrap(); - key_count += 1; + kn_pairs[tick(key_count)].0 = left.get_key(i).unwrap(); } for i in 0..lsize { - kn_pairs[nxt_count].1 = left.next()[i].load(Ordering::Relaxed, guard); - nxt_count += 1; + kn_pairs[tick(nxt_count)].1 = left.load_next(i, guard); } - kn_pairs[key_count].0 = parent.keys[left_idx].get().unwrap(); - key_count += 1; + kn_pairs[tick(key_count)].0 = parent.get_key(left_idx).unwrap(); for i in 0..right.key_count() { - kn_pairs[key_count].0 = right.keys[i].get().unwrap(); - key_count += 1; + kn_pairs[tick(key_count)].0 = right.get_key(i).unwrap(); } for i in 0..rsize { - kn_pairs[nxt_count].1 = right.next()[i].load(Ordering::Relaxed, guard); - nxt_count += 1; + kn_pairs[tick(nxt_count)].1 = right.load_next(i, guard); } - let kn_pairs = &mut kn_pairs[0..key_count]; + let kn_pairs = &mut kn_pairs[0..*key_count]; - (key_count, nxt_count) = (0, 0); + (*key_count, *nxt_count) = (0, 0); let (new_left, pivot) = { let mut new_internal = - Owned::new(Node::internal(true, left_size, kn_pairs[key_count].0)); + Owned::new(Node::internal(true, left_size, kn_pairs[*key_count].0)); for i in 0..left_size - 1 { - *new_internal.keys[i].get_mut() = Some(kn_pairs[key_count].0); - key_count += 1; + new_internal.init_key(i, Some(kn_pairs[tick(key_count)].0)); } for i in 0..left_size { - new_internal.next_mut()[i] = Atomic::from(kn_pairs[nxt_count].1); - nxt_count += 1; + new_internal.init_next(i, kn_pairs[tick(nxt_count)].1); } - let pivot = kn_pairs[key_count].0; - key_count += 1; + let pivot = kn_pairs[tick(key_count)].0; (new_internal, pivot) }; @@ -1106,14 +1103,12 @@ where let new_right = { let mut new_internal = - Owned::new(Node::internal(true, right_size, kn_pairs[key_count].0)); + Owned::new(Node::internal(true, right_size, kn_pairs[*key_count].0)); for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { - *new_internal.keys[i].get_mut() = Some(kn_pairs[key_count].0); - key_count += 1; + new_internal.init_key(i, Some(kn_pairs[tick(key_count)].0)); } for i in 0..right_size { - new_internal.next_mut()[i] = Atomic::from(kn_pairs[nxt_count].1); - nxt_count += 1; + new_internal.init_next(i, kn_pairs[tick(nxt_count)].1); } new_internal }; @@ -1123,17 +1118,17 @@ where let mut new_parent = Owned::new(Node::internal(parent.weight, psize, parent.search_key)); - cells_clone( + slice_clone( &parent.keys[0..], &mut new_parent.keys[0..], parent.key_count(), ); - ptrs_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); - new_parent.next_mut()[left_idx] = Atomic::from(new_left); - new_parent.next_mut()[right_idx] = Atomic::from(new_right); - *new_parent.keys[left_idx].get_mut() = Some(pivot); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, new_left); + new_parent.init_next(right_idx, new_right); + new_parent.init_key(left_idx, Some(pivot)); - gparent.next()[cursor.gp_p_idx].store(new_parent, Ordering::SeqCst); + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); node.marked.store(true, Ordering::Relaxed); parent.marked.store(true, Ordering::Relaxed); sibling.marked.store(true, Ordering::Relaxed); @@ -1170,16 +1165,17 @@ impl Drop for ElimABTree { } } +/// Equivalent to `x++`. #[inline] -fn ptrs_clone(src: &[Atomic], dst: &mut [Atomic], len: usize) { - dst[0..len].clone_from_slice(&src[0..len]); +fn tick(counter: &mut usize) -> usize { + let val = *counter; + *counter += 1; + return val; } #[inline] -fn cells_clone(src: &[Cell], dst: &mut [Cell], len: usize) { - for i in 0..len { - *dst[i].get_mut() = src[i].get(); - } +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); } impl ConcurrentMap for ElimABTree From 41f12a7b1e1e33639308c561537d7917a7429fcd Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 29 Oct 2024 08:09:30 +0000 Subject: [PATCH 45/84] Make ElimAbTree more Rusty with iterators --- Cargo.lock | 7 +- Cargo.toml | 1 + src/ds_impl/ebr/elim_ab_tree.rs | 222 +++++++++++++++----------------- 3 files changed, 111 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7877a97c..f4421d9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "atomic" @@ -369,7 +369,7 @@ dependencies = [ name = "hp-brcu" version = "0.1.0" dependencies = [ - "arrayvec 0.7.4", + "arrayvec 0.7.6", "atomic", "bitflags 2.5.0", "cfg-if 1.0.0", @@ -807,6 +807,7 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" name = "smr-benchmark" version = "0.1.0" dependencies = [ + "arrayvec 0.7.6", "bitflags 2.5.0", "cdrc", "cfg-if 1.0.0", diff --git a/Cargo.toml b/Cargo.toml index 47c77721..17378bf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ csv = "1.3.0" rand = "0.8" typenum = "1.17" num = "0.4.3" +arrayvec = "0.7.6" hp_pp = { path = "./smrs/hp-pp" } nbr = { path = "./smrs/nbr" } cdrc = { path = "./smrs/cdrc" } diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs index 1bb4dffd..02e0c05c 100644 --- a/src/ds_impl/ebr/elim_ab_tree.rs +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -1,8 +1,10 @@ use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use arrayvec::ArrayVec; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared}; use std::cell::{Cell, UnsafeCell}; use std::hint::spin_loop; +use std::iter::once; use std::ptr::{eq, null, null_mut}; use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; @@ -123,6 +125,9 @@ struct Node { keys: [Cell>; DEGREE], search_key: K, lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). size: AtomicUsize, weight: bool, marked: AtomicBool, @@ -418,6 +423,42 @@ where (next, keys) } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + guard: &'g Guard, + ) -> impl Iterator, Shared<'g, Self>)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i, guard))) + .chain(once((None, self.load_next(self.key_count(), guard)))) + } } struct WriteGuard<'g, K, V> { @@ -578,15 +619,10 @@ where // We do not have a room for this key. We need to make new nodes. try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); - let mut kv_pairs: [(K, V); DEGREE + 1] = [Default::default(); DEGREE + 1]; - let count = &mut 0; - for i in 0..DEGREE { - let key = some_or!(node.get_key(i), continue); - let value = node.get_value(i).unwrap(); - kv_pairs[tick(count)] = (key, value); - } - kv_pairs[tick(count)] = (*key, *value); - let kv_pairs = &mut kv_pairs[0..*count]; + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); kv_pairs.sort_by_key(|(k, _)| *k); // Create new node(s). @@ -594,7 +630,7 @@ where // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. // The array contents are then split between the two new leaves. - let left_size = *count / 2; + let left_size = kv_pairs.len() / 2; let right_size = DEGREE + 1 - left_size; let mut left = Node::leaf(true, left_size, kv_pairs[0].0); @@ -908,38 +944,30 @@ where if size < 2 * Self::UNDERFULL_THRESHOLD { // AbsorbSibling - let (key_count, next_count) = (&mut 0, &mut 0); let new_node = if left.is_leaf() { - let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); - for i in 0..DEGREE { - let key = some_or!(left.get_key(i), continue); - let value = left.get_value(i).unwrap(); - new_leaf.init_key(tick(key_count), Some(key)); - new_leaf.init_value(tick(next_count), value); - } debug_assert!(right.is_leaf()); - for i in 0..DEGREE { - let key = some_or!(right.get_key(i), continue); - let value = right.get_value(i).unwrap(); - new_leaf.init_key(tick(key_count), Some(key)); - new_leaf.init_value(tick(next_count), value); + let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); } new_leaf } else { - let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); - for i in 0..left.key_count() { - new_internal.init_key(tick(key_count), left.get_key(i)); - } - new_internal.init_key(tick(key_count), parent.get_key(left_idx)); - for i in 0..lsize { - new_internal.init_next(tick(next_count), left.load_next(i, guard)); - } debug_assert!(!right.is_leaf()); - for i in 0..right.key_count() { - new_internal.init_key(tick(key_count), right.get_key(i)); - } - for i in 0..rsize { - new_internal.init_next(tick(next_count), right.load_next(i, guard)); + let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); } new_internal } @@ -1008,111 +1036,80 @@ where assert!(left.is_leaf() == right.is_leaf()); + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). let (new_left, new_right, pivot) = if left.is_leaf() { - let mut kv_pairs: [(K, V); 2 * DEGREE] = [Default::default(); 2 * DEGREE]; - - // Combine the contents of `l` and `s` - // (and one key from `p` if `l` and `s` are internal). - let (key_count, val_count) = (&mut 0, &mut 0); - - for i in 0..DEGREE { - let key = some_or!(left.get_key(i), continue); - let val = left.get_value(i).unwrap(); - kv_pairs[tick(key_count)].0 = key; - kv_pairs[tick(val_count)].1 = val; - } - - for i in 0..DEGREE { - let key = some_or!(right.get_key(i), continue); - let val = right.get_value(i).unwrap(); - kv_pairs[tick(key_count)].0 = key; - kv_pairs[tick(val_count)].1 = val; - } - - let kv_pairs = &mut kv_pairs[0..*key_count]; + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); kv_pairs.sort_by_key(|(k, _)| *k); - (*key_count, *val_count) = (0, 0); + let mut kv_iter = kv_pairs.iter().copied(); - let (new_left, pivot) = { + let new_left = { let mut new_leaf = - Owned::new(Node::leaf(true, left_size, kv_pairs[*key_count].0)); + Owned::new(Node::leaf(true, left_size, Default::default())); for i in 0..left_size { - new_leaf.init_key(i, Some(kv_pairs[tick(key_count)].0)); - new_leaf.init_value(i, kv_pairs[tick(val_count)].1); + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); } - (new_leaf, kv_pairs[*key_count].0) + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf }; - // Reserve one key for the parent (to go between `new_left` and `new_right`). - - let new_right = { + let (new_right, pivot) = { debug_assert!(left.is_leaf()); let mut new_leaf = - Owned::new(Node::leaf(true, right_size, kv_pairs[*key_count].0)); - for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { - new_leaf.init_key(i, Some(kv_pairs[tick(key_count)].0)); - } + Owned::new(Node::leaf(true, right_size, Default::default())); for i in 0..right_size { - new_leaf.init_value(i, kv_pairs[tick(val_count)].1); + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); } - new_leaf + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) }; + debug_assert!(kv_iter.next().is_none()); (new_left, new_right, pivot) } else { - let mut kn_pairs: [(K, Shared<'g, Node>); 2 * DEGREE] = - [Default::default(); 2 * DEGREE]; - // Combine the contents of `l` and `s` // (and one key from `p` if `l` and `s` are internal). - let (key_count, nxt_count) = (&mut 0, &mut 0); - - for i in 0..left.key_count() { - kn_pairs[tick(key_count)].0 = left.get_key(i).unwrap(); - } - for i in 0..lsize { - kn_pairs[tick(nxt_count)].1 = left.load_next(i, guard); - } - - kn_pairs[tick(key_count)].0 = parent.get_key(left_idx).unwrap(); - - for i in 0..right.key_count() { - kn_pairs[tick(key_count)].0 = right.get_key(i).unwrap(); - } - for i in 0..rsize { - kn_pairs[tick(nxt_count)].1 = right.load_next(i, guard); - } - let kn_pairs = &mut kn_pairs[0..*key_count]; - - (*key_count, *nxt_count) = (0, 0); + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)); let (new_left, pivot) = { let mut new_internal = - Owned::new(Node::internal(true, left_size, kn_pairs[*key_count].0)); - for i in 0..left_size - 1 { - new_internal.init_key(i, Some(kn_pairs[tick(key_count)].0)); - } + Owned::new(Node::internal(true, left_size, Default::default())); for i in 0..left_size { - new_internal.init_next(i, kn_pairs[tick(nxt_count)].1); + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); } - let pivot = kn_pairs[tick(key_count)].0; + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); (new_internal, pivot) }; - // Reserve one key for the parent (to go between `new_left` and `new_right`). - let new_right = { let mut new_internal = - Owned::new(Node::internal(true, right_size, kn_pairs[*key_count].0)); - for i in 0..right_size - (if left.is_leaf() { 0 } else { 1 }) { - new_internal.init_key(i, Some(kn_pairs[tick(key_count)].0)); - } + Owned::new(Node::internal(true, right_size, Default::default())); for i in 0..right_size { - new_internal.init_next(i, kn_pairs[tick(nxt_count)].1); + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); } + new_internal.search_key = new_internal.get_key(0).unwrap(); new_internal }; + debug_assert!(kn_iter.next().is_none()); (new_left, new_right, pivot) }; @@ -1165,14 +1162,7 @@ impl Drop for ElimABTree { } } -/// Equivalent to `x++`. -#[inline] -fn tick(counter: &mut usize) -> usize { - let val = *counter; - *counter += 1; - return val; -} - +/// Similar to `memcpy`, but for `Clone` types. #[inline] fn slice_clone(src: &[T], dst: &mut [T], len: usize) { dst[0..len].clone_from_slice(&src[0..len]); From 70b738ab91083213e4aa41713726fdfa0c014101 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 29 Oct 2024 08:31:37 +0000 Subject: [PATCH 46/84] Support returning an owned `V` in NR-based DSs --- src/ds_impl/nr/bonsai_tree.rs | 6 ++--- src/ds_impl/nr/concurrent_map.rs | 26 ++++++++++++++++----- src/ds_impl/nr/ellen_tree.rs | 6 ++--- src/ds_impl/nr/list.rs | 14 ++++++------ src/ds_impl/nr/michael_hash_map.rs | 30 ++++++++----------------- src/ds_impl/nr/natarajan_mittal_tree.rs | 6 ++--- src/ds_impl/nr/skip_list.rs | 10 ++++----- 7 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/ds_impl/nr/bonsai_tree.rs b/src/ds_impl/nr/bonsai_tree.rs index 8e88e243..09d37dab 100644 --- a/src/ds_impl/nr/bonsai_tree.rs +++ b/src/ds_impl/nr/bonsai_tree.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; use std::cmp; @@ -610,7 +610,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.get(key) } #[inline(always)] @@ -618,7 +618,7 @@ where self.insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.remove(key) } } diff --git a/src/ds_impl/nr/concurrent_map.rs b/src/ds_impl/nr/concurrent_map.rs index 277e927c..e7e8733f 100644 --- a/src/ds_impl/nr/concurrent_map.rs +++ b/src/ds_impl/nr/concurrent_map.rs @@ -1,14 +1,30 @@ +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { fn new() -> Self; - fn get(&self, key: &K) -> Option<&'static V>; + fn get(&self, key: &K) -> Option>; fn insert(&self, key: K, value: V) -> bool; - fn remove(&self, key: &K) -> Option<&'static V>; + fn remove(&self, key: &K) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_utils::thread; use rand::prelude::*; @@ -41,7 +57,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.remove(&i).unwrap()); + assert_eq!(i.to_string(), *map.remove(&i).unwrap().output()); } }); } @@ -56,7 +72,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.get(&i).unwrap()); + assert_eq!(i.to_string(), *map.get(&i).unwrap().output()); } }); } diff --git a/src/ds_impl/nr/ellen_tree.rs b/src/ds_impl/nr/ellen_tree.rs index 05755c4a..3f8fe2af 100644 --- a/src/ds_impl/nr/ellen_tree.rs +++ b/src/ds_impl/nr/ellen_tree.rs @@ -1,6 +1,6 @@ use std::sync::atomic::Ordering; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; bitflags! { @@ -468,7 +468,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { match self.find(key) { Some(node) => Some(node.value.as_ref().unwrap()), None => None, @@ -481,7 +481,7 @@ where } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.delete(key) } } diff --git a/src/ds_impl/nr/list.rs b/src/ds_impl/nr/list.rs index 3c06bd94..03c08eb6 100644 --- a/src/ds_impl/nr/list.rs +++ b/src/ds_impl/nr/list.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; use std::cmp::Ordering::{Equal, Greater, Less}; @@ -316,7 +316,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.inner.harris_get(key) } #[inline(always)] @@ -324,7 +324,7 @@ where self.inner.harris_insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.inner.harris_remove(key) } } @@ -343,7 +343,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.inner.harris_michael_get(key) } #[inline(always)] @@ -351,7 +351,7 @@ where self.inner.harris_michael_insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.inner.harris_michael_remove(key) } } @@ -382,7 +382,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.inner.harris_herlihy_shavit_get(key) } #[inline(always)] @@ -390,7 +390,7 @@ where self.inner.harris_insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.inner.harris_remove(key) } } diff --git a/src/ds_impl/nr/michael_hash_map.rs b/src/ds_impl/nr/michael_hash_map.rs index 602565f9..dfa6d123 100644 --- a/src/ds_impl/nr/michael_hash_map.rs +++ b/src/ds_impl/nr/michael_hash_map.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -33,21 +33,6 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get(&self, k: &K) -> Option<&'static V> { - let i = Self::hash(k); - self.get_bucket(i).get(k) - } - - pub fn insert(&self, k: K, v: V) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(k, v) - } - - pub fn remove(&self, k: &K) -> Option<&'static V> { - let i = Self::hash(k); - self.get_bucket(i).remove(k) - } } impl ConcurrentMap for HashMap @@ -60,16 +45,19 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { - self.get(key) + fn get(&self, key: &K) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(key) } #[inline(always)] fn insert(&self, key: K, value: V) -> bool { - self.insert(key, value) + let i = Self::hash(&key); + self.get_bucket(i).insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { - self.remove(key) + fn remove(&self, key: &K) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).remove(key) } } diff --git a/src/ds_impl/nr/natarajan_mittal_tree.rs b/src/ds_impl/nr/natarajan_mittal_tree.rs index bf63c898..0d7f6c5d 100644 --- a/src/ds_impl/nr/natarajan_mittal_tree.rs +++ b/src/ds_impl/nr/natarajan_mittal_tree.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; use std::cmp; use std::sync::atomic::Ordering; @@ -445,7 +445,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.get(key) } #[inline(always)] @@ -453,7 +453,7 @@ where self.insert(key, value).is_ok() } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.remove(key) } } diff --git a/src/ds_impl/nr/skip_list.rs b/src/ds_impl/nr/skip_list.rs index 082507eb..3d66f1f3 100644 --- a/src/ds_impl/nr/skip_list.rs +++ b/src/ds_impl/nr/skip_list.rs @@ -1,7 +1,7 @@ use std::mem::transmute; use std::sync::atomic::Ordering; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; const MAX_HEIGHT: usize = 32; @@ -331,9 +331,9 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { let cursor = self.find_optimistic(key); - unsafe { transmute(cursor.found.map(|node| &node.value)) } + unsafe { transmute::<_, Option<&'static V>>(cursor.found.map(|node| &node.value)) } } #[inline(always)] @@ -342,8 +342,8 @@ where } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { - unsafe { transmute(self.remove(key)) } + fn remove(&self, key: &K) -> Option> { + unsafe { transmute::<_, Option<&'static V>>(self.remove(key)) } } } From 8d07a2ab22a95f3e74d0c31c84bda63b3b69929a Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 29 Oct 2024 09:43:28 +0000 Subject: [PATCH 47/84] Implement NR ElimAbTree --- src/ds_impl/nr/bonsai_tree.rs | 2 +- src/ds_impl/nr/concurrent_map.rs | 14 +- src/ds_impl/nr/elim_ab_tree.rs | 1159 +++++++++++++++++++++++ src/ds_impl/nr/ellen_tree.rs | 2 +- src/ds_impl/nr/list.rs | 6 +- src/ds_impl/nr/michael_hash_map.rs | 2 +- src/ds_impl/nr/mod.rs | 1 + src/ds_impl/nr/natarajan_mittal_tree.rs | 2 +- src/ds_impl/nr/pointers.rs | 8 + src/ds_impl/nr/skip_list.rs | 2 +- 10 files changed, 1186 insertions(+), 12 deletions(-) create mode 100644 src/ds_impl/nr/elim_ab_tree.rs diff --git a/src/ds_impl/nr/bonsai_tree.rs b/src/ds_impl/nr/bonsai_tree.rs index 09d37dab..b06a5985 100644 --- a/src/ds_impl/nr/bonsai_tree.rs +++ b/src/ds_impl/nr/bonsai_tree.rs @@ -630,6 +630,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/concurrent_map.rs b/src/ds_impl/nr/concurrent_map.rs index e7e8733f..3457618a 100644 --- a/src/ds_impl/nr/concurrent_map.rs +++ b/src/ds_impl/nr/concurrent_map.rs @@ -27,11 +27,17 @@ pub mod tests { use super::{ConcurrentMap, OutputHolder}; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -42,7 +48,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(i, i.to_string())); + assert!(map.insert(i, to_value(&i))); } }); } @@ -57,7 +63,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.remove(&i).unwrap().output()); + assert_eq!(to_value(&i), *map.remove(&i).unwrap().output()); } }); } @@ -72,7 +78,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.get(&i).unwrap().output()); + assert_eq!(to_value(&i), *map.get(&i).unwrap().output()); } }); } diff --git a/src/ds_impl/nr/elim_ab_tree.rs b/src/ds_impl/nr/elim_ab_tree.rs new file mode 100644 index 00000000..8404d303 --- /dev/null +++ b/src/ds_impl/nr/elim_ab_tree.rs @@ -0,0 +1,1159 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use super::pointers::{Atomic, Shared}; +use arrayvec::ArrayVec; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next<'g>(&self, index: usize) -> Shared { + self.next()[index].load(Ordering::Acquire) + } + + fn store_next<'g>(&'g self, index: usize, ptr: Shared, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: Shared) { + self.next_mut()[index] = Atomic::from(ptr); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator, Shared)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i))) + .chain(once((None, self.load_next(self.key_count())))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +struct Cursor { + l: Shared>, + p: Shared>, + gp: Shared>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Shared::from_owned(left)); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K) -> Option { + let mut node = unsafe { self.entry.load_next(0).deref() }; + while let NodeKind::Internal { next } = &node.kind { + let next = next[node.child_index(key)].load(Ordering::Acquire); + node = unsafe { next.deref() }; + } + node.read_consistent(key).1 + } + + fn search(&self, key: &K, target: Option>>) -> (bool, Cursor) { + let mut cursor = Cursor { + l: self.entry.load_next(0), + p: Shared::from(&self.entry as *const _ as usize), + gp: Shared::null(), + gp_p_idx: 0, + p_l_idx: 0, + l_key_idx: 0, + val: None, + }; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { cursor.l.deref() }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.l = l_node.load_next(cursor.p_l_idx); + } + + if let Some(target) = target { + (cursor.l == target, cursor) + } else { + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); + cursor.val = value; + cursor.l_key_idx = index; + (value.is_some(), cursor) + } + } + + pub fn insert(&self, key: &K, value: &V) -> Option { + loop { + let (_, cursor) = self.search(key, None); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, &cursor) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner(&self, key: &K, value: &V, cursor: &Cursor) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Shared::from_owned(left)); + internal.init_next(1, Shared::from_owned(right)); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Shared::from_owned(internal); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + self.fix_tag_violation(new_internal); + + Ok(None) + } + } + + fn fix_tag_violation(&self, viol: Shared>) { + loop { + let viol_node = unsafe { viol.deref() }; + if viol_node.weight { + return; + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0) != viol); + + let (found, cursor) = self.search(&viol_node.search_key, Some(viol)); + if !found { + return; + } + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + if !eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + self.fix_tag_violation(cursor.p); + continue; + } + + try_acq_val_or!(node, node_lock, Operation::Balance, None, continue); + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next(cursor.gp_p_idx, Shared::from_owned(absorber), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + return; + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Shared::from_owned(left)); + new_internal.init_next(1, Shared::from_owned(right)); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Shared::from_owned(new_internal); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + drop((node_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(new_internal); + return; + } + } + } + + pub fn remove(&self, key: &K) -> Option { + loop { + let (_, cursor) = self.search(key, None); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner(&self, key: &K, cursor: &Cursor) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation(cursor.l); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation(&self, viol: Shared>) { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + let viol_node = unsafe { viol.deref() }; + loop { + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0) + { + // No degree violation at `viol`. + return; + } + + // Search for `viol`. + let (_, cursor) = self.search(&viol_node.search_key, Some(viol)); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p != self.entry.load_next(0) + { + self.fix_underfull_violation(cursor.p); + continue; + } + + if !eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + let sibling_idx = if cursor.p_l_idx > 0 { + cursor.p_l_idx - 1 + } else { + 1 + }; + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling_sh = parent.load_next(sibling_idx); + let sibling = unsafe { sibling_sh.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if sibling_idx < cursor.p_l_idx { + ((sibling, sibling_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, sibling_idx)) + }; + + try_acq_val_or!(left, left_lock, Operation::Balance, None, continue); + try_acq_val_or!(right, right_lock, Operation::Balance, None, continue); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return; + } + + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight || !node.weight || !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(cursor.p); + self.fix_tag_violation(cursor.l); + self.fix_tag_violation(sibling_sh); + continue; + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Node::leaf(true, size, node.search_key); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Node::internal(true, size, node.search_key); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + }; + let new_node = Shared::from_owned(new_node); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node); + return; + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..sibling_idx { + new_parent.init_next(i, parent.load_next(i)); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next(i)); + } + + new_parent.init_next( + cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 }), + new_node, + ); + let new_parent = Shared::from_owned(new_parent); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node); + self.fix_underfull_violation(new_parent); + return; + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Node::leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Node::leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let (new_left, pivot) = { + let mut new_internal = Node::internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = Node::internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = Node::internal(parent.weight, psize, parent.search_key); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, Shared::from_owned(new_left)); + new_parent.init_next(right_idx, Shared::from_owned(new_right)); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next( + cursor.gp_p_idx, + Shared::from_owned(new_parent), + &gparent_lock, + ); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + return; + } + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let mut stack = vec![]; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self::new() + } + + fn get(&self, key: &K) -> Option> { + self.search_basic(key) + } + + fn insert(&self, key: K, value: V) -> bool { + self.insert(&key, &value).is_none() + } + + fn remove(&self, key: &K) -> Option> { + self.remove(key) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::nr::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/nr/ellen_tree.rs b/src/ds_impl/nr/ellen_tree.rs index 3f8fe2af..6f86733d 100644 --- a/src/ds_impl/nr/ellen_tree.rs +++ b/src/ds_impl/nr/ellen_tree.rs @@ -493,6 +493,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/list.rs b/src/ds_impl/nr/list.rs index 03c08eb6..41abfbaa 100644 --- a/src/ds_impl/nr/list.rs +++ b/src/ds_impl/nr/list.rs @@ -402,17 +402,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/nr/michael_hash_map.rs b/src/ds_impl/nr/michael_hash_map.rs index dfa6d123..63009c2e 100644 --- a/src/ds_impl/nr/michael_hash_map.rs +++ b/src/ds_impl/nr/michael_hash_map.rs @@ -68,6 +68,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/mod.rs b/src/ds_impl/nr/mod.rs index bd66b816..b1eb08a8 100644 --- a/src/ds_impl/nr/mod.rs +++ b/src/ds_impl/nr/mod.rs @@ -3,6 +3,7 @@ pub mod pointers; pub mod bonsai_tree; pub mod double_link; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; diff --git a/src/ds_impl/nr/natarajan_mittal_tree.rs b/src/ds_impl/nr/natarajan_mittal_tree.rs index 0d7f6c5d..c4759ed9 100644 --- a/src/ds_impl/nr/natarajan_mittal_tree.rs +++ b/src/ds_impl/nr/natarajan_mittal_tree.rs @@ -465,6 +465,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/pointers.rs b/src/ds_impl/nr/pointers.rs index 36055b6b..5abbee52 100644 --- a/src/ds_impl/nr/pointers.rs +++ b/src/ds_impl/nr/pointers.rs @@ -83,6 +83,14 @@ impl From> for Atomic { } } +impl Clone for Atomic { + fn clone(&self) -> Self { + Self { + link: AtomicPtr::new(self.link.load(Ordering::Relaxed)), + } + } +} + pub struct Shared { ptr: *mut T, } diff --git a/src/ds_impl/nr/skip_list.rs b/src/ds_impl/nr/skip_list.rs index 3d66f1f3..8df399ff 100644 --- a/src/ds_impl/nr/skip_list.rs +++ b/src/ds_impl/nr/skip_list.rs @@ -354,6 +354,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } From a4bd4c5ad2029de79e3205772cbdb8cd22f824be Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 29 Oct 2024 10:42:39 +0000 Subject: [PATCH 48/84] Support returning an owned `V` in HP(++)-based DSs --- src/ds_impl/hp/bonsai_tree.rs | 16 +++-- src/ds_impl/hp/concurrent_map.rs | 44 ++++++++++-- src/ds_impl/hp/ellen_tree.rs | 21 +++--- src/ds_impl/hp/list.rs | 44 +++++++++--- src/ds_impl/hp/michael_hash_map.rs | 40 +++++------ src/ds_impl/hp/natarajan_mittal_tree.rs | 16 +++-- src/ds_impl/hp/skip_list.rs | 16 +++-- src/ds_impl/hp_pp/bonsai_tree.rs | 27 ++++--- src/ds_impl/hp_pp/ellen_tree.rs | 32 ++++----- src/ds_impl/hp_pp/list.rs | 83 +++++++++++----------- src/ds_impl/hp_pp/michael_hash_map.rs | 51 +++++-------- src/ds_impl/hp_pp/natarajan_mittal_tree.rs | 27 ++++--- src/ds_impl/hp_pp/skip_list.rs | 27 ++++--- 13 files changed, 250 insertions(+), 194 deletions(-) diff --git a/src/ds_impl/hp/bonsai_tree.rs b/src/ds_impl/hp/bonsai_tree.rs index 7abd11cb..a318d867 100644 --- a/src/ds_impl/hp/bonsai_tree.rs +++ b/src/ds_impl/hp/bonsai_tree.rs @@ -1,7 +1,7 @@ use hp_pp::{light_membarrier, Thread}; use hp_pp::{tag, tagged, untagged, HazardPointer, DEFAULT_DOMAIN}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::ptr; @@ -788,7 +788,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } @@ -798,7 +802,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -810,6 +818,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/concurrent_map.rs b/src/ds_impl/hp/concurrent_map.rs index 48b650fb..6c41a2f1 100644 --- a/src/ds_impl/hp/concurrent_map.rs +++ b/src/ds_impl/hp/concurrent_map.rs @@ -1,3 +1,19 @@ +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { type Handle<'domain>; @@ -5,24 +21,38 @@ pub trait ConcurrentMap { fn handle() -> Self::Handle<'static>; - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V>; + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option>; fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool; - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V>; + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -34,7 +64,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(&mut handle, i, i.to_string())); + assert!(map.insert(&mut handle, i, to_value(&i))); } }); } @@ -50,7 +80,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.remove(&mut handle, &i).unwrap()); + assert_eq!(to_value(&i), *map.remove(&mut handle, &i).unwrap().output()); } }); } @@ -66,7 +96,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.get(&mut handle, &i).unwrap()); + assert_eq!(to_value(&i), *map.get(&mut handle, &i).unwrap().output()); } }); } diff --git a/src/ds_impl/hp/ellen_tree.rs b/src/ds_impl/hp/ellen_tree.rs index 72eacb0d..e8822017 100644 --- a/src/ds_impl/hp/ellen_tree.rs +++ b/src/ds_impl/hp/ellen_tree.rs @@ -27,7 +27,7 @@ use hp_pp::{ decompose_ptr, light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, }; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -849,11 +849,12 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - match self.find(key, handle) { - Some(value) => Some(value), - None => None, - } + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.find(key, handle) } #[inline(always)] @@ -862,7 +863,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.delete(key, handle) } } @@ -874,6 +879,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 38dc7e71..cceca5de 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Pointer, Shared}; use core::mem; @@ -532,7 +532,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_get(key, handle) } #[inline(always)] @@ -540,7 +544,11 @@ where self.inner.harris_insert(key, value, handle) } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_remove(key, handle) } } @@ -575,7 +583,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_get(key, handle) } #[inline(always)] @@ -583,7 +595,11 @@ where self.inner.harris_michael_insert(key, value, handle) } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_remove(key, handle) } } @@ -607,7 +623,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_get(key, handle) } #[inline(always)] @@ -615,7 +635,11 @@ where self.inner.harris_michael_insert(key, value, handle) } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_remove(key, handle) } } @@ -627,17 +651,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/hp/michael_hash_map.rs b/src/ds_impl/hp/michael_hash_map.rs index beaef9df..b48adf21 100644 --- a/src/ds_impl/hp/michael_hash_map.rs +++ b/src/ds_impl/hp/michael_hash_map.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -33,21 +33,6 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get<'hp>(&self, handle: &'hp mut Handle<'_>, k: &K) -> Option<&'hp V> { - let i = Self::hash(k); - self.get_bucket(i).get(handle, k) - } - - pub fn insert(&self, handle: &mut Handle<'_>, k: K, v: V) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(handle, k, v) - } - - pub fn remove<'hp>(&self, handle: &'hp mut Handle<'_>, k: &K) -> Option<&'hp V> { - let i = Self::hash(k); - self.get_bucket(i).remove(handle, k) - } } impl ConcurrentMap for HashMap @@ -66,16 +51,27 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - self.get(handle, key) + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(handle, key) } #[inline(always)] fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { - self.insert(handle, key, value) + let i = Self::hash(&key); + self.get_bucket(i).insert(handle, key, value) } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - self.remove(handle, key) + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).remove(handle, key) } } @@ -86,6 +82,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/natarajan_mittal_tree.rs b/src/ds_impl/hp/natarajan_mittal_tree.rs index 714eedd9..cf2b8c3b 100644 --- a/src/ds_impl/hp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp/natarajan_mittal_tree.rs @@ -1,7 +1,7 @@ use hp_pp::{light_membarrier, Thread}; use hp_pp::{tag, tagged, untagged, HazardPointer, DEFAULT_DOMAIN}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; @@ -646,7 +646,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } @@ -656,7 +660,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -668,6 +676,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/skip_list.rs b/src/ds_impl/hp/skip_list.rs index babf32fb..6fab4f7e 100644 --- a/src/ds_impl/hp/skip_list.rs +++ b/src/ds_impl/hp/skip_list.rs @@ -4,7 +4,7 @@ use std::sync::atomic::{fence, AtomicPtr, AtomicUsize, Ordering}; use hp_pp::{light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -506,7 +506,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { let node = unsafe { &*self.find_optimistic(key, handle)? }; if node.key.eq(&key) { Some(&node.value) @@ -521,7 +525,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -533,6 +541,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/bonsai_tree.rs b/src/ds_impl/hp_pp/bonsai_tree.rs index 220bd7c5..dabf0de0 100644 --- a/src/ds_impl/hp_pp/bonsai_tree.rs +++ b/src/ds_impl/hp_pp/bonsai_tree.rs @@ -1,7 +1,7 @@ use hp_pp::{light_membarrier, Invalidate, Thread, Unlink}; use hp_pp::{tag, tagged, untagged, HazardPointer, ProtectError, DEFAULT_DOMAIN}; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::ptr; @@ -854,26 +854,25 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -885,6 +884,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/ellen_tree.rs b/src/ds_impl/hp_pp/ellen_tree.rs index df622fe0..5f569622 100644 --- a/src/ds_impl/hp_pp/ellen_tree.rs +++ b/src/ds_impl/hp_pp/ellen_tree.rs @@ -28,7 +28,7 @@ use hp_pp::{ DEFAULT_DOMAIN, }; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -950,29 +950,25 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { - match self.find(key, handle) { - Some(value) => Some(value), - None => None, - } + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.find(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(&key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.delete(key, handle) } } @@ -984,6 +980,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/list.rs b/src/ds_impl/hp_pp/list.rs index 30b329d6..45551dd6 100644 --- a/src/ds_impl/hp_pp/list.rs +++ b/src/ds_impl/hp_pp/list.rs @@ -1,4 +1,4 @@ -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp::Ordering::{Equal, Greater, Less}; use std::sync::atomic::{AtomicPtr, Ordering}; @@ -582,24 +582,23 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.inner.harris_insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_remove(key, handle) } } @@ -638,24 +637,23 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.inner.harris_michael_insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_remove(key, handle) } } @@ -690,25 +688,24 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { - self.inner.harris_herlihy_shavit_get(key, handle) + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.inner.harris_michael_get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { - self.inner.harris_insert(key, value, handle) + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + self.inner.harris_michael_insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { - self.inner.harris_remove(key, handle) + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.inner.harris_michael_remove(key, handle) } } @@ -719,17 +716,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/hp_pp/michael_hash_map.rs b/src/ds_impl/hp_pp/michael_hash_map.rs index e964e2a7..2d6f70b5 100644 --- a/src/ds_impl/hp_pp/michael_hash_map.rs +++ b/src/ds_impl/hp_pp/michael_hash_map.rs @@ -1,4 +1,4 @@ -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -34,21 +34,6 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get<'domain, 'hp>(&self, handle: &'hp mut Handle<'domain>, k: &K) -> Option<&'hp V> { - let i = Self::hash(k); - self.get_bucket(i).get(handle, k) - } - - pub fn insert<'domain, 'hp>(&self, handle: &'hp mut Handle<'domain>, k: K, v: V) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(handle, k, v) - } - - pub fn remove<'domain, 'hp>(&self, handle: &'hp mut Handle<'domain>, k: &K) -> Option<&'hp V> { - let i = Self::hash(&k); - self.get_bucket(i).remove(handle, k) - } } impl ConcurrentMap for HashMap @@ -67,25 +52,27 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { - self.get(handle, key) + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(handle, key) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { - self.insert(handle, key, value) + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + let i = Self::hash(&key); + self.get_bucket(i).insert(handle, key, value) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { - self.remove(handle, key) + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).remove(handle, key) } } @@ -96,6 +83,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/natarajan_mittal_tree.rs b/src/ds_impl/hp_pp/natarajan_mittal_tree.rs index 556425c5..6cd1f008 100644 --- a/src/ds_impl/hp_pp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp_pp/natarajan_mittal_tree.rs @@ -2,7 +2,7 @@ use hp_pp::{ light_membarrier, tag, tagged, untagged, HazardPointer, ProtectError, Thread, DEFAULT_DOMAIN, }; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::mem; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; @@ -751,26 +751,25 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(key, value, handle).is_ok() } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -782,6 +781,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/skip_list.rs b/src/ds_impl/hp_pp/skip_list.rs index 00c61eec..7ae00040 100644 --- a/src/ds_impl/hp_pp/skip_list.rs +++ b/src/ds_impl/hp_pp/skip_list.rs @@ -6,7 +6,7 @@ use hp_pp::{ decompose_ptr, light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, }; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -526,7 +526,11 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { let node = unsafe { &*self.find_optimistic(key, handle)? }; if node.key.eq(&key) { Some(&node.value) @@ -536,21 +540,16 @@ where } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -562,6 +561,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } From 1e929cb737db161f93a572cee7066037bac06cc9 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 29 Oct 2024 10:53:40 +0000 Subject: [PATCH 49/84] Add NR ElimAbTree to the executable data structures --- src/bin/nr.rs | 5 +++-- src/ds_impl/nr/mod.rs | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/bin/nr.rs b/src/bin/nr.rs index b132fb37..b594c9aa 100644 --- a/src/bin/nr.rs +++ b/src/bin/nr.rs @@ -9,7 +9,8 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::nr::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -36,7 +37,7 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), - DS::ElimAbTree => todo!(), + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); diff --git a/src/ds_impl/nr/mod.rs b/src/ds_impl/nr/mod.rs index b1eb08a8..2d5551d7 100644 --- a/src/ds_impl/nr/mod.rs +++ b/src/ds_impl/nr/mod.rs @@ -14,6 +14,7 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; From 5a621b2d8222784f250212aeed5e615413b1e26b Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 29 Oct 2024 13:05:49 +0000 Subject: [PATCH 50/84] Implement HP ElimAbTree --- run.sh | 9 + src/bin/hp.rs | 5 +- src/ds_impl/hp/elim_ab_tree.rs | 1374 ++++++++++++++++++++++++++++++++ src/ds_impl/hp/mod.rs | 2 + src/ds_impl/hp/pointers.rs | 18 +- test-scripts/sanitize-elim.sh | 10 + 6 files changed, 1415 insertions(+), 3 deletions(-) create mode 100644 run.sh create mode 100644 src/ds_impl/hp/elim_ab_tree.rs create mode 100644 test-scripts/sanitize-elim.sh diff --git a/run.sh b/run.sh new file mode 100644 index 00000000..c1ad55fa --- /dev/null +++ b/run.sh @@ -0,0 +1,9 @@ +cargo run --release --bin nr -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin ebr -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin pebr -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin hp -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin hp-pp -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin nbr -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin hp-brcu -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin vbr -- -dnm-tree -t92 -g0 -r100000 -i10 +cargo run --release --bin hp-rcu -- -dnm-tree -t92 -g0 -r100000 -i10 diff --git a/src/bin/hp.rs b/src/bin/hp.rs index 1eb79d58..bef365ea 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -11,7 +11,8 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -38,7 +39,7 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::NMTree => bench_map::>(config, PrefillStrategy::Random), - _ => panic!("Unsupported(or unimplemented) data structure for HP"), + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); diff --git a/src/ds_impl/hp/elim_ab_tree.rs b/src/ds_impl/hp/elim_ab_tree.rs new file mode 100644 index 00000000..1162cd0e --- /dev/null +++ b/src/ds_impl/hp/elim_ab_tree.rs @@ -0,0 +1,1374 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use super::pointers::{Atomic, Pointer, Shared}; +use arrayvec::ArrayVec; +use hp_pp::{light_membarrier, HazardPointer, Thread, DEFAULT_DOMAIN}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next(&self, index: usize) -> Shared { + self.next()[index].load(Ordering::Acquire) + } + + fn protect_next(&self, index: usize, slot: &mut HazardPointer<'_>) -> Result, ()> { + let atomic = &self.next()[index]; + let mut ptr = atomic.load(Ordering::Relaxed); + loop { + slot.protect_raw(ptr.with_tag(0).into_raw()); + light_membarrier(); + let new = atomic.load(Ordering::Acquire); + if ptr == new { + break; + } + ptr = new; + } + if self.marked.load(Ordering::Acquire) { + return Err(()); + } + Ok(ptr) + } + + fn store_next<'g>(&'g self, index: usize, ptr: impl Pointer, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: impl Pointer) { + self.next_mut()[index] = Atomic::from(ptr.into_raw()); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } + + fn p_s_idx(p_l_idx: usize) -> usize { + if p_l_idx > 0 { + p_l_idx - 1 + } else { + 1 + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator, Shared)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i))) + .chain(once((None, self.load_next(self.key_count())))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +struct Cursor { + l: Shared>, + p: Shared>, + gp: Shared>, + // A pointer to a sibling node. + s: Shared>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + p_s_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +pub struct Handle<'domain> { + l_h: HazardPointer<'domain>, + p_h: HazardPointer<'domain>, + gp_h: HazardPointer<'domain>, + /// A protector for the sibling node. + s_h: HazardPointer<'domain>, + thread: Box>, +} + +impl Default for Handle<'static> { + fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); + Self { + l_h: HazardPointer::new(&mut thread), + p_h: HazardPointer::new(&mut thread), + gp_h: HazardPointer::new(&mut thread), + s_h: HazardPointer::new(&mut thread), + thread, + } + } +} + +impl<'domain> Handle<'domain> { + // bypass E0499-E0503, etc that are supposed to be fixed by polonius + #[inline] + fn launder<'hp2>(&mut self) -> &'hp2 mut Self { + unsafe { core::mem::transmute(self) } + } +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Shared::from_owned(left)); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option { + loop { + match self.search_basic_inner(key, handle) { + Ok(found) => return found, + Err(_) => continue, + } + } + } + + pub fn search_basic_inner<'hp>( + &self, + key: &K, + handle: &'hp mut Handle<'_>, + ) -> Result, ()> { + let mut node = unsafe { self.entry.protect_next(0, &mut handle.p_h)?.deref() }; + while !node.is_leaf() { + let next = node.protect_next(node.child_index(key), &mut handle.l_h)?; + HazardPointer::swap(&mut handle.l_h, &mut handle.p_h); + node = unsafe { next.deref() }; + } + Ok(node.read_consistent(key).1) + } + + fn search<'hp>( + &self, + key: &K, + target: Option>>, + handle: &'hp mut Handle<'_>, + ) -> (bool, Cursor) { + loop { + match self.search_inner(key, target, handle.launder()) { + Ok(found) => return found, + Err(_) => continue, + } + } + } + + fn search_inner<'hp>( + &self, + key: &K, + target: Option>>, + handle: &'hp mut Handle<'_>, + ) -> Result<(bool, Cursor), ()> { + let mut cursor = Cursor { + l: self.entry.protect_next(0, &mut handle.l_h)?, + s: self.entry.protect_next(1, &mut handle.s_h)?, + p: Shared::from(&self.entry as *const _ as usize), + gp: Shared::null(), + gp_p_idx: 0, + p_l_idx: 0, + p_s_idx: 1, + l_key_idx: 0, + val: None, + }; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { cursor.l.deref() }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + HazardPointer::swap(&mut handle.gp_h, &mut handle.p_h); + HazardPointer::swap(&mut handle.p_h, &mut handle.l_h); + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.p_s_idx = Node::::p_s_idx(cursor.p_l_idx); + cursor.l = l_node.protect_next(cursor.p_l_idx, &mut handle.l_h)?; + cursor.s = l_node.protect_next(cursor.p_s_idx, &mut handle.s_h)?; + } + + if let Some(target) = target { + Ok((cursor.l == target, cursor)) + } else { + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); + cursor.val = value; + cursor.l_key_idx = index; + Ok((value.is_some(), cursor)) + } + } + + pub fn insert<'hp>(&self, key: &K, value: &V, handle: &'hp mut Handle<'_>) -> Option { + loop { + let (_, cursor) = self.search(key, None, handle.launder()); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, &cursor, handle) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner<'hp>( + &self, + key: &K, + value: &V, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Shared::from_owned(left)); + internal.init_next(1, Shared::from_owned(right)); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Shared::from_owned(internal); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { + handle.thread.retire(cursor.l.with_tag(0).into_raw()); + } + self.fix_tag_violation(new_internal, handle); + + Ok(None) + } + } + + fn fix_tag_violation<'hp>(&self, viol: Shared>, handle: &'hp mut Handle<'_>) { + let mut stack = vec![(unsafe { viol.deref() }.search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + let (found, cursor) = self.search(&search_key, Some(viol), handle); + if !found || cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + match self.fix_tag_violation_inner(&cursor, handle) { + Ok(()) => continue, + Err(None) => stack.push((search_key, viol)), + Err(Some(pair)) => stack.extend_from_slice(&[(search_key, viol), pair]), + } + } + } + + fn fix_tag_violation_inner<'hp>( + &self, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> Result<(), Option<(K, Shared>)>> { + let viol = cursor.l; + let viol_node = unsafe { cursor.l.deref() }; + if viol_node.weight { + return Ok(()); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0) != viol); + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + return Err(Some((parent.search_key, cursor.p))); + } + + try_acq_val_or!(node, node_lock, Operation::Balance, None, return Err(None)); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return Err(None) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return Err(None) + ); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next(cursor.gp_p_idx, Shared::from_owned(absorber), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { + handle.thread.retire(cursor.l.with_tag(0).into_raw()); + } + unsafe { + handle.thread.retire(cursor.p.with_tag(0).into_raw()); + } + return Ok(()); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Shared::from_owned(left)); + new_internal.init_next(1, Shared::from_owned(right)); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Shared::from_owned(new_internal); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { + handle.thread.retire(cursor.l.with_tag(0).into_raw()); + }; + unsafe { + handle.thread.retire(cursor.p.with_tag(0).into_raw()); + }; + + drop((node_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(new_internal, handle); + return Ok(()); + } + } + + pub fn remove<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option { + loop { + let (_, cursor) = self.search(key, None, handle.launder()); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor, handle) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner<'hp>( + &self, + key: &K, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation(cursor.l, handle); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation<'hp>(&self, viol: Shared>, handle: &'hp mut Handle<'_>) { + let mut stack = vec![(unsafe { viol.deref() }.search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + let (_, cursor) = self.search(&search_key, Some(viol), handle); + if cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + match self.fix_underfull_violation_inner(&cursor, handle) { + Ok(()) => continue, + Err(None) => stack.push((search_key, viol)), + Err(Some(pair)) => stack.extend_from_slice(&[(search_key, viol), pair]), + } + } + } + + fn fix_underfull_violation_inner<'hp>( + &self, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> Result<(), Option<(K, Shared>)>> { + let viol = cursor.l; + let viol_node = unsafe { viol.deref() }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0) + { + // No degree violation at `viol`. + return Ok(()); + } + + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p != self.entry.load_next(0) + { + return Err(Some((parent.search_key, cursor.p))); + } + + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling = unsafe { cursor.s.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if cursor.p_s_idx < cursor.p_l_idx { + ((sibling, cursor.p_s_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, cursor.p_s_idx)) + }; + + try_acq_val_or!(left, left_lock, Operation::Balance, None, return Err(None)); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + return Err(None) + ); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return Ok(()); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return Err(None) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return Err(None) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return Err(Some((parent.search_key, cursor.p))); + } + if !node.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return Err(Some((node.search_key, cursor.l))); + } + if !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return Err(Some((node.search_key, cursor.s))); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Node::leaf(true, size, node.search_key); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Node::internal(true, size, node.search_key); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + }; + let new_node = Shared::from_owned(new_node); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + handle.thread.retire(cursor.l.with_tag(0).into_raw()); + handle.thread.retire(cursor.p.with_tag(0).into_raw()); + handle.thread.retire(cursor.s.with_tag(0).into_raw()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node, handle); + return Ok(()); + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..cursor.p_s_idx { + new_parent.init_next(i, parent.load_next(i)); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next(i)); + } + + new_parent.init_next( + cursor.p_l_idx + - (if cursor.p_l_idx > cursor.p_s_idx { + 1 + } else { + 0 + }), + new_node, + ); + let new_parent = Shared::from_owned(new_parent); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + handle.thread.retire(cursor.l.with_tag(0).into_raw()); + handle.thread.retire(cursor.p.with_tag(0).into_raw()); + handle.thread.retire(cursor.s.with_tag(0).into_raw()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node, handle); + self.fix_underfull_violation(new_parent, handle); + return Ok(()); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Node::leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Node::leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let (new_left, pivot) = { + let mut new_internal = Node::internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = Node::internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = Node::internal(parent.weight, psize, parent.search_key); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, Shared::from_owned(new_left)); + new_parent.init_next(right_idx, Shared::from_owned(new_right)); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next( + cursor.gp_p_idx, + Shared::from_owned(new_parent), + &gparent_lock, + ); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + handle.thread.retire(cursor.l.with_tag(0).into_raw()); + handle.thread.retire(cursor.p.with_tag(0).into_raw()); + handle.thread.retire(cursor.s.with_tag(0).into_raw()); + } + + return Ok(()); + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let mut stack = vec![]; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + type Handle<'domain> = Handle<'domain>; + + fn new() -> Self { + ElimABTree::new() + } + + fn handle() -> Self::Handle<'static> { + Handle::default() + } + + #[inline(always)] + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.search_basic(key, handle) + } + + #[inline(always)] + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + self.insert(&key, &value, handle).is_none() + } + + #[inline(always)] + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.remove(key, handle) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::hp::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/hp/mod.rs b/src/ds_impl/hp/mod.rs index bd66b816..2d5551d7 100644 --- a/src/ds_impl/hp/mod.rs +++ b/src/ds_impl/hp/mod.rs @@ -3,6 +3,7 @@ pub mod pointers; pub mod bonsai_tree; pub mod double_link; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; @@ -13,6 +14,7 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; diff --git a/src/ds_impl/hp/pointers.rs b/src/ds_impl/hp/pointers.rs index 68ff9308..2020c4ed 100644 --- a/src/ds_impl/hp/pointers.rs +++ b/src/ds_impl/hp/pointers.rs @@ -36,7 +36,7 @@ impl Atomic { } #[inline] - pub fn store(&self, ptr: Shared, order: Ordering) { + pub fn store>(&self, ptr: P, order: Ordering) { self.link.store(ptr.into_raw(), order) } @@ -110,6 +110,22 @@ impl From> for Atomic { } } +impl From<*mut T> for Atomic { + #[inline] + fn from(value: *mut T) -> Self { + let link = AtomicPtr::new(value); + Self { link } + } +} + +impl Clone for Atomic { + fn clone(&self) -> Self { + Self { + link: AtomicPtr::new(self.link.load(Ordering::Relaxed)), + } + } +} + pub struct Shared { ptr: *mut T, } diff --git a/test-scripts/sanitize-elim.sh b/test-scripts/sanitize-elim.sh new file mode 100644 index 00000000..d5ec5153 --- /dev/null +++ b/test-scripts/sanitize-elim.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' + +run="cargo run --bin hp --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " + +set -e +for i in {1..5000}; do + $run -delim-ab-tree -i3 -t256 -r100000 -g1 +done From cbd19b414ebbb878464f71f24202ab46c3736972 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 29 Oct 2024 13:14:39 +0000 Subject: [PATCH 51/84] Make `V` customizable in `smoke` of (B)RCU --- src/ds_impl/hp_brcu/bonsai_tree.rs | 2 +- src/ds_impl/hp_brcu/concurrent_map.rs | 12 +++++++++--- src/ds_impl/hp_brcu/list.rs | 6 +++--- src/ds_impl/hp_brcu/list_alter.rs | 6 +++--- src/ds_impl/hp_brcu/michael_hash_map.rs | 2 +- src/ds_impl/hp_brcu/natarajan_mittal_tree.rs | 2 +- src/ds_impl/hp_brcu/skip_list.rs | 2 +- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/ds_impl/hp_brcu/bonsai_tree.rs b/src/ds_impl/hp_brcu/bonsai_tree.rs index 2d6fbec1..266ef1a5 100644 --- a/src/ds_impl/hp_brcu/bonsai_tree.rs +++ b/src/ds_impl/hp_brcu/bonsai_tree.rs @@ -807,6 +807,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_brcu/concurrent_map.rs b/src/ds_impl/hp_brcu/concurrent_map.rs index 2c5a4466..8c97fae9 100644 --- a/src/ds_impl/hp_brcu/concurrent_map.rs +++ b/src/ds_impl/hp_brcu/concurrent_map.rs @@ -26,11 +26,17 @@ pub mod tests { use crossbeam_utils::thread; use hp_brcu::THREAD; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -44,7 +50,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(i, i.to_string(), output, thread)); + assert!(map.insert(i, to_value(&i), output, thread)); } }); }); @@ -83,7 +89,7 @@ pub mod tests { keys.shuffle(&mut rng); for i in keys { assert!(map.get(&i, output, thread)); - assert_eq!(i.to_string(), *output.output()); + assert_eq!(to_value(&i), *output.output()); } }); }); diff --git a/src/ds_impl/hp_brcu/list.rs b/src/ds_impl/hp_brcu/list.rs index c222e4e6..da9065cc 100644 --- a/src/ds_impl/hp_brcu/list.rs +++ b/src/ds_impl/hp_brcu/list.rs @@ -525,15 +525,15 @@ where #[test] fn smoke_h_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } diff --git a/src/ds_impl/hp_brcu/list_alter.rs b/src/ds_impl/hp_brcu/list_alter.rs index 9524b5f9..57321cd8 100644 --- a/src/ds_impl/hp_brcu/list_alter.rs +++ b/src/ds_impl/hp_brcu/list_alter.rs @@ -646,15 +646,15 @@ where #[test] fn smoke_h_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } diff --git a/src/ds_impl/hp_brcu/michael_hash_map.rs b/src/ds_impl/hp_brcu/michael_hash_map.rs index 788f8422..97f5ec55 100644 --- a/src/ds_impl/hp_brcu/michael_hash_map.rs +++ b/src/ds_impl/hp_brcu/michael_hash_map.rs @@ -101,6 +101,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs b/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs index f9179665..5e036700 100644 --- a/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs @@ -562,6 +562,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_brcu/skip_list.rs b/src/ds_impl/hp_brcu/skip_list.rs index e1ec2887..e16a3b02 100644 --- a/src/ds_impl/hp_brcu/skip_list.rs +++ b/src/ds_impl/hp_brcu/skip_list.rs @@ -454,6 +454,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } From cfc9150ad640f384b871f578083cda2fc7680e03 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 30 Oct 2024 09:50:53 +0000 Subject: [PATCH 52/84] Fix a bug in HP ElimAbTree --- src/ds_impl/hp/elim_ab_tree.rs | 132 +++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 55 deletions(-) diff --git a/src/ds_impl/hp/elim_ab_tree.rs b/src/ds_impl/hp/elim_ab_tree.rs index 1162cd0e..dfcc11d0 100644 --- a/src/ds_impl/hp/elim_ab_tree.rs +++ b/src/ds_impl/hp/elim_ab_tree.rs @@ -755,17 +755,20 @@ where // Manually unlock and fix the tag. drop((parent_lock, node_lock)); - unsafe { - handle.thread.retire(cursor.l.with_tag(0).into_raw()); - } - self.fix_tag_violation(new_internal, handle); + unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; + self.fix_tag_violation(kv_pairs[left_size].0, new_internal, handle); Ok(None) } } - fn fix_tag_violation<'hp>(&self, viol: Shared>, handle: &'hp mut Handle<'_>) { - let mut stack = vec![(unsafe { viol.deref() }.search_key, viol)]; + fn fix_tag_violation<'hp>( + &self, + search_key: K, + viol: Shared>, + handle: &'hp mut Handle<'_>, + ) { + let mut stack = vec![(search_key, viol)]; while let Some((search_key, viol)) = stack.pop() { let (found, cursor) = self.search(&search_key, Some(viol), handle); if !found || cursor.l != viol { @@ -773,11 +776,11 @@ where // We hand over responsibility for `viol` to that update. continue; } - match self.fix_tag_violation_inner(&cursor, handle) { - Ok(()) => continue, - Err(None) => stack.push((search_key, viol)), - Err(Some(pair)) => stack.extend_from_slice(&[(search_key, viol), pair]), + let (success, recur) = self.fix_tag_violation_inner(&cursor, handle); + if !success { + stack.push((search_key, viol)); } + stack.extend(recur); } } @@ -785,11 +788,11 @@ where &self, cursor: &Cursor, handle: &'hp mut Handle<'_>, - ) -> Result<(), Option<(K, Shared>)>> { + ) -> (bool, Option<(K, Shared>)>) { let viol = cursor.l; let viol_node = unsafe { cursor.l.deref() }; if viol_node.weight { - return Ok(()); + return (true, None); } // `viol` should be internal because leaves always have weight = 1. @@ -808,23 +811,29 @@ where // We cannot apply this update if p has a weight violation. // So, we check if this is the case, and, if so, try to fix it. if !parent.weight { - return Err(Some((parent.search_key, cursor.p))); + return (false, Some((parent.search_key, cursor.p))); } - try_acq_val_or!(node, node_lock, Operation::Balance, None, return Err(None)); + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + return (false, None) + ); try_acq_val_or!( parent, parent_lock, Operation::Balance, None, - return Err(None) + return (false, None) ); try_acq_val_or!( gparent, gparent_lock, Operation::Balance, None, - return Err(None) + return (false, None) ); let psize = parent.size.load(Ordering::Relaxed); @@ -849,13 +858,9 @@ where node.marked.store(true, Ordering::Relaxed); parent.marked.store(true, Ordering::Relaxed); - unsafe { - handle.thread.retire(cursor.l.with_tag(0).into_raw()); - } - unsafe { - handle.thread.retire(cursor.p.with_tag(0).into_raw()); - } - return Ok(()); + unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; + unsafe { handle.thread.retire(cursor.p.with_tag(0).into_raw()) }; + return (true, None); } else { // Split case. @@ -898,16 +903,14 @@ where node.marked.store(true, Ordering::Relaxed); parent.marked.store(true, Ordering::Relaxed); - unsafe { - handle.thread.retire(cursor.l.with_tag(0).into_raw()); - }; - unsafe { - handle.thread.retire(cursor.p.with_tag(0).into_raw()); - }; + unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; + unsafe { handle.thread.retire(cursor.p.with_tag(0).into_raw()) }; drop((node_lock, parent_lock, gparent_lock)); - self.fix_tag_violation(new_internal, handle); - return Ok(()); + return ( + true, + Some((keys[left_size - 1].get().unwrap(), new_internal)), + ); } } @@ -964,7 +967,7 @@ where if new_size == Self::UNDERFULL_THRESHOLD - 1 { drop(node_lock); - self.fix_underfull_violation(cursor.l, handle); + self.fix_underfull_violation(node.search_key, cursor.l, handle); } return Ok(Some(val)); } @@ -972,8 +975,13 @@ where Err(()) } - fn fix_underfull_violation<'hp>(&self, viol: Shared>, handle: &'hp mut Handle<'_>) { - let mut stack = vec![(unsafe { viol.deref() }.search_key, viol)]; + fn fix_underfull_violation<'hp>( + &self, + search_key: K, + viol: Shared>, + handle: &'hp mut Handle<'_>, + ) { + let mut stack = vec![(search_key, viol)]; while let Some((search_key, viol)) = stack.pop() { // We search for `viol` and try to fix any violation we find there. // This entails performing AbsorbSibling or Distribute. @@ -983,11 +991,11 @@ where // We hand over responsibility for `viol` to that update. continue; } - match self.fix_underfull_violation_inner(&cursor, handle) { - Ok(()) => continue, - Err(None) => stack.push((search_key, viol)), - Err(Some(pair)) => stack.extend_from_slice(&[(search_key, viol), pair]), + let (success, recur) = self.fix_underfull_violation_inner(&cursor, handle); + if !success { + stack.push((search_key, viol)); } + stack.extend(recur); } } @@ -995,7 +1003,7 @@ where &self, cursor: &Cursor, handle: &'hp mut Handle<'_>, - ) -> Result<(), Option<(K, Shared>)>> { + ) -> (bool, ArrayVec<(K, Shared>), 2>) { let viol = cursor.l; let viol_node = unsafe { viol.deref() }; @@ -1007,7 +1015,7 @@ where || viol == self.entry.load_next(0) { // No degree violation at `viol`. - return Ok(()); + return (true, ArrayVec::<_, 2>::new()); } let node = unsafe { cursor.l.deref() }; @@ -1021,7 +1029,10 @@ where && !eq(parent, &self.entry) && cursor.p != self.entry.load_next(0) { - return Err(Some((parent.search_key, cursor.p))); + return ( + false, + ArrayVec::from_iter(once((parent.search_key, cursor.p))), + ); } // Don't need a lock on parent here because if the pointer to sibling changes @@ -1036,19 +1047,25 @@ where ((node, cursor.p_l_idx), (sibling, cursor.p_s_idx)) }; - try_acq_val_or!(left, left_lock, Operation::Balance, None, return Err(None)); + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); try_acq_val_or!( right, right_lock, Operation::Balance, None, - return Err(None) + return (false, ArrayVec::new()) ); // Repeat this check, this might have changed while we locked `viol`. if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { // No degree violation at `viol`. - return Ok(()); + return (true, ArrayVec::new()); } try_acq_val_or!( @@ -1056,14 +1073,14 @@ where parent_lock, Operation::Balance, None, - return Err(None) + return (false, ArrayVec::new()) ); try_acq_val_or!( gparent, gparent_lock, Operation::Balance, None, - return Err(None) + return (false, ArrayVec::new()) ); // We can only apply AbsorbSibling or Distribute if there are no @@ -1071,15 +1088,18 @@ where // So, we first check for any weight violations and fix any that we see. if !parent.weight { drop((left_lock, right_lock, parent_lock, gparent_lock)); - return Err(Some((parent.search_key, cursor.p))); + self.fix_tag_violation(parent.search_key, cursor.p, handle); + return (false, ArrayVec::new()); } if !node.weight { drop((left_lock, right_lock, parent_lock, gparent_lock)); - return Err(Some((node.search_key, cursor.l))); + self.fix_tag_violation(node.search_key, cursor.l, handle); + return (false, ArrayVec::new()); } if !sibling.weight { drop((left_lock, right_lock, parent_lock, gparent_lock)); - return Err(Some((node.search_key, cursor.s))); + self.fix_tag_violation(sibling.search_key, cursor.s, handle); + return (false, ArrayVec::new()); } // There are no weight violations at `parent`, `node` or `sibling`. @@ -1142,8 +1162,7 @@ where } drop((left_lock, right_lock, parent_lock, gparent_lock)); - self.fix_underfull_violation(new_node, handle); - return Ok(()); + return (true, ArrayVec::from_iter(once((node.search_key, new_node)))); } else { debug_assert!(!eq(gparent, &self.entry) || psize > 2); let mut new_parent = Node::internal(true, psize - 1, parent.search_key); @@ -1183,9 +1202,12 @@ where } drop((left_lock, right_lock, parent_lock, gparent_lock)); - self.fix_underfull_violation(new_node, handle); - self.fix_underfull_violation(new_parent, handle); - return Ok(()); + return ( + true, + ArrayVec::from_iter( + [(node.search_key, new_node), (parent.search_key, new_parent)].into_iter(), + ), + ); } } else { // Distribute @@ -1293,7 +1315,7 @@ where handle.thread.retire(cursor.s.with_tag(0).into_raw()); } - return Ok(()); + return (true, ArrayVec::new()); } } } From 336be6ab19d0b7cff93f57a471a9207f7ddcbf06 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 30 Oct 2024 11:42:03 +0000 Subject: [PATCH 53/84] Implement HP-(B)RCU ElimAbTree --- smrs/hp-brcu/src/pointers.rs | 9 + src/bin/hp-brcu.rs | 3 +- src/bin/hp-rcu.rs | 3 +- src/ds_impl/hp_brcu/concurrent_map.rs | 3 +- src/ds_impl/hp_brcu/elim_ab_tree.rs | 1388 +++++++++++++++++++++++++ src/ds_impl/hp_brcu/mod.rs | 2 + 6 files changed, 1404 insertions(+), 4 deletions(-) create mode 100644 src/ds_impl/hp_brcu/elim_ab_tree.rs diff --git a/smrs/hp-brcu/src/pointers.rs b/smrs/hp-brcu/src/pointers.rs index 183cffc2..bdaf0a91 100644 --- a/smrs/hp-brcu/src/pointers.rs +++ b/smrs/hp-brcu/src/pointers.rs @@ -140,6 +140,15 @@ impl Default for Atomic { } } +impl Clone for Atomic { + fn clone(&self) -> Self { + Self { + link: AtomicUsize::new(self.link.load(Ordering::Relaxed)), + _marker: PhantomData, + } + } +} + /// A pointer to a shared object. /// /// This pointer is valid for use only during the lifetime `'r`. diff --git a/src/bin/hp-brcu.rs b/src/bin/hp-brcu.rs index 511fd25c..d484888d 100644 --- a/src/bin/hp-brcu.rs +++ b/src/bin/hp-brcu.rs @@ -10,7 +10,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp_brcu::{ - BonsaiTreeMap, ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -36,6 +36,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::BonsaiTree => { bench_map::>(config, PrefillStrategy::Decreasing) } + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), _ => panic!("Unsupported(or unimplemented) data structure for HP-BRCU"), }; output.write_record(config, &perf); diff --git a/src/bin/hp-rcu.rs b/src/bin/hp-rcu.rs index a5661bd8..95dd481d 100644 --- a/src/bin/hp-rcu.rs +++ b/src/bin/hp-rcu.rs @@ -10,7 +10,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp_brcu::{ - BonsaiTreeMap, ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -37,6 +37,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::BonsaiTree => { bench_map::>(config, PrefillStrategy::Decreasing) } + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), _ => panic!("Unsupported(or unimplemented) data structure for HP-BRCU"), }; output.write_record(config, &perf); diff --git a/src/ds_impl/hp_brcu/concurrent_map.rs b/src/ds_impl/hp_brcu/concurrent_map.rs index 8c97fae9..2c38024a 100644 --- a/src/ds_impl/hp_brcu/concurrent_map.rs +++ b/src/ds_impl/hp_brcu/concurrent_map.rs @@ -15,8 +15,7 @@ pub trait ConcurrentMap { fn new() -> Self; fn get(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool; fn insert(&self, key: K, value: V, output: &mut Self::Output, thread: &mut Thread) -> bool; - fn remove<'domain, 'hp>(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) - -> bool; + fn remove(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool; } #[cfg(test)] diff --git a/src/ds_impl/hp_brcu/elim_ab_tree.rs b/src/ds_impl/hp_brcu/elim_ab_tree.rs new file mode 100644 index 00000000..8d018e29 --- /dev/null +++ b/src/ds_impl/hp_brcu/elim_ab_tree.rs @@ -0,0 +1,1388 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use arrayvec::ArrayVec; +use hp_brcu::{Atomic, Handle, Owned, Pointer, RollbackProof, Shared, Shield, Thread, Unprotected}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next<'g, G: Handle>(&self, index: usize, guard: &'g G) -> Shared<'g, Self> { + self.next()[index].load(Ordering::Acquire, guard) + } + + fn load_next_locked<'g>(&'g self, index: usize, _: &MCSLockGuard<'g, K, V>) -> Shared { + // Safety: the node is locked by this thread. + self.next()[index].load(Ordering::Acquire, &unsafe { Unprotected::new() }) + } + + fn store_next<'g>(&'g self, index: usize, ptr: Shared, _: &MCSLockGuard<'g, K, V>) { + // Safety: the node is locked by this thread. + self.next()[index].store(ptr, Ordering::Release, &unsafe { Unprotected::new() }); + } + + fn init_next<'g>(&mut self, index: usize, ptr: Shared) { + self.next_mut()[index] = Atomic::from(ptr); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } + + fn p_s_idx(p_l_idx: usize) -> usize { + if p_l_idx > 0 { + p_l_idx - 1 + } else { + 1 + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &'g MCSLockGuard<'g, K, V>, + ) -> impl Iterator, Shared)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next_locked(i, lock))) + .chain(once((None, self.load_next_locked(self.key_count(), lock)))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +pub struct Cursor { + l: Shield>, + p: Shield>, + gp: Shield>, + // A pointer to a sibling node. + s: Shield>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + p_s_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +impl Cursor { + fn empty(thread: &mut Thread) -> Self { + Self { + l: Shield::null(thread), + p: Shield::null(thread), + gp: Shield::null(thread), + s: Shield::null(thread), + gp_p_idx: 0, + p_l_idx: 0, + p_s_idx: 0, + l_key_idx: 0, + val: None, + } + } +} + +impl OutputHolder for Cursor { + fn default(thread: &mut Thread) -> Self { + Self::empty(thread) + } + + fn output(&self) -> &V { + self.val.as_ref().unwrap() + } +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Owned::new(left).into_shared()); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, cursor: &mut Cursor, thread: &mut Thread) { + unsafe { + cursor.val = thread.critical_section(|guard| { + let mut node = self.entry.load_next(0, guard).deref(); + while let NodeKind::Internal { next } = &node.kind { + let next = next[node.child_index(key)].load(Ordering::Acquire, guard); + node = next.deref(); + } + node.read_consistent(key).1 + }); + } + } + + fn search( + &self, + key: &K, + target: Option>>, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> bool { + unsafe { + thread.critical_section(|guard| { + let mut l = self.entry.load_next(0, guard); + let mut s = self.entry.load_next(1, guard); + let mut p = Shared::from_usize(&self.entry as *const _ as usize); + let mut gp = Shared::null(); + let mut gp_p_idx = 0; + let mut p_l_idx = 0; + let mut p_s_idx = 1; + let mut l_key_idx = 0; + + while !l.deref().is_leaf() && target.map(|target| target != l).unwrap_or(true) { + let l_node = l.deref(); + gp = p; + p = l; + gp_p_idx = p_l_idx; + p_l_idx = l_node.child_index(key); + p_s_idx = Node::::p_s_idx(p_l_idx); + l = l_node.load_next(p_l_idx, guard); + s = l_node.load_next(p_s_idx, guard); + } + + let found = if let Some(target) = target { + l == target + } else { + let (index, value) = l.deref().read_consistent(key); + cursor.val = value; + l_key_idx = index; + value.is_some() + }; + cursor.l.protect(l); + cursor.s.protect(s); + cursor.p.protect(p); + cursor.gp.protect(gp); + cursor.gp_p_idx = gp_p_idx; + cursor.p_l_idx = p_l_idx; + cursor.p_s_idx = p_s_idx; + cursor.l_key_idx = l_key_idx; + found + }) + } + } + + /// TODO: start from here... + pub fn insert( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> bool { + loop { + self.search(key, None, cursor, thread); + if cursor.val.is_some() { + return false; + } + if let Ok(result) = self.insert_inner(key, value, cursor, thread) { + cursor.val = result; + return result.is_none(); + } + } + } + + fn insert_inner( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Owned::new(left).into_shared()); + internal.init_next(1, Owned::new(right).into_shared()); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Owned::new(internal).into_shared(); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { thread.retire(cursor.l.shared()) }; + self.fix_tag_violation(kv_pairs[left_size].0, new_internal.as_raw(), cursor, thread); + + Ok(None) + } + } + + fn fix_tag_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + thread: &mut Thread, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + let found = self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + thread, + ); + if !found || cursor.l.as_raw() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_tag_violation_inner(cursor, thread); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_tag_violation_inner<'c>( + &self, + cursor: &Cursor, + thread: &mut Thread, + ) -> (bool, Option<(K, usize)>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { cursor.l.deref() }; + if viol_node.weight { + return (true, None); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!( + !eq(viol_node, &self.entry) + && self.entry.load_next(0, unsafe { &Unprotected::new() }) != viol + ); + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + return (false, Some((parent.search_key, cursor.p.as_raw()))); + } + + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, None) + ); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next( + cursor.gp_p_idx, + Owned::new(absorber).into_shared(), + &gparent_lock, + ); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { thread.retire(cursor.l.shared()) }; + unsafe { thread.retire(cursor.p.shared()) }; + return (true, None); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Owned::new(left).into_shared()); + new_internal.init_next(1, Owned::new(right).into_shared()); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Owned::new(new_internal).into_shared(); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { thread.retire(cursor.l.shared()) }; + unsafe { thread.retire(cursor.p.shared()) }; + + drop((node_lock, parent_lock, gparent_lock)); + return ( + true, + Some((keys[left_size - 1].get().unwrap(), new_internal.as_raw())), + ); + } + } + + pub fn remove(&self, key: &K, cursor: &mut Cursor, thread: &mut Thread) -> bool { + loop { + self.search(key, None, cursor, thread); + if cursor.val.is_none() { + return false; + } + if let Ok(result) = self.remove_inner(key, cursor, thread) { + cursor.val = result; + return result.is_some(); + } + } + } + + fn remove_inner( + &self, + key: &K, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = cursor.gp.as_ref(); + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation( + node.search_key, + cursor.l.as_raw(), + cursor, + thread, + ); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + thread: &mut Thread, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + thread, + ); + if cursor.l.as_raw() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_underfull_violation_inner(cursor, thread); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_underfull_violation_inner( + &self, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> (bool, ArrayVec<(K, usize), 2>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { viol.deref() }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0, unsafe { &Unprotected::new() }) + { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p.shared() != self.entry.load_next(0, unsafe { &Unprotected::new() }) + { + return ( + false, + ArrayVec::from_iter(once((parent.search_key, cursor.p.as_raw()))), + ); + } + + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling = unsafe { cursor.s.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if cursor.p_s_idx < cursor.p_l_idx { + ((sibling, cursor.p_s_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, cursor.p_s_idx)) + }; + + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(parent.search_key, cursor.p.as_raw(), cursor, thread); + return (false, ArrayVec::new()); + } + if !node.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(node.search_key, cursor.l.as_raw(), cursor, thread); + return (false, ArrayVec::new()); + } + if !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(sibling.search_key, cursor.s.as_raw(), cursor, thread); + return (false, ArrayVec::new()); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Node::leaf(true, size, node.search_key); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Node::internal(true, size, node.search_key); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + }; + let new_node = Owned::new(new_node).into_shared(); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + thread.retire(cursor.l.shared()); + thread.retire(cursor.p.shared()); + thread.retire(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter(once((node.search_key, new_node.as_raw()))), + ); + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..cursor.p_s_idx { + new_parent.init_next(i, parent.load_next(i, unsafe { &Unprotected::new() })); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent + .init_next(i - 1, parent.load_next(i, unsafe { &Unprotected::new() })); + } + + new_parent.init_next( + cursor.p_l_idx + - (if cursor.p_l_idx > cursor.p_s_idx { + 1 + } else { + 0 + }), + new_node, + ); + let new_parent = Owned::new(new_parent).into_shared(); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + thread.retire(cursor.l.shared()); + thread.retire(cursor.p.shared()); + thread.retire(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter( + [ + (node.search_key, new_node.as_raw()), + (parent.search_key, new_parent.as_raw()), + ] + .into_iter(), + ), + ); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Node::leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Node::leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let (new_left, pivot) = { + let mut new_internal = Node::internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = Node::internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = Node::internal(parent.weight, psize, parent.search_key); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, Owned::new(new_left).into_shared()); + new_parent.init_next(right_idx, Owned::new(new_right).into_shared()); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next( + cursor.gp_p_idx, + Owned::new(new_parent).into_shared(), + &gparent_lock, + ); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + thread.retire(cursor.l.shared()); + thread.retire(cursor.p.shared()); + thread.retire(cursor.s.shared()); + } + + return (true, ArrayVec::new()); + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let guard = unsafe { &Unprotected::new() }; + let mut stack = vec![]; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + type Output = Cursor; + + fn new() -> Self { + Self::new() + } + + fn get(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.search_basic(key, output, thread); + output.val.is_some() + } + + fn insert(&self, key: K, value: V, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.insert(&key, &value, output, thread) + } + + fn remove(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.remove(key, output, thread) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::hp_brcu::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/hp_brcu/mod.rs b/src/ds_impl/hp_brcu/mod.rs index fb1937d7..72f79678 100644 --- a/src/ds_impl/hp_brcu/mod.rs +++ b/src/ds_impl/hp_brcu/mod.rs @@ -1,6 +1,7 @@ pub mod concurrent_map; mod bonsai_tree; +mod elim_ab_tree; mod list; pub mod list_alter; mod michael_hash_map; @@ -9,6 +10,7 @@ mod skip_list; pub use self::concurrent_map::ConcurrentMap; pub use bonsai_tree::BonsaiTreeMap; +pub use elim_ab_tree::ElimABTree; pub use list::{HHSList, HList, HMList}; pub use michael_hash_map::HashMap; pub use natarajan_mittal_tree::NMTreeMap; From 0c28af546ac4edd36f825e0773afca9680309938 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 30 Oct 2024 11:42:26 +0000 Subject: [PATCH 54/84] Use `Release` orderings for marking in HP ElimAbTree --- src/ds_impl/hp/elim_ab_tree.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/ds_impl/hp/elim_ab_tree.rs b/src/ds_impl/hp/elim_ab_tree.rs index dfcc11d0..072607c1 100644 --- a/src/ds_impl/hp/elim_ab_tree.rs +++ b/src/ds_impl/hp/elim_ab_tree.rs @@ -855,8 +855,8 @@ where slice_clone(&keys, &mut absorber.keys, DEGREE); gparent.store_next(cursor.gp_p_idx, Shared::from_owned(absorber), &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; unsafe { handle.thread.retire(cursor.p.with_tag(0).into_raw()) }; @@ -900,8 +900,8 @@ where let new_internal = Shared::from_owned(new_internal); gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; unsafe { handle.thread.retire(cursor.p.with_tag(0).into_raw()) }; @@ -1151,9 +1151,9 @@ where if eq(gparent, &self.entry) && psize == 2 { debug_assert!(cursor.gp_p_idx == 0); gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); - sibling.marked.store(true, Ordering::Relaxed); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + sibling.marked.store(true, Ordering::Release); unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()); @@ -1191,9 +1191,9 @@ where let new_parent = Shared::from_owned(new_parent); gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); - sibling.marked.store(true, Ordering::Relaxed); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + sibling.marked.store(true, Ordering::Release); unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()); @@ -1305,9 +1305,9 @@ where Shared::from_owned(new_parent), &gparent_lock, ); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); - sibling.marked.store(true, Ordering::Relaxed); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + sibling.marked.store(true, Ordering::Release); unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()); From 80fbe7f24c37923543542c459b701af8b25ec5a4 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 30 Oct 2024 14:17:05 +0000 Subject: [PATCH 55/84] Update the bench script of HP-revisted --- bench-scripts/hp-revisited/bench.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bench-scripts/hp-revisited/bench.py b/bench-scripts/hp-revisited/bench.py index 26f078b1..9e341321 100644 --- a/bench-scripts/hp-revisited/bench.py +++ b/bench-scripts/hp-revisited/bench.py @@ -6,9 +6,9 @@ RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") -dss = ['h-list', 'hm-list', 'hhs-list', 'hash-map', 'nm-tree', 'skip-list'] +dss = ['h-list', 'hhs-list', 'hash-map', 'nm-tree', 'skip-list', 'elim-ab-tree'] # "-large" suffix if it uses a large garbage bag. -mms = ['nr', 'ebr', 'pebr', 'hp', 'hp-pp', 'nbr', 'hp-brcu', 'vbr'] +mms = ['nr', 'ebr', 'pebr', 'hp', 'hp-pp', 'hp-brcu', 'vbr'] i = 10 cpu_count = os.cpu_count() if not cpu_count or cpu_count <= 24: @@ -54,6 +54,8 @@ def invalid(mm, ds, g): is_invalid |= g == 0 # HHSList is just HList with faster get() if mm == 'nbr': is_invalid |= ds in ["hm-list", "skip-list"] + if ds == 'elim-ab-tree': + is_invalid |= mm in ["pebr", "hp-pp", "vbr"] return is_invalid cmds = [] From bac2af2641429a157541baf436e69a1371a11de2 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 1 Nov 2024 04:25:31 +0000 Subject: [PATCH 56/84] Add ElimAbTree in the plot script --- bench-scripts/hp-revisited/plot.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bench-scripts/hp-revisited/plot.py b/bench-scripts/hp-revisited/plot.py index 9b36deb0..b13d0a50 100644 --- a/bench-scripts/hp-revisited/plot.py +++ b/bench-scripts/hp-revisited/plot.py @@ -29,6 +29,7 @@ HASHMAP = "hash-map" NMTREE = "nm-tree" SKIPLIST = "skip-list" +ELIMABTREE = "elim-ab-tree" FORMAL_NAMES = { HLIST: "HList", @@ -36,12 +37,13 @@ HASHMAP: "HashMap", NMTREE: "NMTree", SKIPLIST: "SkipList", + ELIMABTREE: "ElimAbTRee", } # DS with read-dominated bench & write-only bench -dss_all = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST] -dss_read = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST] -dss_write = [HLIST, HASHMAP, NMTREE, SKIPLIST] +dss_all = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] +dss_read = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] +dss_write = [HLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] WRITE, HALF, READ = "write", "half", "read" From 490e30b4ad047e9b8e7f1326b4cd33da871a34af Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 1 Nov 2024 04:37:25 +0000 Subject: [PATCH 57/84] Fix a small typo error in `plot.py` --- bench-scripts/hp-revisited/plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-scripts/hp-revisited/plot.py b/bench-scripts/hp-revisited/plot.py index b13d0a50..90f4b70e 100644 --- a/bench-scripts/hp-revisited/plot.py +++ b/bench-scripts/hp-revisited/plot.py @@ -37,7 +37,7 @@ HASHMAP: "HashMap", NMTREE: "NMTree", SKIPLIST: "SkipList", - ELIMABTREE: "ElimAbTRee", + ELIMABTREE: "ElimAbTree", } # DS with read-dominated bench & write-only bench From 327046f8f8f74660a4a189ae5f3cf1e7783dfd69 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 1 Nov 2024 12:05:26 +0000 Subject: [PATCH 58/84] Fix bugs in HP(++) HHSList --- src/ds_impl/hp/list.rs | 6 +++--- src/ds_impl/hp_pp/list.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index cceca5de..2d9e1084 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -628,11 +628,11 @@ where handle: &'hp mut Self::Handle<'_>, key: &'hp K, ) -> Option> { - self.inner.harris_michael_get(key, handle) + self.inner.harris_herlihy_shavit_get(key, handle) } #[inline(always)] fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { - self.inner.harris_michael_insert(key, value, handle) + self.inner.harris_insert(key, value, handle) } #[inline(always)] fn remove<'hp>( @@ -640,7 +640,7 @@ where handle: &'hp mut Self::Handle<'_>, key: &'hp K, ) -> Option> { - self.inner.harris_michael_remove(key, handle) + self.inner.harris_remove(key, handle) } } diff --git a/src/ds_impl/hp_pp/list.rs b/src/ds_impl/hp_pp/list.rs index 45551dd6..a0e568ce 100644 --- a/src/ds_impl/hp_pp/list.rs +++ b/src/ds_impl/hp_pp/list.rs @@ -693,11 +693,11 @@ where handle: &'hp mut Self::Handle<'_>, key: &'hp K, ) -> Option> { - self.inner.harris_michael_get(key, handle) + self.inner.harris_herlihy_shavit_get(key, handle) } #[inline(always)] fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { - self.inner.harris_michael_insert(key, value, handle) + self.inner.harris_insert(key, value, handle) } #[inline(always)] fn remove<'hp>( @@ -705,7 +705,7 @@ where handle: &'hp mut Self::Handle<'_>, key: &'hp K, ) -> Option> { - self.inner.harris_michael_remove(key, handle) + self.inner.harris_remove(key, handle) } } From e1cd97495b7fc3f7eebc5ac54c44902dc1498fb3 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 5 Nov 2024 11:53:44 +0000 Subject: [PATCH 59/84] Implement CrystallineL and its lists --- Cargo.lock | 11 + Cargo.toml | 2 + smrs/crystalline-l/Cargo.toml | 10 + smrs/crystalline-l/src/lib.rs | 715 ++++++++++++++++++++ src/bin/crystalline-l.rs | 211 ++++++ src/ds_impl/crystalline_l/concurrent_map.rs | 107 +++ src/ds_impl/crystalline_l/list.rs | 682 +++++++++++++++++++ src/ds_impl/crystalline_l/mod.rs | 7 + src/ds_impl/mod.rs | 1 + 9 files changed, 1746 insertions(+) create mode 100644 smrs/crystalline-l/Cargo.toml create mode 100644 smrs/crystalline-l/src/lib.rs create mode 100644 src/bin/crystalline-l.rs create mode 100644 src/ds_impl/crystalline_l/concurrent_map.rs create mode 100644 src/ds_impl/crystalline_l/list.rs create mode 100644 src/ds_impl/crystalline_l/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f4421d9a..30c84452 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -296,6 +296,16 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crystalline-l" +version = "0.1.0" +dependencies = [ + "atomic", + "crossbeam-utils 0.8.20", + "rustc-hash", + "static_assertions 1.1.0", +] + [[package]] name = "csv" version = "1.3.0" @@ -816,6 +826,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-pebr-epoch", "crossbeam-utils 0.8.20", + "crystalline-l", "csv", "hp-brcu", "hp_pp", diff --git a/Cargo.toml b/Cargo.toml index 17378bf9..905c982e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "./smrs/hp-brcu", "./smrs/vbr", "./smrs/circ", + "./smrs/crystalline-l", ] [package] @@ -33,6 +34,7 @@ cdrc = { path = "./smrs/cdrc" } hp-brcu = { path = "./smrs/hp-brcu" } vbr = { path = "./smrs/vbr" } circ = { path = "./smrs/circ" } +crystalline-l = { path = "./smrs/crystalline-l" } [target.'cfg(target_os = "linux")'.dependencies] tikv-jemallocator = "0.5" diff --git a/smrs/crystalline-l/Cargo.toml b/smrs/crystalline-l/Cargo.toml new file mode 100644 index 00000000..4a3535b9 --- /dev/null +++ b/smrs/crystalline-l/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "crystalline-l" +version = "0.1.0" +edition = "2021" + +[dependencies] +crossbeam-utils = "0.8.14" +rustc-hash = "1.1.0" +atomic = "0.5" +static_assertions = "1.1.0" diff --git a/smrs/crystalline-l/src/lib.rs b/smrs/crystalline-l/src/lib.rs new file mode 100644 index 00000000..91e49ca0 --- /dev/null +++ b/smrs/crystalline-l/src/lib.rs @@ -0,0 +1,715 @@ +use std::cell::{Cell, RefCell, RefMut, UnsafeCell}; +use std::mem::{align_of, size_of, swap, ManuallyDrop}; +use std::ptr::null_mut; +use std::sync::atomic::{AtomicIsize, AtomicPtr, AtomicUsize, Ordering}; +use std::sync::Mutex; + +use crossbeam_utils::CachePadded; +use static_assertions::const_assert_eq; + +const SLOTS_CAP: usize = 16; +const CACHE_CAP: usize = 12; +static COLLECT_FREQ: AtomicUsize = AtomicUsize::new(110); + +pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); + +pub fn set_collect_frequency(freq: usize) { + COLLECT_FREQ.store(freq, Ordering::Relaxed); +} + +const fn invalid_ptr() -> *mut T { + usize::MAX as *mut T +} + +#[repr(C)] +struct Node { + item: T, + info: UnsafeCell>, +} + +impl Node { + fn new(item: T, birth_era: usize) -> Self { + Self { + item, + info: UnsafeCell::new(NodeInfo::alive(birth_era)), + } + } + + unsafe fn birth_era(&self) -> usize { + (*self.info.get()).birth_era + } + + unsafe fn next(&self) -> &AtomicPtr> { + &(*self.info.get()).ret_info.ns.next + } + + unsafe fn set_next(&self, ptr: *mut Node, order: Ordering) { + (*self.info.get()).ret_info.ns.next.store(ptr, order); + } + + unsafe fn slot(&self) -> *mut AtomicPtr> { + (*self.info.get()).ret_info.ns.slot + } + + unsafe fn set_slot(&self, ptr: *mut AtomicPtr>) { + (*(*self.info.get()).ret_info).ns.slot = ptr + } + + unsafe fn batch_link(&self) -> *mut Node { + (*self.info.get()).ret_info.batch_link + } + + unsafe fn set_batch_link(&self, ptr: *mut Node) { + (*(*self.info.get()).ret_info).batch_link = ptr; + } + + unsafe fn refs(&self) -> &AtomicIsize { + &(*self.info.get()).ret_info.rb.refs + } + + unsafe fn batch_next(&self) -> *mut Node { + (*self.info.get()).ret_info.rb.batch_next + } + + unsafe fn set_batch_next(&self, ptr: *mut Node) { + (*(*self.info.get()).ret_info).rb.batch_next = ptr; + } + + #[inline] + fn free_list(&self) { + let mut curr = (self as *const Node).cast_mut(); + unsafe { + while let Some(curr_node) = curr.as_ref() { + let mut start = curr_node.batch_link(); + curr = curr_node.next().load(Ordering::SeqCst); + + let mut count = 0; + loop { + let obj = start; + start = (*obj).batch_next(); + count += 1; + drop(Box::from_raw(obj)); + if start.is_null() { + break; + } + } + GLOBAL_GARBAGE_COUNT.fetch_sub(count, Ordering::AcqRel); + } + } + } +} + +#[repr(C)] +union NodeInfo { + birth_era: usize, + ret_info: ManuallyDrop>, +} + +#[repr(C)] +struct RetiredInfo { + ns: NextOrSlot, + batch_link: *mut Node, + rb: RefsOrBatNxt, +} + +#[repr(C)] +union NextOrSlot { + next: ManuallyDrop>>, + slot: *mut AtomicPtr>, +} + +#[repr(C)] +union RefsOrBatNxt { + refs: ManuallyDrop, + batch_next: *mut Node, +} + +impl NodeInfo { + fn alive(birth_era: usize) -> Self { + Self { birth_era } + } +} + +struct Batch { + min_era: usize, + first: *mut Node, + last: *mut Node, + count: usize, + list_count: usize, + list: *mut Node, +} + +impl Default for Batch { + fn default() -> Self { + Self { + min_era: 0, + first: null_mut(), + last: null_mut(), + count: 0, + list_count: 0, + list: null_mut(), + } + } +} + +impl Batch { + fn traverse(&mut self, next: *mut Node) { + let mut curr = next; + unsafe { + while let Some(curr_node) = curr.as_ref() { + curr = curr_node.next().load(Ordering::Acquire); + let refs = &*curr_node.batch_link(); + if refs.refs().fetch_sub(1, Ordering::AcqRel) == 1 { + // Here, we have exclusive ownership. + refs.set_next(self.list, Ordering::SeqCst); + self.list = refs as *const _ as *mut _; + } + } + } + } + + fn traverse_cache(&mut self, next: *mut Node) { + if next.is_null() { + return; + } + if self.list_count == CACHE_CAP { + unsafe { self.list.as_ref().map(|list| list.free_list()) }; + self.list = null_mut(); + self.list_count = 0; + } + self.list_count += 1; + self.traverse(next); + } +} + +#[repr(C)] +struct EraSlots { + first: [AtomicPtr>; SLOTS_CAP], + era: [AtomicUsize; SLOTS_CAP], +} + +// Some static checks for a hack in `try_retire`. +const_assert_eq!(size_of::>>(), size_of::()); +const_assert_eq!( + size_of::>(), + size_of::() * SLOTS_CAP * 2 +); + +impl Default for EraSlots { + fn default() -> Self { + // Derive macro requires `T: Default`, which is unnecessary. + Self { + first: [(); SLOTS_CAP].map(|_| AtomicPtr::new(invalid_ptr())), + era: Default::default(), + } + } +} + +pub struct Domain { + slots: Vec>>, + batches: Vec>>>, + era: CachePadded, + avail_hidx: Mutex>, +} + +unsafe impl Sync for Domain {} +unsafe impl Send for Domain {} + +impl Drop for Domain { + fn drop(&mut self) { + debug_assert_eq!(self.avail_hidx.lock().unwrap().len(), self.slots.len()); + let handles = (0..self.slots.len()) + .map(|_| self.register()) + .collect::>(); + for handle in handles { + unsafe { handle.clear_all() }; + } + } +} + +impl Domain { + pub fn new(handles: usize) -> Self { + Self { + slots: (0..handles).map(|_| Default::default()).collect(), + batches: (0..handles).map(|_| Default::default()).collect(), + era: CachePadded::new(AtomicUsize::new(1)), + avail_hidx: Mutex::new((0..handles).collect()), + } + } + + fn load_era(&self) -> usize { + self.era.load(Ordering::Acquire) + } + + pub fn register(&self) -> Handle<'_, T> { + Handle { + domain: self, + hidx: self.avail_hidx.lock().unwrap().pop().unwrap(), + alloc_count: Cell::new(0), + avail_sidx: RefCell::new((0..SLOTS_CAP).collect()), + } + } +} + +pub struct Handle<'d, T> { + domain: &'d Domain, + hidx: usize, + alloc_count: Cell, + avail_sidx: RefCell>, +} + +impl<'d, T> Handle<'d, T> { + fn incr_alloc(&self) { + let next_count = self.alloc_count.get() + 1; + self.alloc_count.set(next_count); + if next_count % COLLECT_FREQ.load(Ordering::Relaxed) == 0 { + self.domain.era.fetch_add(1, Ordering::AcqRel); + } + } + + pub fn global_era(&self) -> usize { + self.domain.load_era() + } + + fn slots(&self) -> &EraSlots { + &self.domain.slots[self.hidx] + } + + fn batch_mut(&self) -> RefMut<'_, Batch> { + self.domain.batches[self.hidx].borrow_mut() + } + + pub unsafe fn retire(&self, ptr: Shared) { + debug_assert!(!ptr.is_null()); + let node = unsafe { ptr.ptr.deref() }; + let mut batch = self.batch_mut(); + if batch.first.is_null() { + batch.min_era = unsafe { node.birth_era() }; + batch.last = node as *const _ as *mut _; + } else { + let birth_era = unsafe { node.birth_era() }; + if batch.min_era > birth_era { + batch.min_era = birth_era; + } + unsafe { node.set_batch_link(batch.last) }; + } + + // Implicitly initialize refs to 0 for the last node. + unsafe { node.set_batch_next(batch.first) }; + + let node_ptr = node as *const _ as *mut Node; + batch.first = node_ptr; + batch.count += 1; + if batch.count % COLLECT_FREQ.load(Ordering::Relaxed) == 0 { + unsafe { (*batch.last).set_batch_link(node_ptr) }; + GLOBAL_GARBAGE_COUNT.fetch_add(batch.count, Ordering::AcqRel); + self.try_retire(batch); + } + } + + fn try_retire(&self, mut batch: RefMut<'_, Batch>) { + let mut curr = batch.first; + let refs = batch.last; + let min_era = batch.min_era; + + // Find available slots. + let mut last = curr; + for slot in &self.domain.slots { + for i in 0..SLOTS_CAP { + let first = slot.first[i].load(Ordering::Acquire); + if first == invalid_ptr() { + continue; + } + let era = slot.era[i].load(Ordering::Acquire); + if era < min_era { + continue; + } + if last == refs { + return; + } + unsafe { (*last).set_slot(&slot.first[i] as *const _ as *mut _) }; + last = unsafe { (*last).batch_next() }; + } + } + // Retire if successful. + let mut adjs = 0; + while curr != last { + 'body: { + unsafe { + let slot_first = (*curr).slot(); + // HACK: Get a corresponding era. Hint: See the layout of `EraSlots`... + let slot_era = slot_first.offset(SLOTS_CAP as isize) as *mut AtomicUsize; + let mut prev = (*slot_first).load(Ordering::Acquire); + + loop { + if prev == invalid_ptr() { + break 'body; + } + let era = (*slot_era).load(Ordering::Acquire); + if era < min_era { + break 'body; + } + (*curr).set_next(prev, Ordering::Relaxed); + match (*slot_first).compare_exchange_weak( + prev, + curr, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break, + Err(new) => prev = new, + } + } + adjs += 1; + } + } + curr = unsafe { (*curr).batch_next() }; + } + + // Adjust the reference count. + unsafe { + if (*refs).refs().fetch_add(adjs, Ordering::AcqRel) == -adjs { + (*refs).set_next(null_mut(), Ordering::SeqCst); + (*refs).free_list(); + } + } + batch.first = null_mut(); + batch.count = 0; + } + + pub unsafe fn clear_all(&self) { + let mut first = [null_mut(); SLOTS_CAP]; + for i in 0..SLOTS_CAP { + first[i] = self.slots().first[i].swap(invalid_ptr(), Ordering::AcqRel); + } + for i in 0..SLOTS_CAP { + if first[i] != invalid_ptr() { + self.batch_mut().traverse(first[i]); + } + } + let mut batch = self.batch_mut(); + unsafe { batch.list.as_ref().map(|list| list.free_list()) }; + batch.list = null_mut(); + batch.list_count = 0; + } +} + +impl<'d, T> Drop for Handle<'d, T> { + fn drop(&mut self) { + self.domain.avail_hidx.lock().unwrap().push(self.hidx); + } +} + +pub struct HazardEra<'d, 'h, T> { + handle: &'h Handle<'d, T>, + sidx: usize, +} + +impl<'d, 'h, T> HazardEra<'d, 'h, T> { + pub fn new(handle: &'h Handle<'d, T>) -> Self { + let sidx = handle.avail_sidx.borrow_mut().pop().unwrap(); + Self { handle, sidx } + } + + pub fn era(&self) -> usize { + self.handle.slots().era[self.sidx].load(Ordering::Acquire) + } + + pub fn update_era(&mut self, mut curr_era: usize) -> usize { + let first_link = &self.handle.slots().first[self.sidx]; + if !first_link.load(Ordering::Acquire).is_null() { + let first = first_link.swap(invalid_ptr(), Ordering::AcqRel); + if first != invalid_ptr() { + self.handle.batch_mut().traverse_cache(first); + } + first_link.store(null_mut(), Ordering::SeqCst); + curr_era = self.handle.global_era(); + } + self.handle.slots().era[self.sidx].store(curr_era, Ordering::SeqCst); + return curr_era; + } + + pub fn swap(h1: &mut Self, h2: &mut Self) { + swap(&mut h1.sidx, &mut h2.sidx); + } +} + +impl<'d, 'h, T> Drop for HazardEra<'d, 'h, T> { + fn drop(&mut self) { + self.handle.avail_sidx.borrow_mut().push(self.sidx); + } +} + +pub struct Shared { + ptr: TaggedPtr>, +} + +impl<'d, T> From for Shared { + fn from(value: usize) -> Self { + Self { + ptr: TaggedPtr::from(value as *const _), + } + } +} + +impl<'d, T> Clone for Shared { + fn clone(&self) -> Self { + Self { ptr: self.ptr } + } +} + +impl<'d, T> Copy for Shared {} + +impl<'d, T> Shared { + pub fn new(item: T, handle: &Handle<'d, T>) -> Self { + handle.incr_alloc(); + let era = handle.global_era(); + Self { + ptr: TaggedPtr::from(Box::into_raw(Box::new(Node::new(item, era)))), + } + } + + pub fn null() -> Self { + Self { + ptr: TaggedPtr::null(), + } + } + + pub fn tag(&self) -> usize { + self.ptr.tag() + } + + pub fn is_null(&self) -> bool { + self.ptr.is_null() + } + + pub fn with_tag(&self, tag: usize) -> Self { + Self::from_raw(self.ptr.with_tag(tag)) + } + + pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { + self.ptr.as_ref().map(|node| &node.item) + } + + pub unsafe fn deref<'g>(&self) -> &'g T { + &self.ptr.deref().item + } + + pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { + &mut self.ptr.deref_mut().item + } + + fn from_raw(ptr: TaggedPtr>) -> Self { + Self { ptr } + } + + pub unsafe fn into_owned(self) -> T { + Box::from_raw(self.ptr.as_raw()).item + } + + /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, + /// are identical. + pub fn ptr_eq(self, other: Self) -> bool { + self.ptr.ptr_eq(other.ptr) + } +} + +// Technically, `Atomic` and `Shared` should depend on the lifetime of the domain `'d`, +// but it can create a circular lifetime dependency because it will make `T` in `Domain` +// also depend on `'d`. +pub struct Atomic { + link: atomic::Atomic>>, +} + +const_assert_eq!( + size_of::>>>(), + size_of::() +); + +unsafe impl Sync for Atomic {} +unsafe impl Send for Atomic {} + +impl Default for Atomic { + fn default() -> Self { + Self::null() + } +} + +impl From> for Atomic { + fn from(value: Shared) -> Self { + Self { + link: atomic::Atomic::new(value.ptr), + } + } +} + +impl Atomic { + pub fn new(init: T, handle: &Handle) -> Self { + Self { + link: atomic::Atomic::new(Shared::new(init, handle).ptr), + } + } + + pub fn null() -> Self { + Self { + link: atomic::Atomic::new(TaggedPtr::null()), + } + } + + pub fn load(&self, order: Ordering) -> Shared { + Shared::from_raw(self.link.load(order)) + } + + pub fn store(&self, ptr: Shared, order: Ordering) { + self.link.store(ptr.ptr, order); + } + + pub fn fetch_or(&self, val: usize, order: Ordering) -> Shared { + let prev = unsafe { &*(&self.link as *const _ as *const AtomicUsize) }.fetch_or(val, order); + Shared::from_raw(TaggedPtr::from(prev as *const _)) + } + + /// Loads and protects the pointer by the original CrystallineL's way. + /// + /// Hint: For traversal based structures where need to check tag, the guarnatee is similar to + /// `load` + `HazardPointer::set`. For Tstack, the guarnatee is similar to + /// `HazardPointer::protect`. + pub fn protect<'d, 'h>(&self, he: &mut HazardEra<'d, 'h, T>) -> Shared { + let mut prev_era = he.era(); + loop { + // Somewhat similar to atomically load and protect in Hazard pointers. + let ptr = self.link.load(Ordering::Acquire); + let curr_era = he.handle.global_era(); + if curr_era == prev_era { + return Shared::from_raw(ptr); + } + prev_era = he.update_era(curr_era); + } + } + + pub fn compare_exchange( + &self, + current: Shared, + new: Shared, + success: Ordering, + failure: Ordering, + ) -> Result, CompareExchangeError> { + match self + .link + .compare_exchange(current.ptr, new.ptr, success, failure) + { + Ok(current) => Ok(Shared::from_raw(current)), + Err(current) => Err(CompareExchangeError { + new, + current: Shared::from_raw(current), + }), + } + } + + pub unsafe fn into_owned(self) -> T { + Box::from_raw(self.link.into_inner().as_raw()).item + } +} + +pub struct CompareExchangeError { + pub new: Shared, + pub current: Shared, +} + +struct TaggedPtr { + ptr: *mut T, +} + +impl Default for TaggedPtr { + fn default() -> Self { + Self { ptr: null_mut() } + } +} + +impl Clone for TaggedPtr { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for TaggedPtr {} + +impl From<*const T> for TaggedPtr { + fn from(value: *const T) -> Self { + Self { + ptr: value.cast_mut(), + } + } +} + +impl From<*mut T> for TaggedPtr { + fn from(value: *mut T) -> Self { + Self { ptr: value } + } +} + +impl TaggedPtr { + pub fn null() -> Self { + Self { ptr: null_mut() } + } + + pub fn is_null(&self) -> bool { + self.as_raw().is_null() + } + + pub fn tag(&self) -> usize { + let ptr = self.ptr as usize; + ptr & low_bits::() + } + + /// Converts this pointer to a raw pointer (without the tag). + pub fn as_raw(&self) -> *mut T { + let ptr = self.ptr as usize; + (ptr & !low_bits::()) as *mut T + } + + pub fn with_tag(&self, tag: usize) -> Self { + Self::from(with_tag(self.ptr, tag)) + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn deref<'g>(&self) -> &'g T { + &*self.as_raw() + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { + &mut *self.as_raw() + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { + if self.is_null() { + None + } else { + Some(self.deref()) + } + } + + /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, + /// are identical. + pub fn ptr_eq(self, other: Self) -> bool { + self.ptr == other.ptr + } +} + +/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. +const fn low_bits() -> usize { + (1 << align_of::().trailing_zeros()) - 1 +} + +/// Returns the pointer with the given tag +fn with_tag(ptr: *mut T, tag: usize) -> *mut T { + ((ptr as usize & !low_bits::()) | (tag & low_bits::())) as *mut T +} diff --git a/src/bin/crystalline-l.rs b/src/bin/crystalline-l.rs new file mode 100644 index 00000000..e13bb1ac --- /dev/null +++ b/src/bin/crystalline-l.rs @@ -0,0 +1,211 @@ +use crystalline_l::{set_collect_frequency, Domain, GLOBAL_GARBAGE_COUNT}; + +use crossbeam_utils::thread::scope; +use rand::prelude::*; +use std::cmp::max; +use std::io::{stdout, Write}; +use std::path::Path; +use std::sync::atomic::Ordering; +use std::sync::{mpsc, Arc, Barrier}; +use std::thread::available_parallelism; +use std::time::Instant; + +use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; +use smr_benchmark::ds_impl::crystalline_l::{ConcurrentMap, HHSList, HList, HMList}; + +fn main() { + let (config, output) = setup( + Path::new(file!()) + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .unwrap(), + ); + bench(&config, output) +} + +fn bench(config: &Config, output: BenchWriter) { + println!("{}", config); + let perf = match config.ds { + DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), + _ => panic!("Unsupported(or unimplemented) data structure for CrystallineL"), + }; + output.write_record(config, &perf); + println!("{}", perf); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. + Random, + /// Inserts keys in an increasing order, with a single thread. + Decreasing, +} + +impl PrefillStrategy { + fn prefill + Send + Sync>( + self, + config: &Config, + map: &M, + domain: &Domain, + ) { + match self { + PrefillStrategy::Random => { + let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); + print!("prefilling with {threads} threads... "); + stdout().flush().unwrap(); + scope(|s| { + for t in 0..threads { + s.spawn(move |_| { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let rng = &mut rand::thread_rng(); + let count = config.prefill / threads + + if t < config.prefill % threads { 1 } else { 0 }; + for _ in 0..count { + let key = config.key_dist.sample(rng); + let value = key; + map.insert(key, value, &mut shields, &handle); + } + }); + } + }) + .unwrap(); + } + PrefillStrategy::Decreasing => { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let rng = &mut rand::thread_rng(); + let mut keys = Vec::with_capacity(config.prefill); + for _ in 0..config.prefill { + keys.push(config.key_dist.sample(rng)); + } + keys.sort_by(|a, b| b.cmp(a)); + for key in keys.drain(..) { + let value = key; + map.insert(key, value, &mut shields, &handle); + } + } + } + print!("prefilled... "); + stdout().flush().unwrap(); + } +} + +fn bench_map + Send + Sync>( + config: &Config, + strategy: PrefillStrategy, +) -> Perf { + match config.bag_size { + BagSize::Small => set_collect_frequency(512), + BagSize::Large => set_collect_frequency(4096), + } + let max_threads = available_parallelism() + .map(|v| v.get()) + .unwrap_or(1) + .max(config.threads) + + 2; + let domain = &Domain::new(max_threads); + let mut handle = domain.register(); + let map = &M::new(&mut handle); + strategy.prefill(config, map, domain); + + let barrier = &Arc::new(Barrier::new(config.threads + config.aux_thread)); + let (ops_sender, ops_receiver) = mpsc::channel(); + let (mem_sender, mem_receiver) = mpsc::channel(); + + scope(|s| { + // sampling & interference thread + if config.aux_thread > 0 { + let mem_sender = mem_sender.clone(); + s.spawn(move |_| { + let mut samples = 0usize; + let mut acc = 0usize; + let mut peak = 0usize; + let mut garb_acc = 0usize; + let mut garb_peak = 0usize; + barrier.clone().wait(); + + let start = Instant::now(); + let mut next_sampling = start + config.sampling_period; + while start.elapsed() < config.duration { + let now = Instant::now(); + if now > next_sampling { + let allocated = config.mem_sampler.sample(); + samples += 1; + + acc += allocated; + peak = max(peak, allocated); + + let garbages = GLOBAL_GARBAGE_COUNT.load(Ordering::Acquire); + garb_acc += garbages; + garb_peak = max(garb_peak, garbages); + + next_sampling = now + config.sampling_period; + } + std::thread::sleep(config.aux_thread_period); + } + + if config.sampling { + mem_sender + .send((peak, acc / samples, garb_peak, garb_acc / samples)) + .unwrap(); + } else { + mem_sender.send((0, 0, 0, 0)).unwrap(); + } + }); + } else { + mem_sender.send((0, 0, 0, 0)).unwrap(); + } + + for _ in 0..config.threads { + let ops_sender = ops_sender.clone(); + s.spawn(move |_| { + let mut ops: u64 = 0; + let mut rng = &mut rand::thread_rng(); + let handle = domain.register(); + let mut shields = M::shields(&handle); + barrier.clone().wait(); + let start = Instant::now(); + + while start.elapsed() < config.duration { + let key = config.key_dist.sample(rng); + match Op::OPS[config.op_dist.sample(&mut rng)] { + Op::Get => { + map.get(&key, &mut shields, &handle); + } + Op::Insert => { + let value = key; + map.insert(key, value, &mut shields, &handle); + } + Op::Remove => { + map.remove(&key, &mut shields, &handle); + } + } + ops += 1; + } + + ops_sender.send(ops).unwrap(); + }); + } + }) + .unwrap(); + println!("end"); + + let mut ops = 0; + for _ in 0..config.threads { + let local_ops = ops_receiver.recv().unwrap(); + ops += local_ops; + } + let ops_per_sec = ops / config.interval; + let (peak_mem, avg_mem, peak_garb, avg_garb) = mem_receiver.recv().unwrap(); + Perf { + ops_per_sec, + peak_mem, + avg_mem, + peak_garb, + avg_garb, + } +} diff --git a/src/ds_impl/crystalline_l/concurrent_map.rs b/src/ds_impl/crystalline_l/concurrent_map.rs new file mode 100644 index 00000000..3c08a9f4 --- /dev/null +++ b/src/ds_impl/crystalline_l/concurrent_map.rs @@ -0,0 +1,107 @@ +use crystalline_l::Handle; + +pub trait ConcurrentMap { + type Node; + type Shields<'d, 'h> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h>; + fn new<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self; + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V>; + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> bool; + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V>; +} + +#[cfg(test)] +pub mod tests { + extern crate rand; + use super::ConcurrentMap; + use crossbeam_utils::thread; + use crystalline_l::Domain; + use rand::prelude::*; + + const THREADS: i32 = 30; + const ELEMENTS_PER_THREADS: i32 = 1000; + + pub fn smoke + Send + Sync>() { + let domain = &Domain::new((THREADS + 1) as usize); + // Drop the map before the domain. + smoke_inner::(domain); + } + + fn smoke_inner + Send + Sync>(domain: &Domain) { + let mut handle = domain.register(); + let map = &M::new(&mut handle); + + thread::scope(|s| { + for t in 0..THREADS { + s.spawn(move |_| { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert!(map.insert(i, i.to_string(), &mut shields, &handle)); + } + }); + } + }) + .unwrap(); + + thread::scope(|s| { + for t in 0..(THREADS / 2) { + s.spawn(move |_| { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert_eq!( + i.to_string(), + *map.remove(&i, &mut shields, &handle).unwrap() + ); + } + }); + } + }) + .unwrap(); + + thread::scope(|s| { + for t in (THREADS / 2)..THREADS { + s.spawn(move |_| { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert_eq!(i.to_string(), *map.get(&i, &mut shields, &handle).unwrap()); + } + }); + } + }) + .unwrap(); + } +} diff --git a/src/ds_impl/crystalline_l/list.rs b/src/ds_impl/crystalline_l/list.rs new file mode 100644 index 00000000..5394dc78 --- /dev/null +++ b/src/ds_impl/crystalline_l/list.rs @@ -0,0 +1,682 @@ +use super::concurrent_map::ConcurrentMap; + +use std::cmp::Ordering::{Equal, Greater, Less}; +use std::sync::atomic::Ordering; + +use crystalline_l::{Atomic, Handle, HazardEra, Shared}; + +// `#[repr(C)]` is used to ensure the first field +// is also the first data in the memory alignment. +#[repr(C)] +pub struct Node { + /// Mark: tag(), Tag: not needed + next: Atomic, + key: K, + value: V, +} + +pub struct List { + head: Atomic>, +} + +impl Default for List +where + K: Ord, +{ + fn default() -> Self { + Self::new() + } +} + +impl Drop for List { + fn drop(&mut self) { + let mut curr = self.head.load(Ordering::Relaxed); + + while !curr.is_null() { + curr = unsafe { curr.into_owned() }.next.load(Ordering::Relaxed); + } + } +} + +pub struct EraShields<'d, 'h, K, V> { + prev_h: HazardEra<'d, 'h, Node>, + curr_h: HazardEra<'d, 'h, Node>, + next_h: HazardEra<'d, 'h, Node>, + // `anchor_h` and `anchor_next_h` are used for `find_harris` + anchor_h: HazardEra<'d, 'h, Node>, + anchor_next_h: HazardEra<'d, 'h, Node>, +} + +impl<'d, 'h, K, V> EraShields<'d, 'h, K, V> { + fn new(handle: &'h Handle<'d, Node>) -> Self { + Self { + prev_h: HazardEra::new(handle), + curr_h: HazardEra::new(handle), + next_h: HazardEra::new(handle), + anchor_h: HazardEra::new(handle), + anchor_next_h: HazardEra::new(handle), + } + } + + // bypass E0499-E0503, etc that are supposed to be fixed by polonius + #[inline] + fn launder<'he2>(&mut self) -> &'he2 mut Self { + unsafe { core::mem::transmute(self) } + } +} + +pub struct Cursor { + prev: Shared>, + // For harris, this keeps the mark bit. Don't mix harris and harris-michael. + curr: Shared>, + // `anchor` is used for `find_harris` + // anchor and anchor_next are non-null iff exist + anchor: Shared>, + anchor_next: Shared>, +} + +impl Cursor { + pub fn new<'d, 'h>(head: &Atomic>, shields: &mut EraShields<'d, 'h, K, V>) -> Self { + Self { + prev: Shared::from(head as *const _ as usize), + curr: head.protect(&mut shields.curr_h), + anchor: Shared::null(), + anchor_next: Shared::null(), + } + } +} + +impl Cursor +where + K: Ord, +{ + /// Optimistically traverses while maintaining `anchor` and `anchor_next`. + /// It is used for both Harris and Harris-Herlihy-Shavit traversals. + #[inline] + fn traverse_with_anchor<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h, K, V>, + ) -> Result { + // Invariants: + // anchor, anchor_next: protected if they are not null. + // prev: always protected with prev_sh + // curr: not protected. + // curr: also has tag value when it is obtained from prev. + Ok(loop { + let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { + break false; + }; + + // Validation depending on the state of `self.curr`. + // + // - If it is marked, validate on anchor. + // - If it is not marked, it is already protected safely by the Crystalline. + if self.curr.tag() != 0 { + // Validate on anchor. + + debug_assert!(!self.anchor.is_null()); + debug_assert!(!self.anchor_next.is_null()); + let an_new = unsafe { self.anchor.deref() }.next.load(Ordering::Acquire); + + if an_new.tag() != 0 { + return Err(()); + } else if !an_new.ptr_eq(self.anchor_next) { + // Anchor is updated but clear, so can restart from anchor. + + self.prev = self.anchor; + self.curr = an_new; + self.anchor = Shared::null(); + + // Set prev HP as anchor HP, since prev should always be protected. + HazardEra::swap(&mut shields.prev_h, &mut shields.anchor_h); + continue; + } + } + + let next = curr_node.next.protect(&mut shields.next_h); + if next.tag() == 0 { + if curr_node.key < *key { + self.prev = self.curr; + self.curr = next; + self.anchor = Shared::null(); + HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + } else { + break curr_node.key == *key; + } + } else { + if self.anchor.is_null() { + self.anchor = self.prev; + self.anchor_next = self.curr; + HazardEra::swap(&mut shields.anchor_h, &mut shields.prev_h); + } else if self.anchor_next.ptr_eq(self.prev) { + HazardEra::swap(&mut shields.anchor_next_h, &mut shields.prev_h); + } + self.prev = self.curr; + self.curr = next; + HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + } + }) + } + + #[inline] + fn find_harris<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Result { + // Finding phase + // - cursor.curr: first unmarked node w/ key >= search key (4) + // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) + // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) + + let found = self.traverse_with_anchor(key, shields)?; + + if self.anchor.is_null() { + self.prev = self.prev.with_tag(0); + self.curr = self.curr.with_tag(0); + Ok(found) + } else { + debug_assert_eq!(self.anchor_next.tag(), 0); + // TODO: on CAS failure, if anchor is not tagged, we can restart from anchor. + unsafe { &self.anchor.deref().next } + .compare_exchange( + self.anchor_next, + self.curr.with_tag(0), + Ordering::AcqRel, + Ordering::Relaxed, + ) + .map_err(|_| { + self.curr = self.curr.with_tag(0); + () + })?; + + let mut node = self.anchor_next; + while !node.with_tag(0).ptr_eq(self.curr.with_tag(0)) { + // NOTE: It may seem like this can be done with a NA load, but we do a `fetch_or` in remove, which always does an write. + // This can be a NA load if the `fetch_or` in delete is changed to a CAS, but it is not clear if it is worth it. + let next = unsafe { node.deref().next.load(Ordering::Relaxed) }; + debug_assert!(next.tag() != 0); + unsafe { handle.retire(node) }; + node = next; + } + self.prev = self.anchor.with_tag(0); + self.curr = self.curr.with_tag(0); + Ok(found) + } + } + + #[inline] + fn find_harris_michael<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Result { + loop { + debug_assert_eq!(self.curr.tag(), 0); + let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { + return Ok(false); + }; + let mut next = curr_node.next.protect(&mut shields.next_h); + + if next.tag() != 0 { + next = next.with_tag(0); + unsafe { self.prev.deref() } + .next + .compare_exchange(self.curr, next, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| ())?; + unsafe { handle.retire(self.curr) }; + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + self.curr = next; + continue; + } + + match curr_node.key.cmp(key) { + Less => { + HazardEra::swap(&mut shields.prev_h, &mut shields.curr_h); + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + self.prev = self.curr; + self.curr = next; + } + Equal => return Ok(true), + Greater => return Ok(false), + } + } + } + + #[inline] + fn find_harris_herlihy_shavit<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h, K, V>, + _handle: &'h Handle<'d, Node>, + ) -> Result { + let found = self.traverse_with_anchor(key, shields)?; + // Return only the found `curr` node. + // Others are not necessary because we are not going to do insertion or deletion + // with this Harris-Herlihy-Shavit traversal. + self.curr = self.curr.with_tag(0); + Ok(found) + } +} + +impl List +where + K: Ord, +{ + /// Creates a new list. + pub fn new() -> Self { + List { + head: Atomic::null(), + } + } + + #[inline] + fn get<'d, 'h, 'he, F>( + &self, + key: &K, + find: F, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Option<&'he V> + where + F: Fn( + &mut Cursor, + &K, + &mut EraShields<'d, 'h, K, V>, + &'h Handle<'d, Node>, + ) -> Result, + { + loop { + let mut cursor = Cursor::new(&self.head, shields); + match find(&mut cursor, key, shields, handle) { + Ok(true) => return unsafe { Some(&(cursor.curr.deref().value)) }, + Ok(false) => return None, + Err(_) => continue, + } + } + } + + fn insert_inner<'d, 'h, 'he, F>( + &self, + node: Shared>, + find: &F, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Result + where + F: Fn( + &mut Cursor, + &K, + &mut EraShields<'d, 'h, K, V>, + &'h Handle<'d, Node>, + ) -> Result, + { + loop { + let mut cursor = Cursor::new(&self.head, shields); + let found = find(&mut cursor, unsafe { &node.deref().key }, shields, handle)?; + if found { + drop(unsafe { node.into_owned() }); + return Ok(false); + } + + unsafe { node.deref() } + .next + .store(cursor.curr, Ordering::Relaxed); + if unsafe { cursor.prev.deref() } + .next + .compare_exchange(cursor.curr, node, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + return Ok(true); + } + } + } + + #[inline] + fn insert<'d, 'h, 'he, F>( + &self, + key: K, + value: V, + find: F, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> bool + where + F: Fn( + &mut Cursor, + &K, + &mut EraShields<'d, 'h, K, V>, + &'h Handle<'d, Node>, + ) -> Result, + { + let node = Shared::new( + Node { + key, + value, + next: Atomic::null(), + }, + handle, + ); + + loop { + match self.insert_inner(node, &find, shields, handle) { + Ok(r) => return r, + Err(()) => continue, + } + } + } + + fn remove_inner<'d, 'h, 'he, F>( + &self, + key: &K, + find: &F, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Result, ()> + where + F: Fn( + &mut Cursor, + &K, + &mut EraShields<'d, 'h, K, V>, + &'h Handle<'d, Node>, + ) -> Result, + { + loop { + let mut cursor = Cursor::new(&self.head, shields); + let found = find(&mut cursor, key, shields, handle)?; + if !found { + return Ok(None); + } + + let curr_node = unsafe { cursor.curr.deref() }; + let next = curr_node.next.fetch_or(1, Ordering::AcqRel); + if next.tag() == 1 { + continue; + } + + if unsafe { &cursor.prev.deref().next } + .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + unsafe { handle.retire(cursor.curr) }; + } + + return Ok(Some(&curr_node.value)); + } + } + + #[inline] + fn remove<'d, 'h, 'he, F>( + &self, + key: &K, + find: F, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Option<&'he V> + where + F: Fn( + &mut Cursor, + &K, + &mut EraShields<'d, 'h, K, V>, + &'h Handle<'d, Node>, + ) -> Result, + { + loop { + match self.remove_inner(key, &find, shields.launder(), handle) { + Ok(r) => return r, + Err(_) => continue, + } + } + } + + pub fn harris_get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Option<&'he V> { + self.get(key, Cursor::find_harris, shields, handle) + } + + pub fn harris_insert<'d, 'h, 'he>( + &self, + key: K, + value: V, + shields: &mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> bool { + self.insert(key, value, Cursor::find_harris, shields, handle) + } + + pub fn harris_remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Option<&'he V> { + self.remove(key, Cursor::find_harris, shields, handle) + } + + pub fn harris_michael_get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Option<&'he V> { + self.get(key, Cursor::find_harris_michael, shields, handle) + } + + pub fn harris_michael_insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> bool { + self.insert(key, value, Cursor::find_harris_michael, shields, handle) + } + + pub fn harris_michael_remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Option<&'he V> { + self.remove(key, Cursor::find_harris_michael, shields, handle) + } + + pub fn harris_herlihy_shavit_get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &'h Handle<'d, Node>, + ) -> Option<&'he V> { + self.get(key, Cursor::find_harris_herlihy_shavit, shields, handle) + } +} + +pub struct HList { + inner: List, +} + +impl ConcurrentMap for HList +where + K: Ord + 'static, + V: 'static, +{ + type Node = Node; + + type Shields<'d, 'h> + = EraShields<'d, 'h, K, V> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(_: &'h Handle<'d, Self::Node>) -> Self { + Self { inner: List::new() } + } + + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.inner.harris_get(key, shields, handle) + } + + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> bool { + self.inner.harris_insert(key, value, shields, handle) + } + + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.inner.harris_remove(key, shields, handle) + } +} + +pub struct HMList { + inner: List, +} + +impl ConcurrentMap for HMList +where + K: Ord + 'static, + V: 'static, +{ + type Node = Node; + + type Shields<'d, 'h> + = EraShields<'d, 'h, K, V> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(_: &'h Handle<'d, Self::Node>) -> Self { + Self { inner: List::new() } + } + + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.inner.harris_michael_get(key, shields, handle) + } + + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> bool { + self.inner + .harris_michael_insert(key, value, shields, handle) + } + + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.inner.harris_michael_remove(key, shields, handle) + } +} + +pub struct HHSList { + inner: List, +} + +impl ConcurrentMap for HHSList +where + K: Ord + 'static, + V: 'static, +{ + type Node = Node; + + type Shields<'d, 'h> + = EraShields<'d, 'h, K, V> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(_: &'h Handle<'d, Self::Node>) -> Self { + Self { inner: List::new() } + } + + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.inner.harris_herlihy_shavit_get(key, shields, handle) + } + + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> bool { + self.inner.harris_insert(key, value, shields, handle) + } + + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.inner.harris_remove(key, shields, handle) + } +} + +#[cfg(test)] +mod tests { + use super::{HHSList, HList, HMList}; + use crate::ds_impl::crystalline_l::concurrent_map; + + #[test] + fn smoke_h_list() { + concurrent_map::tests::smoke::>(); + } + + #[test] + fn smoke_hm_list() { + concurrent_map::tests::smoke::>(); + } + + #[test] + fn smoke_hhs_list() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/crystalline_l/mod.rs b/src/ds_impl/crystalline_l/mod.rs new file mode 100644 index 00000000..c88a58e4 --- /dev/null +++ b/src/ds_impl/crystalline_l/mod.rs @@ -0,0 +1,7 @@ +pub mod concurrent_map; + +pub mod list; + +pub use self::concurrent_map::ConcurrentMap; + +pub use self::list::{HHSList, HList, HMList}; diff --git a/src/ds_impl/mod.rs b/src/ds_impl/mod.rs index d7917775..4e95d3f3 100644 --- a/src/ds_impl/mod.rs +++ b/src/ds_impl/mod.rs @@ -1,6 +1,7 @@ pub mod cdrc; pub mod circ_ebr; pub mod circ_hp; +pub mod crystalline_l; pub mod ebr; pub mod hp; pub mod hp_brcu; From 7cbb2b1b6f934e5a61034deb23bc9a73ee8ce72a Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 5 Nov 2024 13:27:33 +0000 Subject: [PATCH 60/84] Reclaim thread-local bags during `Domain::drop` --- smrs/crystalline-l/src/lib.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/smrs/crystalline-l/src/lib.rs b/smrs/crystalline-l/src/lib.rs index 91e49ca0..5013a20f 100644 --- a/smrs/crystalline-l/src/lib.rs +++ b/smrs/crystalline-l/src/lib.rs @@ -173,7 +173,9 @@ impl Batch { return; } if self.list_count == CACHE_CAP { - unsafe { self.list.as_ref().map(|list| list.free_list()) }; + if let Some(list) = unsafe { self.list.as_ref() } { + list.free_list(); + } self.list = null_mut(); self.list_count = 0; } @@ -223,6 +225,16 @@ impl Drop for Domain { .collect::>(); for handle in handles { unsafe { handle.clear_all() }; + let batch = handle.batch_mut(); + if !batch.first.is_null() { + let mut curr = batch.first; + while curr != batch.last { + let next = unsafe { (*curr).batch_next() }; + unsafe { drop(Box::from_raw(curr)) }; + curr = next; + } + unsafe { drop(Box::from_raw(batch.last)) }; + } } } } @@ -388,7 +400,9 @@ impl<'d, T> Handle<'d, T> { } } let mut batch = self.batch_mut(); - unsafe { batch.list.as_ref().map(|list| list.free_list()) }; + if let Some(list) = batch.list.as_ref() { + list.free_list(); + } batch.list = null_mut(); batch.list_count = 0; } From 1cad22da8c36803b70210291e06f1153f87f785c Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 5 Nov 2024 14:47:28 +0000 Subject: [PATCH 61/84] Implement CrystallineL HashMap --- Cargo.lock | 1 + Cargo.toml | 1 + smrs/crystalline-l/src/lib.rs | 13 ++- src/bin/crystalline-l.rs | 4 +- src/ds_impl/crystalline_l/list.rs | 9 +- src/ds_impl/crystalline_l/michael_hash_map.rs | 104 ++++++++++++++++++ src/ds_impl/crystalline_l/mod.rs | 2 + 7 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 src/ds_impl/crystalline_l/michael_hash_map.rs diff --git a/Cargo.lock b/Cargo.lock index 30c84452..d95c0895 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -833,6 +833,7 @@ dependencies = [ "nbr", "num", "rand", + "scopeguard", "tikv-jemalloc-ctl", "tikv-jemallocator", "typenum", diff --git a/Cargo.toml b/Cargo.toml index 905c982e..bcb2de90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ rand = "0.8" typenum = "1.17" num = "0.4.3" arrayvec = "0.7.6" +scopeguard = "1" hp_pp = { path = "./smrs/hp-pp" } nbr = { path = "./smrs/nbr" } cdrc = { path = "./smrs/cdrc" } diff --git a/smrs/crystalline-l/src/lib.rs b/smrs/crystalline-l/src/lib.rs index 5013a20f..df6a64a9 100644 --- a/smrs/crystalline-l/src/lib.rs +++ b/smrs/crystalline-l/src/lib.rs @@ -224,7 +224,7 @@ impl Drop for Domain { .map(|_| self.register()) .collect::>(); for handle in handles { - unsafe { handle.clear_all() }; + handle.clear_all(); let batch = handle.batch_mut(); if !batch.first.is_null() { let mut curr = batch.first; @@ -314,7 +314,6 @@ impl<'d, T> Handle<'d, T> { batch.count += 1; if batch.count % COLLECT_FREQ.load(Ordering::Relaxed) == 0 { unsafe { (*batch.last).set_batch_link(node_ptr) }; - GLOBAL_GARBAGE_COUNT.fetch_add(batch.count, Ordering::AcqRel); self.try_retire(batch); } } @@ -343,7 +342,9 @@ impl<'d, T> Handle<'d, T> { last = unsafe { (*last).batch_next() }; } } + // Retire if successful. + GLOBAL_GARBAGE_COUNT.fetch_add(batch.count, Ordering::AcqRel); let mut adjs = 0; while curr != last { 'body: { @@ -389,7 +390,7 @@ impl<'d, T> Handle<'d, T> { batch.count = 0; } - pub unsafe fn clear_all(&self) { + pub fn clear_all(&self) { let mut first = [null_mut(); SLOTS_CAP]; for i in 0..SLOTS_CAP { first[i] = self.slots().first[i].swap(invalid_ptr(), Ordering::AcqRel); @@ -400,7 +401,7 @@ impl<'d, T> Handle<'d, T> { } } let mut batch = self.batch_mut(); - if let Some(list) = batch.list.as_ref() { + if let Some(list) = unsafe { batch.list.as_ref() } { list.free_list(); } batch.list = null_mut(); @@ -446,6 +447,10 @@ impl<'d, 'h, T> HazardEra<'d, 'h, T> { pub fn swap(h1: &mut Self, h2: &mut Self) { swap(&mut h1.sidx, &mut h2.sidx); } + + pub fn clear(&mut self) { + self.handle.slots().era[self.sidx].store(0, Ordering::Release); + } } impl<'d, 'h, T> Drop for HazardEra<'d, 'h, T> { diff --git a/src/bin/crystalline-l.rs b/src/bin/crystalline-l.rs index e13bb1ac..0b49ab87 100644 --- a/src/bin/crystalline-l.rs +++ b/src/bin/crystalline-l.rs @@ -11,7 +11,7 @@ use std::thread::available_parallelism; use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; -use smr_benchmark::ds_impl::crystalline_l::{ConcurrentMap, HHSList, HList, HMList}; +use smr_benchmark::ds_impl::crystalline_l::{ConcurrentMap, HHSList, HList, HMList, HashMap}; fn main() { let (config, output) = setup( @@ -30,6 +30,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), _ => panic!("Unsupported(or unimplemented) data structure for CrystallineL"), }; output.write_record(config, &perf); @@ -184,6 +185,7 @@ fn bench_map + Send + Sync>( map.remove(&key, &mut shields, &handle); } } + handle.clear_all(); ops += 1; } diff --git a/src/ds_impl/crystalline_l/list.rs b/src/ds_impl/crystalline_l/list.rs index 5394dc78..1cbdeaec 100644 --- a/src/ds_impl/crystalline_l/list.rs +++ b/src/ds_impl/crystalline_l/list.rs @@ -48,7 +48,7 @@ pub struct EraShields<'d, 'h, K, V> { } impl<'d, 'h, K, V> EraShields<'d, 'h, K, V> { - fn new(handle: &'h Handle<'d, Node>) -> Self { + pub fn new(handle: &'h Handle<'d, Node>) -> Self { Self { prev_h: HazardEra::new(handle), curr_h: HazardEra::new(handle), @@ -175,6 +175,11 @@ where let found = self.traverse_with_anchor(key, shields)?; + scopeguard::defer! { + shields.anchor_h.clear(); + shields.anchor_next_h.clear(); + } + if self.anchor.is_null() { self.prev = self.prev.with_tag(0); self.curr = self.curr.with_tag(0); @@ -260,6 +265,8 @@ where // Others are not necessary because we are not going to do insertion or deletion // with this Harris-Herlihy-Shavit traversal. self.curr = self.curr.with_tag(0); + shields.anchor_h.clear(); + shields.anchor_next_h.clear(); Ok(found) } } diff --git a/src/ds_impl/crystalline_l/michael_hash_map.rs b/src/ds_impl/crystalline_l/michael_hash_map.rs new file mode 100644 index 00000000..eeabac10 --- /dev/null +++ b/src/ds_impl/crystalline_l/michael_hash_map.rs @@ -0,0 +1,104 @@ +use super::concurrent_map::ConcurrentMap; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use super::list::HHSList; +pub use super::list::{Cursor, EraShields, Node}; + +use crystalline_l::Handle; + +pub struct HashMap { + buckets: Vec>, +} + +impl HashMap +where + K: Ord + Hash + 'static, + V: 'static, +{ + pub fn with_capacity<'d, 'h>(n: usize, handle: &'h Handle<'d, Node>) -> Self { + let mut buckets = Vec::with_capacity(n); + for _ in 0..n { + buckets.push(HHSList::new(handle)); + } + + HashMap { buckets } + } + + #[inline] + pub fn get_bucket(&self, index: usize) -> &HHSList { + unsafe { self.buckets.get_unchecked(index % self.buckets.len()) } + } + + #[inline] + fn hash(k: &K) -> usize { + let mut s = DefaultHasher::new(); + k.hash(&mut s); + s.finish() as usize + } +} + +impl ConcurrentMap for HashMap +where + K: Ord + Hash + Send + 'static, + V: Send + 'static, +{ + type Node = Node; + + type Shields<'d, 'h> = EraShields<'d, 'h, K, V> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self { + Self::with_capacity(30000, handle) + } + + #[inline(always)] + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + let i = Self::hash(key); + self.get_bucket(i).get(key, shields, handle) + } + + #[inline(always)] + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> bool { + let i = Self::hash(&key); + self.get_bucket(i).insert(key, value, shields, handle) + } + + #[inline(always)] + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + let i = Self::hash(key); + self.get_bucket(i).remove(key, shields, handle) + } +} + +#[cfg(test)] +mod tests { + use super::HashMap; + use crate::ds_impl::crystalline_l::concurrent_map; + + #[test] + fn smoke_hashmap() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/crystalline_l/mod.rs b/src/ds_impl/crystalline_l/mod.rs index c88a58e4..6e548ea0 100644 --- a/src/ds_impl/crystalline_l/mod.rs +++ b/src/ds_impl/crystalline_l/mod.rs @@ -1,7 +1,9 @@ pub mod concurrent_map; pub mod list; +pub mod michael_hash_map; pub use self::concurrent_map::ConcurrentMap; pub use self::list::{HHSList, HList, HMList}; +pub use self::michael_hash_map::HashMap; From a5a4d2416098978f03afd0c40268c0bdebb6ddca Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Tue, 5 Nov 2024 15:43:59 +0000 Subject: [PATCH 62/84] Revert removing `unsafe` from `clear_all` --- smrs/crystalline-l/src/lib.rs | 4 ++-- src/bin/crystalline-l.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/smrs/crystalline-l/src/lib.rs b/smrs/crystalline-l/src/lib.rs index df6a64a9..577f17e0 100644 --- a/smrs/crystalline-l/src/lib.rs +++ b/smrs/crystalline-l/src/lib.rs @@ -224,7 +224,7 @@ impl Drop for Domain { .map(|_| self.register()) .collect::>(); for handle in handles { - handle.clear_all(); + unsafe { handle.clear_all() }; let batch = handle.batch_mut(); if !batch.first.is_null() { let mut curr = batch.first; @@ -390,7 +390,7 @@ impl<'d, T> Handle<'d, T> { batch.count = 0; } - pub fn clear_all(&self) { + pub unsafe fn clear_all(&self) { let mut first = [null_mut(); SLOTS_CAP]; for i in 0..SLOTS_CAP { first[i] = self.slots().first[i].swap(invalid_ptr(), Ordering::AcqRel); diff --git a/src/bin/crystalline-l.rs b/src/bin/crystalline-l.rs index 0b49ab87..c72b76e8 100644 --- a/src/bin/crystalline-l.rs +++ b/src/bin/crystalline-l.rs @@ -185,7 +185,6 @@ fn bench_map + Send + Sync>( map.remove(&key, &mut shields, &handle); } } - handle.clear_all(); ops += 1; } From 2d530875ad5201eb10801103a7e4b16537172d5f Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 6 Nov 2024 04:37:43 +0000 Subject: [PATCH 63/84] Remove a redundant unsafe block --- smrs/crystalline-l/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smrs/crystalline-l/src/lib.rs b/smrs/crystalline-l/src/lib.rs index 577f17e0..32815e21 100644 --- a/smrs/crystalline-l/src/lib.rs +++ b/smrs/crystalline-l/src/lib.rs @@ -401,7 +401,7 @@ impl<'d, T> Handle<'d, T> { } } let mut batch = self.batch_mut(); - if let Some(list) = unsafe { batch.list.as_ref() } { + if let Some(list) = batch.list.as_ref() { list.free_list(); } batch.list = null_mut(); From b3682de4d27505077bb5ed28093f33deae86c2d8 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 6 Nov 2024 08:00:35 +0000 Subject: [PATCH 64/84] Add a caution of `clear_all` on each operation --- src/bin/crystalline-l.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bin/crystalline-l.rs b/src/bin/crystalline-l.rs index c72b76e8..5baad220 100644 --- a/src/bin/crystalline-l.rs +++ b/src/bin/crystalline-l.rs @@ -185,6 +185,8 @@ fn bench_map + Send + Sync>( map.remove(&key, &mut shields, &handle); } } + // Original author's implementation of `clear_all` has an use-after-free error. + // unsafe { handle.clear_all() }; ops += 1; } From e136587214db1c32c5e687b8ad2e7f5ccb8bf432 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 6 Nov 2024 15:59:26 +0000 Subject: [PATCH 65/84] Implement CrystallineL NMTree --- src/bin/crystalline-l.rs | 5 +- src/ds_impl/crystalline_l/mod.rs | 2 + .../crystalline_l/natarajan_mittal_tree.rs | 689 ++++++++++++++++++ test-scripts/sanitize-crystalline-l.sh | 12 + 4 files changed, 707 insertions(+), 1 deletion(-) create mode 100644 src/ds_impl/crystalline_l/natarajan_mittal_tree.rs create mode 100644 test-scripts/sanitize-crystalline-l.sh diff --git a/src/bin/crystalline-l.rs b/src/bin/crystalline-l.rs index 5baad220..a8fb9389 100644 --- a/src/bin/crystalline-l.rs +++ b/src/bin/crystalline-l.rs @@ -11,7 +11,9 @@ use std::thread::available_parallelism; use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; -use smr_benchmark::ds_impl::crystalline_l::{ConcurrentMap, HHSList, HList, HMList, HashMap}; +use smr_benchmark::ds_impl::crystalline_l::{ + ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, +}; fn main() { let (config, output) = setup( @@ -31,6 +33,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), + DS::NMTree => bench_map::>(config, PrefillStrategy::Random), _ => panic!("Unsupported(or unimplemented) data structure for CrystallineL"), }; output.write_record(config, &perf); diff --git a/src/ds_impl/crystalline_l/mod.rs b/src/ds_impl/crystalline_l/mod.rs index 6e548ea0..f6d449ed 100644 --- a/src/ds_impl/crystalline_l/mod.rs +++ b/src/ds_impl/crystalline_l/mod.rs @@ -2,8 +2,10 @@ pub mod concurrent_map; pub mod list; pub mod michael_hash_map; +pub mod natarajan_mittal_tree; pub use self::concurrent_map::ConcurrentMap; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; +pub use self::natarajan_mittal_tree::NMTreeMap; diff --git a/src/ds_impl/crystalline_l/natarajan_mittal_tree.rs b/src/ds_impl/crystalline_l/natarajan_mittal_tree.rs new file mode 100644 index 00000000..dbd16816 --- /dev/null +++ b/src/ds_impl/crystalline_l/natarajan_mittal_tree.rs @@ -0,0 +1,689 @@ +use super::concurrent_map::ConcurrentMap; + +use std::cmp; +use std::sync::atomic::Ordering; + +use crystalline_l::{Atomic, Handle, HazardEra, Shared}; + +bitflags! { + /// TODO + /// A remove operation is registered by marking the corresponding edges: the (parent, target) + /// edge is _flagged_ and the (parent, sibling) edge is _tagged_. + struct Marks: usize { + const FLAG = 1usize.wrapping_shl(1); + const TAG = 1usize.wrapping_shl(0); + } +} + +impl Marks { + fn new(flag: bool, tag: bool) -> Self { + (if flag { Marks::FLAG } else { Marks::empty() }) + | (if tag { Marks::TAG } else { Marks::empty() }) + } + + fn flag(self) -> bool { + !(self & Marks::FLAG).is_empty() + } + + fn tag(self) -> bool { + !(self & Marks::TAG).is_empty() + } + + fn marked(self) -> bool { + !(self & (Marks::TAG | Marks::FLAG)).is_empty() + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +enum Key { + Fin(K), + Inf, +} + +impl PartialOrd for Key +where + K: PartialOrd, +{ + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Key::Fin(k1), Key::Fin(k2)) => k1.partial_cmp(k2), + (Key::Fin(_), Key::Inf) => Some(std::cmp::Ordering::Less), + (Key::Inf, Key::Fin(_)) => Some(std::cmp::Ordering::Greater), + (Key::Inf, Key::Inf) => Some(std::cmp::Ordering::Equal), + } + } +} + +impl PartialEq for Key +where + K: PartialEq, +{ + fn eq(&self, rhs: &K) -> bool { + match self { + Key::Fin(k) => k == rhs, + _ => false, + } + } +} + +impl PartialOrd for Key +where + K: PartialOrd, +{ + fn partial_cmp(&self, rhs: &K) -> Option { + match self { + Key::Fin(k) => k.partial_cmp(rhs), + _ => Some(std::cmp::Ordering::Greater), + } + } +} + +impl Key +where + K: Ord, +{ + fn cmp(&self, rhs: &K) -> std::cmp::Ordering { + match self { + Key::Fin(k) => k.cmp(rhs), + _ => std::cmp::Ordering::Greater, + } + } +} + +pub struct Node { + key: Key, + value: Option, + left: Atomic>, + right: Atomic>, +} + +impl Node +where + K: Clone, + V: Clone, +{ + fn new_leaf(key: Key, value: Option) -> Node { + Node { + key, + value, + left: Atomic::null(), + right: Atomic::null(), + } + } + + /// Make a new internal node, consuming the given left and right nodes, + /// using the right node's key. + fn new_internal( + left: Node, + right: Node, + handle: &Handle<'_, Node>, + ) -> Node { + Node { + key: right.key.clone(), + value: None, + left: Atomic::new(left, handle), + right: Atomic::new(right, handle), + } + } +} + +#[derive(Clone, Copy)] +enum Direction { + L, + R, +} + +pub struct EraShields<'d, 'h, K, V> { + ancestor_h: HazardEra<'d, 'h, Node>, + successor_h: HazardEra<'d, 'h, Node>, + parent_h: HazardEra<'d, 'h, Node>, + leaf_h: HazardEra<'d, 'h, Node>, + curr_h: HazardEra<'d, 'h, Node>, +} + +impl<'d, 'h, K, V> EraShields<'d, 'h, K, V> { + pub fn new(handle: &'h Handle<'d, Node>) -> Self { + Self { + ancestor_h: HazardEra::new(handle), + successor_h: HazardEra::new(handle), + parent_h: HazardEra::new(handle), + leaf_h: HazardEra::new(handle), + curr_h: HazardEra::new(handle), + } + } + + // bypass E0499-E0503, etc that are supposed to be fixed by polonius + #[inline] + fn launder<'he2>(&mut self) -> &'he2 mut Self { + unsafe { core::mem::transmute(self) } + } +} + +/// All Shared<_> are unmarked. +/// +/// All of the edges of path from `successor` to `parent` are in the process of removal. +pub struct SeekRecord { + /// Parent of `successor` + ancestor: Shared>, + /// The first internal node with a marked outgoing edge + successor: Shared>, + /// The direction of successor from ancestor. + successor_dir: Direction, + /// Parent of `leaf` + parent: Shared>, + /// The end of the access path. + leaf: Shared>, + /// The direction of leaf from parent. + leaf_dir: Direction, +} + +impl SeekRecord { + fn new() -> Self { + Self { + ancestor: Shared::null(), + successor: Shared::null(), + successor_dir: Direction::L, + parent: Shared::null(), + leaf: Shared::null(), + leaf_dir: Direction::L, + } + } + + fn successor_addr(&self) -> &Atomic> { + match self.successor_dir { + Direction::L => unsafe { &self.ancestor.deref().left }, + Direction::R => unsafe { &self.ancestor.deref().right }, + } + } + + fn leaf_addr(&self) -> &Atomic> { + match self.leaf_dir { + Direction::L => unsafe { &self.parent.deref().left }, + Direction::R => unsafe { &self.parent.deref().right }, + } + } + + fn leaf_sibling_addr(&self) -> &Atomic> { + match self.leaf_dir { + Direction::L => unsafe { &self.parent.deref().right }, + Direction::R => unsafe { &self.parent.deref().left }, + } + } +} + +pub struct NMTreeMap { + r: Node, +} + +impl Default for NMTreeMap +where + K: Ord + Clone, + V: Clone, +{ + fn default() -> Self { + todo!("new") + } +} + +impl Drop for NMTreeMap { + fn drop(&mut self) { + unsafe { + let mut stack = vec![ + self.r.left.load(Ordering::Relaxed), + self.r.right.load(Ordering::Relaxed), + ]; + assert!(self.r.value.is_none()); + + while let Some(node) = stack.pop() { + if node.is_null() { + continue; + } + + let node_ref = node.deref(); + + stack.push(node_ref.left.load(Ordering::Relaxed)); + stack.push(node_ref.right.load(Ordering::Relaxed)); + drop(node.into_owned()); + } + } + } +} + +impl NMTreeMap +where + K: Ord + Clone, + V: Clone, +{ + pub fn new<'d>(handle: &Handle<'d, Node>) -> Self { + // An empty tree has 5 default nodes with infinite keys so that the SeekRecord is allways + // well-defined. + // r + // / \ + // s inf2 + // / \ + // inf0 inf1 + let inf0 = Node::new_leaf(Key::Inf, None); + let inf1 = Node::new_leaf(Key::Inf, None); + let inf2 = Node::new_leaf(Key::Inf, None); + let s = Node::new_internal(inf0, inf1, handle); + let r = Node::new_internal(s, inf2, handle); + NMTreeMap { r } + } + + fn seek<'d, 'h>( + &self, + key: &K, + record: &mut SeekRecord, + shields: &mut EraShields<'d, 'h, K, V>, + ) -> Result<(), ()> { + let s = self.r.left.load(Ordering::Relaxed).with_tag(0); + let s_node = unsafe { s.deref() }; + + // The root node is always alive; we do not have to protect it. + record.ancestor = Shared::from(&self.r as *const _ as usize); + record.successor = s; // TODO: should preserve tag? + + record.successor_dir = Direction::L; + // The `s` node is always alive; we do not have to protect it. + record.parent = s; + + record.leaf = s_node.left.protect(&mut shields.leaf_h); + record.leaf_dir = Direction::L; + + let mut prev_tag = Marks::from_bits_truncate(record.leaf.tag()).tag(); + let mut curr_dir = Direction::L; + let mut curr = unsafe { record.leaf.deref() } + .left + .protect(&mut shields.curr_h); + + // `ancestor` always points untagged node. + while !curr.is_null() { + if !prev_tag { + // untagged edge: advance ancestor and successor pointers + record.ancestor = record.parent; + record.successor = record.leaf; + record.successor_dir = record.leaf_dir; + // `ancestor` and `successor` are already protected by + // hazard pointers of `parent` and `leaf`. + + // Advance the parent and leaf pointers when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (parent), ancestor -> O (ancestor) -> O + // / \ / \ + // (leaf), successor -> O O => (parent), successor -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardEra::swap(&mut shields.ancestor_h, &mut shields.parent_h); + HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); + } else if record.successor.ptr_eq(record.parent) { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O (ancestor) -> O + // / \ / \ + // (parent), successor -> O O (successor) -> O O + // / \ => / \ + // (leaf) -> O O (parent) -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardEra::swap(&mut shields.successor_h, &mut shields.parent_h); + HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); + } else { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O + // / \ + // (successor) -> O O + // ... ... + // (parent) -> O + // / \ + // (leaf) -> O O + record.parent = record.leaf; + HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); + } + debug_assert_eq!(record.successor.tag(), 0); + + if Marks::from_bits_truncate(curr.tag()).marked() { + // `curr` is marked. Validate by `ancestor`. + let succ_new = record.successor_addr().load(Ordering::Acquire); + if Marks::from_bits_truncate(succ_new.tag()).marked() + || !record.successor.ptr_eq(succ_new) + { + // Validation is failed. Let's restart from the root. + // TODO: Maybe it can be optimized (by restarting from the anchor), but + // it would require a serious reasoning (including shield swapping, etc). + return Err(()); + } + } + + record.leaf = curr; + record.leaf_dir = curr_dir; + + // update other variables + prev_tag = Marks::from_bits_truncate(curr.tag()).tag(); + let curr_node = unsafe { curr.deref() }; + if curr_node.key.cmp(key) == cmp::Ordering::Greater { + curr_dir = Direction::L; + curr = curr_node.left.load(Ordering::Acquire); + } else { + curr_dir = Direction::R; + curr = curr_node.right.load(Ordering::Acquire); + } + } + Ok(()) + } + + /// Physically removes node. + /// + /// Returns true if it successfully unlinks the flagged node in `record`. + fn cleanup<'d>(&self, record: &mut SeekRecord, handle: &Handle<'d, Node>) -> bool { + // Identify the node(subtree) that will replace `successor`. + let leaf_marked = record.leaf_addr().load(Ordering::Acquire); + let leaf_flag = Marks::from_bits_truncate(leaf_marked.tag()).flag(); + let target_sibling_addr = if leaf_flag { + record.leaf_sibling_addr() + } else { + record.leaf_addr() + }; + + // NOTE: the ibr implementation uses CAS + // tag (parent, sibling) edge -> all of the parent's edges can't change now + // TODO: Is Release enough? + target_sibling_addr.fetch_or(Marks::TAG.bits(), Ordering::AcqRel); + + // Try to replace (ancestor, successor) w/ (ancestor, sibling). + // Since (parent, sibling) might have been concurrently flagged, copy + // the flag to the new edge (ancestor, sibling). + let target_sibling = target_sibling_addr.load(Ordering::Acquire); + let flag = Marks::from_bits_truncate(target_sibling.tag()).flag(); + let is_unlinked = record + .successor_addr() + .compare_exchange( + record.successor.with_tag(0), + target_sibling.with_tag(Marks::new(flag, false).bits()), + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok(); + + if is_unlinked { + unsafe { + // destroy the subtree of successor except target_sibling + let mut stack = vec![record.successor]; + + while let Some(node) = stack.pop() { + if node.is_null() || node.with_tag(0).ptr_eq(target_sibling.with_tag(0)) { + continue; + } + + let node_ref = node.deref(); + + stack.push(node_ref.left.load(Ordering::Relaxed)); + stack.push(node_ref.right.load(Ordering::Relaxed)); + handle.retire(node); + } + } + } + + is_unlinked + } + + fn get_inner<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + ) -> Result, ()> { + let mut record = SeekRecord::new(); + + self.seek(key, &mut record, shields)?; + let leaf_node = unsafe { record.leaf.deref() }; + + if leaf_node.key.cmp(key) != cmp::Ordering::Equal { + return Ok(None); + } + + Ok(Some(leaf_node.value.as_ref().unwrap())) + } + + pub fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + ) -> Option<&'he V> { + loop { + if let Ok(r) = self.get_inner(key, shields.launder()) { + return r; + } + } + } + + fn insert_inner<'d, 'h, 'he>( + &self, + key: &K, + value: V, + record: &mut SeekRecord, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &Handle<'d, Node>, + ) -> Result<(), Result> { + let mut new_leaf = Shared::new(Node::new_leaf(Key::Fin(key.clone()), Some(value)), handle); + + let mut new_internal = Shared::new( + Node:: { + key: Key::Inf, // temporary placeholder + value: None, + left: Atomic::null(), + right: Atomic::null(), + }, + handle, + ); + + loop { + self.seek(key, record, shields).map_err(|_| unsafe { + let value = new_leaf.deref_mut().value.take().unwrap(); + drop(new_leaf.into_owned()); + drop(new_internal.into_owned()); + Err(value) + })?; + let leaf = record.leaf.with_tag(0); + + let (new_left, new_right) = match unsafe { leaf.deref() }.key.cmp(key) { + cmp::Ordering::Equal => { + // Newly created nodes that failed to be inserted are free'd here. + let value = unsafe { new_leaf.deref_mut() }.value.take().unwrap(); + unsafe { + drop(new_leaf.into_owned()); + drop(new_internal.into_owned()); + } + return Err(Ok(value)); + } + cmp::Ordering::Greater => (new_leaf, leaf), + cmp::Ordering::Less => (leaf, new_leaf), + }; + + let new_internal_node = unsafe { new_internal.deref_mut() }; + new_internal_node.key = unsafe { new_right.deref().key.clone() }; + new_internal_node.left.store(new_left, Ordering::Relaxed); + new_internal_node.right.store(new_right, Ordering::Relaxed); + + // NOTE: record.leaf_addr is called childAddr in the paper. + match record.leaf_addr().compare_exchange( + leaf, + new_internal, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(()), + Err(e) => { + // Insertion failed. Help the conflicting remove operation if needed. + // NOTE: The paper version checks if any of the mark is set, which is redundant. + if e.current.with_tag(0).ptr_eq(leaf) { + self.cleanup(record, handle); + } + } + } + } + } + + pub fn insert<'d, 'h, 'he>( + &self, + key: K, + mut value: V, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &Handle<'d, Node>, + ) -> Result<(), (K, V)> { + loop { + let mut record = SeekRecord::new(); + match self.insert_inner(&key, value, &mut record, shields, handle) { + Ok(()) => return Ok(()), + Err(Ok(v)) => return Err((key, v)), + Err(Err(v)) => value = v, + } + } + } + + fn remove_inner<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &Handle<'d, Node>, + ) -> Result, ()> { + // `leaf` and `value` are the snapshot of the node to be deleted. + // NOTE: The paper version uses one big loop for both phases. + // injection phase + let mut record = SeekRecord::new(); + let (leaf, value) = loop { + self.seek(key, &mut record, shields)?; + + // candidates + let leaf = record.leaf.with_tag(0); + let leaf_node = unsafe { record.leaf.deref() }; + + if leaf_node.key.cmp(key) != cmp::Ordering::Equal { + return Ok(None); + } + + let value = leaf_node.value.as_ref().unwrap(); + + // Try injecting the deletion flag. + match record.leaf_addr().compare_exchange( + leaf, + leaf.with_tag(Marks::new(true, false).bits()), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + // Finalize the node to be removed + if self.cleanup(&mut record, handle) { + return Ok(Some(value)); + } + // In-place cleanup failed. Enter the cleanup phase. + break (leaf, value); + } + Err(e) => { + // Flagging failed. + // case 1. record.leaf_addr(e.current) points to another node: restart. + // case 2. Another thread flagged/tagged the edge to leaf: help and restart + // NOTE: The paper version checks if any of the mark is set, which is redundant. + if leaf.ptr_eq(e.current.with_tag(Marks::empty().bits())) { + self.cleanup(&mut record, handle); + } + } + } + }; + + let leaf = leaf.with_tag(0); + + // cleanup phase + loop { + self.seek(key, &mut record, shields)?; + if !record.leaf.with_tag(0).ptr_eq(leaf) { + // The edge to leaf flagged for deletion was removed by a helping thread + return Ok(Some(value)); + } + + // leaf is still present in the tree. + if self.cleanup(&mut record, handle) { + return Ok(Some(value)); + } + } + } + + pub fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h, K, V>, + handle: &Handle<'d, Node>, + ) -> Option<&'he V> { + loop { + if let Ok(r) = self.remove_inner(key, shields.launder(), handle) { + return r; + } + } + } +} + +impl ConcurrentMap for NMTreeMap +where + K: Ord + Clone + 'static, + V: Clone + 'static, +{ + type Node = Node; + + type Shields<'d, 'h> = EraShields<'d, 'h, K, V> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self { + Self::new(handle) + } + + #[inline(always)] + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + _handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.get(key, shields) + } + + #[inline(always)] + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> bool { + self.insert(key, value, shields, handle).is_ok() + } + + #[inline(always)] + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d, Self::Node>, + ) -> Option<&'he V> { + self.remove(key, shields, handle) + } +} + +#[cfg(test)] +mod tests { + use super::NMTreeMap; + use crate::ds_impl::crystalline_l::concurrent_map; + + #[test] + fn smoke_nm_tree() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/test-scripts/sanitize-crystalline-l.sh b/test-scripts/sanitize-crystalline-l.sh new file mode 100644 index 00000000..f0a158f2 --- /dev/null +++ b/test-scripts/sanitize-crystalline-l.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' + +he="cargo run --bin crystalline-l --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " + +set -e +for i in {1..5000}; do + $he -dh-list -i3 -t48 -r10 -g1 + $he -dhm-list -i3 -t48 -r10 -g1 + $he -dhhs-list -i3 -t48 -r10 -g1 +done From fbf09afc4feedba85ac82a91b874e8b13739eb5e Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 6 Nov 2024 17:12:03 +0000 Subject: [PATCH 66/84] Implement hazard eras --- Cargo.lock | 11 + Cargo.toml | 2 + smrs/he/Cargo.toml | 10 + smrs/he/src/lib.rs | 565 ++++++++++++++++++++ src/bin/he.rs | 215 ++++++++ src/ds_impl/he/concurrent_map.rs | 98 ++++ src/ds_impl/he/list.rs | 658 +++++++++++++++++++++++ src/ds_impl/he/michael_hash_map.rs | 102 ++++ src/ds_impl/he/mod.rs | 11 + src/ds_impl/he/natarajan_mittal_tree.rs | 683 ++++++++++++++++++++++++ src/ds_impl/mod.rs | 1 + test-scripts/sanitize-he.sh | 12 + 12 files changed, 2368 insertions(+) create mode 100644 smrs/he/Cargo.toml create mode 100644 smrs/he/src/lib.rs create mode 100644 src/bin/he.rs create mode 100644 src/ds_impl/he/concurrent_map.rs create mode 100644 src/ds_impl/he/list.rs create mode 100644 src/ds_impl/he/michael_hash_map.rs create mode 100644 src/ds_impl/he/mod.rs create mode 100644 src/ds_impl/he/natarajan_mittal_tree.rs create mode 100644 test-scripts/sanitize-he.sh diff --git a/Cargo.lock b/Cargo.lock index d95c0895..9c115545 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -360,6 +360,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "he" +version = "0.1.0" +dependencies = [ + "atomic", + "crossbeam-utils 0.8.20", + "rustc-hash", + "static_assertions 1.1.0", +] + [[package]] name = "heck" version = "0.5.0" @@ -828,6 +838,7 @@ dependencies = [ "crossbeam-utils 0.8.20", "crystalline-l", "csv", + "he", "hp-brcu", "hp_pp", "nbr", diff --git a/Cargo.toml b/Cargo.toml index bcb2de90..6a33523f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "./smrs/vbr", "./smrs/circ", "./smrs/crystalline-l", + "./smrs/he", ] [package] @@ -36,6 +37,7 @@ hp-brcu = { path = "./smrs/hp-brcu" } vbr = { path = "./smrs/vbr" } circ = { path = "./smrs/circ" } crystalline-l = { path = "./smrs/crystalline-l" } +he = { path = "./smrs/he" } [target.'cfg(target_os = "linux")'.dependencies] tikv-jemallocator = "0.5" diff --git a/smrs/he/Cargo.toml b/smrs/he/Cargo.toml new file mode 100644 index 00000000..16b25cc8 --- /dev/null +++ b/smrs/he/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "he" +version = "0.1.0" +edition = "2021" + +[dependencies] +crossbeam-utils = "0.8.14" +rustc-hash = "1.1.0" +atomic = "0.5" +static_assertions = "1.1.0" diff --git a/smrs/he/src/lib.rs b/smrs/he/src/lib.rs new file mode 100644 index 00000000..2735b573 --- /dev/null +++ b/smrs/he/src/lib.rs @@ -0,0 +1,565 @@ +use std::cell::{Cell, RefCell, RefMut}; +use std::mem::{align_of, replace, size_of, swap}; +use std::ptr::null_mut; +use std::sync::atomic::{fence, AtomicPtr, AtomicUsize, Ordering}; +use std::sync::Mutex; + +use crossbeam_utils::CachePadded; +use static_assertions::const_assert_eq; + +const SLOTS_CAP: usize = 16; +static BATCH_CAP: AtomicUsize = AtomicUsize::new(128); + +pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); + +pub fn set_batch_capacity(cap: usize) { + BATCH_CAP.store(cap, Ordering::SeqCst); +} + +fn batch_capacity() -> usize { + BATCH_CAP.load(Ordering::Relaxed) +} + +fn collect_period() -> usize { + BATCH_CAP.load(Ordering::Relaxed) * 2 +} + +struct Node { + item: T, + birth: usize, + retire: Cell, +} + +impl Node { + fn new(item: T, birth: usize) -> Self { + Self { + item, + birth, + retire: Cell::new(0), + } + } +} + +struct Retired { + ptr: *mut Node<()>, + lifespan: unsafe fn(ptr: *mut Node<()>) -> (usize, usize), + deleter: unsafe fn(ptr: *mut Node<()>), +} + +unsafe impl Send for Retired {} + +impl Retired { + fn new(ptr: *mut Node) -> Self { + Self { + ptr: ptr as *mut Node<()>, + lifespan: lifespan::, + deleter: free::, + } + } + + unsafe fn execute(self) { + (self.deleter)(self.ptr) + } + + unsafe fn lifespan(&self) -> (usize, usize) { + (self.lifespan)(self.ptr) + } +} + +unsafe fn lifespan(ptr: *mut Node<()>) -> (usize, usize) { + let node = &*(ptr as *mut Node); + (node.birth, node.retire.get()) +} + +unsafe fn free(ptr: *mut Node<()>) { + drop(Box::from_raw(ptr as *mut Node)) +} + +pub(crate) struct RetiredList { + head: AtomicPtr, +} + +struct RetiredListNode { + retireds: Vec, + next: *const RetiredListNode, +} + +impl RetiredList { + pub(crate) const fn new() -> Self { + Self { + head: AtomicPtr::new(null_mut()), + } + } + + pub(crate) fn push(&self, retireds: Vec) { + let new = Box::leak(Box::new(RetiredListNode { + retireds, + next: null_mut(), + })); + + let mut head = self.head.load(Ordering::Relaxed); + loop { + new.next = head; + match self + .head + .compare_exchange(head, new, Ordering::Release, Ordering::Relaxed) + { + Ok(_) => return, + Err(head_new) => head = head_new, + } + } + } + + pub(crate) fn pop_all(&self) -> Vec { + let mut cur = self.head.swap(null_mut(), Ordering::AcqRel); + let mut retireds = Vec::new(); + while !cur.is_null() { + let mut cur_box = unsafe { Box::from_raw(cur) }; + retireds.append(&mut cur_box.retireds); + cur = cur_box.next.cast_mut(); + } + retireds + } +} + +pub struct Domain { + slots: Vec>, + batches: Vec>>>, + global_batch: CachePadded, + era: CachePadded, + avail_hidx: Mutex>, +} + +unsafe impl Sync for Domain {} +unsafe impl Send for Domain {} + +impl Drop for Domain { + fn drop(&mut self) { + debug_assert_eq!(self.avail_hidx.lock().unwrap().len(), self.slots.len()); + let retireds = self + .batches + .iter_mut() + .flat_map(|batch| batch.get_mut().drain(..)) + .chain(self.global_batch.pop_all().into_iter()); + for retired in retireds { + unsafe { retired.execute() }; + } + } +} + +impl Domain { + pub fn new(handles: usize) -> Self { + Self { + slots: (0..handles).map(|_| Default::default()).collect(), + batches: (0..handles).map(|_| Default::default()).collect(), + global_batch: CachePadded::new(RetiredList::new()), + era: CachePadded::new(AtomicUsize::new(1)), + avail_hidx: Mutex::new((0..handles).collect()), + } + } + + fn load_era(&self, order: Ordering) -> usize { + self.era.load(order) + } + + pub fn register(&self) -> Handle<'_> { + Handle { + domain: self, + hidx: self.avail_hidx.lock().unwrap().pop().unwrap(), + alloc_count: Cell::new(0), + retire_count: Cell::new(0), + avail_sidx: RefCell::new((0..SLOTS_CAP).collect()), + } + } + + fn iter_eras(&self) -> impl Iterator + '_ { + self.slots + .iter() + .flat_map(|slots| slots.iter().map(|slot| slot.load(Ordering::Acquire))) + } +} + +pub struct Handle<'d> { + domain: &'d Domain, + hidx: usize, + alloc_count: Cell, + retire_count: Cell, + avail_sidx: RefCell>, +} + +impl<'d> Handle<'d> { + fn incr_alloc(&self) { + let next_count = self.alloc_count.get() + 1; + self.alloc_count.set(next_count); + if next_count % batch_capacity() == 0 { + self.domain.era.fetch_add(1, Ordering::AcqRel); + } + } + + pub fn global_era(&self, order: Ordering) -> usize { + self.domain.load_era(order) + } + + fn slot(&self, idx: usize) -> &AtomicUsize { + &self.domain.slots[self.hidx][idx] + } + + fn batch_mut(&self) -> RefMut<'_, Vec> { + self.domain.batches[self.hidx].borrow_mut() + } + + pub unsafe fn retire(&self, ptr: Shared) { + let curr_era = self.global_era(Ordering::SeqCst); + ptr.ptr.deref().retire.set(curr_era); + let mut batch = self.batch_mut(); + batch.push(Retired::new(ptr.ptr.as_raw())); + let count = self.retire_count.get() + 1; + self.retire_count.set(count); + + if batch.len() >= batch_capacity() { + GLOBAL_GARBAGE_COUNT.fetch_add(batch.len(), Ordering::AcqRel); + self.domain + .global_batch + .push(replace(&mut *batch, Vec::with_capacity(batch_capacity()))); + } + if count % collect_period() == 0 { + self.collect(); + } + } + + fn collect(&self) { + let retireds = self.domain.global_batch.pop_all(); + let retireds_len = retireds.len(); + fence(Ordering::SeqCst); + + let mut eras = self.domain.iter_eras().collect::>(); + eras.sort_unstable(); + + let not_freed: Vec<_> = retireds + .into_iter() + .filter_map(|ret| { + let (birth, retire) = unsafe { ret.lifespan() }; + let ge_idx = eras.partition_point(|&era| era < birth); + if eras.len() != ge_idx && eras[ge_idx] <= retire { + Some(ret) + } else { + unsafe { ret.execute() }; + None + } + }) + .collect(); + + let freed_count = retireds_len - not_freed.len(); + GLOBAL_GARBAGE_COUNT.fetch_sub(freed_count, Ordering::AcqRel); + + self.domain.global_batch.push(not_freed); + } +} + +impl<'d> Drop for Handle<'d> { + fn drop(&mut self) { + self.domain.avail_hidx.lock().unwrap().push(self.hidx); + } +} + +pub struct HazardEra<'d, 'h> { + handle: &'h Handle<'d>, + sidx: usize, +} + +impl<'d, 'h> HazardEra<'d, 'h> { + pub fn new(handle: &'h Handle<'d>) -> Self { + let sidx = handle.avail_sidx.borrow_mut().pop().unwrap(); + Self { handle, sidx } + } + + pub fn era(&self) -> &AtomicUsize { + &self.handle.slot(self.sidx) + } + + pub fn swap(h1: &mut Self, h2: &mut Self) { + swap(&mut h1.sidx, &mut h2.sidx); + } + + pub fn clear(&mut self) { + self.handle.slot(self.sidx).store(0, Ordering::Release); + } +} + +impl<'d, 'h> Drop for HazardEra<'d, 'h> { + fn drop(&mut self) { + self.handle.avail_sidx.borrow_mut().push(self.sidx); + } +} + +pub struct Shared { + ptr: TaggedPtr>, +} + +impl<'d, T> From for Shared { + fn from(value: usize) -> Self { + Self { + ptr: TaggedPtr::from(value as *const _), + } + } +} + +impl<'d, T> Clone for Shared { + fn clone(&self) -> Self { + Self { ptr: self.ptr } + } +} + +impl<'d, T> Copy for Shared {} + +impl<'d, T> Shared { + pub fn new(item: T, handle: &Handle<'d>) -> Self { + handle.incr_alloc(); + let era = handle.global_era(Ordering::SeqCst); + Self { + ptr: TaggedPtr::from(Box::into_raw(Box::new(Node::new(item, era)))), + } + } + + pub fn null() -> Self { + Self { + ptr: TaggedPtr::null(), + } + } + + pub fn tag(&self) -> usize { + self.ptr.tag() + } + + pub fn is_null(&self) -> bool { + self.ptr.is_null() + } + + pub fn with_tag(&self, tag: usize) -> Self { + Self::from_raw(self.ptr.with_tag(tag)) + } + + pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { + self.ptr.as_ref().map(|node| &node.item) + } + + pub unsafe fn deref<'g>(&self) -> &'g T { + &self.ptr.deref().item + } + + pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { + &mut self.ptr.deref_mut().item + } + + fn from_raw(ptr: TaggedPtr>) -> Self { + Self { ptr } + } + + pub unsafe fn into_owned(self) -> T { + Box::from_raw(self.ptr.as_raw()).item + } + + /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, + /// are identical. + pub fn ptr_eq(self, other: Self) -> bool { + self.ptr.ptr_eq(other.ptr) + } +} + +pub struct Atomic { + link: atomic::Atomic>>, +} + +const_assert_eq!( + size_of::>>>(), + size_of::() +); + +unsafe impl<'d, T: Sync> Sync for Atomic {} +unsafe impl<'d, T: Send> Send for Atomic {} + +impl<'d, T> Default for Atomic { + fn default() -> Self { + Self::null() + } +} + +impl<'d, T> From> for Atomic { + fn from(value: Shared) -> Self { + Self { + link: atomic::Atomic::new(value.ptr), + } + } +} + +impl<'d, T> Atomic { + pub fn new(init: T, handle: &Handle<'d>) -> Self { + Self { + link: atomic::Atomic::new(Shared::new(init, handle).ptr), + } + } + + pub fn null() -> Self { + Self { + link: atomic::Atomic::new(TaggedPtr::null()), + } + } + + pub fn load(&self, order: Ordering) -> Shared { + Shared::from_raw(self.link.load(order)) + } + + pub fn store(&self, ptr: Shared, order: Ordering) { + self.link.store(ptr.ptr, order); + } + + pub fn fetch_or(&self, val: usize, order: Ordering) -> Shared { + let prev = unsafe { &*(&self.link as *const _ as *const AtomicUsize) }.fetch_or(val, order); + Shared::from_raw(TaggedPtr::from(prev as *const _)) + } + + /// Loads and protects the pointer by the original CrystallineL's way. + /// + /// Hint: For traversal based structures where need to check tag, the guarnatee is similar to + /// `load` + `HazardPointer::set`. For Tstack, the guarnatee is similar to + /// `HazardPointer::protect`. + pub fn protect<'h>(&self, he: &mut HazardEra<'d, 'h>) -> Shared { + let mut prev_era = he.era().load(Ordering::Relaxed); + loop { + // Somewhat similar to atomically load and protect in Hazard pointers. + let ptr = self.link.load(Ordering::Acquire); + let curr_era = he.handle.global_era(Ordering::Acquire); + if curr_era == prev_era { + return Shared::from_raw(ptr); + } + he.era().store(curr_era, Ordering::SeqCst); + prev_era = curr_era; + } + } + + pub fn compare_exchange( + &self, + current: Shared, + new: Shared, + success: Ordering, + failure: Ordering, + ) -> Result, CompareExchangeError> { + match self + .link + .compare_exchange(current.ptr, new.ptr, success, failure) + { + Ok(current) => Ok(Shared::from_raw(current)), + Err(current) => Err(CompareExchangeError { + new, + current: Shared::from_raw(current), + }), + } + } + + pub unsafe fn into_owned(self) -> T { + Box::from_raw(self.link.into_inner().as_raw()).item + } +} + +pub struct CompareExchangeError { + pub new: Shared, + pub current: Shared, +} + +struct TaggedPtr { + ptr: *mut T, +} + +impl Default for TaggedPtr { + fn default() -> Self { + Self { ptr: null_mut() } + } +} + +impl Clone for TaggedPtr { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for TaggedPtr {} + +impl From<*const T> for TaggedPtr { + fn from(value: *const T) -> Self { + Self { + ptr: value.cast_mut(), + } + } +} + +impl From<*mut T> for TaggedPtr { + fn from(value: *mut T) -> Self { + Self { ptr: value } + } +} + +impl TaggedPtr { + pub fn null() -> Self { + Self { ptr: null_mut() } + } + + pub fn is_null(&self) -> bool { + self.as_raw().is_null() + } + + pub fn tag(&self) -> usize { + let ptr = self.ptr as usize; + ptr & low_bits::() + } + + /// Converts this pointer to a raw pointer (without the tag). + pub fn as_raw(&self) -> *mut T { + let ptr = self.ptr as usize; + (ptr & !low_bits::()) as *mut T + } + + pub fn with_tag(&self, tag: usize) -> Self { + Self::from(with_tag(self.ptr, tag)) + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn deref<'g>(&self) -> &'g T { + &*self.as_raw() + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { + &mut *self.as_raw() + } + + /// # Safety + /// + /// The pointer (without high and low tag bits) must be a valid location to dereference. + pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { + if self.is_null() { + None + } else { + Some(self.deref()) + } + } + + /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, + /// are identical. + pub fn ptr_eq(self, other: Self) -> bool { + self.ptr == other.ptr + } +} + +/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. +const fn low_bits() -> usize { + (1 << align_of::().trailing_zeros()) - 1 +} + +/// Returns the pointer with the given tag +fn with_tag(ptr: *mut T, tag: usize) -> *mut T { + ((ptr as usize & !low_bits::()) | (tag & low_bits::())) as *mut T +} diff --git a/src/bin/he.rs b/src/bin/he.rs new file mode 100644 index 00000000..b5d58355 --- /dev/null +++ b/src/bin/he.rs @@ -0,0 +1,215 @@ +use he::{set_batch_capacity, Domain, GLOBAL_GARBAGE_COUNT}; + +use crossbeam_utils::thread::scope; +use rand::prelude::*; +use std::cmp::max; +use std::io::{stdout, Write}; +use std::path::Path; +use std::sync::atomic::Ordering; +use std::sync::{mpsc, Arc, Barrier}; +use std::thread::available_parallelism; +use std::time::Instant; + +use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; +use smr_benchmark::ds_impl::he::{ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap}; + +fn main() { + let (config, output) = setup( + Path::new(file!()) + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .unwrap(), + ); + bench(&config, output) +} + +fn bench(config: &Config, output: BenchWriter) { + println!("{}", config); + let perf = match config.ds { + DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), + DS::NMTree => bench_map::>(config, PrefillStrategy::Random), + _ => panic!("Unsupported(or unimplemented) data structure for CrystallineL"), + }; + output.write_record(config, &perf); + println!("{}", perf); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. + Random, + /// Inserts keys in an increasing order, with a single thread. + Decreasing, +} + +impl PrefillStrategy { + fn prefill + Send + Sync>( + self, + config: &Config, + map: &M, + domain: &Domain, + ) { + match self { + PrefillStrategy::Random => { + let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); + print!("prefilling with {threads} threads... "); + stdout().flush().unwrap(); + scope(|s| { + for t in 0..threads { + s.spawn(move |_| { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let rng = &mut rand::thread_rng(); + let count = config.prefill / threads + + if t < config.prefill % threads { 1 } else { 0 }; + for _ in 0..count { + let key = config.key_dist.sample(rng); + let value = key; + map.insert(key, value, &mut shields, &handle); + } + }); + } + }) + .unwrap(); + } + PrefillStrategy::Decreasing => { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let rng = &mut rand::thread_rng(); + let mut keys = Vec::with_capacity(config.prefill); + for _ in 0..config.prefill { + keys.push(config.key_dist.sample(rng)); + } + keys.sort_by(|a, b| b.cmp(a)); + for key in keys.drain(..) { + let value = key; + map.insert(key, value, &mut shields, &handle); + } + } + } + print!("prefilled... "); + stdout().flush().unwrap(); + } +} + +fn bench_map + Send + Sync>( + config: &Config, + strategy: PrefillStrategy, +) -> Perf { + match config.bag_size { + BagSize::Small => set_batch_capacity(512), + BagSize::Large => set_batch_capacity(4096), + } + let max_threads = available_parallelism() + .map(|v| v.get()) + .unwrap_or(1) + .max(config.threads) + + 2; + let domain = &Domain::new(max_threads); + let mut handle = domain.register(); + let map = &M::new(&mut handle); + strategy.prefill(config, map, domain); + + let barrier = &Arc::new(Barrier::new(config.threads + config.aux_thread)); + let (ops_sender, ops_receiver) = mpsc::channel(); + let (mem_sender, mem_receiver) = mpsc::channel(); + + scope(|s| { + // sampling & interference thread + if config.aux_thread > 0 { + let mem_sender = mem_sender.clone(); + s.spawn(move |_| { + let mut samples = 0usize; + let mut acc = 0usize; + let mut peak = 0usize; + let mut garb_acc = 0usize; + let mut garb_peak = 0usize; + barrier.clone().wait(); + + let start = Instant::now(); + let mut next_sampling = start + config.sampling_period; + while start.elapsed() < config.duration { + let now = Instant::now(); + if now > next_sampling { + let allocated = config.mem_sampler.sample(); + samples += 1; + + acc += allocated; + peak = max(peak, allocated); + + let garbages = GLOBAL_GARBAGE_COUNT.load(Ordering::Acquire); + garb_acc += garbages; + garb_peak = max(garb_peak, garbages); + + next_sampling = now + config.sampling_period; + } + std::thread::sleep(config.aux_thread_period); + } + + if config.sampling { + mem_sender + .send((peak, acc / samples, garb_peak, garb_acc / samples)) + .unwrap(); + } else { + mem_sender.send((0, 0, 0, 0)).unwrap(); + } + }); + } else { + mem_sender.send((0, 0, 0, 0)).unwrap(); + } + + for _ in 0..config.threads { + let ops_sender = ops_sender.clone(); + s.spawn(move |_| { + let mut ops: u64 = 0; + let mut rng = &mut rand::thread_rng(); + let handle = domain.register(); + let mut shields = M::shields(&handle); + barrier.clone().wait(); + let start = Instant::now(); + + while start.elapsed() < config.duration { + let key = config.key_dist.sample(rng); + match Op::OPS[config.op_dist.sample(&mut rng)] { + Op::Get => { + map.get(&key, &mut shields, &handle); + } + Op::Insert => { + let value = key; + map.insert(key, value, &mut shields, &handle); + } + Op::Remove => { + map.remove(&key, &mut shields, &handle); + } + } + // Original author's implementation of `clear_all` has an use-after-free error. + // unsafe { handle.clear_all() }; + ops += 1; + } + + ops_sender.send(ops).unwrap(); + }); + } + }) + .unwrap(); + println!("end"); + + let mut ops = 0; + for _ in 0..config.threads { + let local_ops = ops_receiver.recv().unwrap(); + ops += local_ops; + } + let ops_per_sec = ops / config.interval; + let (peak_mem, avg_mem, peak_garb, avg_garb) = mem_receiver.recv().unwrap(); + Perf { + ops_per_sec, + peak_mem, + avg_mem, + peak_garb, + avg_garb, + } +} diff --git a/src/ds_impl/he/concurrent_map.rs b/src/ds_impl/he/concurrent_map.rs new file mode 100644 index 00000000..1eee3911 --- /dev/null +++ b/src/ds_impl/he/concurrent_map.rs @@ -0,0 +1,98 @@ +use he::Handle; + +pub trait ConcurrentMap { + type Shields<'d, 'h> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h>; + fn new<'d, 'h>(handle: &'h Handle<'d>) -> Self; + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V>; + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool; + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V>; +} + +#[cfg(test)] +pub mod tests { + extern crate rand; + use super::ConcurrentMap; + use he::Domain; + use rand::prelude::*; + use std::thread; + + const THREADS: i32 = 30; + const ELEMENTS_PER_THREADS: i32 = 1000; + + pub fn smoke + Send + Sync>() { + let domain = &Domain::new((THREADS + 1) as usize); + let handle = domain.register(); + let map = &M::new(&handle); + + thread::scope(|s| { + for t in 0..THREADS { + s.spawn(move || { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert!(map.insert(i, i.to_string(), &mut shields, &handle)); + } + }); + } + }); + + thread::scope(|s| { + for t in 0..(THREADS / 2) { + s.spawn(move || { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert_eq!( + i.to_string(), + *map.remove(&i, &mut shields, &handle).unwrap() + ); + } + }); + } + }); + + thread::scope(|s| { + for t in (THREADS / 2)..THREADS { + s.spawn(move || { + let handle = domain.register(); + let mut shields = M::shields(&handle); + let mut rng = rand::thread_rng(); + let mut keys: Vec = + (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); + keys.shuffle(&mut rng); + for i in keys { + assert_eq!(i.to_string(), *map.get(&i, &mut shields, &handle).unwrap()); + } + }); + } + }); + } +} diff --git a/src/ds_impl/he/list.rs b/src/ds_impl/he/list.rs new file mode 100644 index 00000000..70e904f5 --- /dev/null +++ b/src/ds_impl/he/list.rs @@ -0,0 +1,658 @@ +use super::concurrent_map::ConcurrentMap; + +use std::cmp::Ordering::{Equal, Greater, Less}; +use std::sync::atomic::Ordering; + +use he::{Atomic, Handle, HazardEra, Shared}; + +// `#[repr(C)]` is used to ensure the first field +// is also the first data in the memory alignment. +#[repr(C)] +pub struct Node { + /// Mark: tag(), Tag: not needed + next: Atomic, + key: K, + value: V, +} + +pub struct List { + head: Atomic>, +} + +impl Default for List +where + K: Ord + 'static, +{ + fn default() -> Self { + Self::new() + } +} + +impl Drop for List { + fn drop(&mut self) { + let mut curr = self.head.load(Ordering::Relaxed); + + while !curr.is_null() { + curr = unsafe { curr.into_owned() }.next.load(Ordering::Relaxed); + } + } +} + +pub struct EraShields<'d, 'h> { + prev_h: HazardEra<'d, 'h>, + curr_h: HazardEra<'d, 'h>, + next_h: HazardEra<'d, 'h>, + // `anchor_h` and `anchor_next_h` are used for `find_harris` + anchor_h: HazardEra<'d, 'h>, + anchor_next_h: HazardEra<'d, 'h>, +} + +impl<'d, 'h> EraShields<'d, 'h> { + pub fn new(handle: &'h Handle<'d>) -> Self { + Self { + prev_h: HazardEra::new(handle), + curr_h: HazardEra::new(handle), + next_h: HazardEra::new(handle), + anchor_h: HazardEra::new(handle), + anchor_next_h: HazardEra::new(handle), + } + } + + // bypass E0499-E0503, etc that are supposed to be fixed by polonius + #[inline] + fn launder<'he2>(&mut self) -> &'he2 mut Self { + unsafe { core::mem::transmute(self) } + } +} + +pub struct Cursor { + prev: Shared>, + // For harris, this keeps the mark bit. Don't mix harris and harris-michael. + curr: Shared>, + // `anchor` is used for `find_harris` + // anchor and anchor_next are non-null iff exist + anchor: Shared>, + anchor_next: Shared>, +} + +impl Cursor { + pub fn new<'d, 'h>(head: &Atomic>, shields: &mut EraShields<'d, 'h>) -> Self { + Self { + prev: Shared::from(head as *const _ as usize), + curr: head.protect(&mut shields.curr_h), + anchor: Shared::null(), + anchor_next: Shared::null(), + } + } +} + +impl Cursor +where + K: Ord, +{ + /// Optimistically traverses while maintaining `anchor` and `anchor_next`. + /// It is used for both Harris and Harris-Herlihy-Shavit traversals. + #[inline] + fn traverse_with_anchor<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h>, + ) -> Result { + // Invariants: + // anchor, anchor_next: protected if they are not null. + // prev: always protected with prev_sh + // curr: not protected. + // curr: also has tag value when it is obtained from prev. + Ok(loop { + let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { + break false; + }; + + // Validation depending on the state of `self.curr`. + // + // - If it is marked, validate on anchor. + // - If it is not marked, it is already protected safely by the Crystalline. + if self.curr.tag() != 0 { + // Validate on anchor. + + debug_assert!(!self.anchor.is_null()); + debug_assert!(!self.anchor_next.is_null()); + let an_new = unsafe { self.anchor.deref() }.next.load(Ordering::Acquire); + + if an_new.tag() != 0 { + return Err(()); + } else if !an_new.ptr_eq(self.anchor_next) { + // Anchor is updated but clear, so can restart from anchor. + + self.prev = self.anchor; + self.curr = an_new; + self.anchor = Shared::null(); + + // Set prev HP as anchor HP, since prev should always be protected. + HazardEra::swap(&mut shields.prev_h, &mut shields.anchor_h); + continue; + } + } + + let next = curr_node.next.protect(&mut shields.next_h); + if next.tag() == 0 { + if curr_node.key < *key { + self.prev = self.curr; + self.curr = next; + self.anchor = Shared::null(); + HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + } else { + break curr_node.key == *key; + } + } else { + if self.anchor.is_null() { + self.anchor = self.prev; + self.anchor_next = self.curr; + HazardEra::swap(&mut shields.anchor_h, &mut shields.prev_h); + } else if self.anchor_next.ptr_eq(self.prev) { + HazardEra::swap(&mut shields.anchor_next_h, &mut shields.prev_h); + } + self.prev = self.curr; + self.curr = next; + HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + } + }) + } + + #[inline] + fn find_harris<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Result { + // Finding phase + // - cursor.curr: first unmarked node w/ key >= search key (4) + // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) + // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) + + let found = self.traverse_with_anchor(key, shields)?; + + scopeguard::defer! { + shields.anchor_h.clear(); + shields.anchor_next_h.clear(); + } + + if self.anchor.is_null() { + self.prev = self.prev.with_tag(0); + self.curr = self.curr.with_tag(0); + Ok(found) + } else { + debug_assert_eq!(self.anchor_next.tag(), 0); + // TODO: on CAS failure, if anchor is not tagged, we can restart from anchor. + unsafe { &self.anchor.deref().next } + .compare_exchange( + self.anchor_next, + self.curr.with_tag(0), + Ordering::AcqRel, + Ordering::Relaxed, + ) + .map_err(|_| { + self.curr = self.curr.with_tag(0); + () + })?; + + let mut node = self.anchor_next; + while !node.with_tag(0).ptr_eq(self.curr.with_tag(0)) { + // NOTE: It may seem like this can be done with a NA load, but we do a `fetch_or` in remove, which always does an write. + // This can be a NA load if the `fetch_or` in delete is changed to a CAS, but it is not clear if it is worth it. + let next = unsafe { node.deref().next.load(Ordering::Relaxed) }; + debug_assert!(next.tag() != 0); + unsafe { handle.retire(node) }; + node = next; + } + self.prev = self.anchor.with_tag(0); + self.curr = self.curr.with_tag(0); + Ok(found) + } + } + + #[inline] + fn find_harris_michael<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Result { + loop { + debug_assert_eq!(self.curr.tag(), 0); + let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { + return Ok(false); + }; + let mut next = curr_node.next.protect(&mut shields.next_h); + + if next.tag() != 0 { + next = next.with_tag(0); + unsafe { self.prev.deref() } + .next + .compare_exchange(self.curr, next, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| ())?; + unsafe { handle.retire(self.curr) }; + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + self.curr = next; + continue; + } + + match curr_node.key.cmp(key) { + Less => { + HazardEra::swap(&mut shields.prev_h, &mut shields.curr_h); + HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); + self.prev = self.curr; + self.curr = next; + } + Equal => return Ok(true), + Greater => return Ok(false), + } + } + } + + #[inline] + fn find_harris_herlihy_shavit<'d, 'h>( + &mut self, + key: &K, + shields: &mut EraShields<'d, 'h>, + _handle: &'h Handle<'d>, + ) -> Result { + let found = self.traverse_with_anchor(key, shields)?; + // Return only the found `curr` node. + // Others are not necessary because we are not going to do insertion or deletion + // with this Harris-Herlihy-Shavit traversal. + self.curr = self.curr.with_tag(0); + shields.anchor_h.clear(); + shields.anchor_next_h.clear(); + Ok(found) + } +} + +impl List +where + K: Ord + 'static, +{ + /// Creates a new list. + pub fn new() -> Self { + List { + head: Atomic::null(), + } + } + + #[inline] + fn get<'d, 'h, 'he, F>( + &self, + key: &K, + find: F, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> + where + F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, + { + loop { + let mut cursor = Cursor::new(&self.head, shields); + match find(&mut cursor, key, shields, handle) { + Ok(true) => return unsafe { Some(&(cursor.curr.deref().value)) }, + Ok(false) => return None, + Err(_) => continue, + } + } + } + + fn insert_inner<'d, 'h, 'he, F>( + &self, + node: Shared>, + find: &F, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Result + where + F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, + { + loop { + let mut cursor = Cursor::new(&self.head, shields); + let found = find(&mut cursor, unsafe { &node.deref().key }, shields, handle)?; + if found { + drop(unsafe { node.into_owned() }); + return Ok(false); + } + + unsafe { node.deref() } + .next + .store(cursor.curr, Ordering::Relaxed); + if unsafe { cursor.prev.deref() } + .next + .compare_exchange(cursor.curr, node, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + return Ok(true); + } + } + } + + #[inline] + fn insert<'d, 'h, 'he, F>( + &self, + key: K, + value: V, + find: F, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool + where + F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, + { + let node = Shared::new( + Node { + key, + value, + next: Atomic::null(), + }, + handle, + ); + + loop { + match self.insert_inner(node, &find, shields, handle) { + Ok(r) => return r, + Err(()) => continue, + } + } + } + + fn remove_inner<'d, 'h, 'he, F>( + &self, + key: &K, + find: &F, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Result, ()> + where + F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, + { + loop { + let mut cursor = Cursor::new(&self.head, shields); + let found = find(&mut cursor, key, shields, handle)?; + if !found { + return Ok(None); + } + + let curr_node = unsafe { cursor.curr.deref() }; + let next = curr_node.next.fetch_or(1, Ordering::AcqRel); + if next.tag() == 1 { + continue; + } + + if unsafe { &cursor.prev.deref().next } + .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) + .is_ok() + { + unsafe { handle.retire(cursor.curr) }; + } + + return Ok(Some(&curr_node.value)); + } + } + + #[inline] + fn remove<'d, 'h, 'he, F>( + &self, + key: &K, + find: F, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> + where + F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, + { + loop { + match self.remove_inner(key, &find, shields.launder(), handle) { + Ok(r) => return r, + Err(_) => continue, + } + } + } + + pub fn harris_get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.get(key, Cursor::find_harris, shields, handle) + } + + pub fn harris_insert<'d, 'h, 'he>( + &self, + key: K, + value: V, + shields: &mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool { + self.insert(key, value, Cursor::find_harris, shields, handle) + } + + pub fn harris_remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.remove(key, Cursor::find_harris, shields, handle) + } + + pub fn harris_michael_get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.get(key, Cursor::find_harris_michael, shields, handle) + } + + pub fn harris_michael_insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool { + self.insert(key, value, Cursor::find_harris_michael, shields, handle) + } + + pub fn harris_michael_remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.remove(key, Cursor::find_harris_michael, shields, handle) + } + + pub fn harris_herlihy_shavit_get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.get(key, Cursor::find_harris_herlihy_shavit, shields, handle) + } +} + +pub struct HList { + inner: List, +} + +impl ConcurrentMap for HList +where + K: Ord + 'static, + V: 'static, +{ + type Shields<'d, 'h> + = EraShields<'d, 'h> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(_: &'h Handle<'d>) -> Self { + Self { inner: List::new() } + } + + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.inner.harris_get(key, shields, handle) + } + + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool { + self.inner.harris_insert(key, value, shields, handle) + } + + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.inner.harris_remove(key, shields, handle) + } +} + +pub struct HMList { + inner: List, +} + +impl ConcurrentMap for HMList +where + K: Ord + 'static, + V: 'static, +{ + type Shields<'d, 'h> + = EraShields<'d, 'h> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(_: &'h Handle<'d>) -> Self { + Self { inner: List::new() } + } + + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.inner.harris_michael_get(key, shields, handle) + } + + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool { + self.inner + .harris_michael_insert(key, value, shields, handle) + } + + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.inner.harris_michael_remove(key, shields, handle) + } +} + +pub struct HHSList { + inner: List, +} + +impl ConcurrentMap for HHSList +where + K: Ord + 'static, + V: 'static, +{ + type Shields<'d, 'h> + = EraShields<'d, 'h> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(_: &'h Handle<'d>) -> Self { + Self { inner: List::new() } + } + + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.inner.harris_herlihy_shavit_get(key, shields, handle) + } + + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool { + self.inner.harris_insert(key, value, shields, handle) + } + + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.inner.harris_remove(key, shields, handle) + } +} + +#[cfg(test)] +mod tests { + use super::{HHSList, HList, HMList}; + use crate::ds_impl::he::concurrent_map; + + #[test] + fn smoke_h_list() { + concurrent_map::tests::smoke::>(); + } + + #[test] + fn smoke_hm_list() { + concurrent_map::tests::smoke::>(); + } + + #[test] + fn smoke_hhs_list() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/he/michael_hash_map.rs b/src/ds_impl/he/michael_hash_map.rs new file mode 100644 index 00000000..c3e6a5d6 --- /dev/null +++ b/src/ds_impl/he/michael_hash_map.rs @@ -0,0 +1,102 @@ +use super::concurrent_map::ConcurrentMap; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use super::list::HHSList; +pub use super::list::{Cursor, EraShields, Node}; + +use he::Handle; + +pub struct HashMap { + buckets: Vec>, +} + +impl HashMap +where + K: Ord + Hash + 'static, + V: 'static, +{ + pub fn with_capacity<'d, 'h>(n: usize, handle: &'h Handle<'d>) -> Self { + let mut buckets = Vec::with_capacity(n); + for _ in 0..n { + buckets.push(HHSList::new(handle)); + } + + HashMap { buckets } + } + + #[inline] + pub fn get_bucket(&self, index: usize) -> &HHSList { + unsafe { self.buckets.get_unchecked(index % self.buckets.len()) } + } + + #[inline] + fn hash(k: &K) -> usize { + let mut s = DefaultHasher::new(); + k.hash(&mut s); + s.finish() as usize + } +} + +impl ConcurrentMap for HashMap +where + K: Ord + Hash + Send + 'static, + V: Send + 'static, +{ + type Shields<'d, 'h> = EraShields<'d, 'h> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(handle: &'h Handle<'d>) -> Self { + Self::with_capacity(30000, handle) + } + + #[inline(always)] + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + let i = Self::hash(key); + self.get_bucket(i).get(key, shields, handle) + } + + #[inline(always)] + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool { + let i = Self::hash(&key); + self.get_bucket(i).insert(key, value, shields, handle) + } + + #[inline(always)] + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + let i = Self::hash(key); + self.get_bucket(i).remove(key, shields, handle) + } +} + +#[cfg(test)] +mod tests { + use super::HashMap; + use crate::ds_impl::he::concurrent_map; + + #[test] + fn smoke_hashmap() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/he/mod.rs b/src/ds_impl/he/mod.rs new file mode 100644 index 00000000..f6d449ed --- /dev/null +++ b/src/ds_impl/he/mod.rs @@ -0,0 +1,11 @@ +pub mod concurrent_map; + +pub mod list; +pub mod michael_hash_map; +pub mod natarajan_mittal_tree; + +pub use self::concurrent_map::ConcurrentMap; + +pub use self::list::{HHSList, HList, HMList}; +pub use self::michael_hash_map::HashMap; +pub use self::natarajan_mittal_tree::NMTreeMap; diff --git a/src/ds_impl/he/natarajan_mittal_tree.rs b/src/ds_impl/he/natarajan_mittal_tree.rs new file mode 100644 index 00000000..12d7d9a6 --- /dev/null +++ b/src/ds_impl/he/natarajan_mittal_tree.rs @@ -0,0 +1,683 @@ +use super::concurrent_map::ConcurrentMap; + +use std::cmp; +use std::sync::atomic::Ordering; + +use he::{Atomic, Handle, HazardEra, Shared}; + +bitflags! { + /// TODO + /// A remove operation is registered by marking the corresponding edges: the (parent, target) + /// edge is _flagged_ and the (parent, sibling) edge is _tagged_. + struct Marks: usize { + const FLAG = 1usize.wrapping_shl(1); + const TAG = 1usize.wrapping_shl(0); + } +} + +impl Marks { + fn new(flag: bool, tag: bool) -> Self { + (if flag { Marks::FLAG } else { Marks::empty() }) + | (if tag { Marks::TAG } else { Marks::empty() }) + } + + fn flag(self) -> bool { + !(self & Marks::FLAG).is_empty() + } + + fn tag(self) -> bool { + !(self & Marks::TAG).is_empty() + } + + fn marked(self) -> bool { + !(self & (Marks::TAG | Marks::FLAG)).is_empty() + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +enum Key { + Fin(K), + Inf, +} + +impl PartialOrd for Key +where + K: PartialOrd, +{ + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Key::Fin(k1), Key::Fin(k2)) => k1.partial_cmp(k2), + (Key::Fin(_), Key::Inf) => Some(std::cmp::Ordering::Less), + (Key::Inf, Key::Fin(_)) => Some(std::cmp::Ordering::Greater), + (Key::Inf, Key::Inf) => Some(std::cmp::Ordering::Equal), + } + } +} + +impl PartialEq for Key +where + K: PartialEq, +{ + fn eq(&self, rhs: &K) -> bool { + match self { + Key::Fin(k) => k == rhs, + _ => false, + } + } +} + +impl PartialOrd for Key +where + K: PartialOrd, +{ + fn partial_cmp(&self, rhs: &K) -> Option { + match self { + Key::Fin(k) => k.partial_cmp(rhs), + _ => Some(std::cmp::Ordering::Greater), + } + } +} + +impl Key +where + K: Ord, +{ + fn cmp(&self, rhs: &K) -> std::cmp::Ordering { + match self { + Key::Fin(k) => k.cmp(rhs), + _ => std::cmp::Ordering::Greater, + } + } +} + +pub struct Node { + key: Key, + value: Option, + left: Atomic>, + right: Atomic>, +} + +impl Node +where + K: Clone, + V: Clone, +{ + fn new_leaf(key: Key, value: Option) -> Node { + Node { + key, + value, + left: Atomic::null(), + right: Atomic::null(), + } + } + + /// Make a new internal node, consuming the given left and right nodes, + /// using the right node's key. + fn new_internal(left: Node, right: Node, handle: &Handle<'_>) -> Node { + Node { + key: right.key.clone(), + value: None, + left: Atomic::new(left, handle), + right: Atomic::new(right, handle), + } + } +} + +#[derive(Clone, Copy)] +enum Direction { + L, + R, +} + +pub struct EraShields<'d, 'h> { + ancestor_h: HazardEra<'d, 'h>, + successor_h: HazardEra<'d, 'h>, + parent_h: HazardEra<'d, 'h>, + leaf_h: HazardEra<'d, 'h>, + curr_h: HazardEra<'d, 'h>, +} + +impl<'d, 'h> EraShields<'d, 'h> { + pub fn new(handle: &'h Handle<'d>) -> Self { + Self { + ancestor_h: HazardEra::new(handle), + successor_h: HazardEra::new(handle), + parent_h: HazardEra::new(handle), + leaf_h: HazardEra::new(handle), + curr_h: HazardEra::new(handle), + } + } + + // bypass E0499-E0503, etc that are supposed to be fixed by polonius + #[inline] + fn launder<'he2>(&mut self) -> &'he2 mut Self { + unsafe { core::mem::transmute(self) } + } +} + +/// All Shared<_> are unmarked. +/// +/// All of the edges of path from `successor` to `parent` are in the process of removal. +pub struct SeekRecord { + /// Parent of `successor` + ancestor: Shared>, + /// The first internal node with a marked outgoing edge + successor: Shared>, + /// The direction of successor from ancestor. + successor_dir: Direction, + /// Parent of `leaf` + parent: Shared>, + /// The end of the access path. + leaf: Shared>, + /// The direction of leaf from parent. + leaf_dir: Direction, +} + +impl SeekRecord { + fn new() -> Self { + Self { + ancestor: Shared::null(), + successor: Shared::null(), + successor_dir: Direction::L, + parent: Shared::null(), + leaf: Shared::null(), + leaf_dir: Direction::L, + } + } + + fn successor_addr(&self) -> &Atomic> { + match self.successor_dir { + Direction::L => unsafe { &self.ancestor.deref().left }, + Direction::R => unsafe { &self.ancestor.deref().right }, + } + } + + fn leaf_addr(&self) -> &Atomic> { + match self.leaf_dir { + Direction::L => unsafe { &self.parent.deref().left }, + Direction::R => unsafe { &self.parent.deref().right }, + } + } + + fn leaf_sibling_addr(&self) -> &Atomic> { + match self.leaf_dir { + Direction::L => unsafe { &self.parent.deref().right }, + Direction::R => unsafe { &self.parent.deref().left }, + } + } +} + +pub struct NMTreeMap { + r: Node, +} + +impl Default for NMTreeMap +where + K: Ord + Clone, + V: Clone, +{ + fn default() -> Self { + todo!("new") + } +} + +impl Drop for NMTreeMap { + fn drop(&mut self) { + unsafe { + let mut stack = vec![ + self.r.left.load(Ordering::Relaxed), + self.r.right.load(Ordering::Relaxed), + ]; + assert!(self.r.value.is_none()); + + while let Some(node) = stack.pop() { + if node.is_null() { + continue; + } + + let node_ref = node.deref(); + + stack.push(node_ref.left.load(Ordering::Relaxed)); + stack.push(node_ref.right.load(Ordering::Relaxed)); + drop(node.into_owned()); + } + } + } +} + +impl NMTreeMap +where + K: Ord + Clone + 'static, + V: Clone + 'static, +{ + pub fn new<'d>(handle: &Handle<'d>) -> Self { + // An empty tree has 5 default nodes with infinite keys so that the SeekRecord is allways + // well-defined. + // r + // / \ + // s inf2 + // / \ + // inf0 inf1 + let inf0 = Node::new_leaf(Key::Inf, None); + let inf1 = Node::new_leaf(Key::Inf, None); + let inf2 = Node::new_leaf(Key::Inf, None); + let s = Node::new_internal(inf0, inf1, handle); + let r = Node::new_internal(s, inf2, handle); + NMTreeMap { r } + } + + fn seek<'d, 'h>( + &self, + key: &K, + record: &mut SeekRecord, + shields: &mut EraShields<'d, 'h>, + ) -> Result<(), ()> { + let s = self.r.left.load(Ordering::Relaxed).with_tag(0); + let s_node = unsafe { s.deref() }; + + // The root node is always alive; we do not have to protect it. + record.ancestor = Shared::from(&self.r as *const _ as usize); + record.successor = s; // TODO: should preserve tag? + + record.successor_dir = Direction::L; + // The `s` node is always alive; we do not have to protect it. + record.parent = s; + + record.leaf = s_node.left.protect(&mut shields.leaf_h); + record.leaf_dir = Direction::L; + + let mut prev_tag = Marks::from_bits_truncate(record.leaf.tag()).tag(); + let mut curr_dir = Direction::L; + let mut curr = unsafe { record.leaf.deref() } + .left + .protect(&mut shields.curr_h); + + // `ancestor` always points untagged node. + while !curr.is_null() { + if !prev_tag { + // untagged edge: advance ancestor and successor pointers + record.ancestor = record.parent; + record.successor = record.leaf; + record.successor_dir = record.leaf_dir; + // `ancestor` and `successor` are already protected by + // hazard pointers of `parent` and `leaf`. + + // Advance the parent and leaf pointers when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (parent), ancestor -> O (ancestor) -> O + // / \ / \ + // (leaf), successor -> O O => (parent), successor -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardEra::swap(&mut shields.ancestor_h, &mut shields.parent_h); + HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); + } else if record.successor.ptr_eq(record.parent) { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O (ancestor) -> O + // / \ / \ + // (parent), successor -> O O (successor) -> O O + // / \ => / \ + // (leaf) -> O O (parent) -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardEra::swap(&mut shields.successor_h, &mut shields.parent_h); + HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); + } else { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O + // / \ + // (successor) -> O O + // ... ... + // (parent) -> O + // / \ + // (leaf) -> O O + record.parent = record.leaf; + HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); + } + debug_assert_eq!(record.successor.tag(), 0); + + if Marks::from_bits_truncate(curr.tag()).marked() { + // `curr` is marked. Validate by `ancestor`. + let succ_new = record.successor_addr().load(Ordering::Acquire); + if Marks::from_bits_truncate(succ_new.tag()).marked() + || !record.successor.ptr_eq(succ_new) + { + // Validation is failed. Let's restart from the root. + // TODO: Maybe it can be optimized (by restarting from the anchor), but + // it would require a serious reasoning (including shield swapping, etc). + return Err(()); + } + } + + record.leaf = curr; + record.leaf_dir = curr_dir; + + // update other variables + prev_tag = Marks::from_bits_truncate(curr.tag()).tag(); + let curr_node = unsafe { curr.deref() }; + if curr_node.key.cmp(key) == cmp::Ordering::Greater { + curr_dir = Direction::L; + curr = curr_node.left.load(Ordering::Acquire); + } else { + curr_dir = Direction::R; + curr = curr_node.right.load(Ordering::Acquire); + } + } + Ok(()) + } + + /// Physically removes node. + /// + /// Returns true if it successfully unlinks the flagged node in `record`. + fn cleanup<'d>(&self, record: &mut SeekRecord, handle: &Handle<'d>) -> bool { + // Identify the node(subtree) that will replace `successor`. + let leaf_marked = record.leaf_addr().load(Ordering::Acquire); + let leaf_flag = Marks::from_bits_truncate(leaf_marked.tag()).flag(); + let target_sibling_addr = if leaf_flag { + record.leaf_sibling_addr() + } else { + record.leaf_addr() + }; + + // NOTE: the ibr implementation uses CAS + // tag (parent, sibling) edge -> all of the parent's edges can't change now + // TODO: Is Release enough? + target_sibling_addr.fetch_or(Marks::TAG.bits(), Ordering::AcqRel); + + // Try to replace (ancestor, successor) w/ (ancestor, sibling). + // Since (parent, sibling) might have been concurrently flagged, copy + // the flag to the new edge (ancestor, sibling). + let target_sibling = target_sibling_addr.load(Ordering::Acquire); + let flag = Marks::from_bits_truncate(target_sibling.tag()).flag(); + let is_unlinked = record + .successor_addr() + .compare_exchange( + record.successor.with_tag(0), + target_sibling.with_tag(Marks::new(flag, false).bits()), + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok(); + + if is_unlinked { + unsafe { + // destroy the subtree of successor except target_sibling + let mut stack = vec![record.successor]; + + while let Some(node) = stack.pop() { + if node.is_null() || node.with_tag(0).ptr_eq(target_sibling.with_tag(0)) { + continue; + } + + let node_ref = node.deref(); + + stack.push(node_ref.left.load(Ordering::Relaxed)); + stack.push(node_ref.right.load(Ordering::Relaxed)); + handle.retire(node); + } + } + } + + is_unlinked + } + + fn get_inner<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + ) -> Result, ()> { + let mut record = SeekRecord::new(); + + self.seek(key, &mut record, shields)?; + let leaf_node = unsafe { record.leaf.deref() }; + + if leaf_node.key.cmp(key) != cmp::Ordering::Equal { + return Ok(None); + } + + Ok(Some(leaf_node.value.as_ref().unwrap())) + } + + pub fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + ) -> Option<&'he V> { + loop { + if let Ok(r) = self.get_inner(key, shields.launder()) { + return r; + } + } + } + + fn insert_inner<'d, 'h, 'he>( + &self, + key: &K, + value: V, + record: &mut SeekRecord, + shields: &'he mut EraShields<'d, 'h>, + handle: &Handle<'d>, + ) -> Result<(), Result> { + let mut new_leaf = Shared::new(Node::new_leaf(Key::Fin(key.clone()), Some(value)), handle); + + let mut new_internal = Shared::new( + Node:: { + key: Key::Inf, // temporary placeholder + value: None, + left: Atomic::null(), + right: Atomic::null(), + }, + handle, + ); + + loop { + self.seek(key, record, shields).map_err(|_| unsafe { + let value = new_leaf.deref_mut().value.take().unwrap(); + drop(new_leaf.into_owned()); + drop(new_internal.into_owned()); + Err(value) + })?; + let leaf = record.leaf.with_tag(0); + + let (new_left, new_right) = match unsafe { leaf.deref() }.key.cmp(key) { + cmp::Ordering::Equal => { + // Newly created nodes that failed to be inserted are free'd here. + let value = unsafe { new_leaf.deref_mut() }.value.take().unwrap(); + unsafe { + drop(new_leaf.into_owned()); + drop(new_internal.into_owned()); + } + return Err(Ok(value)); + } + cmp::Ordering::Greater => (new_leaf, leaf), + cmp::Ordering::Less => (leaf, new_leaf), + }; + + let new_internal_node = unsafe { new_internal.deref_mut() }; + new_internal_node.key = unsafe { new_right.deref().key.clone() }; + new_internal_node.left.store(new_left, Ordering::Relaxed); + new_internal_node.right.store(new_right, Ordering::Relaxed); + + // NOTE: record.leaf_addr is called childAddr in the paper. + match record.leaf_addr().compare_exchange( + leaf, + new_internal, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(()), + Err(e) => { + // Insertion failed. Help the conflicting remove operation if needed. + // NOTE: The paper version checks if any of the mark is set, which is redundant. + if e.current.with_tag(0).ptr_eq(leaf) { + self.cleanup(record, handle); + } + } + } + } + } + + pub fn insert<'d, 'h, 'he>( + &self, + key: K, + mut value: V, + shields: &'he mut EraShields<'d, 'h>, + handle: &Handle<'d>, + ) -> Result<(), (K, V)> { + loop { + let mut record = SeekRecord::new(); + match self.insert_inner(&key, value, &mut record, shields, handle) { + Ok(()) => return Ok(()), + Err(Ok(v)) => return Err((key, v)), + Err(Err(v)) => value = v, + } + } + } + + fn remove_inner<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + handle: &Handle<'d>, + ) -> Result, ()> { + // `leaf` and `value` are the snapshot of the node to be deleted. + // NOTE: The paper version uses one big loop for both phases. + // injection phase + let mut record = SeekRecord::new(); + let (leaf, value) = loop { + self.seek(key, &mut record, shields)?; + + // candidates + let leaf = record.leaf.with_tag(0); + let leaf_node = unsafe { record.leaf.deref() }; + + if leaf_node.key.cmp(key) != cmp::Ordering::Equal { + return Ok(None); + } + + let value = leaf_node.value.as_ref().unwrap(); + + // Try injecting the deletion flag. + match record.leaf_addr().compare_exchange( + leaf, + leaf.with_tag(Marks::new(true, false).bits()), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + // Finalize the node to be removed + if self.cleanup(&mut record, handle) { + return Ok(Some(value)); + } + // In-place cleanup failed. Enter the cleanup phase. + break (leaf, value); + } + Err(e) => { + // Flagging failed. + // case 1. record.leaf_addr(e.current) points to another node: restart. + // case 2. Another thread flagged/tagged the edge to leaf: help and restart + // NOTE: The paper version checks if any of the mark is set, which is redundant. + if leaf.ptr_eq(e.current.with_tag(Marks::empty().bits())) { + self.cleanup(&mut record, handle); + } + } + } + }; + + let leaf = leaf.with_tag(0); + + // cleanup phase + loop { + self.seek(key, &mut record, shields)?; + if !record.leaf.with_tag(0).ptr_eq(leaf) { + // The edge to leaf flagged for deletion was removed by a helping thread + return Ok(Some(value)); + } + + // leaf is still present in the tree. + if self.cleanup(&mut record, handle) { + return Ok(Some(value)); + } + } + } + + pub fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut EraShields<'d, 'h>, + handle: &Handle<'d>, + ) -> Option<&'he V> { + loop { + if let Ok(r) = self.remove_inner(key, shields.launder(), handle) { + return r; + } + } + } +} + +impl ConcurrentMap for NMTreeMap +where + K: Ord + Clone + 'static, + V: Clone + 'static, +{ + type Shields<'d, 'h> = EraShields<'d, 'h> + where + 'd: 'h; + + fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { + EraShields::new(handle) + } + + fn new<'d, 'h>(handle: &'h Handle<'d>) -> Self { + Self::new(handle) + } + + #[inline(always)] + fn get<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + _handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.get(key, shields) + } + + #[inline(always)] + fn insert<'d, 'h>( + &self, + key: K, + value: V, + shields: &mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> bool { + self.insert(key, value, shields, handle).is_ok() + } + + #[inline(always)] + fn remove<'d, 'h, 'he>( + &self, + key: &K, + shields: &'he mut Self::Shields<'d, 'h>, + handle: &'h Handle<'d>, + ) -> Option<&'he V> { + self.remove(key, shields, handle) + } +} + +#[cfg(test)] +mod tests { + use super::NMTreeMap; + use crate::ds_impl::he::concurrent_map; + + #[test] + fn smoke_nm_tree() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/mod.rs b/src/ds_impl/mod.rs index 4e95d3f3..320a67f6 100644 --- a/src/ds_impl/mod.rs +++ b/src/ds_impl/mod.rs @@ -3,6 +3,7 @@ pub mod circ_ebr; pub mod circ_hp; pub mod crystalline_l; pub mod ebr; +pub mod he; pub mod hp; pub mod hp_brcu; pub mod hp_pp; diff --git a/test-scripts/sanitize-he.sh b/test-scripts/sanitize-he.sh new file mode 100644 index 00000000..bc05db07 --- /dev/null +++ b/test-scripts/sanitize-he.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' + +he="cargo run --bin he --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " + +set -e +for i in {1..5000}; do + $he -dh-list -i3 -t48 -r10 -g1 + $he -dhm-list -i3 -t48 -r10 -g1 + $he -dhhs-list -i3 -t48 -r10 -g1 +done From 22b04ffa33c086908e4013695f29912124ddc239 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 7 Nov 2024 11:32:19 +0000 Subject: [PATCH 67/84] Support returning an owned `V` in PEBR-based DSs --- src/ds_impl/pebr/bonsai_tree.rs | 11 +++++-- src/ds_impl/pebr/concurrent_map.rs | 31 +++++++++++++++--- src/ds_impl/pebr/ellen_tree.rs | 11 +++++-- src/ds_impl/pebr/list.rs | 35 ++++++++++++++------ src/ds_impl/pebr/michael_hash_map.rs | 40 ++++++++--------------- src/ds_impl/pebr/natarajan_mittal_tree.rs | 11 +++++-- src/ds_impl/pebr/skip_list.rs | 11 +++++-- 7 files changed, 97 insertions(+), 53 deletions(-) diff --git a/src/ds_impl/pebr/bonsai_tree.rs b/src/ds_impl/pebr/bonsai_tree.rs index 4353810f..5279a316 100644 --- a/src/ds_impl/pebr/bonsai_tree.rs +++ b/src/ds_impl/pebr/bonsai_tree.rs @@ -1,6 +1,6 @@ use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::shield_pool::ShieldPool; use std::cmp; @@ -783,7 +783,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.get(key, handle, guard) } @@ -793,7 +793,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.remove(key, handle, guard) } } diff --git a/src/ds_impl/pebr/concurrent_map.rs b/src/ds_impl/pebr/concurrent_map.rs index 72a0e176..2957ff5e 100644 --- a/src/ds_impl/pebr/concurrent_map.rs +++ b/src/ds_impl/pebr/concurrent_map.rs @@ -1,5 +1,21 @@ use crossbeam_pebr::Guard; +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { type Handle; @@ -12,15 +28,20 @@ pub trait ConcurrentMap { handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V>; + ) -> Option>; fn insert(&self, handle: &mut Self::Handle, key: K, value: V, guard: &mut Guard) -> bool; - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option; + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_pebr::pin; use crossbeam_utils::thread; use rand::prelude::*; @@ -58,7 +79,7 @@ pub mod tests { for i in keys { assert_eq!( i.to_string(), - map.remove(&mut handle, &i, &mut pin()).unwrap() + *map.remove(&mut handle, &i, &mut pin()).unwrap().output() ); } }); @@ -77,7 +98,7 @@ pub mod tests { for i in keys { assert_eq!( i.to_string(), - *map.get(&mut handle, &i, &mut pin()).unwrap() + *map.get(&mut handle, &i, &mut pin()).unwrap().output() ); } }); diff --git a/src/ds_impl/pebr/ellen_tree.rs b/src/ds_impl/pebr/ellen_tree.rs index 24669a91..9471b5ed 100644 --- a/src/ds_impl/pebr/ellen_tree.rs +++ b/src/ds_impl/pebr/ellen_tree.rs @@ -3,7 +3,7 @@ use std::sync::atomic::Ordering; use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -808,7 +808,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { match self.find(key, handle, guard) { Some(node) => Some(node), None => None, @@ -821,7 +821,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.delete(key, handle, guard) } } diff --git a/src/ds_impl/pebr/list.rs b/src/ds_impl/pebr/list.rs index 5d2c8967..7c80d06f 100644 --- a/src/ds_impl/pebr/list.rs +++ b/src/ds_impl/pebr/list.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; use std::cmp::Ordering::{Equal, Greater, Less}; @@ -549,9 +549,9 @@ where fn get<'g>( &'g self, handle: &'g mut Self::Handle, - key: &K, + key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.inner.harris_get(key, handle, guard) } #[inline(always)] @@ -559,7 +559,12 @@ where self.inner.harris_insert(key, value, handle, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.inner.harris_remove(key, handle, guard) } } @@ -606,9 +611,9 @@ where fn get<'g>( &'g self, handle: &'g mut Self::Handle, - key: &K, + key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.inner.harris_michael_get(key, handle, guard) } #[inline(always)] @@ -616,7 +621,12 @@ where self.inner.harris_michael_insert(key, value, handle, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.inner.harris_michael_remove(key, handle, guard) } } @@ -658,9 +668,9 @@ where fn get<'g>( &'g self, handle: &'g mut Self::Handle, - key: &K, + key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.inner.harris_herlihy_shavit_get(key, handle, guard) } #[inline(always)] @@ -668,7 +678,12 @@ where self.inner.harris_insert(key, value, handle, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.inner.harris_remove(key, handle, guard) } } diff --git a/src/ds_impl/pebr/michael_hash_map.rs b/src/ds_impl/pebr/michael_hash_map.rs index 66ef494f..3f6ade98 100644 --- a/src/ds_impl/pebr/michael_hash_map.rs +++ b/src/ds_impl/pebr/michael_hash_map.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_pebr::Guard; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -35,26 +35,6 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get<'g>( - &'g self, - cursor: &'g mut Cursor, - k: &'g K, - guard: &'g mut Guard, - ) -> Option<&'g V> { - let i = Self::hash(k); - self.get_bucket(i).get(cursor, k, guard) - } - - pub fn insert(&self, cursor: &mut Cursor, k: K, v: V, guard: &mut Guard) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(cursor, k, v, guard) - } - - pub fn remove(&self, cursor: &mut Cursor, k: &K, guard: &mut Guard) -> Option { - let i = Self::hash(&k); - self.get_bucket(i).remove(cursor, k, guard) - } } impl ConcurrentMap for HashMap @@ -81,16 +61,24 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { - self.get(handle, key, guard) + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(handle, key, guard) } #[inline(always)] fn insert(&self, handle: &mut Self::Handle, key: K, value: V, guard: &mut Guard) -> bool { - self.insert(handle, key, value, guard) + let i = Self::hash(&key); + self.get_bucket(i).insert(handle, key, value, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { - self.remove(handle, key, guard) + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { + let i = Self::hash(&key); + self.get_bucket(i).remove(handle, key, guard) } } diff --git a/src/ds_impl/pebr/natarajan_mittal_tree.rs b/src/ds_impl/pebr/natarajan_mittal_tree.rs index d4a5199d..c0fe97a9 100644 --- a/src/ds_impl/pebr/natarajan_mittal_tree.rs +++ b/src/ds_impl/pebr/natarajan_mittal_tree.rs @@ -1,6 +1,6 @@ use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::mem; use std::sync::atomic::Ordering; @@ -601,7 +601,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.get(key, handle, guard) } @@ -611,7 +611,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.remove(key, handle, guard) } } diff --git a/src/ds_impl/pebr/skip_list.rs b/src/ds_impl/pebr/skip_list.rs index 6f233263..d3e11d67 100644 --- a/src/ds_impl/pebr/skip_list.rs +++ b/src/ds_impl/pebr/skip_list.rs @@ -5,7 +5,7 @@ use std::{ use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -474,7 +474,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { let cursor = self.find_optimistic(key, handle, guard)?; let node = unsafe { cursor.found?.deref() }; if node.key.eq(&key) { @@ -490,7 +490,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.remove(key, handle, guard) } } From 0443df5f05cc60f5085bd5b964a440a0d8fbcf56 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 7 Nov 2024 11:45:56 +0000 Subject: [PATCH 68/84] Make `V` customizable in `smoke` of PEBR --- src/ds_impl/pebr/bonsai_tree.rs | 2 +- src/ds_impl/pebr/concurrent_map.rs | 14 ++++++++++---- src/ds_impl/pebr/ellen_tree.rs | 2 +- src/ds_impl/pebr/list.rs | 6 +++--- src/ds_impl/pebr/michael_hash_map.rs | 2 +- src/ds_impl/pebr/natarajan_mittal_tree.rs | 2 +- src/ds_impl/pebr/skip_list.rs | 2 +- 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/ds_impl/pebr/bonsai_tree.rs b/src/ds_impl/pebr/bonsai_tree.rs index 5279a316..c45148ff 100644 --- a/src/ds_impl/pebr/bonsai_tree.rs +++ b/src/ds_impl/pebr/bonsai_tree.rs @@ -810,6 +810,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/concurrent_map.rs b/src/ds_impl/pebr/concurrent_map.rs index 2957ff5e..388ee3b6 100644 --- a/src/ds_impl/pebr/concurrent_map.rs +++ b/src/ds_impl/pebr/concurrent_map.rs @@ -45,11 +45,17 @@ pub mod tests { use crossbeam_pebr::pin; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -61,7 +67,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(&mut handle, i, i.to_string(), &mut pin())); + assert!(map.insert(&mut handle, i, to_value(&i), &mut pin())); } }); } @@ -78,7 +84,7 @@ pub mod tests { keys.shuffle(&mut rng); for i in keys { assert_eq!( - i.to_string(), + to_value(&i), *map.remove(&mut handle, &i, &mut pin()).unwrap().output() ); } @@ -97,7 +103,7 @@ pub mod tests { keys.shuffle(&mut rng); for i in keys { assert_eq!( - i.to_string(), + to_value(&i), *map.get(&mut handle, &i, &mut pin()).unwrap().output() ); } diff --git a/src/ds_impl/pebr/ellen_tree.rs b/src/ds_impl/pebr/ellen_tree.rs index 9471b5ed..358427a0 100644 --- a/src/ds_impl/pebr/ellen_tree.rs +++ b/src/ds_impl/pebr/ellen_tree.rs @@ -838,6 +838,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/list.rs b/src/ds_impl/pebr/list.rs index 7c80d06f..ee366e4e 100644 --- a/src/ds_impl/pebr/list.rs +++ b/src/ds_impl/pebr/list.rs @@ -695,17 +695,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/pebr/michael_hash_map.rs b/src/ds_impl/pebr/michael_hash_map.rs index 3f6ade98..40dcde06 100644 --- a/src/ds_impl/pebr/michael_hash_map.rs +++ b/src/ds_impl/pebr/michael_hash_map.rs @@ -89,6 +89,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/natarajan_mittal_tree.rs b/src/ds_impl/pebr/natarajan_mittal_tree.rs index c0fe97a9..c7982ad7 100644 --- a/src/ds_impl/pebr/natarajan_mittal_tree.rs +++ b/src/ds_impl/pebr/natarajan_mittal_tree.rs @@ -628,6 +628,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/skip_list.rs b/src/ds_impl/pebr/skip_list.rs index d3e11d67..ccabf7b4 100644 --- a/src/ds_impl/pebr/skip_list.rs +++ b/src/ds_impl/pebr/skip_list.rs @@ -507,6 +507,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } From e99c109bdcfd5debade7f17ff5f3ecb56423b4a1 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 8 Nov 2024 09:45:54 +0000 Subject: [PATCH 69/84] Implement PEBR ElimAbTree --- src/bin/pebr.rs | 22 +- src/ds_impl/pebr/elim_ab_tree.rs | 1446 ++++++++++++++++++++++++++++++ src/ds_impl/pebr/mod.rs | 2 + 3 files changed, 1459 insertions(+), 11 deletions(-) create mode 100644 src/ds_impl/pebr/elim_ab_tree.rs diff --git a/src/bin/pebr.rs b/src/bin/pebr.rs index 4097699e..9a4a1e20 100644 --- a/src/bin/pebr.rs +++ b/src/bin/pebr.rs @@ -12,7 +12,8 @@ use typenum::{Unsigned, U1, U4}; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, OpsPerCs, Perf, DS}; use smr_benchmark::ds_impl::pebr::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -38,15 +39,12 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::, N>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::, N>(config, PrefillStrategy::Random), DS::BonsaiTree => { - // Note: Using the `Random` strategy with the Bonsai tree is unsafe - // because it involves multiple threads with unprotected guards. - // It is safe for many other data structures that don't retire elements - // during insertion, but this is not the case for the Bonsai tree. + // For Bonsai Tree, it would be faster to use a single thread to prefill. bench_map::, N>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::, N>(config, PrefillStrategy::Random), DS::SkipList => bench_map::, N>(config, PrefillStrategy::Decreasing), - _ => panic!("Unsupported(or unimplemented) data structure for PEBR"), + DS::ElimAbTree => bench_map::, N>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); @@ -62,6 +60,9 @@ pub enum PrefillStrategy { impl PrefillStrategy { fn prefill + Send + Sync>(self, config: &Config, map: &M) { + // Some data structures (e.g., Bonsai tree, Elim AB-Tree) need SMR's retirement + // functionality even during insertions. + let collector = &crossbeam_pebr::Collector::new(); match self { PrefillStrategy::Random => { let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); @@ -70,10 +71,8 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { - // Safety: We assume that the insert operation does not retire - // any elements. Note that this assumption may not hold for all - // data structures (e.g., Bonsai tree). - let guard = unsafe { crossbeam_pebr::unprotected() }; + let handle = collector.register(); + let guard = &mut handle.pin(); let mut handle = M::handle(guard); let rng = &mut rand::thread_rng(); let count = config.prefill / threads @@ -89,7 +88,8 @@ impl PrefillStrategy { .unwrap(); } PrefillStrategy::Decreasing => { - let guard = unsafe { crossbeam_pebr::unprotected() }; + let handle = collector.register(); + let guard = &mut handle.pin(); let mut handle = M::handle(guard); let rng = &mut rand::thread_rng(); let mut keys = Vec::with_capacity(config.prefill); diff --git a/src/ds_impl/pebr/elim_ab_tree.rs b/src/ds_impl/pebr/elim_ab_tree.rs new file mode 100644 index 00000000..a8abcecf --- /dev/null +++ b/src/ds_impl/pebr/elim_ab_tree.rs @@ -0,0 +1,1446 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use arrayvec::ArrayVec; +use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::mem::swap; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next<'g>(&self, index: usize, guard: &'g Guard) -> Shared<'g, Self> { + self.next()[index].load(Ordering::Acquire, guard) + } + + fn protect_next<'g>( + &self, + index: usize, + slot: &mut Shield, + guard: &'g Guard, + ) -> Result, ShieldError> { + let ptr = self.next()[index].load(Ordering::Acquire, guard); + slot.defend(ptr, guard).map(|_| ptr) + } + + fn store_next<'g>(&'g self, index: usize, ptr: impl Pointer, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: impl Pointer) { + self.next_mut()[index] = Atomic::from(ptr.into_usize() as *const Self); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } + + fn p_s_idx(p_l_idx: usize) -> usize { + if p_l_idx > 0 { + p_l_idx - 1 + } else { + 1 + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + guard: &'g Guard, + ) -> impl Iterator, Shared<'g, Self>)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i, guard))) + .chain(once((None, self.load_next(self.key_count(), guard)))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +pub struct Cursor { + l: Shield>, + p: Shield>, + gp: Shield>, + /// A protector for the sibling node. + s: Shield>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + p_s_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +impl Cursor { + fn new(guard: &Guard) -> Self { + Self { + l: Shield::null(guard), + p: Shield::null(guard), + gp: Shield::null(guard), + s: Shield::null(guard), + gp_p_idx: 0, + p_l_idx: 0, + p_s_idx: 0, + l_key_idx: 0, + val: None, + } + } + + fn release(&mut self) { + self.l.release(); + self.p.release(); + self.gp.release(); + self.s.release(); + } +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Owned::new(left)); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, cursor: &mut Cursor, guard: &mut Guard) -> Option { + loop { + match self.search_basic_inner(key, cursor, guard) { + Ok(found) => return found, + Err(_) => guard.repin(), + } + } + } + + fn search_basic_inner( + &self, + key: &K, + cursor: &mut Cursor, + guard: &Guard, + ) -> Result, ShieldError> { + let mut node = unsafe { self.entry.protect_next(0, &mut cursor.p, guard)?.deref() }; + while !node.is_leaf() { + let next = node.protect_next(node.child_index(key), &mut cursor.l, guard)?; + swap(&mut cursor.l, &mut cursor.p); + node = unsafe { next.deref() }; + } + Ok(node.read_consistent(key).1) + } + + fn search<'g>( + &self, + key: &K, + target: Option>>, + cursor: &mut Cursor, + guard: &'g mut Guard, + ) -> bool { + loop { + match self.search_inner(key, target, cursor, guard) { + Ok(found) => return found, + Err(_) => guard.repin(), + } + } + } + + fn search_inner<'g>( + &self, + key: &K, + target: Option>>, + cursor: &mut Cursor, + guard: &'g Guard, + ) -> Result { + self.entry.protect_next(0, &mut cursor.l, guard)?; + self.entry.protect_next(1, &mut cursor.s, guard)?; + unsafe { + cursor + .p + .defend_fake(Shared::from_usize(&self.entry as *const _ as usize)) + }; + cursor.gp.release(); + cursor.gp_p_idx = 0; + cursor.p_l_idx = 0; + cursor.p_s_idx = 1; + cursor.l_key_idx = 0; + cursor.val = None; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target + .map(|target| target != cursor.l.shared()) + .unwrap_or(true) + { + swap(&mut cursor.gp, &mut cursor.p); + swap(&mut cursor.p, &mut cursor.l); + let l_node = unsafe { cursor.p.deref() }; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.p_s_idx = Node::::p_s_idx(cursor.p_l_idx); + l_node.protect_next(cursor.p_l_idx, &mut cursor.l, guard)?; + l_node.protect_next(cursor.p_s_idx, &mut cursor.s, guard)?; + } + + if let Some(target) = target { + Ok(cursor.l.shared() == target) + } else { + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); + cursor.val = value; + cursor.l_key_idx = index; + Ok(value.is_some()) + } + } + + pub fn insert( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> Option { + loop { + self.search(key, None, cursor, guard); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, cursor, guard) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Owned::new(left)); + internal.init_next(1, Owned::new(right)); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Owned::new(internal).into_shared(guard); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { guard.defer_destroy(cursor.l.shared()) }; + self.fix_tag_violation( + kv_pairs[left_size].0, + new_internal.into_usize(), + cursor, + guard, + ); + + Ok(None) + } + } + + fn fix_tag_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + guard: &mut Guard, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + let found = self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + guard, + ); + if !found || cursor.l.shared().into_usize() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_tag_violation_inner(cursor, guard); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_tag_violation_inner<'g>( + &self, + cursor: &mut Cursor, + guard: &'g Guard, + ) -> (bool, Option<(K, usize)>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { viol.deref() }; + if viol_node.weight { + return (true, None); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0, guard) != viol); + + debug_assert!(!cursor.gp.shared().is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + return ( + false, + Some((parent.search_key, cursor.p.shared().into_usize())), + ); + } + + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, None) + ); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next(cursor.gp_p_idx, Owned::new(absorber), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l.shared()) }; + unsafe { guard.defer_destroy(cursor.p.shared()) }; + return (true, None); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Owned::new(left)); + new_internal.init_next(1, Owned::new(right)); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Owned::new(new_internal).into_shared(guard); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l.shared()) }; + unsafe { guard.defer_destroy(cursor.p.shared()) }; + + drop((node_lock, parent_lock, gparent_lock)); + return ( + true, + Some(( + keys[left_size - 1].get().unwrap(), + new_internal.into_usize(), + )), + ); + } + } + + pub fn remove(&self, key: &K, cursor: &mut Cursor, guard: &mut Guard) -> Option { + loop { + self.search(key, None, cursor, guard); + if cursor.val.is_none() { + return None; + } + if let Ok(result) = self.remove_inner(key, cursor, guard) { + cursor.val = result; + return result; + } + } + } + + fn remove_inner( + &self, + key: &K, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation( + node.search_key, + cursor.l.shared().into_usize(), + cursor, + guard, + ); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + guard: &mut Guard, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + guard, + ); + if cursor.l.shared().into_usize() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_underfull_violation_inner(cursor, guard); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_underfull_violation_inner( + &self, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> (bool, ArrayVec<(K, usize), 2>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { viol.deref() }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0, guard) + { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.shared().is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p.shared() != self.entry.load_next(0, guard) + { + return ( + false, + ArrayVec::from_iter(once((parent.search_key, cursor.p.shared().into_usize()))), + ); + } + + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling = unsafe { cursor.s.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if cursor.p_s_idx < cursor.p_l_idx { + ((sibling, cursor.p_s_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, cursor.p_s_idx)) + }; + + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation( + parent.search_key, + cursor.p.shared().into_usize(), + cursor, + guard, + ); + return (false, ArrayVec::new()); + } + if !node.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation( + node.search_key, + cursor.l.shared().into_usize(), + cursor, + guard, + ); + return (false, ArrayVec::new()); + } + if !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation( + sibling.search_key, + cursor.s.shared().into_usize(), + cursor, + guard, + ); + return (false, ArrayVec::new()); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + } + .into_shared(guard); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l.shared()); + guard.defer_destroy(cursor.p.shared()); + guard.defer_destroy(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter(once((node.search_key, new_node.into_usize()))), + ); + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..cursor.p_s_idx { + new_parent.init_next(i, parent.load_next(i, guard)); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next(i, guard)); + } + + new_parent.init_next( + cursor.p_l_idx + - (if cursor.p_l_idx > cursor.p_s_idx { + 1 + } else { + 0 + }), + new_node, + ); + let new_parent = Owned::new(new_parent).into_shared(guard); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l.shared()); + guard.defer_destroy(cursor.p.shared()); + guard.defer_destroy(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter( + [ + (node.search_key, new_node.into_usize()), + (parent.search_key, new_parent.into_usize()), + ] + .into_iter(), + ), + ); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Owned::new(Node::leaf(true, left_size, Default::default())); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Owned::new(Node::leaf(true, right_size, Default::default())); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)); + + let (new_left, pivot) = { + let mut new_internal = + Owned::new(Node::internal(true, left_size, Default::default())); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = + Owned::new(Node::internal(true, right_size, Default::default())); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = + Owned::new(Node::internal(parent.weight, psize, parent.search_key)); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, new_left); + new_parent.init_next(right_idx, new_right); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l.shared()); + guard.defer_destroy(cursor.p.shared()); + guard.defer_destroy(cursor.s.shared()); + } + + return (true, ArrayVec::new()); + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let mut stack = vec![]; + let guard = unsafe { unprotected() }; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + type Handle = Cursor; + + fn new() -> Self { + Self::new() + } + + fn handle<'g>(guard: &'g Guard) -> Self::Handle { + Cursor::new(guard) + } + + fn clear(handle: &mut Self::Handle) { + handle.release(); + } + + fn get<'g>( + &'g self, + handle: &'g mut Self::Handle, + key: &'g K, + guard: &'g mut Guard, + ) -> Option> { + self.search_basic(key, handle, guard) + } + + fn insert(&self, handle: &mut Self::Handle, key: K, value: V, guard: &mut Guard) -> bool { + self.insert(&key, &value, handle, guard).is_none() + } + + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { + self.remove(key, handle, guard) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::pebr::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/pebr/mod.rs b/src/ds_impl/pebr/mod.rs index 33b22bef..b12f32a2 100644 --- a/src/ds_impl/pebr/mod.rs +++ b/src/ds_impl/pebr/mod.rs @@ -3,6 +3,7 @@ pub mod shield_pool; pub mod concurrent_map; pub mod bonsai_tree; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; @@ -12,6 +13,7 @@ pub mod skip_list; pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; From 489f649fb3d754fc837031a77725a051358efdf9 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 8 Nov 2024 10:24:08 +0000 Subject: [PATCH 70/84] Initialize VBR objects by `Default` --- smrs/vbr/src/lib.rs | 39 +++++++++------ src/ds_impl/vbr/list.rs | 62 +++++++++++++++--------- src/ds_impl/vbr/michael_hash_map.rs | 12 ++--- src/ds_impl/vbr/mod.rs | 1 + src/ds_impl/vbr/natarajan_mittal_tree.rs | 43 ++++++++++------ src/ds_impl/vbr/skip_list.rs | 44 +++++++++++------ 6 files changed, 127 insertions(+), 74 deletions(-) diff --git a/smrs/vbr/src/lib.rs b/smrs/vbr/src/lib.rs index 5fad7de9..d2aeb8d0 100644 --- a/smrs/vbr/src/lib.rs +++ b/smrs/vbr/src/lib.rs @@ -1,11 +1,6 @@ use std::{ - cell::RefCell, - collections::VecDeque, - fmt::Display, - marker::PhantomData, - mem::{align_of, zeroed}, - ptr::null_mut, - sync::atomic::AtomicU64, + cell::RefCell, collections::VecDeque, fmt::Display, marker::PhantomData, mem::align_of, + ptr::null_mut, sync::atomic::AtomicU64, }; use atomic::{Atomic, Ordering}; @@ -31,6 +26,7 @@ pub fn bag_capacity() -> usize { ENTRIES_PER_BAG.load(Ordering::Relaxed) } +#[derive(Default)] pub struct Inner { birth: AtomicU64, retire: AtomicU64, @@ -45,7 +41,7 @@ pub struct Global { unsafe impl Sync for Global {} unsafe impl Send for Global {} -impl Global { +impl Global { pub fn new(capacity: usize) -> Self { let avail = BagStack::new(); let count = capacity / bag_capacity() + if capacity % bag_capacity() > 0 { 1 } else { 0 }; @@ -170,7 +166,7 @@ pub struct Bag { entries: Vec<*mut T>, } -impl Bag { +impl Bag { fn new() -> Self { Self { next: AtomicU128::new(0), @@ -181,7 +177,7 @@ impl Bag { fn new_with_alloc() -> Self { let mut entries = vec![null_mut(); bag_capacity()]; for ptr in &mut entries { - *ptr = unsafe { Box::into_raw(Box::new(zeroed())) }; + *ptr = Box::into_raw(Box::new(T::default())); } Self { next: AtomicU128::new(0), @@ -208,7 +204,7 @@ pub struct Local { retired: RefCell>>>, } -impl Local { +impl Local { fn global(&self) -> &Global { unsafe { &*self.global } } @@ -290,7 +286,7 @@ pub struct Guard { epoch: u64, } -impl Guard { +impl Guard { fn global(&self) -> &Global { unsafe { &*self.local().global } } @@ -310,6 +306,8 @@ impl Guard { let ptr = self.local().pop_avail(); debug_assert!(!ptr.is_null()); let slot_ref = unsafe { &*ptr }; + // If the retire epoch is (greater than or) equal to the current epoch, + // try advance the global epoch. if self.epoch <= slot_ref.retire.load(Ordering::SeqCst) { self.local().return_avail(ptr); let _ = self.global().advance(self.epoch); @@ -437,10 +435,19 @@ pub struct MutAtomic { _marker: PhantomData, } +impl Default for MutAtomic { + fn default() -> Self { + Self { + link: AtomicU128::new(0), + _marker: PhantomData, + } + } +} + unsafe impl Sync for MutAtomic {} unsafe impl Send for MutAtomic {} -impl MutAtomic { +impl MutAtomic { pub fn null() -> Self { Self { link: AtomicU128::new(0), @@ -524,7 +531,7 @@ pub struct Entry { unsafe impl Sync for Entry {} unsafe impl Send for Entry {} -impl Entry { +impl Entry { pub fn new(init: Shared) -> Self { Self { link: init.as_raw(), @@ -559,14 +566,14 @@ pub struct ImmAtomic { unsafe impl Sync for ImmAtomic {} unsafe impl Send for ImmAtomic {} -impl ImmAtomic { +impl ImmAtomic { pub fn new(v: T) -> Self { Self { data: Atomic::new(v), } } - pub fn get(&self, guard: &Guard) -> Result { + pub fn get(&self, guard: &Guard) -> Result { let value = unsafe { self.get_unchecked() }; compiler_fence(Ordering::SeqCst); guard.validate_epoch()?; diff --git a/src/ds_impl/vbr/list.rs b/src/ds_impl/vbr/list.rs index d7447d68..199a8864 100644 --- a/src/ds_impl/vbr/list.rs +++ b/src/ds_impl/vbr/list.rs @@ -7,8 +7,8 @@ use std::sync::atomic::Ordering; pub struct Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { /// Mark: tag(), Tag: not needed next: MutAtomic>, @@ -16,18 +16,32 @@ where value: ImmAtomic, } +impl Default for Node +where + K: 'static + Copy + Default + Default, + V: 'static + Copy + Default + Default, +{ + fn default() -> Self { + Self { + next: MutAtomic::null(), + key: ImmAtomic::new(Default::default()), + value: ImmAtomic::new(Default::default()), + } + } +} + struct List where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { head: Entry>, } struct Cursor<'g, K, V> where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { prev: Shared<'g, Node>, // Tag of `curr` should always be zero so when `curr` is stored in a `prev`, we don't store a @@ -37,8 +51,8 @@ where impl<'g, K, V> Cursor<'g, K, V> where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { /// Creates the head cursor. #[inline] @@ -57,8 +71,8 @@ where impl List where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { /// Creates a new list. #[inline] @@ -417,16 +431,16 @@ where pub struct HList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { inner: List, } impl ConcurrentMap for HList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; @@ -461,16 +475,16 @@ where pub struct HMList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { inner: List, } impl ConcurrentMap for HMList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; @@ -505,16 +519,16 @@ where pub struct HHSList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { inner: List, } impl HHSList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { /// Pop the first element efficiently. /// This method is used for only the fine grained benchmark (src/bin/long_running). @@ -525,8 +539,8 @@ where impl ConcurrentMap for HHSList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; diff --git a/src/ds_impl/vbr/michael_hash_map.rs b/src/ds_impl/vbr/michael_hash_map.rs index b984d860..e3918b21 100644 --- a/src/ds_impl/vbr/michael_hash_map.rs +++ b/src/ds_impl/vbr/michael_hash_map.rs @@ -8,16 +8,16 @@ use super::list::{HHSList, Node}; pub struct HashMap where - K: 'static + Ord + Hash + Copy, - V: 'static + Copy, + K: 'static + Ord + Hash + Copy + Default, + V: 'static + Copy + Default, { buckets: Vec>, } impl HashMap where - K: 'static + Ord + Hash + Copy, - V: 'static + Copy, + K: 'static + Ord + Hash + Copy + Default, + V: 'static + Copy + Default, { pub fn with_capacity(n: usize, local: &Local>) -> Self { let mut buckets = Vec::with_capacity(n); @@ -59,8 +59,8 @@ where impl ConcurrentMap for HashMap where - K: 'static + Ord + Hash + Copy, - V: 'static + Copy, + K: 'static + Ord + Hash + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; diff --git a/src/ds_impl/vbr/mod.rs b/src/ds_impl/vbr/mod.rs index 37f66d63..9d5088f0 100644 --- a/src/ds_impl/vbr/mod.rs +++ b/src/ds_impl/vbr/mod.rs @@ -4,6 +4,7 @@ pub mod list; pub mod michael_hash_map; pub mod natarajan_mittal_tree; pub mod skip_list; +// pub mod elim_ab_tree; pub use self::concurrent_map::ConcurrentMap; diff --git a/src/ds_impl/vbr/natarajan_mittal_tree.rs b/src/ds_impl/vbr/natarajan_mittal_tree.rs index 863d6db8..03774415 100644 --- a/src/ds_impl/vbr/natarajan_mittal_tree.rs +++ b/src/ds_impl/vbr/natarajan_mittal_tree.rs @@ -34,8 +34,8 @@ impl Marks { pub struct Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { key: ImmAtomic, value: ImmAtomic, @@ -43,10 +43,25 @@ where right: MutAtomic>, } +impl Default for Node +where + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, +{ + fn default() -> Self { + Self { + key: ImmAtomic::new(Default::default()), + value: ImmAtomic::new(Default::default()), + left: MutAtomic::null(), + right: MutAtomic::null(), + } + } +} + impl Node where - K: 'static + Copy + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, { fn new_leaf<'g>( key: K, @@ -90,8 +105,8 @@ enum Direction { /// All of the edges of path from `successor` to `parent` are in the process of removal. struct SeekRecord<'g, K, V> where - K: 'static + Copy + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, { /// Parent of `successor` ancestor: Shared<'g, Node>, @@ -109,8 +124,8 @@ where impl<'g, K, V> SeekRecord<'g, K, V> where - K: 'static + Copy + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, { fn successor_addr(&self) -> &MutAtomic> { match self.successor_dir { @@ -136,16 +151,16 @@ where pub struct NMTreeMap where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { r: Entry>, } impl NMTreeMap where - K: 'static + Copy + Ord + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Ord + Bounded, + V: 'static + Copy + Default, { pub fn new(local: &Local>) -> Self { // An empty tree has 5 default nodes with infinite keys so that the SeekRecord is allways @@ -498,8 +513,8 @@ where impl ConcurrentMap for NMTreeMap where - K: 'static + Copy + Ord + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Ord + Bounded, + V: 'static + Copy + Default, { type Global = Global>; diff --git a/src/ds_impl/vbr/skip_list.rs b/src/ds_impl/vbr/skip_list.rs index 32a1ac34..cfca91a1 100644 --- a/src/ds_impl/vbr/skip_list.rs +++ b/src/ds_impl/vbr/skip_list.rs @@ -14,8 +14,8 @@ type Tower = [MutAtomic>; MAX_HEIGHT]; pub struct Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { key: ImmAtomic, value: ImmAtomic, @@ -24,6 +24,22 @@ where refs: AtomicUsize, } +impl Default for Node +where + K: 'static + Copy + Default, + V: 'static + Copy + Default, +{ + fn default() -> Self { + Self { + key: ImmAtomic::new(Default::default()), + value: ImmAtomic::new(Default::default()), + next: unsafe { zeroed() }, + height: ImmAtomic::new(0), + refs: AtomicUsize::new(0), + } + } +} + fn generate_height() -> usize { // returns 1 with probability 3/4 if rand::random::() % 4 < 3 { @@ -39,8 +55,8 @@ fn generate_height() -> usize { impl Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { pub fn decrement(ptr: Shared>, guard: &Guard>) { let prev = unsafe { ptr.deref() }.refs.fetch_sub(1, Ordering::SeqCst); @@ -92,8 +108,8 @@ where struct Cursor<'g, K, V> where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { found: Option>>, preds: [Shared<'g, Node>; MAX_HEIGHT], @@ -102,8 +118,8 @@ where impl<'g, K, V> Cursor<'g, K, V> where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { fn new(head: Shared<'g, Node>) -> Self { Self { @@ -116,16 +132,16 @@ where pub struct SkipList where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { head: Entry>, } impl SkipList where - K: 'static + Copy + Ord, - V: 'static + Copy, + K: 'static + Copy + Default + Ord, + V: 'static + Copy + Default, { pub fn new(local: &Local>) -> Self { let guard = &local.guard(); @@ -392,8 +408,8 @@ where impl ConcurrentMap for SkipList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; From 7a5a121946969f5a24b682818a5c347ab521cca2 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Sun, 10 Nov 2024 13:36:54 +0000 Subject: [PATCH 71/84] [WIP] Implementing VBR ElimAbTree (currently buggy) --- smrs/vbr/src/lib.rs | 39 +- src/bin/vbr.rs | 3 +- src/ds_impl/vbr/elim_ab_tree.rs | 1546 +++++++++++++++++++++++++++++++ src/ds_impl/vbr/mod.rs | 3 +- 4 files changed, 1583 insertions(+), 8 deletions(-) create mode 100644 src/ds_impl/vbr/elim_ab_tree.rs diff --git a/smrs/vbr/src/lib.rs b/smrs/vbr/src/lib.rs index d2aeb8d0..ff5b787b 100644 --- a/smrs/vbr/src/lib.rs +++ b/smrs/vbr/src/lib.rs @@ -1,6 +1,6 @@ use std::{ cell::RefCell, collections::VecDeque, fmt::Display, marker::PhantomData, mem::align_of, - ptr::null_mut, sync::atomic::AtomicU64, + ops::Deref, ptr::null_mut, sync::atomic::AtomicU64, }; use atomic::{Atomic, Ordering}; @@ -33,6 +33,14 @@ pub struct Inner { data: T, } +impl Deref for Inner { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + pub struct Global { epoch: CachePadded, avail: BagStack>, @@ -299,9 +307,9 @@ impl Guard { self.epoch = self.global().epoch(); } - pub fn allocate<'g, F>(&'g self, init: F) -> Result, ()> + pub fn allocate<'g, F>(&'g self, mut init: F) -> Result, ()> where - F: Fn(Shared<'g, T>), + F: FnMut(Shared<'g, T>), { let ptr = self.local().pop_avail(); debug_assert!(!ptr.is_null()); @@ -327,12 +335,27 @@ impl Guard { Ok(result) } + /// # Safety + /// + /// The current must conceptually have exclusive permission to retire this pointer + /// (e.g., after successful physical deletion in a lock-free data structure). pub unsafe fn retire(&self, ptr: Shared) { let inner = ptr.as_inner().expect("Attempted to retire a null pointer."); + if inner.birth.load(Ordering::SeqCst) > ptr.birth { + // It is already retired and reclaimed. + return; + } + self.retire_raw(inner as *const _ as *mut _); + } - if inner.birth.load(Ordering::SeqCst) > ptr.birth - || inner.retire.load(Ordering::SeqCst) != NOT_RETIRED - { + /// # Safety + /// + /// The pointee must not be relcaimed yet, and the current must conceptually have exclusive + /// permission to retire this pointer (e.g., after successful physical deletion in a lock-free + /// data structure). + pub unsafe fn retire_raw(&self, ptr: *mut Inner) { + let inner = unsafe { &*ptr }; + if inner.retire.load(Ordering::SeqCst) != NOT_RETIRED { return; } @@ -557,6 +580,10 @@ impl Entry { }) } } + + pub fn load_raw(&self) -> *mut Inner { + self.link + } } pub struct ImmAtomic { diff --git a/src/bin/vbr.rs b/src/bin/vbr.rs index 6b2a1b1a..4aea8c1d 100644 --- a/src/bin/vbr.rs +++ b/src/bin/vbr.rs @@ -9,7 +9,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::vbr::{ - ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + ConcurrentMap, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -32,6 +32,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), _ => panic!("Unsupported(or unimplemented) data structure for VBR"), }; output.write_record(config, &perf); diff --git a/src/ds_impl/vbr/elim_ab_tree.rs b/src/ds_impl/vbr/elim_ab_tree.rs new file mode 100644 index 00000000..e05f112d --- /dev/null +++ b/src/ds_impl/vbr/elim_ab_tree.rs @@ -0,0 +1,1546 @@ +use super::concurrent_map::ConcurrentMap; +use arrayvec::ArrayVec; +use vbr::{Entry, Global, Guard, Inner, Local}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $guard:ident, $acq_val_err:expr, $epoch_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + if $guard.validate_epoch().is_err() { + $epoch_val_err; + } + }; +} + +struct MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +#[derive(Default)] +pub struct Node +where + K: Copy + Default, + V: Copy + Default, +{ + _keys: [Cell>; DEGREE], + search_key: Cell, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: Cell, + marked: AtomicBool, + // Leaf node data + values: [Cell>; DEGREE], + write_version: AtomicUsize, + // Internal node data + next: [AtomicPtr>>; DEGREE], + is_leaf: Cell, +} + +impl Node +where + K: Default + Copy, + V: Default + Copy, +{ + fn weight(&self, guard: &Guard) -> Result { + let result = self.weight.get(); + guard.validate_epoch().map(|_| result) + } + + fn search_key(&self, guard: &Guard) -> Result { + let result = self.search_key.get(); + guard.validate_epoch().map(|_| result) + } + + fn is_leaf(&self, guard: &Guard) -> Result { + let result = self.is_leaf.get(); + guard.validate_epoch().map(|_| result) + } + + fn load_next(&self, index: usize, guard: &Guard) -> Result<*mut Inner, ()> { + let result = self.next[index].load(Ordering::Acquire); + guard.validate_epoch().map(|_| result) + } + + fn load_next_locked<'l>( + &'l self, + index: usize, + _: &MCSLockGuard<'l, K, V>, + ) -> *mut Inner { + self.next[index].load(Ordering::Acquire) + } + + fn store_next<'l>(&self, index: usize, ptr: *mut Inner, _: &MCSLockGuard<'l, K, V>) { + self.next[index].store(ptr, Ordering::Release); + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_next(&self, index: usize, ptr: *mut Inner) { + self.next[index].store(ptr, Ordering::Release); + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let init_version = self.write_version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + // It is safe to skip the epoch validation, as we are grabbing the lock. + self.write_version + .store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self, guard: &Guard) -> Result { + let result = if self.is_leaf.get() { + self.size.load(Ordering::Acquire) + } else { + self.size.load(Ordering::Acquire) - 1 + }; + guard.validate_epoch().map(|_| result) + } + + fn key_count_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> usize { + if self.is_leaf.get() { + self.size.load(Ordering::Acquire) + } else { + self.size.load(Ordering::Acquire) - 1 + } + } + + fn get_value(&self, index: usize, guard: &Guard) -> Result, ()> { + let value = self.values[index].get(); + guard.validate_epoch().map(|_| value) + } + + fn get_value_locked<'l>(&'l self, index: usize, _: &MCSLockGuard<'l, K, V>) -> Option { + self.values[index].get() + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_value(&self, index: usize, val: V) { + self.values[index].set(Some(val)); + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + self.values[index].set(Some(val)); + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn get_key_unchecked(&self, index: usize) -> Option { + self._keys[index].get() + } + + fn get_key(&self, index: usize, guard: &Guard) -> Result, ()> { + let result = self._keys[index].get(); + guard.validate_epoch().map(|_| result) + } + + fn get_key_locked<'l>(&self, index: usize, _: &MCSLockGuard<'l, K, V>) -> Option { + self._keys[index].get() + } + + fn key_slice<'l>(&self, _: &MCSLockGuard<'l, K, V>) -> &[Cell>] { + &self._keys + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn key_slice_unchecked<'l>(&self) -> &[Cell>] { + &self._keys + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_key(&self, index: usize, key: Option) { + self._keys[index].set(key); + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self._keys[index].set(key); + } + + unsafe fn init_on_allocate(&self, weight: bool, size: usize, search_key: K) { + for key in &self._keys { + key.set(Default::default()); + } + self.search_key.set(search_key); + self.size.store(size, Ordering::Release); + self.weight.set(weight); + self.marked.store(false, Ordering::Release); + for next in &self.next { + next.store(null_mut(), Ordering::Release); + } + for value in &self.values { + value.set(Default::default()); + } + self.write_version.store(0, Ordering::Release); + } + + unsafe fn init_for_internal(&self, weight: bool, size: usize, search_key: K) { + self.init_on_allocate(weight, size, search_key); + self.is_leaf.set(false); + } + + unsafe fn init_for_leaf(&self, weight: bool, size: usize, search_key: K) { + self.init_on_allocate(weight, size, search_key); + self.is_leaf.set(true); + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn child_index(&self, key: &K, guard: &Guard) -> Result { + let key_count = self.key_count(guard)?; + let mut index = 0; + while index < key_count && !(key < &self.get_key(index, guard)?.unwrap()) { + index += 1; + } + guard.validate_epoch().map(|_| index) + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K, guard: &Guard) -> Result<(usize, Option), ()> { + loop { + guard.validate_epoch()?; + let mut version = self.write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = self.write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.get_key(key_index, guard)? != Some(*key) { + key_index += 1; + } + let value = self.values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == self.write_version.load(Ordering::Acquire) { + return guard.validate_epoch().map(|_| (key_index, value)); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child<'l1, 'l2>( + &'l1 self, + child: &'l2 Self, + child_idx: usize, + self_lock: &MCSLockGuard<'l1, K, V>, + child_lock: &MCSLockGuard<'l2, K, V>, + ) -> ( + [AtomicPtr>>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let next: [AtomicPtr>>; DEGREE * 2] = Default::default(); + let keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + atomic_clone(&self.next[0..], &next[0..], child_idx); + atomic_clone(&child.next[0..], &next[child_idx..], nsize); + atomic_clone( + &self.next[child_idx + 1..], + &next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + cell_clone(&self.key_slice(self_lock)[0..], &keys[0..], child_idx); + cell_clone( + &child.key_slice(child_lock)[0..], + &keys[child_idx..], + child.key_count_locked(child_lock), + ); + // Safety: Both `parent` and `child` is locked, so they cannot be reclaimed. + cell_clone( + &self.key_slice(self_lock)[child_idx..], + &keys[child_idx + child.key_count_locked(child_lock)..], + self.key_count_locked(self_lock) - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.key_slice(lock) + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &'g MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value_locked(i, lock).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &'g MCSLockGuard<'g, K, V>, + ) -> impl Iterator, *mut Inner)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next_locked(i, lock))) + .chain(once(( + None, + self.load_next_locked(self.key_count_locked(lock), lock), + ))) + } +} + +struct WriteGuard<'g, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + fn drop(&mut self) { + self.node + .write_version + .store(self.init_version + 2, Ordering::Release); + } +} + +struct Cursor +where + K: Default + Copy, + V: Default + Copy, +{ + l: *mut Inner>, + p: *mut Inner>, + gp: *mut Inner>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +pub struct ElimABTree +where + K: Default + Copy, + V: Default + Copy, +{ + entry: Entry>, +} + +unsafe impl Sync for ElimABTree +where + K: Sync + Default + Copy, + V: Sync + Default + Copy, +{ +} +unsafe impl Send for ElimABTree +where + K: Send + Default + Copy, + V: Send + Default + Copy, +{ +} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new(local: &Local>) -> Self { + loop { + let guard = &local.guard(); + let left = ok_or!( + guard.allocate(|node| unsafe { node.deref().init_for_leaf(true, 0, K::default()) }), + continue + ); + let entry = ok_or!( + guard.allocate(|node| unsafe { + node.deref().init_for_internal(true, 1, K::default()); + node.deref().init_next(0, left.as_raw()); + }), + unsafe { + guard.retire(left); + continue; + } + ); + return Self { + entry: Entry::new(entry), + }; + } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, local: &Local>) -> Option { + loop { + let guard = &local.guard(); + return ok_or!(self.search_basic_inner(key, guard), continue); + } + } + + fn search_basic_inner(&self, key: &K, guard: &Guard>) -> Result, ()> { + let entry = unsafe { self.entry.load(guard)?.deref() }; + let mut node = unsafe { &*entry.load_next(0, guard)? }; + while !node.is_leaf(guard)? { + let next = node.load_next(node.child_index(key, guard)?, guard)?; + node = unsafe { &*next }; + } + Ok(node.read_consistent(key, guard)?.1) + } + + fn search( + &self, + key: &K, + target: Option<*mut Inner>>, + guard: &mut Guard>, + ) -> (bool, Cursor) { + loop { + if let Ok(result) = self.search_inner(key, target, guard) { + return result; + } + guard.refresh(); + } + } + + fn search_inner( + &self, + key: &K, + target: Option<*mut Inner>>, + guard: &Guard>, + ) -> Result<(bool, Cursor), ()> { + let entry_sh = self.entry.load(guard)?; + let entry = unsafe { entry_sh.deref() }; + let mut cursor = Cursor { + l: entry.load_next(0, guard)?, + p: entry_sh.as_raw(), + gp: null_mut(), + gp_p_idx: 0, + p_l_idx: 0, + l_key_idx: 0, + val: None, + }; + + while !unsafe { &*cursor.l }.is_leaf(guard)? + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { &*cursor.l }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key, guard)?; + cursor.l = l_node.load_next(cursor.p_l_idx, guard)?; + } + + if let Some(target) = target { + Ok((cursor.l == target, cursor)) + } else { + let (index, value) = unsafe { &*cursor.l }.read_consistent(key, guard)?; + cursor.val = value; + cursor.l_key_idx = index; + Ok((value.is_some(), cursor)) + } + } + + pub fn insert(&self, key: &K, value: &V, local: &Local>) -> Option { + loop { + let guard = &mut local.guard(); + let (_, cursor) = self.search(key, None, guard); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, &cursor, guard) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner<'g>( + &self, + key: &K, + value: &V, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result, ()> { + let node = unsafe { &*cursor.l }; + let parent = unsafe { &*cursor.p }; + + debug_assert!(node.is_leaf(guard)?); + debug_assert!(!parent.is_leaf(guard)?); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key_locked(i, &node_lock) == Some(*key) { + return Ok(Some(node.get_value(i, guard)?.unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key_locked(i, &node_lock).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!( + parent, + parent_lock, + Operation::Insert, + None, + guard, + return Err(()), + return Err(()) + ); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let left = guard.allocate(|left| unsafe { + left.deref().init_for_leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.deref().init_key(i, Some(kv_pairs[i].0)); + left.deref().init_value(i, kv_pairs[i].1); + } + })?; + + let Ok(right) = guard.allocate(|right| unsafe { + right + .deref() + .init_for_leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.deref().init_key(i, Some(kv_pairs[i + left_size].0)); + right.deref().init_value(i, kv_pairs[i + left_size].1); + } + }) else { + unsafe { guard.retire(left) }; + return Err(()); + }; + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let Ok(internal) = guard.allocate(|internal| unsafe { + internal.deref().init_for_internal( + eq(parent, self.entry.load_raw()), + 2, + kv_pairs[left_size].0, + ); + internal.deref().init_key(0, Some(kv_pairs[left_size].0)); + internal.deref().init_next(0, left.as_raw()); + internal.deref().init_next(1, right.as_raw()); + }) else { + unsafe { guard.retire(left) }; + unsafe { guard.retire(right) }; + return Err(()); + }; + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + parent.store_next(cursor.p_l_idx, internal.as_raw(), &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { guard.retire_raw(cursor.l) }; + self.fix_tag_violation(kv_pairs[left_size].0, internal.as_raw(), guard); + + Ok(None) + } + } + + fn fix_tag_violation( + &self, + search_key: K, + viol: *mut Inner>, + guard: &mut Guard>, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + guard.refresh(); + let (found, cursor) = self.search(&search_key, Some(viol), guard); + if !found || cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let Ok((success, recur)) = self.fix_tag_violation_inner(&cursor, guard) else { + stack.push((search_key, viol)); + continue; + }; + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_tag_violation_inner( + &self, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result<(bool, Option<(K, *mut Inner>)>), ()> { + let viol = cursor.l; + let viol_node = unsafe { &*viol }; + if viol_node.weight(guard)? { + return Ok((true, None)); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf(guard)?); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!( + !eq(viol_node, self.entry.load_raw()) + && unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? != viol + ); + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { &*cursor.l }; + let parent = unsafe { &*cursor.p }; + let gparent = unsafe { &*cursor.gp }; + debug_assert!(!node.is_leaf(guard)?); + debug_assert!(!parent.is_leaf(guard)?); + debug_assert!(!gparent.is_leaf(guard)?); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight(guard)? { + return Ok((false, Some((parent.search_key(guard)?, cursor.p)))); + } + + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + guard, + return Ok((false, None)), + return Err(()) + ); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + guard, + return Ok((false, None)), + return Err(()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + guard, + return Ok((false, None)), + return Err(()) + ); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx, &parent_lock, &node_lock); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let absorber = guard.allocate(|absorber| unsafe { + absorber.deref().init_for_internal( + true, + size, + parent.get_key_locked(0, &parent_lock).unwrap(), + ); + atomic_clone(&next, &absorber.deref().next, DEGREE); + cell_clone(&keys, &absorber.deref().key_slice_unchecked(), DEGREE); + })?; + + gparent.store_next(cursor.gp_p_idx, absorber.as_raw(), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.retire_raw(cursor.l) }; + unsafe { guard.retire_raw(cursor.p) }; + return Ok((true, None)); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let left = guard.allocate(|left| unsafe { + left.deref() + .init_for_internal(true, left_size, keys[0].get().unwrap()); + cell_clone(&keys, &left.deref().key_slice_unchecked(), left_size - 1); + atomic_clone(&next, &left.deref().next, left_size); + })?; + + let right_size = size - left_size; + let Ok(right) = guard.allocate(|right| unsafe { + right + .deref() + .init_for_internal(true, right_size, keys[left_size].get().unwrap()); + cell_clone( + &keys[left_size..], + &right.deref().key_slice_unchecked()[0..], + right_size - 1, + ); + atomic_clone(&next[left_size..], &right.deref().next[0..], right_size); + }) else { + unsafe { guard.retire(left) }; + return Err(()); + }; + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let Ok(new_internal) = guard.allocate(|new_internal| unsafe { + new_internal.deref().init_for_internal( + eq(gparent, self.entry.load_raw()), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.deref().init_key(0, keys[left_size - 1].get()); + new_internal.deref().init_next(0, left.as_raw()); + new_internal.deref().init_next(1, right.as_raw()); + }) else { + unsafe { guard.retire(left) }; + unsafe { guard.retire(right) }; + return Err(()); + }; + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + gparent.store_next(cursor.gp_p_idx, new_internal.as_raw(), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.retire_raw(cursor.l) }; + unsafe { guard.retire_raw(cursor.p) }; + + drop((node_lock, parent_lock, gparent_lock)); + return Ok(( + true, + Some((keys[left_size - 1].get().unwrap(), new_internal.as_raw())), + )); + } + } + + pub fn remove(&self, key: &K, local: &Local>) -> Option { + loop { + let guard = &mut local.guard(); + let (_, cursor) = self.search(key, None, guard); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor, guard) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner( + &self, + key: &K, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result, ()> { + let node = unsafe { &*cursor.l }; + let parent = unsafe { &*cursor.p }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf(guard)?); + debug_assert!(!parent.is_leaf(guard)?); + debug_assert!(if let Some(gp) = gparent { + !gp.is_leaf(guard)? + } else { + true + }); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + guard, + return Err(()), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key_locked(i, &node_lock) == Some(*key) { + // Safety of raw `get`: `node` is locked. + let val = node.values[i].get().unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + let search_key = node.search_key.get(); + drop(node_lock); + self.fix_underfull_violation(search_key, cursor.l, guard); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation( + &self, + search_key: K, + viol: *mut Inner>, + guard: &mut Guard>, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + guard.refresh(); + let (_, cursor) = self.search(&search_key, Some(viol), guard); + if cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let Ok((success, recur)) = self.fix_underfull_violation_inner(&cursor, guard) else { + stack.push((search_key, viol)); + continue; + }; + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_underfull_violation_inner( + &self, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result<(bool, ArrayVec<(K, *mut Inner>), 2>), ()> { + let viol = cursor.l; + let viol_node = unsafe { &*viol }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, self.entry.load_raw()) + || viol == unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? + { + // No degree violation at `viol`. + return Ok((true, ArrayVec::<_, 2>::new())); + } + + let node = unsafe { &*cursor.l }; + let parent = unsafe { &*cursor.p }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { &*cursor.gp }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, self.entry.load_raw()) + && cursor.p != unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? + { + return Ok(( + false, + ArrayVec::from_iter(once((parent.search_key(guard)?, cursor.p))), + )); + } + + let sibling_idx = if cursor.p_l_idx > 0 { + cursor.p_l_idx - 1 + } else { + 1 + }; + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling_sh = parent.load_next(sibling_idx, guard)?; + let sibling = unsafe { &*sibling_sh }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if sibling_idx < cursor.p_l_idx { + ((sibling, sibling_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, sibling_idx)) + }; + + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return Ok((true, ArrayVec::new())); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight.get() { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(parent.search_key.get(), cursor.p, guard); + return Ok((false, ArrayVec::new())); + } + if !node.weight.get() { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(node.search_key.get(), cursor.l, guard); + return Ok((false, ArrayVec::new())); + } + if !sibling.weight.get() { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(sibling.search_key.get(), sibling_sh, guard); + return Ok((false, ArrayVec::new())); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight.get() && node.weight.get() && sibling.weight.get()); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf.get() && sibling.is_leaf.get()) + || (!node.is_leaf.get() && !sibling.is_leaf.get()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = guard.allocate(|new_node| unsafe { + if left.is_leaf.get() { + debug_assert!(right.is_leaf.get()); + let new_leaf = new_node.deref(); + new_leaf.init_for_leaf(true, size, node.search_key.get()); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + } else { + debug_assert!(!right.is_leaf.get()); + let new_internal = new_node.deref(); + new_internal.init_for_internal(true, size, node.search_key.get()); + let key_btw = parent.get_key_locked(left_idx, &parent_lock).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + } + })?; + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, self.entry.load_raw()) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node.as_raw(), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.retire_raw(cursor.l); + guard.retire_raw(cursor.p); + guard.retire_raw(sibling_sh); + } + + let search_key = node.search_key.get(); + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return Ok(( + true, + ArrayVec::from_iter(once((search_key, new_node.as_raw()))), + )); + } else { + debug_assert!(!eq(gparent, self.entry.load_raw()) || psize > 2); + let Ok(new_parent) = guard.allocate(|new_parent| unsafe { + let new_parent = new_parent.deref(); + new_parent.init_for_internal(true, psize - 1, parent.search_key.get()); + + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key_locked(i, &parent_lock)); + } + for i in 0..sibling_idx { + new_parent.init_next(i, parent.load_next_locked(i, &parent_lock)); + } + for i in left_idx + 1..parent.key_count_locked(&parent_lock) { + new_parent.init_key(i - 1, parent.get_key_locked(i, &parent_lock)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next_locked(i, &parent_lock)); + } + + new_parent.init_next( + cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 }), + new_node.as_raw(), + ); + }) else { + unsafe { guard.retire(new_node) }; + return Err(()); + }; + + gparent.store_next(cursor.gp_p_idx, new_parent.as_raw(), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.retire_raw(cursor.l); + guard.retire_raw(cursor.p); + guard.retire_raw(sibling_sh); + } + + let node_key = node.search_key.get(); + let parent_key = parent.search_key.get(); + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return Ok(( + true, + ArrayVec::from_iter( + [ + (node_key, new_node.as_raw()), + (parent_key, new_parent.as_raw()), + ] + .into_iter(), + ), + )); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf.get() == right.is_leaf.get()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf.get() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = guard.allocate(|new_leaf| unsafe { + let new_leaf = new_leaf.deref(); + new_leaf.init_for_leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf + .search_key + .set(new_leaf.get_key_unchecked(0).unwrap()); + })?; + + let Ok(new_right) = guard.allocate(|new_leaf| unsafe { + debug_assert!(left.is_leaf.get()); + let new_leaf = new_leaf.deref(); + new_leaf.init_for_leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf + .search_key + .set(new_leaf.get_key_unchecked(0).unwrap()); + }) else { + unsafe { guard.retire(new_left) }; + return Err(()); + }; + let pivot = unsafe { new_right.deref() }.search_key.get(); + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key_locked(left_idx, &parent_lock).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let new_left = guard.allocate(|new_internal| unsafe { + let new_internal = new_internal.deref(); + new_internal.init_for_internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal + .search_key + .set(new_internal.get_key_unchecked(0).unwrap()); + })?; + let pivot = unsafe { new_left.deref().get_key_unchecked(left_size - 1) } + .take() + .unwrap(); + + let Ok(new_right) = guard.allocate(|new_internal| unsafe { + let new_internal = new_internal.deref(); + new_internal.init_for_internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal + .search_key + .set(new_internal.get_key_unchecked(0).unwrap()); + }) else { + unsafe { guard.retire(new_left) }; + return Err(()); + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let Ok(new_parent) = guard.allocate(|new_parent| unsafe { + let new_parent = new_parent.deref(); + new_parent.init_for_internal(parent.weight.get(), psize, parent.search_key.get()); + cell_clone( + &parent.key_slice(&parent_lock)[0..], + &new_parent.key_slice_unchecked()[0..], + parent.key_count_locked(&parent_lock), + ); + atomic_clone(&parent.next[0..], &new_parent.next[0..], psize); + new_parent.init_next(left_idx, new_left.as_raw()); + new_parent.init_next(right_idx, new_right.as_raw()); + new_parent.init_key(left_idx, Some(pivot)); + }) else { + unsafe { guard.retire(new_left) }; + unsafe { guard.retire(new_right) }; + return Err(()); + }; + + gparent.store_next(cursor.gp_p_idx, new_parent.as_raw(), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.retire_raw(cursor.l); + guard.retire_raw(cursor.p); + guard.retire_raw(sibling_sh); + } + + return Ok((true, ArrayVec::new())); + } + } +} + +/// Similar to `memcpy`, but for `Cell` types. +#[inline] +fn cell_clone(src: &[Cell], dst: &[Cell], len: usize) { + for i in 0..len { + dst[i].set(src[i].get()); + } +} + +/// Similar to `memcpy`, but for `AtomicPtr` types. +#[inline] +fn atomic_clone(src: &[AtomicPtr], dst: &[AtomicPtr], len: usize) { + for i in 0..len { + dst[i].store(src[i].load(Ordering::Relaxed), Ordering::Relaxed); + } +} + +impl ConcurrentMap for ElimABTree +where + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, +{ + type Global = Global>; + type Local = Local>; + + fn global(key_range_hint: usize) -> Self::Global { + Global::new(key_range_hint) + } + + fn local(global: &Self::Global) -> Self::Local { + Local::new(global) + } + + fn new(local: &Self::Local) -> Self { + ElimABTree::new(local) + } + + #[inline(always)] + fn get(&self, key: &K, local: &Self::Local) -> Option { + self.search_basic(key, local) + } + + #[inline(always)] + fn insert(&self, key: K, value: V, local: &Self::Local) -> bool { + self.insert(&key, &value, local).is_none() + } + + #[inline(always)] + fn remove(&self, key: &K, local: &Self::Local) -> Option { + self.remove(key, local) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::vbr::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/vbr/mod.rs b/src/ds_impl/vbr/mod.rs index 9d5088f0..7bfded52 100644 --- a/src/ds_impl/vbr/mod.rs +++ b/src/ds_impl/vbr/mod.rs @@ -1,13 +1,14 @@ pub mod concurrent_map; +pub mod elim_ab_tree; pub mod list; pub mod michael_hash_map; pub mod natarajan_mittal_tree; pub mod skip_list; -// pub mod elim_ab_tree; pub use self::concurrent_map::ConcurrentMap; +pub use elim_ab_tree::ElimABTree; pub use list::{HHSList, HList, HMList}; pub use michael_hash_map::HashMap; pub use natarajan_mittal_tree::NMTreeMap; From f1671393e2c7617367da1bfcb62da8c4bc739cf1 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Sun, 10 Nov 2024 15:08:19 +0000 Subject: [PATCH 72/84] Fix a bug in VBR ElimAbTree (missing validation) --- src/ds_impl/vbr/elim_ab_tree.rs | 342 ++++++++++++++++++++------------ 1 file changed, 217 insertions(+), 125 deletions(-) diff --git a/src/ds_impl/vbr/elim_ab_tree.rs b/src/ds_impl/vbr/elim_ab_tree.rs index e05f112d..00ac4a98 100644 --- a/src/ds_impl/vbr/elim_ab_tree.rs +++ b/src/ds_impl/vbr/elim_ab_tree.rs @@ -17,7 +17,7 @@ macro_rules! try_acq_val_or { let __slot = UnsafeCell::new(MCSLockSlot::new()); let $lock = match ( $node.acquire($op, $key, &__slot), - $node.marked.load(Ordering::Acquire), + $node._marked.load(Ordering::Acquire), ) { (AcqResult::Acquired(lock), false) => lock, _ => $acq_val_err, @@ -151,20 +151,20 @@ where V: Copy + Default, { _keys: [Cell>; DEGREE], - search_key: Cell, + _search_key: Cell, lock: AtomicPtr>, /// The number of next pointers (for an internal node) or values (for a leaf node). /// Note that it may not be equal to the number of keys, because the last next pointer /// is mapped by a bottom key (i.e., `None`). - size: AtomicUsize, - weight: Cell, - marked: AtomicBool, + _size: AtomicUsize, + _weight: Cell, + _marked: AtomicBool, // Leaf node data - values: [Cell>; DEGREE], - write_version: AtomicUsize, + _values: [Cell>; DEGREE], + _write_version: AtomicUsize, // Internal node data - next: [AtomicPtr>>; DEGREE], - is_leaf: Cell, + _next: [AtomicPtr>>; DEGREE], + _is_leaf: Cell, } impl Node @@ -172,23 +172,59 @@ where K: Default + Copy, V: Default + Copy, { + fn get_marked_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> bool { + self._marked.load(Ordering::Acquire) + } + + fn set_marked_locked<'l>(&'l self, mark: bool, _: &MCSLockGuard<'l, K, V>) { + self._marked.store(mark, Ordering::Release); + } + fn weight(&self, guard: &Guard) -> Result { - let result = self.weight.get(); + let result = self._weight.get(); guard.validate_epoch().map(|_| result) } + fn weight_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> bool { + self._weight.get() + } + fn search_key(&self, guard: &Guard) -> Result { - let result = self.search_key.get(); + let result = self._search_key.get(); guard.validate_epoch().map(|_| result) } + fn search_key_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> K { + self._search_key.get() + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn search_key_unchecked(&self) -> K { + self._search_key.get() + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_search_key(&self, key: K) { + self._search_key.set(key); + } + fn is_leaf(&self, guard: &Guard) -> Result { - let result = self.is_leaf.get(); + let result = self._is_leaf.get(); guard.validate_epoch().map(|_| result) } + fn is_leaf_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> bool { + self._is_leaf.get() + } + fn load_next(&self, index: usize, guard: &Guard) -> Result<*mut Inner, ()> { - let result = self.next[index].load(Ordering::Acquire); + let result = self._next[index].load(Ordering::Acquire); guard.validate_epoch().map(|_| result) } @@ -197,11 +233,11 @@ where index: usize, _: &MCSLockGuard<'l, K, V>, ) -> *mut Inner { - self.next[index].load(Ordering::Acquire) + self._next[index].load(Ordering::Acquire) } - fn store_next<'l>(&self, index: usize, ptr: *mut Inner, _: &MCSLockGuard<'l, K, V>) { - self.next[index].store(ptr, Ordering::Release); + fn store_next<'l>(&'l self, index: usize, ptr: *mut Inner, _: &MCSLockGuard<'l, K, V>) { + self._next[index].store(ptr, Ordering::Release); } /// # Safety @@ -209,15 +245,27 @@ where /// The current thread has an exclusive ownership and it is not exposed to the shared memory. /// (e.g., in the `init` closure of `allocate` function) unsafe fn init_next(&self, index: usize, ptr: *mut Inner) { - self.next[index].store(ptr, Ordering::Release); + self._next[index].store(ptr, Ordering::Release); + } + + fn next_slice<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> &[AtomicPtr>>] { + &self._next + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn next_slice_unchecked(&self) -> &[AtomicPtr>>] { + &self._next } fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { debug_assert!(eq(unsafe { lock.owner_node() }, self)); - let init_version = self.write_version.load(Ordering::Acquire); + let init_version = self._write_version.load(Ordering::Acquire); debug_assert!(init_version % 2 == 0); // It is safe to skip the epoch validation, as we are grabbing the lock. - self.write_version + self._write_version .store(init_version + 1, Ordering::Release); compiler_fence(Ordering::SeqCst); @@ -227,30 +275,51 @@ where }; } + fn get_size(&self, guard: &Guard) -> Result { + let result = self._size.load(Ordering::Acquire); + guard.validate_epoch().map(|_| result) + } + + fn get_size_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> usize { + self._size.load(Ordering::Acquire) + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn get_size_unchecked(&self) -> usize { + self._size.load(Ordering::Acquire) + } + + fn set_size_locked<'l>(&'l self, size: usize, _: &MCSLockGuard<'l, K, V>) { + self._size.store(size, Ordering::Release); + } + fn key_count(&self, guard: &Guard) -> Result { - let result = if self.is_leaf.get() { - self.size.load(Ordering::Acquire) + let result = if self.is_leaf(guard)? { + self.get_size(guard)? } else { - self.size.load(Ordering::Acquire) - 1 + self.get_size(guard)? - 1 }; guard.validate_epoch().map(|_| result) } - fn key_count_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> usize { - if self.is_leaf.get() { - self.size.load(Ordering::Acquire) + fn key_count_locked<'l>(&'l self, lock: &MCSLockGuard<'l, K, V>) -> usize { + if self._is_leaf.get() { + self.get_size_locked(lock) } else { - self.size.load(Ordering::Acquire) - 1 + self.get_size_locked(lock) - 1 } } fn get_value(&self, index: usize, guard: &Guard) -> Result, ()> { - let value = self.values[index].get(); + let value = self._values[index].get(); guard.validate_epoch().map(|_| value) } fn get_value_locked<'l>(&'l self, index: usize, _: &MCSLockGuard<'l, K, V>) -> Option { - self.values[index].get() + self._values[index].get() } /// # Safety @@ -258,11 +327,11 @@ where /// The current thread has an exclusive ownership and it is not exposed to the shared memory. /// (e.g., in the `init` closure of `allocate` function) unsafe fn init_value(&self, index: usize, val: V) { - self.values[index].set(Some(val)); + self._values[index].set(Some(val)); } fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { - self.values[index].set(Some(val)); + self._values[index].set(Some(val)); } /// # Safety @@ -278,11 +347,11 @@ where guard.validate_epoch().map(|_| result) } - fn get_key_locked<'l>(&self, index: usize, _: &MCSLockGuard<'l, K, V>) -> Option { + fn get_key_locked<'l>(&'l self, index: usize, _: &MCSLockGuard<'l, K, V>) -> Option { self._keys[index].get() } - fn key_slice<'l>(&self, _: &MCSLockGuard<'l, K, V>) -> &[Cell>] { + fn key_slice<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> &[Cell>] { &self._keys } @@ -290,7 +359,7 @@ where /// /// The current thread has an exclusive ownership and it is not exposed to the shared memory. /// (e.g., in the `init` closure of `allocate` function) - unsafe fn key_slice_unchecked<'l>(&self) -> &[Cell>] { + unsafe fn key_slice_unchecked(&self) -> &[Cell>] { &self._keys } @@ -310,27 +379,27 @@ where for key in &self._keys { key.set(Default::default()); } - self.search_key.set(search_key); - self.size.store(size, Ordering::Release); - self.weight.set(weight); - self.marked.store(false, Ordering::Release); - for next in &self.next { + self._search_key.set(search_key); + self._size.store(size, Ordering::Release); + self._weight.set(weight); + self._marked.store(false, Ordering::Release); + for next in &self._next { next.store(null_mut(), Ordering::Release); } - for value in &self.values { + for value in &self._values { value.set(Default::default()); } - self.write_version.store(0, Ordering::Release); + self._write_version.store(0, Ordering::Release); } unsafe fn init_for_internal(&self, weight: bool, size: usize, search_key: K) { self.init_on_allocate(weight, size, search_key); - self.is_leaf.set(false); + self._is_leaf.set(false); } unsafe fn init_for_leaf(&self, weight: bool, size: usize, search_key: K) { self.init_on_allocate(weight, size, search_key); - self.is_leaf.set(true); + self._is_leaf.set(true); } } @@ -352,18 +421,18 @@ where fn read_consistent(&self, key: &K, guard: &Guard) -> Result<(usize, Option), ()> { loop { guard.validate_epoch()?; - let mut version = self.write_version.load(Ordering::Acquire); + let mut version = self._write_version.load(Ordering::Acquire); while version & 1 > 0 { - version = self.write_version.load(Ordering::Acquire); + version = self._write_version.load(Ordering::Acquire); } let mut key_index = 0; while key_index < DEGREE && self.get_key(key_index, guard)? != Some(*key) { key_index += 1; } - let value = self.values.get(key_index).and_then(|value| value.get()); + let value = self._values.get(key_index).and_then(|value| value.get()); compiler_fence(Ordering::SeqCst); - if version == self.write_version.load(Ordering::Acquire) { + if version == self._write_version.load(Ordering::Acquire) { return guard.validate_epoch().map(|_| (key_index, value)); } } @@ -458,13 +527,17 @@ where ) { let next: [AtomicPtr>>; DEGREE * 2] = Default::default(); let keys: [Cell>; DEGREE * 2] = Default::default(); - let psize = self.size.load(Ordering::Relaxed); - let nsize = child.size.load(Ordering::Relaxed); + let psize = self.get_size_locked(self_lock); + let nsize = child.get_size_locked(child_lock); - atomic_clone(&self.next[0..], &next[0..], child_idx); - atomic_clone(&child.next[0..], &next[child_idx..], nsize); + atomic_clone(&self.next_slice(self_lock)[0..], &next[0..], child_idx); atomic_clone( - &self.next[child_idx + 1..], + &child.next_slice(child_lock)[0..], + &next[child_idx..], + nsize, + ); + atomic_clone( + &self.next_slice(self_lock)[child_idx + 1..], &next[child_idx + nsize..], psize - (child_idx + 1), ); @@ -540,7 +613,7 @@ where { fn drop(&mut self) { self.node - .write_version + ._write_version .store(self.init_version + 2, Ordering::Release); } } @@ -719,7 +792,8 @@ where AcqResult::Acquired(lock) => lock, AcqResult::Eliminated(value) => return Ok(Some(value)), }; - if node.marked.load(Ordering::SeqCst) { + guard.validate_epoch()?; + if node.get_marked_locked(&node_lock) { return Err(()); } for i in 0..DEGREE { @@ -729,7 +803,7 @@ where } // At this point, we are guaranteed key is not in the node. - if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + if node.get_size_locked(&node_lock) < Self::ABSORB_THRESHOLD { // We have the capacity to fit this new key. So let's just find an empty slot. for i in 0..DEGREE { if node.get_key_locked(i, &node_lock).is_some() { @@ -738,8 +812,7 @@ where let wguard = node.start_write(&node_lock); node.set_key(i, Some(*key), &wguard); node.set_value(i, *value, &wguard); - node.size - .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + node.set_size_locked(node.get_size_locked(&node_lock) + 1, &node_lock); node.elim_key_ops(*value, wguard, &node_lock); @@ -816,7 +889,7 @@ where // a node since any update to parent would have deleted node (and hence we would have // returned at the `node.marked` check). parent.store_next(cursor.p_l_idx, internal.as_raw(), &parent_lock); - node.marked.store(true, Ordering::Release); + node.set_marked_locked(true, &node_lock); // Manually unlock and fix the tag. drop((parent_lock, node_lock)); @@ -873,7 +946,7 @@ where ); debug_assert!(!cursor.gp.is_null()); - let node = unsafe { &*cursor.l }; + let node = viol_node; let parent = unsafe { &*cursor.p }; let gparent = unsafe { &*cursor.gp }; debug_assert!(!node.is_leaf(guard)?); @@ -914,8 +987,8 @@ where return Err(()) ); - let psize = parent.size.load(Ordering::Relaxed); - let nsize = viol_node.size.load(Ordering::Relaxed); + let psize = parent.get_size_locked(&parent_lock); + let nsize = viol_node.get_size_locked(&node_lock); // We don't ever change the size of a tag node, so its size should always be 2. debug_assert_eq!(nsize, 2); let c = psize + nsize; @@ -934,13 +1007,13 @@ where size, parent.get_key_locked(0, &parent_lock).unwrap(), ); - atomic_clone(&next, &absorber.deref().next, DEGREE); + atomic_clone(&next, &absorber.deref().next_slice_unchecked(), DEGREE); cell_clone(&keys, &absorber.deref().key_slice_unchecked(), DEGREE); })?; gparent.store_next(cursor.gp_p_idx, absorber.as_raw(), &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); + node.set_marked_locked(true, &node_lock); + parent.set_marked_locked(true, &parent_lock); unsafe { guard.retire_raw(cursor.l) }; unsafe { guard.retire_raw(cursor.p) }; @@ -962,7 +1035,7 @@ where left.deref() .init_for_internal(true, left_size, keys[0].get().unwrap()); cell_clone(&keys, &left.deref().key_slice_unchecked(), left_size - 1); - atomic_clone(&next, &left.deref().next, left_size); + atomic_clone(&next, &left.deref().next_slice_unchecked(), left_size); })?; let right_size = size - left_size; @@ -975,7 +1048,11 @@ where &right.deref().key_slice_unchecked()[0..], right_size - 1, ); - atomic_clone(&next[left_size..], &right.deref().next[0..], right_size); + atomic_clone( + &next[left_size..], + &right.deref().next_slice_unchecked()[0..], + right_size, + ); }) else { unsafe { guard.retire(left) }; return Err(()); @@ -1003,8 +1080,8 @@ where // as this Overflow if `n` will become the root. gparent.store_next(cursor.gp_p_idx, new_internal.as_raw(), &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); + node.set_marked_locked(true, &node_lock); + parent.set_marked_locked(true, &parent_lock); unsafe { guard.retire_raw(cursor.l) }; unsafe { guard.retire_raw(cursor.p) }; @@ -1061,23 +1138,22 @@ where // Bug Fix: Added a check to ensure the node size is greater than 0. // This prevents underflows caused by decrementing the size value. // This check is not present in the original code. - if node.size.load(Ordering::Acquire) == 0 { + if node.get_size_locked(&node_lock) == 0 { return Err(()); } - let new_size = node.size.load(Ordering::Relaxed) - 1; + let new_size = node.get_size_locked(&node_lock) - 1; for i in 0..DEGREE { if node.get_key_locked(i, &node_lock) == Some(*key) { - // Safety of raw `get`: `node` is locked. - let val = node.values[i].get().unwrap(); + let val = node.get_value_locked(i, &node_lock).unwrap(); let wguard = node.start_write(&node_lock); node.set_key(i, None, &wguard); - node.size.store(new_size, Ordering::Relaxed); + node.set_size_locked(new_size, &node_lock); node.elim_key_ops(val, wguard, &node_lock); if new_size == Self::UNDERFULL_THRESHOLD - 1 { - let search_key = node.search_key.get(); + let search_key = node.search_key_locked(&node_lock); drop(node_lock); self.fix_underfull_violation(search_key, cursor.l, guard); } @@ -1126,7 +1202,7 @@ where // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot // "be turned into" the root. The root is only created by the root absorb // operation below, so a node that is not the root will never become the root. - if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + if viol_node.get_size(guard)? >= Self::UNDERFULL_THRESHOLD || eq(viol_node, self.entry.load_raw()) || viol == unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? { @@ -1134,14 +1210,14 @@ where return Ok((true, ArrayVec::<_, 2>::new())); } - let node = unsafe { &*cursor.l }; + let node = viol_node; let parent = unsafe { &*cursor.p }; // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, // then `p` is not the root. debug_assert!(!cursor.gp.is_null()); let gparent = unsafe { &*cursor.gp }; - if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + if parent.get_size(guard)? < Self::UNDERFULL_THRESHOLD && !eq(parent, self.entry.load_raw()) && cursor.p != unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? { @@ -1189,7 +1265,8 @@ where ); // Repeat this check, this might have changed while we locked `viol`. - if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // Safety: `viol_node` is locked by either `left_lock` or `right_lock`. + if unsafe { viol_node.get_size_unchecked() } >= Self::UNDERFULL_THRESHOLD { // No degree violation at `viol`. return Ok((true, ArrayVec::new())); } @@ -1216,43 +1293,51 @@ where // We can only apply AbsorbSibling or Distribute if there are no // weight violations at `parent`, `node`, or `sibling`. // So, we first check for any weight violations and fix any that we see. - if !parent.weight.get() { + if !parent.weight_locked(&parent_lock) { + let search_key = parent.search_key_locked(&parent_lock); drop((left_lock, right_lock, parent_lock, gparent_lock)); - self.fix_tag_violation(parent.search_key.get(), cursor.p, guard); + self.fix_tag_violation(search_key, cursor.p, guard); return Ok((false, ArrayVec::new())); } - if !node.weight.get() { + if !left.weight_locked(&left_lock) { + let search_key = left.search_key_locked(&left_lock); drop((left_lock, right_lock, parent_lock, gparent_lock)); - self.fix_tag_violation(node.search_key.get(), cursor.l, guard); + self.fix_tag_violation(search_key, left as *const _ as *mut _, guard); return Ok((false, ArrayVec::new())); } - if !sibling.weight.get() { + if !right.weight_locked(&right_lock) { + let search_key = right.search_key_locked(&right_lock); drop((left_lock, right_lock, parent_lock, gparent_lock)); - self.fix_tag_violation(sibling.search_key.get(), sibling_sh, guard); + self.fix_tag_violation(search_key, right as *const _ as *mut _, guard); return Ok((false, ArrayVec::new())); } // There are no weight violations at `parent`, `node` or `sibling`. - debug_assert!(parent.weight.get() && node.weight.get() && sibling.weight.get()); + debug_assert!( + parent.weight_locked(&parent_lock) + && left.weight_locked(&left_lock) + && right.weight_locked(&right_lock) + ); // l and s are either both leaves or both internal nodes, // because there are no weight violations at these nodes. debug_assert!( - (node.is_leaf.get() && sibling.is_leaf.get()) - || (!node.is_leaf.get() && !sibling.is_leaf.get()) + (left.is_leaf_locked(&left_lock) && right.is_leaf_locked(&right_lock)) + || (!left.is_leaf_locked(&left_lock) && !right.is_leaf_locked(&right_lock)) ); - let lsize = left.size.load(Ordering::Relaxed); - let rsize = right.size.load(Ordering::Relaxed); - let psize = parent.size.load(Ordering::Relaxed); + let lsize = left.get_size_locked(&left_lock); + let rsize = right.get_size_locked(&right_lock); + let psize = parent.get_size_locked(&parent_lock); let size = lsize + rsize; if size < 2 * Self::UNDERFULL_THRESHOLD { // AbsorbSibling let new_node = guard.allocate(|new_node| unsafe { - if left.is_leaf.get() { - debug_assert!(right.is_leaf.get()); + if left.is_leaf_locked(&left_lock) { + debug_assert!(right.is_leaf_locked(&right_lock)); let new_leaf = new_node.deref(); - new_leaf.init_for_leaf(true, size, node.search_key.get()); + // Safety: `node` is locked. + new_leaf.init_for_leaf(true, size, node.search_key_unchecked()); let kv_iter = left .iter_key_value(&left_lock) .chain(right.iter_key_value(&right_lock)) @@ -1262,9 +1347,10 @@ where new_leaf.init_value(i, value); } } else { - debug_assert!(!right.is_leaf.get()); + debug_assert!(!right.is_leaf_locked(&right_lock)); let new_internal = new_node.deref(); - new_internal.init_for_internal(true, size, node.search_key.get()); + // Safety: `node` is locked. + new_internal.init_for_internal(true, size, node.search_key_unchecked()); let key_btw = parent.get_key_locked(left_idx, &parent_lock).unwrap(); let kn_iter = left .iter_key_next(&left_lock) @@ -1283,9 +1369,9 @@ where if eq(gparent, self.entry.load_raw()) && psize == 2 { debug_assert!(cursor.gp_p_idx == 0); gparent.store_next(cursor.gp_p_idx, new_node.as_raw(), &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); - sibling.marked.store(true, Ordering::Relaxed); + left.set_marked_locked(true, &left_lock); + parent.set_marked_locked(true, &parent_lock); + right.set_marked_locked(true, &right_lock); unsafe { guard.retire_raw(cursor.l); @@ -1293,7 +1379,8 @@ where guard.retire_raw(sibling_sh); } - let search_key = node.search_key.get(); + // Safety: `node` is locked. + let search_key = unsafe { node.search_key_unchecked() }; drop((left_lock, right_lock, parent_lock, gparent_lock)); return Ok(( true, @@ -1303,7 +1390,11 @@ where debug_assert!(!eq(gparent, self.entry.load_raw()) || psize > 2); let Ok(new_parent) = guard.allocate(|new_parent| unsafe { let new_parent = new_parent.deref(); - new_parent.init_for_internal(true, psize - 1, parent.search_key.get()); + new_parent.init_for_internal( + true, + psize - 1, + parent.search_key_locked(&parent_lock), + ); for i in 0..left_idx { new_parent.init_key(i, parent.get_key_locked(i, &parent_lock)); @@ -1328,9 +1419,9 @@ where }; gparent.store_next(cursor.gp_p_idx, new_parent.as_raw(), &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); - sibling.marked.store(true, Ordering::Relaxed); + left.set_marked_locked(true, &left_lock); + parent.set_marked_locked(true, &parent_lock); + right.set_marked_locked(true, &right_lock); unsafe { guard.retire_raw(cursor.l); @@ -1338,8 +1429,9 @@ where guard.retire_raw(sibling_sh); } - let node_key = node.search_key.get(); - let parent_key = parent.search_key.get(); + // Safety: `node` is locked. + let node_key = unsafe { node.search_key_unchecked() }; + let parent_key = parent.search_key_locked(&parent_lock); drop((left_lock, right_lock, parent_lock, gparent_lock)); return Ok(( true, @@ -1357,11 +1449,11 @@ where let left_size = size / 2; let right_size = size - left_size; - assert!(left.is_leaf.get() == right.is_leaf.get()); + assert!(left.is_leaf_locked(&left_lock) == right.is_leaf_locked(&right_lock)); // `pivot`: Reserve one key for the parent // (to go between `new_left` and `new_right`). - let (new_left, new_right, pivot) = if left.is_leaf.get() { + let (new_left, new_right, pivot) = if left.is_leaf_locked(&left_lock) { // Combine the contents of `l` and `s`. let mut kv_pairs = left .iter_key_value(&left_lock) @@ -1378,13 +1470,11 @@ where new_leaf.init_key(i, Some(k)); new_leaf.init_value(i, v); } - new_leaf - .search_key - .set(new_leaf.get_key_unchecked(0).unwrap()); + new_leaf.init_search_key(new_leaf.get_key_unchecked(0).unwrap()); })?; let Ok(new_right) = guard.allocate(|new_leaf| unsafe { - debug_assert!(left.is_leaf.get()); + debug_assert!(left.is_leaf_locked(&left_lock)); let new_leaf = new_leaf.deref(); new_leaf.init_for_leaf(true, right_size, Default::default()); for i in 0..right_size { @@ -1392,14 +1482,12 @@ where new_leaf.init_key(i, Some(k)); new_leaf.init_value(i, v); } - new_leaf - .search_key - .set(new_leaf.get_key_unchecked(0).unwrap()); + new_leaf.init_search_key(new_leaf.get_key_unchecked(0).unwrap()); }) else { unsafe { guard.retire(new_left) }; return Err(()); }; - let pivot = unsafe { new_right.deref() }.search_key.get(); + let pivot = unsafe { new_right.deref().search_key_unchecked() }; debug_assert!(kv_iter.next().is_none()); (new_left, new_right, pivot) @@ -1420,9 +1508,7 @@ where new_internal.init_key(i, k); new_internal.init_next(i, n); } - new_internal - .search_key - .set(new_internal.get_key_unchecked(0).unwrap()); + new_internal.init_search_key(new_internal.get_key_unchecked(0).unwrap()); })?; let pivot = unsafe { new_left.deref().get_key_unchecked(left_size - 1) } .take() @@ -1436,9 +1522,7 @@ where new_internal.init_key(i, k); new_internal.init_next(i, n); } - new_internal - .search_key - .set(new_internal.get_key_unchecked(0).unwrap()); + new_internal.init_search_key(new_internal.get_key_unchecked(0).unwrap()); }) else { unsafe { guard.retire(new_left) }; return Err(()); @@ -1450,13 +1534,21 @@ where let Ok(new_parent) = guard.allocate(|new_parent| unsafe { let new_parent = new_parent.deref(); - new_parent.init_for_internal(parent.weight.get(), psize, parent.search_key.get()); + new_parent.init_for_internal( + parent.weight_locked(&parent_lock), + psize, + parent.search_key_locked(&parent_lock), + ); cell_clone( &parent.key_slice(&parent_lock)[0..], &new_parent.key_slice_unchecked()[0..], parent.key_count_locked(&parent_lock), ); - atomic_clone(&parent.next[0..], &new_parent.next[0..], psize); + atomic_clone( + &parent.next_slice(&parent_lock)[0..], + &new_parent.next_slice_unchecked()[0..], + psize, + ); new_parent.init_next(left_idx, new_left.as_raw()); new_parent.init_next(right_idx, new_right.as_raw()); new_parent.init_key(left_idx, Some(pivot)); @@ -1467,9 +1559,9 @@ where }; gparent.store_next(cursor.gp_p_idx, new_parent.as_raw(), &gparent_lock); - node.marked.store(true, Ordering::Relaxed); - parent.marked.store(true, Ordering::Relaxed); - sibling.marked.store(true, Ordering::Relaxed); + left.set_marked_locked(true, &left_lock); + parent.set_marked_locked(true, &parent_lock); + right.set_marked_locked(true, &right_lock); unsafe { guard.retire_raw(cursor.l); From c8645c26ac5f111f7f8adbf3dd08d4b8d81a2b1f Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Sun, 10 Nov 2024 17:40:39 +0000 Subject: [PATCH 73/84] Remove incorrect `debug_assert`s in HP SkipList --- src/ds_impl/hp/skip_list.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ds_impl/hp/skip_list.rs b/src/ds_impl/hp/skip_list.rs index 6fab4f7e..9dc0f5f2 100644 --- a/src/ds_impl/hp/skip_list.rs +++ b/src/ds_impl/hp/skip_list.rs @@ -206,8 +206,6 @@ where if tag(curr) != 0 { // Validate on anchor. - debug_assert_ne!(untagged(anchor), untagged(pred)); - let an_new = unsafe { &*untagged(anchor) }.next[level].load(Ordering::Acquire); @@ -225,8 +223,6 @@ where } } else { // Validate on prev. - debug_assert_eq!(untagged(anchor), untagged(pred)); - let curr_new = unsafe { &*untagged(pred) }.next[level].load(Ordering::Acquire); From 2a5ea905908cb7a196e1084cf0bf98ccee17dbad Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Sun, 10 Nov 2024 19:37:28 +0000 Subject: [PATCH 74/84] Revise `search_basic` implementation in HP ElimAbTree --- src/ds_impl/hp/elim_ab_tree.rs | 148 +++++++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 26 deletions(-) diff --git a/src/ds_impl/hp/elim_ab_tree.rs b/src/ds_impl/hp/elim_ab_tree.rs index 072607c1..16a8c33f 100644 --- a/src/ds_impl/hp/elim_ab_tree.rs +++ b/src/ds_impl/hp/elim_ab_tree.rs @@ -172,11 +172,15 @@ impl Node { self.next()[index].load(Ordering::Acquire) } - fn protect_next(&self, index: usize, slot: &mut HazardPointer<'_>) -> Result, ()> { + fn protect_next_pess( + &self, + index: usize, + slot: &mut HazardPointer<'_>, + ) -> Result, ()> { let atomic = &self.next()[index]; let mut ptr = atomic.load(Ordering::Relaxed); loop { - slot.protect_raw(ptr.with_tag(0).into_raw()); + slot.protect_raw(ptr.into_raw()); light_membarrier(); let new = atomic.load(Ordering::Acquire); if ptr == new { @@ -190,6 +194,27 @@ impl Node { Ok(ptr) } + /// Stores the next pointer into the given slot. + /// It retries if the marked state or the pointer changes. + fn protect_next_consistent( + &self, + index: usize, + slot: &mut HazardPointer<'_>, + ) -> (Shared, bool) { + let mut marked = self.marked.load(Ordering::Acquire); + let mut ptr = self.load_next(index); + loop { + slot.protect_raw(ptr.into_raw()); + light_membarrier(); + let marked_new = self.marked.load(Ordering::Acquire); + let ptr_new = self.load_next(index); + if marked_new == marked && ptr_new == ptr { + return (ptr, marked); + } + (marked, ptr) = (marked_new, ptr_new); + } + } + fn store_next<'g>(&'g self, index: usize, ptr: impl Pointer, _: &MCSLockGuard<'g, K, V>) { self.next()[index].store(ptr, Ordering::Release); } @@ -520,6 +545,8 @@ pub struct Handle<'domain> { gp_h: HazardPointer<'domain>, /// A protector for the sibling node. s_h: HazardPointer<'domain>, + /// Used in `search_basic`, which does optimistic traversal. + c_h: HazardPointer<'domain>, thread: Box>, } @@ -531,6 +558,7 @@ impl Default for Handle<'static> { p_h: HazardPointer::new(&mut thread), gp_h: HazardPointer::new(&mut thread), s_h: HazardPointer::new(&mut thread), + c_h: HazardPointer::new(&mut thread), thread, } } @@ -583,13 +611,81 @@ where key: &K, handle: &'hp mut Handle<'_>, ) -> Result, ()> { - let mut node = unsafe { self.entry.protect_next(0, &mut handle.p_h)?.deref() }; - while !node.is_leaf() { - let next = node.protect_next(node.child_index(key), &mut handle.l_h)?; - HazardPointer::swap(&mut handle.l_h, &mut handle.p_h); - node = unsafe { next.deref() }; + let (a_h, an_h, p_h, l_h, n_h) = ( + &mut handle.gp_h, + &mut handle.s_h, + &mut handle.p_h, + &mut handle.l_h, + &mut handle.c_h, + ); + + let mut p = Shared::>::from(&self.entry as *const _ as usize); + let mut p_l_idx = 0; + let (mut l, mut l_marked) = self.entry.protect_next_consistent(p_l_idx, l_h); + + let mut a = Shared::>::null(); + let mut a_an_idx = 0; + let mut an = Shared::>::null(); + + loop { + // Validation depending on the state of `p`. + // + // - If it is marked, validate on anchor. + // - If it is not marked, it is already protected safely. + if l_marked { + // Validate on anchor. + debug_assert!(!a.is_null()); + debug_assert!(!an.is_null()); + let an_new = unsafe { a.deref() }.load_next(a_an_idx); + + if an != an_new { + if unsafe { a.deref() }.marked.load(Ordering::Acquire) { + return Err(()); + } + // Anchor is updated but clear, so can restart from anchor. + p = a; + p_l_idx = a_an_idx; + l = an_new; + l_marked = false; + a = Shared::null(); + + HazardPointer::swap(p_h, a_h); + continue; + } + } + + let l_node = unsafe { l.deref() }; + if l_node.is_leaf() { + return Ok(l_node.read_consistent(key).1); + } + + let l_n_idx = l_node.child_index(key); + let (n, n_marked) = l_node.protect_next_consistent(l_n_idx, n_h); + if n_marked { + if a.is_null() { + a = p; + a_an_idx = p_l_idx; + an = l; + HazardPointer::swap(a_h, p_h); + } else if an == p { + HazardPointer::swap(an_h, p_h); + } + p = l; + p_l_idx = l_n_idx; + l = n; + l_marked = n_marked; + HazardPointer::swap(l_h, p_h); + HazardPointer::swap(l_h, n_h); + } else { + p = l; + p_l_idx = l_n_idx; + l = n; + l_marked = n_marked; + a = Shared::null(); + HazardPointer::swap(l_h, p_h); + HazardPointer::swap(l_h, n_h); + } } - Ok(node.read_consistent(key).1) } fn search<'hp>( @@ -613,8 +709,8 @@ where handle: &'hp mut Handle<'_>, ) -> Result<(bool, Cursor), ()> { let mut cursor = Cursor { - l: self.entry.protect_next(0, &mut handle.l_h)?, - s: self.entry.protect_next(1, &mut handle.s_h)?, + l: self.entry.protect_next_pess(0, &mut handle.l_h)?, + s: self.entry.protect_next_pess(1, &mut handle.s_h)?, p: Shared::from(&self.entry as *const _ as usize), gp: Shared::null(), gp_p_idx: 0, @@ -635,8 +731,8 @@ where cursor.gp_p_idx = cursor.p_l_idx; cursor.p_l_idx = l_node.child_index(key); cursor.p_s_idx = Node::::p_s_idx(cursor.p_l_idx); - cursor.l = l_node.protect_next(cursor.p_l_idx, &mut handle.l_h)?; - cursor.s = l_node.protect_next(cursor.p_s_idx, &mut handle.s_h)?; + cursor.l = l_node.protect_next_pess(cursor.p_l_idx, &mut handle.l_h)?; + cursor.s = l_node.protect_next_pess(cursor.p_s_idx, &mut handle.s_h)?; } if let Some(target) = target { @@ -755,7 +851,7 @@ where // Manually unlock and fix the tag. drop((parent_lock, node_lock)); - unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; + unsafe { handle.thread.retire(cursor.l.into_raw()) }; self.fix_tag_violation(kv_pairs[left_size].0, new_internal, handle); Ok(None) @@ -858,8 +954,8 @@ where node.marked.store(true, Ordering::Release); parent.marked.store(true, Ordering::Release); - unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; - unsafe { handle.thread.retire(cursor.p.with_tag(0).into_raw()) }; + unsafe { handle.thread.retire(cursor.l.into_raw()) }; + unsafe { handle.thread.retire(cursor.p.into_raw()) }; return (true, None); } else { // Split case. @@ -903,8 +999,8 @@ where node.marked.store(true, Ordering::Release); parent.marked.store(true, Ordering::Release); - unsafe { handle.thread.retire(cursor.l.with_tag(0).into_raw()) }; - unsafe { handle.thread.retire(cursor.p.with_tag(0).into_raw()) }; + unsafe { handle.thread.retire(cursor.l.into_raw()) }; + unsafe { handle.thread.retire(cursor.p.into_raw()) }; drop((node_lock, parent_lock, gparent_lock)); return ( @@ -1156,9 +1252,9 @@ where sibling.marked.store(true, Ordering::Release); unsafe { - handle.thread.retire(cursor.l.with_tag(0).into_raw()); - handle.thread.retire(cursor.p.with_tag(0).into_raw()); - handle.thread.retire(cursor.s.with_tag(0).into_raw()); + handle.thread.retire(cursor.l.into_raw()); + handle.thread.retire(cursor.p.into_raw()); + handle.thread.retire(cursor.s.into_raw()); } drop((left_lock, right_lock, parent_lock, gparent_lock)); @@ -1196,9 +1292,9 @@ where sibling.marked.store(true, Ordering::Release); unsafe { - handle.thread.retire(cursor.l.with_tag(0).into_raw()); - handle.thread.retire(cursor.p.with_tag(0).into_raw()); - handle.thread.retire(cursor.s.with_tag(0).into_raw()); + handle.thread.retire(cursor.l.into_raw()); + handle.thread.retire(cursor.p.into_raw()); + handle.thread.retire(cursor.s.into_raw()); } drop((left_lock, right_lock, parent_lock, gparent_lock)); @@ -1310,9 +1406,9 @@ where sibling.marked.store(true, Ordering::Release); unsafe { - handle.thread.retire(cursor.l.with_tag(0).into_raw()); - handle.thread.retire(cursor.p.with_tag(0).into_raw()); - handle.thread.retire(cursor.s.with_tag(0).into_raw()); + handle.thread.retire(cursor.l.into_raw()); + handle.thread.retire(cursor.p.into_raw()); + handle.thread.retire(cursor.s.into_raw()); } return (true, ArrayVec::new()); From 52b59646460981143e023f89b1d20c9d3302b0c9 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Sun, 10 Nov 2024 19:42:53 +0000 Subject: [PATCH 75/84] Remove `run.sh` and add `+x` to all bash scripts --- run.sh | 9 --------- test-scripts/sanitize-crystalline-l.sh | 0 test-scripts/sanitize-elim.sh | 0 test-scripts/sanitize-he.sh | 0 4 files changed, 9 deletions(-) delete mode 100644 run.sh mode change 100644 => 100755 test-scripts/sanitize-crystalline-l.sh mode change 100644 => 100755 test-scripts/sanitize-elim.sh mode change 100644 => 100755 test-scripts/sanitize-he.sh diff --git a/run.sh b/run.sh deleted file mode 100644 index c1ad55fa..00000000 --- a/run.sh +++ /dev/null @@ -1,9 +0,0 @@ -cargo run --release --bin nr -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin ebr -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin pebr -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin hp -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin hp-pp -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin nbr -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin hp-brcu -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin vbr -- -dnm-tree -t92 -g0 -r100000 -i10 -cargo run --release --bin hp-rcu -- -dnm-tree -t92 -g0 -r100000 -i10 diff --git a/test-scripts/sanitize-crystalline-l.sh b/test-scripts/sanitize-crystalline-l.sh old mode 100644 new mode 100755 diff --git a/test-scripts/sanitize-elim.sh b/test-scripts/sanitize-elim.sh old mode 100644 new mode 100755 diff --git a/test-scripts/sanitize-he.sh b/test-scripts/sanitize-he.sh old mode 100644 new mode 100755 From d32eb248f9c37c3684af458291cf1d203569ae0a Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Fri, 1 Nov 2024 12:34:29 +0000 Subject: [PATCH 76/84] Update bench scripts --- .../hp-revisited/bench-short-lists.py | 99 +++++++++++++++++++ bench-scripts/hp-revisited/bench.py | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 bench-scripts/hp-revisited/bench-short-lists.py diff --git a/bench-scripts/hp-revisited/bench-short-lists.py b/bench-scripts/hp-revisited/bench-short-lists.py new file mode 100644 index 00000000..b2f6b047 --- /dev/null +++ b/bench-scripts/hp-revisited/bench-short-lists.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import subprocess +import os + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") +BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") + +dss = ['hhs-list', 'hm-list'] +# "-large" suffix if it uses a large garbage bag. +mms = ['hp', 'hp-pp'] +i = 10 +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = list(map(str, [1] + list(range(4, 33, 4)))) +elif cpu_count <= 64: + ts = list(map(str, [1] + list(range(8, 129, 8)))) +else: + ts = list(map(str, [1] + list(range(12, 193, 12)))) +runs = 2 +gs = [0, 1, 2] + +subprocess.run(['cargo', 'build', '--release']) + +def key_ranges(ds): + return ["16"] + +def is_suffix(orig, suf): + return len(suf) <= len(orig) and mm[-len(suf):] == suf + +def make_cmd(mm, i, ds, g, t, kr): + bag = "small" + if is_suffix(mm, "-large"): + mm = mm[:len(mm)-len("-large")] + bag = "large" + + return [os.path.join(BIN_PATH, mm), + '-i', str(i), + '-d', str(ds), + '-g', str(g), + '-t', str(t), + '-r', str(kr), + '-b', bag, + '-o', os.path.join(RESULTS_PATH, f'{ds}.csv')] + +def invalid(mm, ds, g): + is_invalid = False + if ds == 'hhs-list': + is_invalid |= g == 0 # HHSList is just HList with faster get() + if mm == 'nbr': + is_invalid |= ds in ["hm-list", "skip-list"] + if ds == 'elim-ab-tree': + is_invalid |= mm in ["pebr", "hp-pp", "vbr"] + return is_invalid + +cmds = [] + +for ds in dss: + for kr in key_ranges(ds): + for mm in mms: + for g in gs: + if invalid(mm, ds, g): + continue + for t in ts: + cmds.append(make_cmd(mm, i, ds, g, t, kr)) + +print('number of configurations: ', len(cmds)) +print('estimated time: ', (len(cmds) * i * 1.1) // 60, ' min *', runs, 'times\n') + +for i, cmd in enumerate(cmds): + try: + print(f"\rdry-running commands... ({i+1}/{len(cmds)})", end="") + subprocess.run(cmd + ['--dry-run']) + except: + print(f"A dry-run for the following command is failed:\n{' '.join(cmd)}") + exit(1) +print("\nAll dry-runs passed!\n") + +os.makedirs(RESULTS_PATH, exist_ok=True) +failed = [] +for run in range(runs): + for i, cmd in enumerate(cmds): + print("run {}/{}, bench {}/{}: '{}'".format(run + 1, runs, i + 1, len(cmds), ' '.join(cmd))) + try: + subprocess.run(cmd, timeout=i+30) + except subprocess.TimeoutExpired: + print("timeout") + failed.append(' '.join(cmd)) + except KeyboardInterrupt: + if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) + exit(0) + except: + failed.append(' '.join(cmd)) + +if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) diff --git a/bench-scripts/hp-revisited/bench.py b/bench-scripts/hp-revisited/bench.py index 9e341321..3381889c 100644 --- a/bench-scripts/hp-revisited/bench.py +++ b/bench-scripts/hp-revisited/bench.py @@ -6,7 +6,7 @@ RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") -dss = ['h-list', 'hhs-list', 'hash-map', 'nm-tree', 'skip-list', 'elim-ab-tree'] +dss = ['h-list', 'hm-list', 'hhs-list', 'hash-map', 'nm-tree', 'skip-list', 'elim-ab-tree'] # "-large" suffix if it uses a large garbage bag. mms = ['nr', 'ebr', 'pebr', 'hp', 'hp-pp', 'hp-brcu', 'vbr'] i = 10 From f176f0abaa1264351aaf28e3a7a514dffe3fc01a Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Thu, 7 Nov 2024 07:11:22 +0000 Subject: [PATCH 77/84] Update bench and plot scripts --- bench-scripts/hp-revisited/legends.py | 6 + .../hp-revisited/plot-short-lists.py | 151 ++++++++++++++++++ bench-scripts/hp-revisited/plot.py | 10 +- 3 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 bench-scripts/hp-revisited/plot-short-lists.py diff --git a/bench-scripts/hp-revisited/legends.py b/bench-scripts/hp-revisited/legends.py index f038e4a7..db77815b 100644 --- a/bench-scripts/hp-revisited/legends.py +++ b/bench-scripts/hp-revisited/legends.py @@ -64,6 +64,12 @@ "color": "orange", "linestyle": (0, (2, 1)), }, + # Used in `plot-short-lists` + "PESSIM_HP": { + "marker": "v", + "color": "#828282", + "linestyle": "dotted", + }, } # Add some common or derivable properties. diff --git a/bench-scripts/hp-revisited/plot-short-lists.py b/bench-scripts/hp-revisited/plot-short-lists.py new file mode 100644 index 00000000..8de0fe8c --- /dev/null +++ b/bench-scripts/hp-revisited/plot-short-lists.py @@ -0,0 +1,151 @@ +import pandas as pd +import warnings +import os, math +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from legends import * + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +warnings.filterwarnings("ignore") +pd.set_option('display.max_rows', None) + +# avoid Type 3 fonts +matplotlib.rcParams['pdf.fonttype'] = 42 +matplotlib.rcParams['ps.fonttype'] = 42 + +# raw column names +THREADS = "threads" +THROUGHPUT = "throughput" +PEAK_MEM = "peak_mem" +AVG_GARB = "avg_garb" +PEAK_GARB = "peak_garb" + +# legend +SMR_ONLY = "SMR\n" + +HMLIST = "hm-list" +HHSLIST = "hhs-list" + +# DS with read-dominated bench & write-only bench +dss_all = [HHSLIST, HMLIST] + +WRITE, HALF, READ = "write", "half", "read" + +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = [1] + list(range(4, 33, 4)) +elif cpu_count <= 64: + ts = [1] + list(range(8, 129, 8)) +else: + ts = [1] + list(range(12, 193, 12)) +n_map = {0: ''} + +(label_size, xtick_size, ytick_size, marker_size) = (24, 20, 18, 20) + +SMRs = [HP] +COMBs = [f"{HP}_{HHSLIST}", f"{HP}_{HMLIST}"] + +HHSLIST_SHAPE = line_shapes[HP] +HMLIST_SHAPE = line_shapes["PESSIM_HP"] + +def plot_title(bench): + return 'HHSList v.s. HMList' + +def range_to_str(kr: int): + UNITS = ["K", "M", "B", "T"] + for i in range(len(UNITS)): + unit = pow(10, 3 * (i+1)) + div = kr // unit + if div < 1000 or i == len(UNITS) - 1: + return f"{div}{UNITS[i]}" + +def draw(title, name, data, y_value, y_label=None, y_max=None, y_from_zero=False): + print(name) + plt.figure(figsize=(10, 7)) + plt.title(title, fontsize=36, pad=15) + + d = data[data.mm_ds == f"{HP}_{HHSLIST}"].sort_values(by=[THREADS], axis=0) + h1, = plt.plot(d[THREADS], d[y_value], label="HP; HHSList", + linewidth=3, markersize=marker_size, **HHSLIST_SHAPE, zorder=30) + + d = data[data.mm_ds == f"{HP}_{HMLIST}"].sort_values(by=[THREADS], axis=0) + h2, = plt.plot(d[THREADS], d[y_value], label="HP; HMList", + linewidth=3, markersize=marker_size, **HMLIST_SHAPE, zorder=30) + + plt.legend(handles=[h1, h2], fontsize=label_size, loc="lower right") + + plt.xlabel("Threads", fontsize=label_size) + plt.ylabel(y_label, fontsize=label_size) + plt.yticks(fontsize=ytick_size) + plt.xticks(ts, fontsize=xtick_size, rotation=90) + plt.grid(alpha=0.5) + + if data.threads.max() >= cpu_count: + left, right = plt.xlim() + plt.axvspan(cpu_count, right, facecolor="#FF00000A") + plt.xlim(left, right) + + y_max = min(y_max, data[y_value].max()) if y_max else data[y_value].max() + y_min = 0 if y_from_zero else data[y_value].min() + y_margin = (y_max - y_min) * 0.05 + plt.ylim(y_min, y_max+y_margin) + + plt.savefig(name, bbox_inches='tight') + + +def draw_throughput(data, bench): + data = data.copy() + y_label = 'Throughput (M op/s)' + y_max = data.throughput.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/short-lists_{bench}_throughput.pdf', + data, THROUGHPUT, y_label, y_max, True) + + +def draw_peak_garb(data, bench): + data = data.copy() + y_label = 'Peak unreclaimed blocks (×10⁴)' + y_max = data.peak_garb.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/short-lists_{bench}_peak_garb.pdf', + data, PEAK_GARB, y_label, y_max) + + +raw_data = {} +# averaged data for write:read = 100:0, 50:50, 10:90 +avg_data = { WRITE: {}, HALF: {}, READ: {} } + +# preprocess +csvs = [] +for ds in dss_all: + csvs.append(pd.read_csv(f'{RESULTS_PATH}/' + ds + '.csv')) + +data = pd.concat(csvs) +data.throughput = data.throughput.map(lambda x: x / 1000_000) +data.peak_mem = data.peak_mem.map(lambda x: x / (2 ** 20)) +data.avg_mem = data.avg_mem.map(lambda x: x / (2 ** 20)) +data.peak_garb = data.peak_garb.map(lambda x: x / 10000) +data.avg_garb = data.avg_garb.map(lambda x: x / 10000) +data["mm_ds"] = list(map(lambda p: p[0] + "_" + p[1], zip(data.mm, data.ds))) +data.mm = list(map(lambda tup: tup[0] if tup[1] == "small" else tup[0] + "-large", zip(data.mm, data.bag_size))) +data = data[data.mm.isin(SMRs)] +data = data[data.key_range == 16] +data = data.drop(["bag_size", "ds", "mm"], axis=1) + +# take average of each runs +avg = data.groupby(['mm_ds', 'threads', 'non_coop', 'get_rate', 'key_range']).mean().reset_index() + +avg[SMR_ONLY] = pd.Categorical(avg.mm_ds.map(str), COMBs) +avg.sort_values(by=SMR_ONLY, inplace=True) +for i, bench in [(0, WRITE), (1, HALF), (2, READ)]: + avg_data[bench] = avg[avg.get_rate == i] + +# 1. throughput graphs, 3 lines (SMR_ONLY) each. +draw_throughput(avg_data[WRITE], WRITE) +draw_throughput(avg_data[HALF], HALF) +draw_throughput(avg_data[READ], READ) + +# 2. peak garbage graph +draw_peak_garb(avg_data[WRITE], WRITE) +draw_peak_garb(avg_data[HALF], HALF) +draw_peak_garb(avg_data[READ], READ) diff --git a/bench-scripts/hp-revisited/plot.py b/bench-scripts/hp-revisited/plot.py index 90f4b70e..ee9cedb9 100644 --- a/bench-scripts/hp-revisited/plot.py +++ b/bench-scripts/hp-revisited/plot.py @@ -25,6 +25,7 @@ SMR_ONLY = "SMR\n" HLIST = "h-list" +HMLIST = "hm-list" HHSLIST = "hhs-list" HASHMAP = "hash-map" NMTREE = "nm-tree" @@ -33,6 +34,7 @@ FORMAL_NAMES = { HLIST: "HList", + HMLIST: "HMList", HHSLIST: "HHSList", HASHMAP: "HashMap", NMTREE: "NMTree", @@ -41,9 +43,9 @@ } # DS with read-dominated bench & write-only bench -dss_all = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] -dss_read = [HLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] -dss_write = [HLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] +dss_all = [HLIST, HMLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] +dss_read = [HLIST, HMLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] +dss_write = [HLIST, HMLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] WRITE, HALF, READ = "write", "half", "read" @@ -74,7 +76,7 @@ def plot_title(ds, bench): return FORMAL_NAMES[ds] def key_ranges(ds): - if ds in [HLIST, HHSLIST]: + if ds in [HLIST, HMLIST, HHSLIST]: return [1000, 10000] else: return [100000, 100000000] From 2e30fcb83d3b424809242c1826c1f1db4c9ee5ff Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 11 Nov 2024 04:29:27 +0000 Subject: [PATCH 78/84] Add PEBR- and VBR-based ElimAbTree --- bench-scripts/hp-revisited/bench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-scripts/hp-revisited/bench.py b/bench-scripts/hp-revisited/bench.py index 3381889c..f164404a 100644 --- a/bench-scripts/hp-revisited/bench.py +++ b/bench-scripts/hp-revisited/bench.py @@ -55,7 +55,7 @@ def invalid(mm, ds, g): if mm == 'nbr': is_invalid |= ds in ["hm-list", "skip-list"] if ds == 'elim-ab-tree': - is_invalid |= mm in ["pebr", "hp-pp", "vbr"] + is_invalid |= mm in ["hp-pp"] return is_invalid cmds = [] From 0f54e919134abc061280e527be79ee7efcdc0f02 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 11 Nov 2024 07:10:19 +0000 Subject: [PATCH 79/84] Check if the key is `Some` in `child_index` of VBR ElimAbTree --- src/ds_impl/vbr/elim_ab_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds_impl/vbr/elim_ab_tree.rs b/src/ds_impl/vbr/elim_ab_tree.rs index 00ac4a98..99b6ac26 100644 --- a/src/ds_impl/vbr/elim_ab_tree.rs +++ b/src/ds_impl/vbr/elim_ab_tree.rs @@ -411,7 +411,7 @@ where fn child_index(&self, key: &K, guard: &Guard) -> Result { let key_count = self.key_count(guard)?; let mut index = 0; - while index < key_count && !(key < &self.get_key(index, guard)?.unwrap()) { + while index < key_count && !(key < &some_or!(self.get_key(index, guard)?, return Err(()))) { index += 1; } guard.validate_epoch().map(|_| index) From 77dc68c92a257244ea8cd1043f9f57d8e461dd5d Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 11 Nov 2024 07:20:14 +0000 Subject: [PATCH 80/84] Add scripts for HP trees experiments --- bench-scripts/hp-revisited/bench-hp-trees.py | 99 ++++++++++++ bench-scripts/hp-revisited/plot-hp-trees.py | 151 +++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 bench-scripts/hp-revisited/bench-hp-trees.py create mode 100644 bench-scripts/hp-revisited/plot-hp-trees.py diff --git a/bench-scripts/hp-revisited/bench-hp-trees.py b/bench-scripts/hp-revisited/bench-hp-trees.py new file mode 100644 index 00000000..50be5525 --- /dev/null +++ b/bench-scripts/hp-revisited/bench-hp-trees.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import subprocess +import os + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") +BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") + +dss = ['nm-tree', 'efrb-tree'] +# "-large" suffix if it uses a large garbage bag. +mms = ['hp'] +i = 10 +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = list(map(str, [1] + list(range(4, 33, 4)))) +elif cpu_count <= 64: + ts = list(map(str, [1] + list(range(8, 129, 8)))) +else: + ts = list(map(str, [1] + list(range(12, 193, 12)))) +runs = 2 +gs = [0, 1, 2] + +subprocess.run(['cargo', 'build', '--release']) + +def key_ranges(ds): + return ["100000"] + +def is_suffix(orig, suf): + return len(suf) <= len(orig) and mm[-len(suf):] == suf + +def make_cmd(mm, i, ds, g, t, kr): + bag = "small" + if is_suffix(mm, "-large"): + mm = mm[:len(mm)-len("-large")] + bag = "large" + + return [os.path.join(BIN_PATH, mm), + '-i', str(i), + '-d', str(ds), + '-g', str(g), + '-t', str(t), + '-r', str(kr), + '-b', bag, + '-o', os.path.join(RESULTS_PATH, f'{ds}.csv')] + +def invalid(mm, ds, g): + is_invalid = False + if ds == 'hhs-list': + is_invalid |= g == 0 # HHSList is just HList with faster get() + if mm == 'nbr': + is_invalid |= ds in ["hm-list", "skip-list"] + if ds == 'elim-ab-tree': + is_invalid |= mm in ["pebr", "hp-pp", "vbr"] + return is_invalid + +cmds = [] + +for ds in dss: + for kr in key_ranges(ds): + for mm in mms: + for g in gs: + if invalid(mm, ds, g): + continue + for t in ts: + cmds.append(make_cmd(mm, i, ds, g, t, kr)) + +print('number of configurations: ', len(cmds)) +print('estimated time: ', (len(cmds) * i * 1.1) // 60, ' min *', runs, 'times\n') + +for i, cmd in enumerate(cmds): + try: + print(f"\rdry-running commands... ({i+1}/{len(cmds)})", end="") + subprocess.run(cmd + ['--dry-run']) + except: + print(f"A dry-run for the following command is failed:\n{' '.join(cmd)}") + exit(1) +print("\nAll dry-runs passed!\n") + +os.makedirs(RESULTS_PATH, exist_ok=True) +failed = [] +for run in range(runs): + for i, cmd in enumerate(cmds): + print("run {}/{}, bench {}/{}: '{}'".format(run + 1, runs, i + 1, len(cmds), ' '.join(cmd))) + try: + subprocess.run(cmd, timeout=i+30) + except subprocess.TimeoutExpired: + print("timeout") + failed.append(' '.join(cmd)) + except KeyboardInterrupt: + if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) + exit(0) + except: + failed.append(' '.join(cmd)) + +if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) diff --git a/bench-scripts/hp-revisited/plot-hp-trees.py b/bench-scripts/hp-revisited/plot-hp-trees.py new file mode 100644 index 00000000..29862194 --- /dev/null +++ b/bench-scripts/hp-revisited/plot-hp-trees.py @@ -0,0 +1,151 @@ +import pandas as pd +import warnings +import os, math +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from legends import * + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +warnings.filterwarnings("ignore") +pd.set_option('display.max_rows', None) + +# avoid Type 3 fonts +matplotlib.rcParams['pdf.fonttype'] = 42 +matplotlib.rcParams['ps.fonttype'] = 42 + +# raw column names +THREADS = "threads" +THROUGHPUT = "throughput" +PEAK_MEM = "peak_mem" +AVG_GARB = "avg_garb" +PEAK_GARB = "peak_garb" + +# legend +SMR_ONLY = "SMR\n" + +EFRBTREE = "efrb-tree" +NMTREE = "nm-tree" + +# DS with read-dominated bench & write-only bench +dss_all = [NMTREE, EFRBTREE] + +WRITE, HALF, READ = "write", "half", "read" + +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = [1] + list(range(4, 33, 4)) +elif cpu_count <= 64: + ts = [1] + list(range(8, 129, 8)) +else: + ts = [1] + list(range(12, 193, 12)) +n_map = {0: ''} + +(label_size, xtick_size, ytick_size, marker_size) = (24, 20, 18, 20) + +SMRs = [HP] +COMBs = [f"{HP}_{NMTREE}", f"{HP}_{EFRBTREE}"] + +NMTREE_SHAPE = line_shapes[HP] +EFRBTREE_SHAPE = line_shapes["PESSIM_HP"] + +def plot_title(bench): + return 'NMTree v.s. EFRBTree' + +def range_to_str(kr: int): + UNITS = ["K", "M", "B", "T"] + for i in range(len(UNITS)): + unit = pow(10, 3 * (i+1)) + div = kr // unit + if div < 1000 or i == len(UNITS) - 1: + return f"{div}{UNITS[i]}" + +def draw(title, name, data, y_value, y_label=None, y_max=None, y_from_zero=False): + print(name) + plt.figure(figsize=(10, 7)) + plt.title(title, fontsize=36, pad=15) + + d = data[data.mm_ds == f"{HP}_{NMTREE}"].sort_values(by=[THREADS], axis=0) + h1, = plt.plot(d[THREADS], d[y_value], label="HP; NMTREE", + linewidth=3, markersize=marker_size, **NMTREE_SHAPE, zorder=30) + + d = data[data.mm_ds == f"{HP}_{EFRBTREE}"].sort_values(by=[THREADS], axis=0) + h2, = plt.plot(d[THREADS], d[y_value], label="HP; EFRBTREE", + linewidth=3, markersize=marker_size, **EFRBTREE_SHAPE, zorder=30) + + plt.legend(handles=[h1, h2], fontsize=label_size, loc="lower right") + + plt.xlabel("Threads", fontsize=label_size) + plt.ylabel(y_label, fontsize=label_size) + plt.yticks(fontsize=ytick_size) + plt.xticks(ts, fontsize=xtick_size, rotation=90) + plt.grid(alpha=0.5) + + if data.threads.max() >= cpu_count: + left, right = plt.xlim() + plt.axvspan(cpu_count, right, facecolor="#FF00000A") + plt.xlim(left, right) + + y_max = min(y_max, data[y_value].max()) if y_max else data[y_value].max() + y_min = 0 if y_from_zero else data[y_value].min() + y_margin = (y_max - y_min) * 0.05 + plt.ylim(y_min, y_max+y_margin) + + plt.savefig(name, bbox_inches='tight') + + +def draw_throughput(data, bench): + data = data.copy() + y_label = 'Throughput (M op/s)' + y_max = data.throughput.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/hp-trees_{bench}_throughput.pdf', + data, THROUGHPUT, y_label, y_max, True) + + +def draw_peak_garb(data, bench): + data = data.copy() + y_label = 'Peak unreclaimed nodes (×10⁴)' + y_max = data.peak_garb.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/hp-trees_{bench}_peak_garb.pdf', + data, PEAK_GARB, y_label, y_max) + + +raw_data = {} +# averaged data for write:read = 100:0, 50:50, 10:90 +avg_data = { WRITE: {}, HALF: {}, READ: {} } + +# preprocess +csvs = [] +for ds in dss_all: + csvs.append(pd.read_csv(f'{RESULTS_PATH}/' + ds + '.csv')) + +data = pd.concat(csvs) +data.throughput = data.throughput.map(lambda x: x / 1000_000) +data.peak_mem = data.peak_mem.map(lambda x: x / (2 ** 20)) +data.avg_mem = data.avg_mem.map(lambda x: x / (2 ** 20)) +data.peak_garb = data.peak_garb.map(lambda x: x / 10000) +data.avg_garb = data.avg_garb.map(lambda x: x / 10000) +data["mm_ds"] = list(map(lambda p: p[0] + "_" + p[1], zip(data.mm, data.ds))) +data.mm = list(map(lambda tup: tup[0] if tup[1] == "small" else tup[0] + "-large", zip(data.mm, data.bag_size))) +data = data[data.mm.isin(SMRs)] +data = data[data.key_range == 100000] +data = data.drop(["bag_size", "ds", "mm"], axis=1) + +# take average of each runs +avg = data.groupby(['mm_ds', 'threads', 'non_coop', 'get_rate', 'key_range']).mean().reset_index() + +avg[SMR_ONLY] = pd.Categorical(avg.mm_ds.map(str), COMBs) +avg.sort_values(by=SMR_ONLY, inplace=True) +for i, bench in [(0, WRITE), (1, HALF), (2, READ)]: + avg_data[bench] = avg[avg.get_rate == i] + +# 1. throughput graphs, 3 lines (SMR_ONLY) each. +draw_throughput(avg_data[WRITE], WRITE) +draw_throughput(avg_data[HALF], HALF) +draw_throughput(avg_data[READ], READ) + +# 2. peak garbage graph +draw_peak_garb(avg_data[WRITE], WRITE) +draw_peak_garb(avg_data[HALF], HALF) +draw_peak_garb(avg_data[READ], READ) From 1f6ad7294b9bee80e4361e908276bdd32ad7becd Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Mon, 11 Nov 2024 07:20:54 +0000 Subject: [PATCH 81/84] Replace "unreclaimed blocks" with "unreclaimed nodes" --- bench-scripts/hp-revisited/plot-short-lists.py | 2 +- bench-scripts/hp-revisited/plot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bench-scripts/hp-revisited/plot-short-lists.py b/bench-scripts/hp-revisited/plot-short-lists.py index 8de0fe8c..a4a54489 100644 --- a/bench-scripts/hp-revisited/plot-short-lists.py +++ b/bench-scripts/hp-revisited/plot-short-lists.py @@ -105,7 +105,7 @@ def draw_throughput(data, bench): def draw_peak_garb(data, bench): data = data.copy() - y_label = 'Peak unreclaimed blocks (×10⁴)' + y_label = 'Peak unreclaimed nodes (×10⁴)' y_max = data.peak_garb.max() * 1.05 draw(plot_title(bench), f'{RESULTS_PATH}/short-lists_{bench}_peak_garb.pdf', data, PEAK_GARB, y_label, y_max) diff --git a/bench-scripts/hp-revisited/plot.py b/bench-scripts/hp-revisited/plot.py index ee9cedb9..550629f0 100644 --- a/bench-scripts/hp-revisited/plot.py +++ b/bench-scripts/hp-revisited/plot.py @@ -133,7 +133,7 @@ def draw_peak_garb(data, ds, bench, key_range): data = data[data.key_range == key_range] data = data[data.mm != NR] data = data[data.mm != VBR] - y_label = 'Peak unreclaimed blocks (×10⁴)' + y_label = 'Peak unreclaimed nodes (×10⁴)' y_max = 0 for cand in [HP_PP, HP_BRCU]: max_garb = data[data.mm == cand].peak_garb.max() From cb17389ae85434f8d509f53d012b55807fe8c71c Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 27 Nov 2024 10:28:15 +0000 Subject: [PATCH 82/84] Remove unused materials --- Cargo.lock | 22 - Cargo.toml | 4 - smrs/crystalline-l/Cargo.toml | 10 - smrs/crystalline-l/src/lib.rs | 734 ------------------ smrs/he/Cargo.toml | 10 - smrs/he/src/lib.rs | 565 -------------- src/bin/crystalline-l.rs | 217 ------ src/bin/he.rs | 215 ----- src/ds_impl/crystalline_l/concurrent_map.rs | 107 --- src/ds_impl/crystalline_l/list.rs | 689 ---------------- src/ds_impl/crystalline_l/michael_hash_map.rs | 104 --- src/ds_impl/crystalline_l/mod.rs | 11 - .../crystalline_l/natarajan_mittal_tree.rs | 689 ---------------- src/ds_impl/he/concurrent_map.rs | 98 --- src/ds_impl/he/list.rs | 658 ---------------- src/ds_impl/he/michael_hash_map.rs | 102 --- src/ds_impl/he/mod.rs | 11 - src/ds_impl/he/natarajan_mittal_tree.rs | 683 ---------------- src/ds_impl/mod.rs | 2 - test-scripts/sanitize-crystalline-l.sh | 12 - test-scripts/sanitize-he.sh | 12 - 21 files changed, 4955 deletions(-) delete mode 100644 smrs/crystalline-l/Cargo.toml delete mode 100644 smrs/crystalline-l/src/lib.rs delete mode 100644 smrs/he/Cargo.toml delete mode 100644 smrs/he/src/lib.rs delete mode 100644 src/bin/crystalline-l.rs delete mode 100644 src/bin/he.rs delete mode 100644 src/ds_impl/crystalline_l/concurrent_map.rs delete mode 100644 src/ds_impl/crystalline_l/list.rs delete mode 100644 src/ds_impl/crystalline_l/michael_hash_map.rs delete mode 100644 src/ds_impl/crystalline_l/mod.rs delete mode 100644 src/ds_impl/crystalline_l/natarajan_mittal_tree.rs delete mode 100644 src/ds_impl/he/concurrent_map.rs delete mode 100644 src/ds_impl/he/list.rs delete mode 100644 src/ds_impl/he/michael_hash_map.rs delete mode 100644 src/ds_impl/he/mod.rs delete mode 100644 src/ds_impl/he/natarajan_mittal_tree.rs delete mode 100755 test-scripts/sanitize-crystalline-l.sh delete mode 100755 test-scripts/sanitize-he.sh diff --git a/Cargo.lock b/Cargo.lock index 9c115545..bf342462 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -296,16 +296,6 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" -[[package]] -name = "crystalline-l" -version = "0.1.0" -dependencies = [ - "atomic", - "crossbeam-utils 0.8.20", - "rustc-hash", - "static_assertions 1.1.0", -] - [[package]] name = "csv" version = "1.3.0" @@ -360,16 +350,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "he" -version = "0.1.0" -dependencies = [ - "atomic", - "crossbeam-utils 0.8.20", - "rustc-hash", - "static_assertions 1.1.0", -] - [[package]] name = "heck" version = "0.5.0" @@ -836,9 +816,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-pebr-epoch", "crossbeam-utils 0.8.20", - "crystalline-l", "csv", - "he", "hp-brcu", "hp_pp", "nbr", diff --git a/Cargo.toml b/Cargo.toml index 6a33523f..202b860e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,6 @@ members = [ "./smrs/hp-brcu", "./smrs/vbr", "./smrs/circ", - "./smrs/crystalline-l", - "./smrs/he", ] [package] @@ -36,8 +34,6 @@ cdrc = { path = "./smrs/cdrc" } hp-brcu = { path = "./smrs/hp-brcu" } vbr = { path = "./smrs/vbr" } circ = { path = "./smrs/circ" } -crystalline-l = { path = "./smrs/crystalline-l" } -he = { path = "./smrs/he" } [target.'cfg(target_os = "linux")'.dependencies] tikv-jemallocator = "0.5" diff --git a/smrs/crystalline-l/Cargo.toml b/smrs/crystalline-l/Cargo.toml deleted file mode 100644 index 4a3535b9..00000000 --- a/smrs/crystalline-l/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "crystalline-l" -version = "0.1.0" -edition = "2021" - -[dependencies] -crossbeam-utils = "0.8.14" -rustc-hash = "1.1.0" -atomic = "0.5" -static_assertions = "1.1.0" diff --git a/smrs/crystalline-l/src/lib.rs b/smrs/crystalline-l/src/lib.rs deleted file mode 100644 index 32815e21..00000000 --- a/smrs/crystalline-l/src/lib.rs +++ /dev/null @@ -1,734 +0,0 @@ -use std::cell::{Cell, RefCell, RefMut, UnsafeCell}; -use std::mem::{align_of, size_of, swap, ManuallyDrop}; -use std::ptr::null_mut; -use std::sync::atomic::{AtomicIsize, AtomicPtr, AtomicUsize, Ordering}; -use std::sync::Mutex; - -use crossbeam_utils::CachePadded; -use static_assertions::const_assert_eq; - -const SLOTS_CAP: usize = 16; -const CACHE_CAP: usize = 12; -static COLLECT_FREQ: AtomicUsize = AtomicUsize::new(110); - -pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); - -pub fn set_collect_frequency(freq: usize) { - COLLECT_FREQ.store(freq, Ordering::Relaxed); -} - -const fn invalid_ptr() -> *mut T { - usize::MAX as *mut T -} - -#[repr(C)] -struct Node { - item: T, - info: UnsafeCell>, -} - -impl Node { - fn new(item: T, birth_era: usize) -> Self { - Self { - item, - info: UnsafeCell::new(NodeInfo::alive(birth_era)), - } - } - - unsafe fn birth_era(&self) -> usize { - (*self.info.get()).birth_era - } - - unsafe fn next(&self) -> &AtomicPtr> { - &(*self.info.get()).ret_info.ns.next - } - - unsafe fn set_next(&self, ptr: *mut Node, order: Ordering) { - (*self.info.get()).ret_info.ns.next.store(ptr, order); - } - - unsafe fn slot(&self) -> *mut AtomicPtr> { - (*self.info.get()).ret_info.ns.slot - } - - unsafe fn set_slot(&self, ptr: *mut AtomicPtr>) { - (*(*self.info.get()).ret_info).ns.slot = ptr - } - - unsafe fn batch_link(&self) -> *mut Node { - (*self.info.get()).ret_info.batch_link - } - - unsafe fn set_batch_link(&self, ptr: *mut Node) { - (*(*self.info.get()).ret_info).batch_link = ptr; - } - - unsafe fn refs(&self) -> &AtomicIsize { - &(*self.info.get()).ret_info.rb.refs - } - - unsafe fn batch_next(&self) -> *mut Node { - (*self.info.get()).ret_info.rb.batch_next - } - - unsafe fn set_batch_next(&self, ptr: *mut Node) { - (*(*self.info.get()).ret_info).rb.batch_next = ptr; - } - - #[inline] - fn free_list(&self) { - let mut curr = (self as *const Node).cast_mut(); - unsafe { - while let Some(curr_node) = curr.as_ref() { - let mut start = curr_node.batch_link(); - curr = curr_node.next().load(Ordering::SeqCst); - - let mut count = 0; - loop { - let obj = start; - start = (*obj).batch_next(); - count += 1; - drop(Box::from_raw(obj)); - if start.is_null() { - break; - } - } - GLOBAL_GARBAGE_COUNT.fetch_sub(count, Ordering::AcqRel); - } - } - } -} - -#[repr(C)] -union NodeInfo { - birth_era: usize, - ret_info: ManuallyDrop>, -} - -#[repr(C)] -struct RetiredInfo { - ns: NextOrSlot, - batch_link: *mut Node, - rb: RefsOrBatNxt, -} - -#[repr(C)] -union NextOrSlot { - next: ManuallyDrop>>, - slot: *mut AtomicPtr>, -} - -#[repr(C)] -union RefsOrBatNxt { - refs: ManuallyDrop, - batch_next: *mut Node, -} - -impl NodeInfo { - fn alive(birth_era: usize) -> Self { - Self { birth_era } - } -} - -struct Batch { - min_era: usize, - first: *mut Node, - last: *mut Node, - count: usize, - list_count: usize, - list: *mut Node, -} - -impl Default for Batch { - fn default() -> Self { - Self { - min_era: 0, - first: null_mut(), - last: null_mut(), - count: 0, - list_count: 0, - list: null_mut(), - } - } -} - -impl Batch { - fn traverse(&mut self, next: *mut Node) { - let mut curr = next; - unsafe { - while let Some(curr_node) = curr.as_ref() { - curr = curr_node.next().load(Ordering::Acquire); - let refs = &*curr_node.batch_link(); - if refs.refs().fetch_sub(1, Ordering::AcqRel) == 1 { - // Here, we have exclusive ownership. - refs.set_next(self.list, Ordering::SeqCst); - self.list = refs as *const _ as *mut _; - } - } - } - } - - fn traverse_cache(&mut self, next: *mut Node) { - if next.is_null() { - return; - } - if self.list_count == CACHE_CAP { - if let Some(list) = unsafe { self.list.as_ref() } { - list.free_list(); - } - self.list = null_mut(); - self.list_count = 0; - } - self.list_count += 1; - self.traverse(next); - } -} - -#[repr(C)] -struct EraSlots { - first: [AtomicPtr>; SLOTS_CAP], - era: [AtomicUsize; SLOTS_CAP], -} - -// Some static checks for a hack in `try_retire`. -const_assert_eq!(size_of::>>(), size_of::()); -const_assert_eq!( - size_of::>(), - size_of::() * SLOTS_CAP * 2 -); - -impl Default for EraSlots { - fn default() -> Self { - // Derive macro requires `T: Default`, which is unnecessary. - Self { - first: [(); SLOTS_CAP].map(|_| AtomicPtr::new(invalid_ptr())), - era: Default::default(), - } - } -} - -pub struct Domain { - slots: Vec>>, - batches: Vec>>>, - era: CachePadded, - avail_hidx: Mutex>, -} - -unsafe impl Sync for Domain {} -unsafe impl Send for Domain {} - -impl Drop for Domain { - fn drop(&mut self) { - debug_assert_eq!(self.avail_hidx.lock().unwrap().len(), self.slots.len()); - let handles = (0..self.slots.len()) - .map(|_| self.register()) - .collect::>(); - for handle in handles { - unsafe { handle.clear_all() }; - let batch = handle.batch_mut(); - if !batch.first.is_null() { - let mut curr = batch.first; - while curr != batch.last { - let next = unsafe { (*curr).batch_next() }; - unsafe { drop(Box::from_raw(curr)) }; - curr = next; - } - unsafe { drop(Box::from_raw(batch.last)) }; - } - } - } -} - -impl Domain { - pub fn new(handles: usize) -> Self { - Self { - slots: (0..handles).map(|_| Default::default()).collect(), - batches: (0..handles).map(|_| Default::default()).collect(), - era: CachePadded::new(AtomicUsize::new(1)), - avail_hidx: Mutex::new((0..handles).collect()), - } - } - - fn load_era(&self) -> usize { - self.era.load(Ordering::Acquire) - } - - pub fn register(&self) -> Handle<'_, T> { - Handle { - domain: self, - hidx: self.avail_hidx.lock().unwrap().pop().unwrap(), - alloc_count: Cell::new(0), - avail_sidx: RefCell::new((0..SLOTS_CAP).collect()), - } - } -} - -pub struct Handle<'d, T> { - domain: &'d Domain, - hidx: usize, - alloc_count: Cell, - avail_sidx: RefCell>, -} - -impl<'d, T> Handle<'d, T> { - fn incr_alloc(&self) { - let next_count = self.alloc_count.get() + 1; - self.alloc_count.set(next_count); - if next_count % COLLECT_FREQ.load(Ordering::Relaxed) == 0 { - self.domain.era.fetch_add(1, Ordering::AcqRel); - } - } - - pub fn global_era(&self) -> usize { - self.domain.load_era() - } - - fn slots(&self) -> &EraSlots { - &self.domain.slots[self.hidx] - } - - fn batch_mut(&self) -> RefMut<'_, Batch> { - self.domain.batches[self.hidx].borrow_mut() - } - - pub unsafe fn retire(&self, ptr: Shared) { - debug_assert!(!ptr.is_null()); - let node = unsafe { ptr.ptr.deref() }; - let mut batch = self.batch_mut(); - if batch.first.is_null() { - batch.min_era = unsafe { node.birth_era() }; - batch.last = node as *const _ as *mut _; - } else { - let birth_era = unsafe { node.birth_era() }; - if batch.min_era > birth_era { - batch.min_era = birth_era; - } - unsafe { node.set_batch_link(batch.last) }; - } - - // Implicitly initialize refs to 0 for the last node. - unsafe { node.set_batch_next(batch.first) }; - - let node_ptr = node as *const _ as *mut Node; - batch.first = node_ptr; - batch.count += 1; - if batch.count % COLLECT_FREQ.load(Ordering::Relaxed) == 0 { - unsafe { (*batch.last).set_batch_link(node_ptr) }; - self.try_retire(batch); - } - } - - fn try_retire(&self, mut batch: RefMut<'_, Batch>) { - let mut curr = batch.first; - let refs = batch.last; - let min_era = batch.min_era; - - // Find available slots. - let mut last = curr; - for slot in &self.domain.slots { - for i in 0..SLOTS_CAP { - let first = slot.first[i].load(Ordering::Acquire); - if first == invalid_ptr() { - continue; - } - let era = slot.era[i].load(Ordering::Acquire); - if era < min_era { - continue; - } - if last == refs { - return; - } - unsafe { (*last).set_slot(&slot.first[i] as *const _ as *mut _) }; - last = unsafe { (*last).batch_next() }; - } - } - - // Retire if successful. - GLOBAL_GARBAGE_COUNT.fetch_add(batch.count, Ordering::AcqRel); - let mut adjs = 0; - while curr != last { - 'body: { - unsafe { - let slot_first = (*curr).slot(); - // HACK: Get a corresponding era. Hint: See the layout of `EraSlots`... - let slot_era = slot_first.offset(SLOTS_CAP as isize) as *mut AtomicUsize; - let mut prev = (*slot_first).load(Ordering::Acquire); - - loop { - if prev == invalid_ptr() { - break 'body; - } - let era = (*slot_era).load(Ordering::Acquire); - if era < min_era { - break 'body; - } - (*curr).set_next(prev, Ordering::Relaxed); - match (*slot_first).compare_exchange_weak( - prev, - curr, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => break, - Err(new) => prev = new, - } - } - adjs += 1; - } - } - curr = unsafe { (*curr).batch_next() }; - } - - // Adjust the reference count. - unsafe { - if (*refs).refs().fetch_add(adjs, Ordering::AcqRel) == -adjs { - (*refs).set_next(null_mut(), Ordering::SeqCst); - (*refs).free_list(); - } - } - batch.first = null_mut(); - batch.count = 0; - } - - pub unsafe fn clear_all(&self) { - let mut first = [null_mut(); SLOTS_CAP]; - for i in 0..SLOTS_CAP { - first[i] = self.slots().first[i].swap(invalid_ptr(), Ordering::AcqRel); - } - for i in 0..SLOTS_CAP { - if first[i] != invalid_ptr() { - self.batch_mut().traverse(first[i]); - } - } - let mut batch = self.batch_mut(); - if let Some(list) = batch.list.as_ref() { - list.free_list(); - } - batch.list = null_mut(); - batch.list_count = 0; - } -} - -impl<'d, T> Drop for Handle<'d, T> { - fn drop(&mut self) { - self.domain.avail_hidx.lock().unwrap().push(self.hidx); - } -} - -pub struct HazardEra<'d, 'h, T> { - handle: &'h Handle<'d, T>, - sidx: usize, -} - -impl<'d, 'h, T> HazardEra<'d, 'h, T> { - pub fn new(handle: &'h Handle<'d, T>) -> Self { - let sidx = handle.avail_sidx.borrow_mut().pop().unwrap(); - Self { handle, sidx } - } - - pub fn era(&self) -> usize { - self.handle.slots().era[self.sidx].load(Ordering::Acquire) - } - - pub fn update_era(&mut self, mut curr_era: usize) -> usize { - let first_link = &self.handle.slots().first[self.sidx]; - if !first_link.load(Ordering::Acquire).is_null() { - let first = first_link.swap(invalid_ptr(), Ordering::AcqRel); - if first != invalid_ptr() { - self.handle.batch_mut().traverse_cache(first); - } - first_link.store(null_mut(), Ordering::SeqCst); - curr_era = self.handle.global_era(); - } - self.handle.slots().era[self.sidx].store(curr_era, Ordering::SeqCst); - return curr_era; - } - - pub fn swap(h1: &mut Self, h2: &mut Self) { - swap(&mut h1.sidx, &mut h2.sidx); - } - - pub fn clear(&mut self) { - self.handle.slots().era[self.sidx].store(0, Ordering::Release); - } -} - -impl<'d, 'h, T> Drop for HazardEra<'d, 'h, T> { - fn drop(&mut self) { - self.handle.avail_sidx.borrow_mut().push(self.sidx); - } -} - -pub struct Shared { - ptr: TaggedPtr>, -} - -impl<'d, T> From for Shared { - fn from(value: usize) -> Self { - Self { - ptr: TaggedPtr::from(value as *const _), - } - } -} - -impl<'d, T> Clone for Shared { - fn clone(&self) -> Self { - Self { ptr: self.ptr } - } -} - -impl<'d, T> Copy for Shared {} - -impl<'d, T> Shared { - pub fn new(item: T, handle: &Handle<'d, T>) -> Self { - handle.incr_alloc(); - let era = handle.global_era(); - Self { - ptr: TaggedPtr::from(Box::into_raw(Box::new(Node::new(item, era)))), - } - } - - pub fn null() -> Self { - Self { - ptr: TaggedPtr::null(), - } - } - - pub fn tag(&self) -> usize { - self.ptr.tag() - } - - pub fn is_null(&self) -> bool { - self.ptr.is_null() - } - - pub fn with_tag(&self, tag: usize) -> Self { - Self::from_raw(self.ptr.with_tag(tag)) - } - - pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { - self.ptr.as_ref().map(|node| &node.item) - } - - pub unsafe fn deref<'g>(&self) -> &'g T { - &self.ptr.deref().item - } - - pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { - &mut self.ptr.deref_mut().item - } - - fn from_raw(ptr: TaggedPtr>) -> Self { - Self { ptr } - } - - pub unsafe fn into_owned(self) -> T { - Box::from_raw(self.ptr.as_raw()).item - } - - /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, - /// are identical. - pub fn ptr_eq(self, other: Self) -> bool { - self.ptr.ptr_eq(other.ptr) - } -} - -// Technically, `Atomic` and `Shared` should depend on the lifetime of the domain `'d`, -// but it can create a circular lifetime dependency because it will make `T` in `Domain` -// also depend on `'d`. -pub struct Atomic { - link: atomic::Atomic>>, -} - -const_assert_eq!( - size_of::>>>(), - size_of::() -); - -unsafe impl Sync for Atomic {} -unsafe impl Send for Atomic {} - -impl Default for Atomic { - fn default() -> Self { - Self::null() - } -} - -impl From> for Atomic { - fn from(value: Shared) -> Self { - Self { - link: atomic::Atomic::new(value.ptr), - } - } -} - -impl Atomic { - pub fn new(init: T, handle: &Handle) -> Self { - Self { - link: atomic::Atomic::new(Shared::new(init, handle).ptr), - } - } - - pub fn null() -> Self { - Self { - link: atomic::Atomic::new(TaggedPtr::null()), - } - } - - pub fn load(&self, order: Ordering) -> Shared { - Shared::from_raw(self.link.load(order)) - } - - pub fn store(&self, ptr: Shared, order: Ordering) { - self.link.store(ptr.ptr, order); - } - - pub fn fetch_or(&self, val: usize, order: Ordering) -> Shared { - let prev = unsafe { &*(&self.link as *const _ as *const AtomicUsize) }.fetch_or(val, order); - Shared::from_raw(TaggedPtr::from(prev as *const _)) - } - - /// Loads and protects the pointer by the original CrystallineL's way. - /// - /// Hint: For traversal based structures where need to check tag, the guarnatee is similar to - /// `load` + `HazardPointer::set`. For Tstack, the guarnatee is similar to - /// `HazardPointer::protect`. - pub fn protect<'d, 'h>(&self, he: &mut HazardEra<'d, 'h, T>) -> Shared { - let mut prev_era = he.era(); - loop { - // Somewhat similar to atomically load and protect in Hazard pointers. - let ptr = self.link.load(Ordering::Acquire); - let curr_era = he.handle.global_era(); - if curr_era == prev_era { - return Shared::from_raw(ptr); - } - prev_era = he.update_era(curr_era); - } - } - - pub fn compare_exchange( - &self, - current: Shared, - new: Shared, - success: Ordering, - failure: Ordering, - ) -> Result, CompareExchangeError> { - match self - .link - .compare_exchange(current.ptr, new.ptr, success, failure) - { - Ok(current) => Ok(Shared::from_raw(current)), - Err(current) => Err(CompareExchangeError { - new, - current: Shared::from_raw(current), - }), - } - } - - pub unsafe fn into_owned(self) -> T { - Box::from_raw(self.link.into_inner().as_raw()).item - } -} - -pub struct CompareExchangeError { - pub new: Shared, - pub current: Shared, -} - -struct TaggedPtr { - ptr: *mut T, -} - -impl Default for TaggedPtr { - fn default() -> Self { - Self { ptr: null_mut() } - } -} - -impl Clone for TaggedPtr { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for TaggedPtr {} - -impl From<*const T> for TaggedPtr { - fn from(value: *const T) -> Self { - Self { - ptr: value.cast_mut(), - } - } -} - -impl From<*mut T> for TaggedPtr { - fn from(value: *mut T) -> Self { - Self { ptr: value } - } -} - -impl TaggedPtr { - pub fn null() -> Self { - Self { ptr: null_mut() } - } - - pub fn is_null(&self) -> bool { - self.as_raw().is_null() - } - - pub fn tag(&self) -> usize { - let ptr = self.ptr as usize; - ptr & low_bits::() - } - - /// Converts this pointer to a raw pointer (without the tag). - pub fn as_raw(&self) -> *mut T { - let ptr = self.ptr as usize; - (ptr & !low_bits::()) as *mut T - } - - pub fn with_tag(&self, tag: usize) -> Self { - Self::from(with_tag(self.ptr, tag)) - } - - /// # Safety - /// - /// The pointer (without high and low tag bits) must be a valid location to dereference. - pub unsafe fn deref<'g>(&self) -> &'g T { - &*self.as_raw() - } - - /// # Safety - /// - /// The pointer (without high and low tag bits) must be a valid location to dereference. - pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { - &mut *self.as_raw() - } - - /// # Safety - /// - /// The pointer (without high and low tag bits) must be a valid location to dereference. - pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { - if self.is_null() { - None - } else { - Some(self.deref()) - } - } - - /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, - /// are identical. - pub fn ptr_eq(self, other: Self) -> bool { - self.ptr == other.ptr - } -} - -/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. -const fn low_bits() -> usize { - (1 << align_of::().trailing_zeros()) - 1 -} - -/// Returns the pointer with the given tag -fn with_tag(ptr: *mut T, tag: usize) -> *mut T { - ((ptr as usize & !low_bits::()) | (tag & low_bits::())) as *mut T -} diff --git a/smrs/he/Cargo.toml b/smrs/he/Cargo.toml deleted file mode 100644 index 16b25cc8..00000000 --- a/smrs/he/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "he" -version = "0.1.0" -edition = "2021" - -[dependencies] -crossbeam-utils = "0.8.14" -rustc-hash = "1.1.0" -atomic = "0.5" -static_assertions = "1.1.0" diff --git a/smrs/he/src/lib.rs b/smrs/he/src/lib.rs deleted file mode 100644 index 2735b573..00000000 --- a/smrs/he/src/lib.rs +++ /dev/null @@ -1,565 +0,0 @@ -use std::cell::{Cell, RefCell, RefMut}; -use std::mem::{align_of, replace, size_of, swap}; -use std::ptr::null_mut; -use std::sync::atomic::{fence, AtomicPtr, AtomicUsize, Ordering}; -use std::sync::Mutex; - -use crossbeam_utils::CachePadded; -use static_assertions::const_assert_eq; - -const SLOTS_CAP: usize = 16; -static BATCH_CAP: AtomicUsize = AtomicUsize::new(128); - -pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); - -pub fn set_batch_capacity(cap: usize) { - BATCH_CAP.store(cap, Ordering::SeqCst); -} - -fn batch_capacity() -> usize { - BATCH_CAP.load(Ordering::Relaxed) -} - -fn collect_period() -> usize { - BATCH_CAP.load(Ordering::Relaxed) * 2 -} - -struct Node { - item: T, - birth: usize, - retire: Cell, -} - -impl Node { - fn new(item: T, birth: usize) -> Self { - Self { - item, - birth, - retire: Cell::new(0), - } - } -} - -struct Retired { - ptr: *mut Node<()>, - lifespan: unsafe fn(ptr: *mut Node<()>) -> (usize, usize), - deleter: unsafe fn(ptr: *mut Node<()>), -} - -unsafe impl Send for Retired {} - -impl Retired { - fn new(ptr: *mut Node) -> Self { - Self { - ptr: ptr as *mut Node<()>, - lifespan: lifespan::, - deleter: free::, - } - } - - unsafe fn execute(self) { - (self.deleter)(self.ptr) - } - - unsafe fn lifespan(&self) -> (usize, usize) { - (self.lifespan)(self.ptr) - } -} - -unsafe fn lifespan(ptr: *mut Node<()>) -> (usize, usize) { - let node = &*(ptr as *mut Node); - (node.birth, node.retire.get()) -} - -unsafe fn free(ptr: *mut Node<()>) { - drop(Box::from_raw(ptr as *mut Node)) -} - -pub(crate) struct RetiredList { - head: AtomicPtr, -} - -struct RetiredListNode { - retireds: Vec, - next: *const RetiredListNode, -} - -impl RetiredList { - pub(crate) const fn new() -> Self { - Self { - head: AtomicPtr::new(null_mut()), - } - } - - pub(crate) fn push(&self, retireds: Vec) { - let new = Box::leak(Box::new(RetiredListNode { - retireds, - next: null_mut(), - })); - - let mut head = self.head.load(Ordering::Relaxed); - loop { - new.next = head; - match self - .head - .compare_exchange(head, new, Ordering::Release, Ordering::Relaxed) - { - Ok(_) => return, - Err(head_new) => head = head_new, - } - } - } - - pub(crate) fn pop_all(&self) -> Vec { - let mut cur = self.head.swap(null_mut(), Ordering::AcqRel); - let mut retireds = Vec::new(); - while !cur.is_null() { - let mut cur_box = unsafe { Box::from_raw(cur) }; - retireds.append(&mut cur_box.retireds); - cur = cur_box.next.cast_mut(); - } - retireds - } -} - -pub struct Domain { - slots: Vec>, - batches: Vec>>>, - global_batch: CachePadded, - era: CachePadded, - avail_hidx: Mutex>, -} - -unsafe impl Sync for Domain {} -unsafe impl Send for Domain {} - -impl Drop for Domain { - fn drop(&mut self) { - debug_assert_eq!(self.avail_hidx.lock().unwrap().len(), self.slots.len()); - let retireds = self - .batches - .iter_mut() - .flat_map(|batch| batch.get_mut().drain(..)) - .chain(self.global_batch.pop_all().into_iter()); - for retired in retireds { - unsafe { retired.execute() }; - } - } -} - -impl Domain { - pub fn new(handles: usize) -> Self { - Self { - slots: (0..handles).map(|_| Default::default()).collect(), - batches: (0..handles).map(|_| Default::default()).collect(), - global_batch: CachePadded::new(RetiredList::new()), - era: CachePadded::new(AtomicUsize::new(1)), - avail_hidx: Mutex::new((0..handles).collect()), - } - } - - fn load_era(&self, order: Ordering) -> usize { - self.era.load(order) - } - - pub fn register(&self) -> Handle<'_> { - Handle { - domain: self, - hidx: self.avail_hidx.lock().unwrap().pop().unwrap(), - alloc_count: Cell::new(0), - retire_count: Cell::new(0), - avail_sidx: RefCell::new((0..SLOTS_CAP).collect()), - } - } - - fn iter_eras(&self) -> impl Iterator + '_ { - self.slots - .iter() - .flat_map(|slots| slots.iter().map(|slot| slot.load(Ordering::Acquire))) - } -} - -pub struct Handle<'d> { - domain: &'d Domain, - hidx: usize, - alloc_count: Cell, - retire_count: Cell, - avail_sidx: RefCell>, -} - -impl<'d> Handle<'d> { - fn incr_alloc(&self) { - let next_count = self.alloc_count.get() + 1; - self.alloc_count.set(next_count); - if next_count % batch_capacity() == 0 { - self.domain.era.fetch_add(1, Ordering::AcqRel); - } - } - - pub fn global_era(&self, order: Ordering) -> usize { - self.domain.load_era(order) - } - - fn slot(&self, idx: usize) -> &AtomicUsize { - &self.domain.slots[self.hidx][idx] - } - - fn batch_mut(&self) -> RefMut<'_, Vec> { - self.domain.batches[self.hidx].borrow_mut() - } - - pub unsafe fn retire(&self, ptr: Shared) { - let curr_era = self.global_era(Ordering::SeqCst); - ptr.ptr.deref().retire.set(curr_era); - let mut batch = self.batch_mut(); - batch.push(Retired::new(ptr.ptr.as_raw())); - let count = self.retire_count.get() + 1; - self.retire_count.set(count); - - if batch.len() >= batch_capacity() { - GLOBAL_GARBAGE_COUNT.fetch_add(batch.len(), Ordering::AcqRel); - self.domain - .global_batch - .push(replace(&mut *batch, Vec::with_capacity(batch_capacity()))); - } - if count % collect_period() == 0 { - self.collect(); - } - } - - fn collect(&self) { - let retireds = self.domain.global_batch.pop_all(); - let retireds_len = retireds.len(); - fence(Ordering::SeqCst); - - let mut eras = self.domain.iter_eras().collect::>(); - eras.sort_unstable(); - - let not_freed: Vec<_> = retireds - .into_iter() - .filter_map(|ret| { - let (birth, retire) = unsafe { ret.lifespan() }; - let ge_idx = eras.partition_point(|&era| era < birth); - if eras.len() != ge_idx && eras[ge_idx] <= retire { - Some(ret) - } else { - unsafe { ret.execute() }; - None - } - }) - .collect(); - - let freed_count = retireds_len - not_freed.len(); - GLOBAL_GARBAGE_COUNT.fetch_sub(freed_count, Ordering::AcqRel); - - self.domain.global_batch.push(not_freed); - } -} - -impl<'d> Drop for Handle<'d> { - fn drop(&mut self) { - self.domain.avail_hidx.lock().unwrap().push(self.hidx); - } -} - -pub struct HazardEra<'d, 'h> { - handle: &'h Handle<'d>, - sidx: usize, -} - -impl<'d, 'h> HazardEra<'d, 'h> { - pub fn new(handle: &'h Handle<'d>) -> Self { - let sidx = handle.avail_sidx.borrow_mut().pop().unwrap(); - Self { handle, sidx } - } - - pub fn era(&self) -> &AtomicUsize { - &self.handle.slot(self.sidx) - } - - pub fn swap(h1: &mut Self, h2: &mut Self) { - swap(&mut h1.sidx, &mut h2.sidx); - } - - pub fn clear(&mut self) { - self.handle.slot(self.sidx).store(0, Ordering::Release); - } -} - -impl<'d, 'h> Drop for HazardEra<'d, 'h> { - fn drop(&mut self) { - self.handle.avail_sidx.borrow_mut().push(self.sidx); - } -} - -pub struct Shared { - ptr: TaggedPtr>, -} - -impl<'d, T> From for Shared { - fn from(value: usize) -> Self { - Self { - ptr: TaggedPtr::from(value as *const _), - } - } -} - -impl<'d, T> Clone for Shared { - fn clone(&self) -> Self { - Self { ptr: self.ptr } - } -} - -impl<'d, T> Copy for Shared {} - -impl<'d, T> Shared { - pub fn new(item: T, handle: &Handle<'d>) -> Self { - handle.incr_alloc(); - let era = handle.global_era(Ordering::SeqCst); - Self { - ptr: TaggedPtr::from(Box::into_raw(Box::new(Node::new(item, era)))), - } - } - - pub fn null() -> Self { - Self { - ptr: TaggedPtr::null(), - } - } - - pub fn tag(&self) -> usize { - self.ptr.tag() - } - - pub fn is_null(&self) -> bool { - self.ptr.is_null() - } - - pub fn with_tag(&self, tag: usize) -> Self { - Self::from_raw(self.ptr.with_tag(tag)) - } - - pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { - self.ptr.as_ref().map(|node| &node.item) - } - - pub unsafe fn deref<'g>(&self) -> &'g T { - &self.ptr.deref().item - } - - pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { - &mut self.ptr.deref_mut().item - } - - fn from_raw(ptr: TaggedPtr>) -> Self { - Self { ptr } - } - - pub unsafe fn into_owned(self) -> T { - Box::from_raw(self.ptr.as_raw()).item - } - - /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, - /// are identical. - pub fn ptr_eq(self, other: Self) -> bool { - self.ptr.ptr_eq(other.ptr) - } -} - -pub struct Atomic { - link: atomic::Atomic>>, -} - -const_assert_eq!( - size_of::>>>(), - size_of::() -); - -unsafe impl<'d, T: Sync> Sync for Atomic {} -unsafe impl<'d, T: Send> Send for Atomic {} - -impl<'d, T> Default for Atomic { - fn default() -> Self { - Self::null() - } -} - -impl<'d, T> From> for Atomic { - fn from(value: Shared) -> Self { - Self { - link: atomic::Atomic::new(value.ptr), - } - } -} - -impl<'d, T> Atomic { - pub fn new(init: T, handle: &Handle<'d>) -> Self { - Self { - link: atomic::Atomic::new(Shared::new(init, handle).ptr), - } - } - - pub fn null() -> Self { - Self { - link: atomic::Atomic::new(TaggedPtr::null()), - } - } - - pub fn load(&self, order: Ordering) -> Shared { - Shared::from_raw(self.link.load(order)) - } - - pub fn store(&self, ptr: Shared, order: Ordering) { - self.link.store(ptr.ptr, order); - } - - pub fn fetch_or(&self, val: usize, order: Ordering) -> Shared { - let prev = unsafe { &*(&self.link as *const _ as *const AtomicUsize) }.fetch_or(val, order); - Shared::from_raw(TaggedPtr::from(prev as *const _)) - } - - /// Loads and protects the pointer by the original CrystallineL's way. - /// - /// Hint: For traversal based structures where need to check tag, the guarnatee is similar to - /// `load` + `HazardPointer::set`. For Tstack, the guarnatee is similar to - /// `HazardPointer::protect`. - pub fn protect<'h>(&self, he: &mut HazardEra<'d, 'h>) -> Shared { - let mut prev_era = he.era().load(Ordering::Relaxed); - loop { - // Somewhat similar to atomically load and protect in Hazard pointers. - let ptr = self.link.load(Ordering::Acquire); - let curr_era = he.handle.global_era(Ordering::Acquire); - if curr_era == prev_era { - return Shared::from_raw(ptr); - } - he.era().store(curr_era, Ordering::SeqCst); - prev_era = curr_era; - } - } - - pub fn compare_exchange( - &self, - current: Shared, - new: Shared, - success: Ordering, - failure: Ordering, - ) -> Result, CompareExchangeError> { - match self - .link - .compare_exchange(current.ptr, new.ptr, success, failure) - { - Ok(current) => Ok(Shared::from_raw(current)), - Err(current) => Err(CompareExchangeError { - new, - current: Shared::from_raw(current), - }), - } - } - - pub unsafe fn into_owned(self) -> T { - Box::from_raw(self.link.into_inner().as_raw()).item - } -} - -pub struct CompareExchangeError { - pub new: Shared, - pub current: Shared, -} - -struct TaggedPtr { - ptr: *mut T, -} - -impl Default for TaggedPtr { - fn default() -> Self { - Self { ptr: null_mut() } - } -} - -impl Clone for TaggedPtr { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for TaggedPtr {} - -impl From<*const T> for TaggedPtr { - fn from(value: *const T) -> Self { - Self { - ptr: value.cast_mut(), - } - } -} - -impl From<*mut T> for TaggedPtr { - fn from(value: *mut T) -> Self { - Self { ptr: value } - } -} - -impl TaggedPtr { - pub fn null() -> Self { - Self { ptr: null_mut() } - } - - pub fn is_null(&self) -> bool { - self.as_raw().is_null() - } - - pub fn tag(&self) -> usize { - let ptr = self.ptr as usize; - ptr & low_bits::() - } - - /// Converts this pointer to a raw pointer (without the tag). - pub fn as_raw(&self) -> *mut T { - let ptr = self.ptr as usize; - (ptr & !low_bits::()) as *mut T - } - - pub fn with_tag(&self, tag: usize) -> Self { - Self::from(with_tag(self.ptr, tag)) - } - - /// # Safety - /// - /// The pointer (without high and low tag bits) must be a valid location to dereference. - pub unsafe fn deref<'g>(&self) -> &'g T { - &*self.as_raw() - } - - /// # Safety - /// - /// The pointer (without high and low tag bits) must be a valid location to dereference. - pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { - &mut *self.as_raw() - } - - /// # Safety - /// - /// The pointer (without high and low tag bits) must be a valid location to dereference. - pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { - if self.is_null() { - None - } else { - Some(self.deref()) - } - } - - /// Returns `true` if the two pointer values, including the tag values set by `with_tag`, - /// are identical. - pub fn ptr_eq(self, other: Self) -> bool { - self.ptr == other.ptr - } -} - -/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. -const fn low_bits() -> usize { - (1 << align_of::().trailing_zeros()) - 1 -} - -/// Returns the pointer with the given tag -fn with_tag(ptr: *mut T, tag: usize) -> *mut T { - ((ptr as usize & !low_bits::()) | (tag & low_bits::())) as *mut T -} diff --git a/src/bin/crystalline-l.rs b/src/bin/crystalline-l.rs deleted file mode 100644 index a8fb9389..00000000 --- a/src/bin/crystalline-l.rs +++ /dev/null @@ -1,217 +0,0 @@ -use crystalline_l::{set_collect_frequency, Domain, GLOBAL_GARBAGE_COUNT}; - -use crossbeam_utils::thread::scope; -use rand::prelude::*; -use std::cmp::max; -use std::io::{stdout, Write}; -use std::path::Path; -use std::sync::atomic::Ordering; -use std::sync::{mpsc, Arc, Barrier}; -use std::thread::available_parallelism; -use std::time::Instant; - -use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; -use smr_benchmark::ds_impl::crystalline_l::{ - ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, -}; - -fn main() { - let (config, output) = setup( - Path::new(file!()) - .file_stem() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()) - .unwrap(), - ); - bench(&config, output) -} - -fn bench(config: &Config, output: BenchWriter) { - println!("{}", config); - let perf = match config.ds { - DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), - DS::NMTree => bench_map::>(config, PrefillStrategy::Random), - _ => panic!("Unsupported(or unimplemented) data structure for CrystallineL"), - }; - output.write_record(config, &perf); - println!("{}", perf); -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PrefillStrategy { - /// Inserts keys in a random order, with multiple threads. - Random, - /// Inserts keys in an increasing order, with a single thread. - Decreasing, -} - -impl PrefillStrategy { - fn prefill + Send + Sync>( - self, - config: &Config, - map: &M, - domain: &Domain, - ) { - match self { - PrefillStrategy::Random => { - let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); - print!("prefilling with {threads} threads... "); - stdout().flush().unwrap(); - scope(|s| { - for t in 0..threads { - s.spawn(move |_| { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let rng = &mut rand::thread_rng(); - let count = config.prefill / threads - + if t < config.prefill % threads { 1 } else { 0 }; - for _ in 0..count { - let key = config.key_dist.sample(rng); - let value = key; - map.insert(key, value, &mut shields, &handle); - } - }); - } - }) - .unwrap(); - } - PrefillStrategy::Decreasing => { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let rng = &mut rand::thread_rng(); - let mut keys = Vec::with_capacity(config.prefill); - for _ in 0..config.prefill { - keys.push(config.key_dist.sample(rng)); - } - keys.sort_by(|a, b| b.cmp(a)); - for key in keys.drain(..) { - let value = key; - map.insert(key, value, &mut shields, &handle); - } - } - } - print!("prefilled... "); - stdout().flush().unwrap(); - } -} - -fn bench_map + Send + Sync>( - config: &Config, - strategy: PrefillStrategy, -) -> Perf { - match config.bag_size { - BagSize::Small => set_collect_frequency(512), - BagSize::Large => set_collect_frequency(4096), - } - let max_threads = available_parallelism() - .map(|v| v.get()) - .unwrap_or(1) - .max(config.threads) - + 2; - let domain = &Domain::new(max_threads); - let mut handle = domain.register(); - let map = &M::new(&mut handle); - strategy.prefill(config, map, domain); - - let barrier = &Arc::new(Barrier::new(config.threads + config.aux_thread)); - let (ops_sender, ops_receiver) = mpsc::channel(); - let (mem_sender, mem_receiver) = mpsc::channel(); - - scope(|s| { - // sampling & interference thread - if config.aux_thread > 0 { - let mem_sender = mem_sender.clone(); - s.spawn(move |_| { - let mut samples = 0usize; - let mut acc = 0usize; - let mut peak = 0usize; - let mut garb_acc = 0usize; - let mut garb_peak = 0usize; - barrier.clone().wait(); - - let start = Instant::now(); - let mut next_sampling = start + config.sampling_period; - while start.elapsed() < config.duration { - let now = Instant::now(); - if now > next_sampling { - let allocated = config.mem_sampler.sample(); - samples += 1; - - acc += allocated; - peak = max(peak, allocated); - - let garbages = GLOBAL_GARBAGE_COUNT.load(Ordering::Acquire); - garb_acc += garbages; - garb_peak = max(garb_peak, garbages); - - next_sampling = now + config.sampling_period; - } - std::thread::sleep(config.aux_thread_period); - } - - if config.sampling { - mem_sender - .send((peak, acc / samples, garb_peak, garb_acc / samples)) - .unwrap(); - } else { - mem_sender.send((0, 0, 0, 0)).unwrap(); - } - }); - } else { - mem_sender.send((0, 0, 0, 0)).unwrap(); - } - - for _ in 0..config.threads { - let ops_sender = ops_sender.clone(); - s.spawn(move |_| { - let mut ops: u64 = 0; - let mut rng = &mut rand::thread_rng(); - let handle = domain.register(); - let mut shields = M::shields(&handle); - barrier.clone().wait(); - let start = Instant::now(); - - while start.elapsed() < config.duration { - let key = config.key_dist.sample(rng); - match Op::OPS[config.op_dist.sample(&mut rng)] { - Op::Get => { - map.get(&key, &mut shields, &handle); - } - Op::Insert => { - let value = key; - map.insert(key, value, &mut shields, &handle); - } - Op::Remove => { - map.remove(&key, &mut shields, &handle); - } - } - // Original author's implementation of `clear_all` has an use-after-free error. - // unsafe { handle.clear_all() }; - ops += 1; - } - - ops_sender.send(ops).unwrap(); - }); - } - }) - .unwrap(); - println!("end"); - - let mut ops = 0; - for _ in 0..config.threads { - let local_ops = ops_receiver.recv().unwrap(); - ops += local_ops; - } - let ops_per_sec = ops / config.interval; - let (peak_mem, avg_mem, peak_garb, avg_garb) = mem_receiver.recv().unwrap(); - Perf { - ops_per_sec, - peak_mem, - avg_mem, - peak_garb, - avg_garb, - } -} diff --git a/src/bin/he.rs b/src/bin/he.rs deleted file mode 100644 index b5d58355..00000000 --- a/src/bin/he.rs +++ /dev/null @@ -1,215 +0,0 @@ -use he::{set_batch_capacity, Domain, GLOBAL_GARBAGE_COUNT}; - -use crossbeam_utils::thread::scope; -use rand::prelude::*; -use std::cmp::max; -use std::io::{stdout, Write}; -use std::path::Path; -use std::sync::atomic::Ordering; -use std::sync::{mpsc, Arc, Barrier}; -use std::thread::available_parallelism; -use std::time::Instant; - -use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; -use smr_benchmark::ds_impl::he::{ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap}; - -fn main() { - let (config, output) = setup( - Path::new(file!()) - .file_stem() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()) - .unwrap(), - ); - bench(&config, output) -} - -fn bench(config: &Config, output: BenchWriter) { - println!("{}", config); - let perf = match config.ds { - DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), - DS::NMTree => bench_map::>(config, PrefillStrategy::Random), - _ => panic!("Unsupported(or unimplemented) data structure for CrystallineL"), - }; - output.write_record(config, &perf); - println!("{}", perf); -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PrefillStrategy { - /// Inserts keys in a random order, with multiple threads. - Random, - /// Inserts keys in an increasing order, with a single thread. - Decreasing, -} - -impl PrefillStrategy { - fn prefill + Send + Sync>( - self, - config: &Config, - map: &M, - domain: &Domain, - ) { - match self { - PrefillStrategy::Random => { - let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); - print!("prefilling with {threads} threads... "); - stdout().flush().unwrap(); - scope(|s| { - for t in 0..threads { - s.spawn(move |_| { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let rng = &mut rand::thread_rng(); - let count = config.prefill / threads - + if t < config.prefill % threads { 1 } else { 0 }; - for _ in 0..count { - let key = config.key_dist.sample(rng); - let value = key; - map.insert(key, value, &mut shields, &handle); - } - }); - } - }) - .unwrap(); - } - PrefillStrategy::Decreasing => { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let rng = &mut rand::thread_rng(); - let mut keys = Vec::with_capacity(config.prefill); - for _ in 0..config.prefill { - keys.push(config.key_dist.sample(rng)); - } - keys.sort_by(|a, b| b.cmp(a)); - for key in keys.drain(..) { - let value = key; - map.insert(key, value, &mut shields, &handle); - } - } - } - print!("prefilled... "); - stdout().flush().unwrap(); - } -} - -fn bench_map + Send + Sync>( - config: &Config, - strategy: PrefillStrategy, -) -> Perf { - match config.bag_size { - BagSize::Small => set_batch_capacity(512), - BagSize::Large => set_batch_capacity(4096), - } - let max_threads = available_parallelism() - .map(|v| v.get()) - .unwrap_or(1) - .max(config.threads) - + 2; - let domain = &Domain::new(max_threads); - let mut handle = domain.register(); - let map = &M::new(&mut handle); - strategy.prefill(config, map, domain); - - let barrier = &Arc::new(Barrier::new(config.threads + config.aux_thread)); - let (ops_sender, ops_receiver) = mpsc::channel(); - let (mem_sender, mem_receiver) = mpsc::channel(); - - scope(|s| { - // sampling & interference thread - if config.aux_thread > 0 { - let mem_sender = mem_sender.clone(); - s.spawn(move |_| { - let mut samples = 0usize; - let mut acc = 0usize; - let mut peak = 0usize; - let mut garb_acc = 0usize; - let mut garb_peak = 0usize; - barrier.clone().wait(); - - let start = Instant::now(); - let mut next_sampling = start + config.sampling_period; - while start.elapsed() < config.duration { - let now = Instant::now(); - if now > next_sampling { - let allocated = config.mem_sampler.sample(); - samples += 1; - - acc += allocated; - peak = max(peak, allocated); - - let garbages = GLOBAL_GARBAGE_COUNT.load(Ordering::Acquire); - garb_acc += garbages; - garb_peak = max(garb_peak, garbages); - - next_sampling = now + config.sampling_period; - } - std::thread::sleep(config.aux_thread_period); - } - - if config.sampling { - mem_sender - .send((peak, acc / samples, garb_peak, garb_acc / samples)) - .unwrap(); - } else { - mem_sender.send((0, 0, 0, 0)).unwrap(); - } - }); - } else { - mem_sender.send((0, 0, 0, 0)).unwrap(); - } - - for _ in 0..config.threads { - let ops_sender = ops_sender.clone(); - s.spawn(move |_| { - let mut ops: u64 = 0; - let mut rng = &mut rand::thread_rng(); - let handle = domain.register(); - let mut shields = M::shields(&handle); - barrier.clone().wait(); - let start = Instant::now(); - - while start.elapsed() < config.duration { - let key = config.key_dist.sample(rng); - match Op::OPS[config.op_dist.sample(&mut rng)] { - Op::Get => { - map.get(&key, &mut shields, &handle); - } - Op::Insert => { - let value = key; - map.insert(key, value, &mut shields, &handle); - } - Op::Remove => { - map.remove(&key, &mut shields, &handle); - } - } - // Original author's implementation of `clear_all` has an use-after-free error. - // unsafe { handle.clear_all() }; - ops += 1; - } - - ops_sender.send(ops).unwrap(); - }); - } - }) - .unwrap(); - println!("end"); - - let mut ops = 0; - for _ in 0..config.threads { - let local_ops = ops_receiver.recv().unwrap(); - ops += local_ops; - } - let ops_per_sec = ops / config.interval; - let (peak_mem, avg_mem, peak_garb, avg_garb) = mem_receiver.recv().unwrap(); - Perf { - ops_per_sec, - peak_mem, - avg_mem, - peak_garb, - avg_garb, - } -} diff --git a/src/ds_impl/crystalline_l/concurrent_map.rs b/src/ds_impl/crystalline_l/concurrent_map.rs deleted file mode 100644 index 3c08a9f4..00000000 --- a/src/ds_impl/crystalline_l/concurrent_map.rs +++ /dev/null @@ -1,107 +0,0 @@ -use crystalline_l::Handle; - -pub trait ConcurrentMap { - type Node; - type Shields<'d, 'h> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h>; - fn new<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self; - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V>; - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> bool; - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V>; -} - -#[cfg(test)] -pub mod tests { - extern crate rand; - use super::ConcurrentMap; - use crossbeam_utils::thread; - use crystalline_l::Domain; - use rand::prelude::*; - - const THREADS: i32 = 30; - const ELEMENTS_PER_THREADS: i32 = 1000; - - pub fn smoke + Send + Sync>() { - let domain = &Domain::new((THREADS + 1) as usize); - // Drop the map before the domain. - smoke_inner::(domain); - } - - fn smoke_inner + Send + Sync>(domain: &Domain) { - let mut handle = domain.register(); - let map = &M::new(&mut handle); - - thread::scope(|s| { - for t in 0..THREADS { - s.spawn(move |_| { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert!(map.insert(i, i.to_string(), &mut shields, &handle)); - } - }); - } - }) - .unwrap(); - - thread::scope(|s| { - for t in 0..(THREADS / 2) { - s.spawn(move |_| { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert_eq!( - i.to_string(), - *map.remove(&i, &mut shields, &handle).unwrap() - ); - } - }); - } - }) - .unwrap(); - - thread::scope(|s| { - for t in (THREADS / 2)..THREADS { - s.spawn(move |_| { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert_eq!(i.to_string(), *map.get(&i, &mut shields, &handle).unwrap()); - } - }); - } - }) - .unwrap(); - } -} diff --git a/src/ds_impl/crystalline_l/list.rs b/src/ds_impl/crystalline_l/list.rs deleted file mode 100644 index 1cbdeaec..00000000 --- a/src/ds_impl/crystalline_l/list.rs +++ /dev/null @@ -1,689 +0,0 @@ -use super::concurrent_map::ConcurrentMap; - -use std::cmp::Ordering::{Equal, Greater, Less}; -use std::sync::atomic::Ordering; - -use crystalline_l::{Atomic, Handle, HazardEra, Shared}; - -// `#[repr(C)]` is used to ensure the first field -// is also the first data in the memory alignment. -#[repr(C)] -pub struct Node { - /// Mark: tag(), Tag: not needed - next: Atomic, - key: K, - value: V, -} - -pub struct List { - head: Atomic>, -} - -impl Default for List -where - K: Ord, -{ - fn default() -> Self { - Self::new() - } -} - -impl Drop for List { - fn drop(&mut self) { - let mut curr = self.head.load(Ordering::Relaxed); - - while !curr.is_null() { - curr = unsafe { curr.into_owned() }.next.load(Ordering::Relaxed); - } - } -} - -pub struct EraShields<'d, 'h, K, V> { - prev_h: HazardEra<'d, 'h, Node>, - curr_h: HazardEra<'d, 'h, Node>, - next_h: HazardEra<'d, 'h, Node>, - // `anchor_h` and `anchor_next_h` are used for `find_harris` - anchor_h: HazardEra<'d, 'h, Node>, - anchor_next_h: HazardEra<'d, 'h, Node>, -} - -impl<'d, 'h, K, V> EraShields<'d, 'h, K, V> { - pub fn new(handle: &'h Handle<'d, Node>) -> Self { - Self { - prev_h: HazardEra::new(handle), - curr_h: HazardEra::new(handle), - next_h: HazardEra::new(handle), - anchor_h: HazardEra::new(handle), - anchor_next_h: HazardEra::new(handle), - } - } - - // bypass E0499-E0503, etc that are supposed to be fixed by polonius - #[inline] - fn launder<'he2>(&mut self) -> &'he2 mut Self { - unsafe { core::mem::transmute(self) } - } -} - -pub struct Cursor { - prev: Shared>, - // For harris, this keeps the mark bit. Don't mix harris and harris-michael. - curr: Shared>, - // `anchor` is used for `find_harris` - // anchor and anchor_next are non-null iff exist - anchor: Shared>, - anchor_next: Shared>, -} - -impl Cursor { - pub fn new<'d, 'h>(head: &Atomic>, shields: &mut EraShields<'d, 'h, K, V>) -> Self { - Self { - prev: Shared::from(head as *const _ as usize), - curr: head.protect(&mut shields.curr_h), - anchor: Shared::null(), - anchor_next: Shared::null(), - } - } -} - -impl Cursor -where - K: Ord, -{ - /// Optimistically traverses while maintaining `anchor` and `anchor_next`. - /// It is used for both Harris and Harris-Herlihy-Shavit traversals. - #[inline] - fn traverse_with_anchor<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h, K, V>, - ) -> Result { - // Invariants: - // anchor, anchor_next: protected if they are not null. - // prev: always protected with prev_sh - // curr: not protected. - // curr: also has tag value when it is obtained from prev. - Ok(loop { - let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { - break false; - }; - - // Validation depending on the state of `self.curr`. - // - // - If it is marked, validate on anchor. - // - If it is not marked, it is already protected safely by the Crystalline. - if self.curr.tag() != 0 { - // Validate on anchor. - - debug_assert!(!self.anchor.is_null()); - debug_assert!(!self.anchor_next.is_null()); - let an_new = unsafe { self.anchor.deref() }.next.load(Ordering::Acquire); - - if an_new.tag() != 0 { - return Err(()); - } else if !an_new.ptr_eq(self.anchor_next) { - // Anchor is updated but clear, so can restart from anchor. - - self.prev = self.anchor; - self.curr = an_new; - self.anchor = Shared::null(); - - // Set prev HP as anchor HP, since prev should always be protected. - HazardEra::swap(&mut shields.prev_h, &mut shields.anchor_h); - continue; - } - } - - let next = curr_node.next.protect(&mut shields.next_h); - if next.tag() == 0 { - if curr_node.key < *key { - self.prev = self.curr; - self.curr = next; - self.anchor = Shared::null(); - HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - } else { - break curr_node.key == *key; - } - } else { - if self.anchor.is_null() { - self.anchor = self.prev; - self.anchor_next = self.curr; - HazardEra::swap(&mut shields.anchor_h, &mut shields.prev_h); - } else if self.anchor_next.ptr_eq(self.prev) { - HazardEra::swap(&mut shields.anchor_next_h, &mut shields.prev_h); - } - self.prev = self.curr; - self.curr = next; - HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - } - }) - } - - #[inline] - fn find_harris<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Result { - // Finding phase - // - cursor.curr: first unmarked node w/ key >= search key (4) - // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) - // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) - - let found = self.traverse_with_anchor(key, shields)?; - - scopeguard::defer! { - shields.anchor_h.clear(); - shields.anchor_next_h.clear(); - } - - if self.anchor.is_null() { - self.prev = self.prev.with_tag(0); - self.curr = self.curr.with_tag(0); - Ok(found) - } else { - debug_assert_eq!(self.anchor_next.tag(), 0); - // TODO: on CAS failure, if anchor is not tagged, we can restart from anchor. - unsafe { &self.anchor.deref().next } - .compare_exchange( - self.anchor_next, - self.curr.with_tag(0), - Ordering::AcqRel, - Ordering::Relaxed, - ) - .map_err(|_| { - self.curr = self.curr.with_tag(0); - () - })?; - - let mut node = self.anchor_next; - while !node.with_tag(0).ptr_eq(self.curr.with_tag(0)) { - // NOTE: It may seem like this can be done with a NA load, but we do a `fetch_or` in remove, which always does an write. - // This can be a NA load if the `fetch_or` in delete is changed to a CAS, but it is not clear if it is worth it. - let next = unsafe { node.deref().next.load(Ordering::Relaxed) }; - debug_assert!(next.tag() != 0); - unsafe { handle.retire(node) }; - node = next; - } - self.prev = self.anchor.with_tag(0); - self.curr = self.curr.with_tag(0); - Ok(found) - } - } - - #[inline] - fn find_harris_michael<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Result { - loop { - debug_assert_eq!(self.curr.tag(), 0); - let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { - return Ok(false); - }; - let mut next = curr_node.next.protect(&mut shields.next_h); - - if next.tag() != 0 { - next = next.with_tag(0); - unsafe { self.prev.deref() } - .next - .compare_exchange(self.curr, next, Ordering::AcqRel, Ordering::Acquire) - .map_err(|_| ())?; - unsafe { handle.retire(self.curr) }; - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - self.curr = next; - continue; - } - - match curr_node.key.cmp(key) { - Less => { - HazardEra::swap(&mut shields.prev_h, &mut shields.curr_h); - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - self.prev = self.curr; - self.curr = next; - } - Equal => return Ok(true), - Greater => return Ok(false), - } - } - } - - #[inline] - fn find_harris_herlihy_shavit<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h, K, V>, - _handle: &'h Handle<'d, Node>, - ) -> Result { - let found = self.traverse_with_anchor(key, shields)?; - // Return only the found `curr` node. - // Others are not necessary because we are not going to do insertion or deletion - // with this Harris-Herlihy-Shavit traversal. - self.curr = self.curr.with_tag(0); - shields.anchor_h.clear(); - shields.anchor_next_h.clear(); - Ok(found) - } -} - -impl List -where - K: Ord, -{ - /// Creates a new list. - pub fn new() -> Self { - List { - head: Atomic::null(), - } - } - - #[inline] - fn get<'d, 'h, 'he, F>( - &self, - key: &K, - find: F, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Option<&'he V> - where - F: Fn( - &mut Cursor, - &K, - &mut EraShields<'d, 'h, K, V>, - &'h Handle<'d, Node>, - ) -> Result, - { - loop { - let mut cursor = Cursor::new(&self.head, shields); - match find(&mut cursor, key, shields, handle) { - Ok(true) => return unsafe { Some(&(cursor.curr.deref().value)) }, - Ok(false) => return None, - Err(_) => continue, - } - } - } - - fn insert_inner<'d, 'h, 'he, F>( - &self, - node: Shared>, - find: &F, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Result - where - F: Fn( - &mut Cursor, - &K, - &mut EraShields<'d, 'h, K, V>, - &'h Handle<'d, Node>, - ) -> Result, - { - loop { - let mut cursor = Cursor::new(&self.head, shields); - let found = find(&mut cursor, unsafe { &node.deref().key }, shields, handle)?; - if found { - drop(unsafe { node.into_owned() }); - return Ok(false); - } - - unsafe { node.deref() } - .next - .store(cursor.curr, Ordering::Relaxed); - if unsafe { cursor.prev.deref() } - .next - .compare_exchange(cursor.curr, node, Ordering::Release, Ordering::Relaxed) - .is_ok() - { - return Ok(true); - } - } - } - - #[inline] - fn insert<'d, 'h, 'he, F>( - &self, - key: K, - value: V, - find: F, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> bool - where - F: Fn( - &mut Cursor, - &K, - &mut EraShields<'d, 'h, K, V>, - &'h Handle<'d, Node>, - ) -> Result, - { - let node = Shared::new( - Node { - key, - value, - next: Atomic::null(), - }, - handle, - ); - - loop { - match self.insert_inner(node, &find, shields, handle) { - Ok(r) => return r, - Err(()) => continue, - } - } - } - - fn remove_inner<'d, 'h, 'he, F>( - &self, - key: &K, - find: &F, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Result, ()> - where - F: Fn( - &mut Cursor, - &K, - &mut EraShields<'d, 'h, K, V>, - &'h Handle<'d, Node>, - ) -> Result, - { - loop { - let mut cursor = Cursor::new(&self.head, shields); - let found = find(&mut cursor, key, shields, handle)?; - if !found { - return Ok(None); - } - - let curr_node = unsafe { cursor.curr.deref() }; - let next = curr_node.next.fetch_or(1, Ordering::AcqRel); - if next.tag() == 1 { - continue; - } - - if unsafe { &cursor.prev.deref().next } - .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) - .is_ok() - { - unsafe { handle.retire(cursor.curr) }; - } - - return Ok(Some(&curr_node.value)); - } - } - - #[inline] - fn remove<'d, 'h, 'he, F>( - &self, - key: &K, - find: F, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Option<&'he V> - where - F: Fn( - &mut Cursor, - &K, - &mut EraShields<'d, 'h, K, V>, - &'h Handle<'d, Node>, - ) -> Result, - { - loop { - match self.remove_inner(key, &find, shields.launder(), handle) { - Ok(r) => return r, - Err(_) => continue, - } - } - } - - pub fn harris_get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Option<&'he V> { - self.get(key, Cursor::find_harris, shields, handle) - } - - pub fn harris_insert<'d, 'h, 'he>( - &self, - key: K, - value: V, - shields: &mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> bool { - self.insert(key, value, Cursor::find_harris, shields, handle) - } - - pub fn harris_remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Option<&'he V> { - self.remove(key, Cursor::find_harris, shields, handle) - } - - pub fn harris_michael_get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Option<&'he V> { - self.get(key, Cursor::find_harris_michael, shields, handle) - } - - pub fn harris_michael_insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> bool { - self.insert(key, value, Cursor::find_harris_michael, shields, handle) - } - - pub fn harris_michael_remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Option<&'he V> { - self.remove(key, Cursor::find_harris_michael, shields, handle) - } - - pub fn harris_herlihy_shavit_get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &'h Handle<'d, Node>, - ) -> Option<&'he V> { - self.get(key, Cursor::find_harris_herlihy_shavit, shields, handle) - } -} - -pub struct HList { - inner: List, -} - -impl ConcurrentMap for HList -where - K: Ord + 'static, - V: 'static, -{ - type Node = Node; - - type Shields<'d, 'h> - = EraShields<'d, 'h, K, V> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(_: &'h Handle<'d, Self::Node>) -> Self { - Self { inner: List::new() } - } - - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.inner.harris_get(key, shields, handle) - } - - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> bool { - self.inner.harris_insert(key, value, shields, handle) - } - - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.inner.harris_remove(key, shields, handle) - } -} - -pub struct HMList { - inner: List, -} - -impl ConcurrentMap for HMList -where - K: Ord + 'static, - V: 'static, -{ - type Node = Node; - - type Shields<'d, 'h> - = EraShields<'d, 'h, K, V> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(_: &'h Handle<'d, Self::Node>) -> Self { - Self { inner: List::new() } - } - - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.inner.harris_michael_get(key, shields, handle) - } - - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> bool { - self.inner - .harris_michael_insert(key, value, shields, handle) - } - - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.inner.harris_michael_remove(key, shields, handle) - } -} - -pub struct HHSList { - inner: List, -} - -impl ConcurrentMap for HHSList -where - K: Ord + 'static, - V: 'static, -{ - type Node = Node; - - type Shields<'d, 'h> - = EraShields<'d, 'h, K, V> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(_: &'h Handle<'d, Self::Node>) -> Self { - Self { inner: List::new() } - } - - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.inner.harris_herlihy_shavit_get(key, shields, handle) - } - - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> bool { - self.inner.harris_insert(key, value, shields, handle) - } - - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.inner.harris_remove(key, shields, handle) - } -} - -#[cfg(test)] -mod tests { - use super::{HHSList, HList, HMList}; - use crate::ds_impl::crystalline_l::concurrent_map; - - #[test] - fn smoke_h_list() { - concurrent_map::tests::smoke::>(); - } - - #[test] - fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); - } - - #[test] - fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); - } -} diff --git a/src/ds_impl/crystalline_l/michael_hash_map.rs b/src/ds_impl/crystalline_l/michael_hash_map.rs deleted file mode 100644 index eeabac10..00000000 --- a/src/ds_impl/crystalline_l/michael_hash_map.rs +++ /dev/null @@ -1,104 +0,0 @@ -use super::concurrent_map::ConcurrentMap; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; - -use super::list::HHSList; -pub use super::list::{Cursor, EraShields, Node}; - -use crystalline_l::Handle; - -pub struct HashMap { - buckets: Vec>, -} - -impl HashMap -where - K: Ord + Hash + 'static, - V: 'static, -{ - pub fn with_capacity<'d, 'h>(n: usize, handle: &'h Handle<'d, Node>) -> Self { - let mut buckets = Vec::with_capacity(n); - for _ in 0..n { - buckets.push(HHSList::new(handle)); - } - - HashMap { buckets } - } - - #[inline] - pub fn get_bucket(&self, index: usize) -> &HHSList { - unsafe { self.buckets.get_unchecked(index % self.buckets.len()) } - } - - #[inline] - fn hash(k: &K) -> usize { - let mut s = DefaultHasher::new(); - k.hash(&mut s); - s.finish() as usize - } -} - -impl ConcurrentMap for HashMap -where - K: Ord + Hash + Send + 'static, - V: Send + 'static, -{ - type Node = Node; - - type Shields<'d, 'h> = EraShields<'d, 'h, K, V> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self { - Self::with_capacity(30000, handle) - } - - #[inline(always)] - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - let i = Self::hash(key); - self.get_bucket(i).get(key, shields, handle) - } - - #[inline(always)] - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> bool { - let i = Self::hash(&key); - self.get_bucket(i).insert(key, value, shields, handle) - } - - #[inline(always)] - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - let i = Self::hash(key); - self.get_bucket(i).remove(key, shields, handle) - } -} - -#[cfg(test)] -mod tests { - use super::HashMap; - use crate::ds_impl::crystalline_l::concurrent_map; - - #[test] - fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); - } -} diff --git a/src/ds_impl/crystalline_l/mod.rs b/src/ds_impl/crystalline_l/mod.rs deleted file mode 100644 index f6d449ed..00000000 --- a/src/ds_impl/crystalline_l/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub mod concurrent_map; - -pub mod list; -pub mod michael_hash_map; -pub mod natarajan_mittal_tree; - -pub use self::concurrent_map::ConcurrentMap; - -pub use self::list::{HHSList, HList, HMList}; -pub use self::michael_hash_map::HashMap; -pub use self::natarajan_mittal_tree::NMTreeMap; diff --git a/src/ds_impl/crystalline_l/natarajan_mittal_tree.rs b/src/ds_impl/crystalline_l/natarajan_mittal_tree.rs deleted file mode 100644 index dbd16816..00000000 --- a/src/ds_impl/crystalline_l/natarajan_mittal_tree.rs +++ /dev/null @@ -1,689 +0,0 @@ -use super::concurrent_map::ConcurrentMap; - -use std::cmp; -use std::sync::atomic::Ordering; - -use crystalline_l::{Atomic, Handle, HazardEra, Shared}; - -bitflags! { - /// TODO - /// A remove operation is registered by marking the corresponding edges: the (parent, target) - /// edge is _flagged_ and the (parent, sibling) edge is _tagged_. - struct Marks: usize { - const FLAG = 1usize.wrapping_shl(1); - const TAG = 1usize.wrapping_shl(0); - } -} - -impl Marks { - fn new(flag: bool, tag: bool) -> Self { - (if flag { Marks::FLAG } else { Marks::empty() }) - | (if tag { Marks::TAG } else { Marks::empty() }) - } - - fn flag(self) -> bool { - !(self & Marks::FLAG).is_empty() - } - - fn tag(self) -> bool { - !(self & Marks::TAG).is_empty() - } - - fn marked(self) -> bool { - !(self & (Marks::TAG | Marks::FLAG)).is_empty() - } -} - -#[derive(Clone, PartialEq, Eq, Debug)] -enum Key { - Fin(K), - Inf, -} - -impl PartialOrd for Key -where - K: PartialOrd, -{ - fn partial_cmp(&self, other: &Self) -> Option { - match (self, other) { - (Key::Fin(k1), Key::Fin(k2)) => k1.partial_cmp(k2), - (Key::Fin(_), Key::Inf) => Some(std::cmp::Ordering::Less), - (Key::Inf, Key::Fin(_)) => Some(std::cmp::Ordering::Greater), - (Key::Inf, Key::Inf) => Some(std::cmp::Ordering::Equal), - } - } -} - -impl PartialEq for Key -where - K: PartialEq, -{ - fn eq(&self, rhs: &K) -> bool { - match self { - Key::Fin(k) => k == rhs, - _ => false, - } - } -} - -impl PartialOrd for Key -where - K: PartialOrd, -{ - fn partial_cmp(&self, rhs: &K) -> Option { - match self { - Key::Fin(k) => k.partial_cmp(rhs), - _ => Some(std::cmp::Ordering::Greater), - } - } -} - -impl Key -where - K: Ord, -{ - fn cmp(&self, rhs: &K) -> std::cmp::Ordering { - match self { - Key::Fin(k) => k.cmp(rhs), - _ => std::cmp::Ordering::Greater, - } - } -} - -pub struct Node { - key: Key, - value: Option, - left: Atomic>, - right: Atomic>, -} - -impl Node -where - K: Clone, - V: Clone, -{ - fn new_leaf(key: Key, value: Option) -> Node { - Node { - key, - value, - left: Atomic::null(), - right: Atomic::null(), - } - } - - /// Make a new internal node, consuming the given left and right nodes, - /// using the right node's key. - fn new_internal( - left: Node, - right: Node, - handle: &Handle<'_, Node>, - ) -> Node { - Node { - key: right.key.clone(), - value: None, - left: Atomic::new(left, handle), - right: Atomic::new(right, handle), - } - } -} - -#[derive(Clone, Copy)] -enum Direction { - L, - R, -} - -pub struct EraShields<'d, 'h, K, V> { - ancestor_h: HazardEra<'d, 'h, Node>, - successor_h: HazardEra<'d, 'h, Node>, - parent_h: HazardEra<'d, 'h, Node>, - leaf_h: HazardEra<'d, 'h, Node>, - curr_h: HazardEra<'d, 'h, Node>, -} - -impl<'d, 'h, K, V> EraShields<'d, 'h, K, V> { - pub fn new(handle: &'h Handle<'d, Node>) -> Self { - Self { - ancestor_h: HazardEra::new(handle), - successor_h: HazardEra::new(handle), - parent_h: HazardEra::new(handle), - leaf_h: HazardEra::new(handle), - curr_h: HazardEra::new(handle), - } - } - - // bypass E0499-E0503, etc that are supposed to be fixed by polonius - #[inline] - fn launder<'he2>(&mut self) -> &'he2 mut Self { - unsafe { core::mem::transmute(self) } - } -} - -/// All Shared<_> are unmarked. -/// -/// All of the edges of path from `successor` to `parent` are in the process of removal. -pub struct SeekRecord { - /// Parent of `successor` - ancestor: Shared>, - /// The first internal node with a marked outgoing edge - successor: Shared>, - /// The direction of successor from ancestor. - successor_dir: Direction, - /// Parent of `leaf` - parent: Shared>, - /// The end of the access path. - leaf: Shared>, - /// The direction of leaf from parent. - leaf_dir: Direction, -} - -impl SeekRecord { - fn new() -> Self { - Self { - ancestor: Shared::null(), - successor: Shared::null(), - successor_dir: Direction::L, - parent: Shared::null(), - leaf: Shared::null(), - leaf_dir: Direction::L, - } - } - - fn successor_addr(&self) -> &Atomic> { - match self.successor_dir { - Direction::L => unsafe { &self.ancestor.deref().left }, - Direction::R => unsafe { &self.ancestor.deref().right }, - } - } - - fn leaf_addr(&self) -> &Atomic> { - match self.leaf_dir { - Direction::L => unsafe { &self.parent.deref().left }, - Direction::R => unsafe { &self.parent.deref().right }, - } - } - - fn leaf_sibling_addr(&self) -> &Atomic> { - match self.leaf_dir { - Direction::L => unsafe { &self.parent.deref().right }, - Direction::R => unsafe { &self.parent.deref().left }, - } - } -} - -pub struct NMTreeMap { - r: Node, -} - -impl Default for NMTreeMap -where - K: Ord + Clone, - V: Clone, -{ - fn default() -> Self { - todo!("new") - } -} - -impl Drop for NMTreeMap { - fn drop(&mut self) { - unsafe { - let mut stack = vec![ - self.r.left.load(Ordering::Relaxed), - self.r.right.load(Ordering::Relaxed), - ]; - assert!(self.r.value.is_none()); - - while let Some(node) = stack.pop() { - if node.is_null() { - continue; - } - - let node_ref = node.deref(); - - stack.push(node_ref.left.load(Ordering::Relaxed)); - stack.push(node_ref.right.load(Ordering::Relaxed)); - drop(node.into_owned()); - } - } - } -} - -impl NMTreeMap -where - K: Ord + Clone, - V: Clone, -{ - pub fn new<'d>(handle: &Handle<'d, Node>) -> Self { - // An empty tree has 5 default nodes with infinite keys so that the SeekRecord is allways - // well-defined. - // r - // / \ - // s inf2 - // / \ - // inf0 inf1 - let inf0 = Node::new_leaf(Key::Inf, None); - let inf1 = Node::new_leaf(Key::Inf, None); - let inf2 = Node::new_leaf(Key::Inf, None); - let s = Node::new_internal(inf0, inf1, handle); - let r = Node::new_internal(s, inf2, handle); - NMTreeMap { r } - } - - fn seek<'d, 'h>( - &self, - key: &K, - record: &mut SeekRecord, - shields: &mut EraShields<'d, 'h, K, V>, - ) -> Result<(), ()> { - let s = self.r.left.load(Ordering::Relaxed).with_tag(0); - let s_node = unsafe { s.deref() }; - - // The root node is always alive; we do not have to protect it. - record.ancestor = Shared::from(&self.r as *const _ as usize); - record.successor = s; // TODO: should preserve tag? - - record.successor_dir = Direction::L; - // The `s` node is always alive; we do not have to protect it. - record.parent = s; - - record.leaf = s_node.left.protect(&mut shields.leaf_h); - record.leaf_dir = Direction::L; - - let mut prev_tag = Marks::from_bits_truncate(record.leaf.tag()).tag(); - let mut curr_dir = Direction::L; - let mut curr = unsafe { record.leaf.deref() } - .left - .protect(&mut shields.curr_h); - - // `ancestor` always points untagged node. - while !curr.is_null() { - if !prev_tag { - // untagged edge: advance ancestor and successor pointers - record.ancestor = record.parent; - record.successor = record.leaf; - record.successor_dir = record.leaf_dir; - // `ancestor` and `successor` are already protected by - // hazard pointers of `parent` and `leaf`. - - // Advance the parent and leaf pointers when the cursor looks like the following: - // (): protected by its dedicated shield. - // - // (parent), ancestor -> O (ancestor) -> O - // / \ / \ - // (leaf), successor -> O O => (parent), successor -> O O - // / \ / \ - // O O (leaf) -> O O - record.parent = record.leaf; - HazardEra::swap(&mut shields.ancestor_h, &mut shields.parent_h); - HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); - } else if record.successor.ptr_eq(record.parent) { - // Advance the parent and leaf pointer when the cursor looks like the following: - // (): protected by its dedicated shield. - // - // (ancestor) -> O (ancestor) -> O - // / \ / \ - // (parent), successor -> O O (successor) -> O O - // / \ => / \ - // (leaf) -> O O (parent) -> O O - // / \ / \ - // O O (leaf) -> O O - record.parent = record.leaf; - HazardEra::swap(&mut shields.successor_h, &mut shields.parent_h); - HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); - } else { - // Advance the parent and leaf pointer when the cursor looks like the following: - // (): protected by its dedicated shield. - // - // (ancestor) -> O - // / \ - // (successor) -> O O - // ... ... - // (parent) -> O - // / \ - // (leaf) -> O O - record.parent = record.leaf; - HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); - } - debug_assert_eq!(record.successor.tag(), 0); - - if Marks::from_bits_truncate(curr.tag()).marked() { - // `curr` is marked. Validate by `ancestor`. - let succ_new = record.successor_addr().load(Ordering::Acquire); - if Marks::from_bits_truncate(succ_new.tag()).marked() - || !record.successor.ptr_eq(succ_new) - { - // Validation is failed. Let's restart from the root. - // TODO: Maybe it can be optimized (by restarting from the anchor), but - // it would require a serious reasoning (including shield swapping, etc). - return Err(()); - } - } - - record.leaf = curr; - record.leaf_dir = curr_dir; - - // update other variables - prev_tag = Marks::from_bits_truncate(curr.tag()).tag(); - let curr_node = unsafe { curr.deref() }; - if curr_node.key.cmp(key) == cmp::Ordering::Greater { - curr_dir = Direction::L; - curr = curr_node.left.load(Ordering::Acquire); - } else { - curr_dir = Direction::R; - curr = curr_node.right.load(Ordering::Acquire); - } - } - Ok(()) - } - - /// Physically removes node. - /// - /// Returns true if it successfully unlinks the flagged node in `record`. - fn cleanup<'d>(&self, record: &mut SeekRecord, handle: &Handle<'d, Node>) -> bool { - // Identify the node(subtree) that will replace `successor`. - let leaf_marked = record.leaf_addr().load(Ordering::Acquire); - let leaf_flag = Marks::from_bits_truncate(leaf_marked.tag()).flag(); - let target_sibling_addr = if leaf_flag { - record.leaf_sibling_addr() - } else { - record.leaf_addr() - }; - - // NOTE: the ibr implementation uses CAS - // tag (parent, sibling) edge -> all of the parent's edges can't change now - // TODO: Is Release enough? - target_sibling_addr.fetch_or(Marks::TAG.bits(), Ordering::AcqRel); - - // Try to replace (ancestor, successor) w/ (ancestor, sibling). - // Since (parent, sibling) might have been concurrently flagged, copy - // the flag to the new edge (ancestor, sibling). - let target_sibling = target_sibling_addr.load(Ordering::Acquire); - let flag = Marks::from_bits_truncate(target_sibling.tag()).flag(); - let is_unlinked = record - .successor_addr() - .compare_exchange( - record.successor.with_tag(0), - target_sibling.with_tag(Marks::new(flag, false).bits()), - Ordering::AcqRel, - Ordering::Acquire, - ) - .is_ok(); - - if is_unlinked { - unsafe { - // destroy the subtree of successor except target_sibling - let mut stack = vec![record.successor]; - - while let Some(node) = stack.pop() { - if node.is_null() || node.with_tag(0).ptr_eq(target_sibling.with_tag(0)) { - continue; - } - - let node_ref = node.deref(); - - stack.push(node_ref.left.load(Ordering::Relaxed)); - stack.push(node_ref.right.load(Ordering::Relaxed)); - handle.retire(node); - } - } - } - - is_unlinked - } - - fn get_inner<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - ) -> Result, ()> { - let mut record = SeekRecord::new(); - - self.seek(key, &mut record, shields)?; - let leaf_node = unsafe { record.leaf.deref() }; - - if leaf_node.key.cmp(key) != cmp::Ordering::Equal { - return Ok(None); - } - - Ok(Some(leaf_node.value.as_ref().unwrap())) - } - - pub fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - ) -> Option<&'he V> { - loop { - if let Ok(r) = self.get_inner(key, shields.launder()) { - return r; - } - } - } - - fn insert_inner<'d, 'h, 'he>( - &self, - key: &K, - value: V, - record: &mut SeekRecord, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &Handle<'d, Node>, - ) -> Result<(), Result> { - let mut new_leaf = Shared::new(Node::new_leaf(Key::Fin(key.clone()), Some(value)), handle); - - let mut new_internal = Shared::new( - Node:: { - key: Key::Inf, // temporary placeholder - value: None, - left: Atomic::null(), - right: Atomic::null(), - }, - handle, - ); - - loop { - self.seek(key, record, shields).map_err(|_| unsafe { - let value = new_leaf.deref_mut().value.take().unwrap(); - drop(new_leaf.into_owned()); - drop(new_internal.into_owned()); - Err(value) - })?; - let leaf = record.leaf.with_tag(0); - - let (new_left, new_right) = match unsafe { leaf.deref() }.key.cmp(key) { - cmp::Ordering::Equal => { - // Newly created nodes that failed to be inserted are free'd here. - let value = unsafe { new_leaf.deref_mut() }.value.take().unwrap(); - unsafe { - drop(new_leaf.into_owned()); - drop(new_internal.into_owned()); - } - return Err(Ok(value)); - } - cmp::Ordering::Greater => (new_leaf, leaf), - cmp::Ordering::Less => (leaf, new_leaf), - }; - - let new_internal_node = unsafe { new_internal.deref_mut() }; - new_internal_node.key = unsafe { new_right.deref().key.clone() }; - new_internal_node.left.store(new_left, Ordering::Relaxed); - new_internal_node.right.store(new_right, Ordering::Relaxed); - - // NOTE: record.leaf_addr is called childAddr in the paper. - match record.leaf_addr().compare_exchange( - leaf, - new_internal, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => return Ok(()), - Err(e) => { - // Insertion failed. Help the conflicting remove operation if needed. - // NOTE: The paper version checks if any of the mark is set, which is redundant. - if e.current.with_tag(0).ptr_eq(leaf) { - self.cleanup(record, handle); - } - } - } - } - } - - pub fn insert<'d, 'h, 'he>( - &self, - key: K, - mut value: V, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &Handle<'d, Node>, - ) -> Result<(), (K, V)> { - loop { - let mut record = SeekRecord::new(); - match self.insert_inner(&key, value, &mut record, shields, handle) { - Ok(()) => return Ok(()), - Err(Ok(v)) => return Err((key, v)), - Err(Err(v)) => value = v, - } - } - } - - fn remove_inner<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &Handle<'d, Node>, - ) -> Result, ()> { - // `leaf` and `value` are the snapshot of the node to be deleted. - // NOTE: The paper version uses one big loop for both phases. - // injection phase - let mut record = SeekRecord::new(); - let (leaf, value) = loop { - self.seek(key, &mut record, shields)?; - - // candidates - let leaf = record.leaf.with_tag(0); - let leaf_node = unsafe { record.leaf.deref() }; - - if leaf_node.key.cmp(key) != cmp::Ordering::Equal { - return Ok(None); - } - - let value = leaf_node.value.as_ref().unwrap(); - - // Try injecting the deletion flag. - match record.leaf_addr().compare_exchange( - leaf, - leaf.with_tag(Marks::new(true, false).bits()), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - // Finalize the node to be removed - if self.cleanup(&mut record, handle) { - return Ok(Some(value)); - } - // In-place cleanup failed. Enter the cleanup phase. - break (leaf, value); - } - Err(e) => { - // Flagging failed. - // case 1. record.leaf_addr(e.current) points to another node: restart. - // case 2. Another thread flagged/tagged the edge to leaf: help and restart - // NOTE: The paper version checks if any of the mark is set, which is redundant. - if leaf.ptr_eq(e.current.with_tag(Marks::empty().bits())) { - self.cleanup(&mut record, handle); - } - } - } - }; - - let leaf = leaf.with_tag(0); - - // cleanup phase - loop { - self.seek(key, &mut record, shields)?; - if !record.leaf.with_tag(0).ptr_eq(leaf) { - // The edge to leaf flagged for deletion was removed by a helping thread - return Ok(Some(value)); - } - - // leaf is still present in the tree. - if self.cleanup(&mut record, handle) { - return Ok(Some(value)); - } - } - } - - pub fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h, K, V>, - handle: &Handle<'d, Node>, - ) -> Option<&'he V> { - loop { - if let Ok(r) = self.remove_inner(key, shields.launder(), handle) { - return r; - } - } - } -} - -impl ConcurrentMap for NMTreeMap -where - K: Ord + Clone + 'static, - V: Clone + 'static, -{ - type Node = Node; - - type Shields<'d, 'h> = EraShields<'d, 'h, K, V> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(handle: &'h Handle<'d, Self::Node>) -> Self { - Self::new(handle) - } - - #[inline(always)] - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - _handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.get(key, shields) - } - - #[inline(always)] - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> bool { - self.insert(key, value, shields, handle).is_ok() - } - - #[inline(always)] - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d, Self::Node>, - ) -> Option<&'he V> { - self.remove(key, shields, handle) - } -} - -#[cfg(test)] -mod tests { - use super::NMTreeMap; - use crate::ds_impl::crystalline_l::concurrent_map; - - #[test] - fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); - } -} diff --git a/src/ds_impl/he/concurrent_map.rs b/src/ds_impl/he/concurrent_map.rs deleted file mode 100644 index 1eee3911..00000000 --- a/src/ds_impl/he/concurrent_map.rs +++ /dev/null @@ -1,98 +0,0 @@ -use he::Handle; - -pub trait ConcurrentMap { - type Shields<'d, 'h> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h>; - fn new<'d, 'h>(handle: &'h Handle<'d>) -> Self; - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V>; - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool; - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V>; -} - -#[cfg(test)] -pub mod tests { - extern crate rand; - use super::ConcurrentMap; - use he::Domain; - use rand::prelude::*; - use std::thread; - - const THREADS: i32 = 30; - const ELEMENTS_PER_THREADS: i32 = 1000; - - pub fn smoke + Send + Sync>() { - let domain = &Domain::new((THREADS + 1) as usize); - let handle = domain.register(); - let map = &M::new(&handle); - - thread::scope(|s| { - for t in 0..THREADS { - s.spawn(move || { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert!(map.insert(i, i.to_string(), &mut shields, &handle)); - } - }); - } - }); - - thread::scope(|s| { - for t in 0..(THREADS / 2) { - s.spawn(move || { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert_eq!( - i.to_string(), - *map.remove(&i, &mut shields, &handle).unwrap() - ); - } - }); - } - }); - - thread::scope(|s| { - for t in (THREADS / 2)..THREADS { - s.spawn(move || { - let handle = domain.register(); - let mut shields = M::shields(&handle); - let mut rng = rand::thread_rng(); - let mut keys: Vec = - (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); - keys.shuffle(&mut rng); - for i in keys { - assert_eq!(i.to_string(), *map.get(&i, &mut shields, &handle).unwrap()); - } - }); - } - }); - } -} diff --git a/src/ds_impl/he/list.rs b/src/ds_impl/he/list.rs deleted file mode 100644 index 70e904f5..00000000 --- a/src/ds_impl/he/list.rs +++ /dev/null @@ -1,658 +0,0 @@ -use super::concurrent_map::ConcurrentMap; - -use std::cmp::Ordering::{Equal, Greater, Less}; -use std::sync::atomic::Ordering; - -use he::{Atomic, Handle, HazardEra, Shared}; - -// `#[repr(C)]` is used to ensure the first field -// is also the first data in the memory alignment. -#[repr(C)] -pub struct Node { - /// Mark: tag(), Tag: not needed - next: Atomic, - key: K, - value: V, -} - -pub struct List { - head: Atomic>, -} - -impl Default for List -where - K: Ord + 'static, -{ - fn default() -> Self { - Self::new() - } -} - -impl Drop for List { - fn drop(&mut self) { - let mut curr = self.head.load(Ordering::Relaxed); - - while !curr.is_null() { - curr = unsafe { curr.into_owned() }.next.load(Ordering::Relaxed); - } - } -} - -pub struct EraShields<'d, 'h> { - prev_h: HazardEra<'d, 'h>, - curr_h: HazardEra<'d, 'h>, - next_h: HazardEra<'d, 'h>, - // `anchor_h` and `anchor_next_h` are used for `find_harris` - anchor_h: HazardEra<'d, 'h>, - anchor_next_h: HazardEra<'d, 'h>, -} - -impl<'d, 'h> EraShields<'d, 'h> { - pub fn new(handle: &'h Handle<'d>) -> Self { - Self { - prev_h: HazardEra::new(handle), - curr_h: HazardEra::new(handle), - next_h: HazardEra::new(handle), - anchor_h: HazardEra::new(handle), - anchor_next_h: HazardEra::new(handle), - } - } - - // bypass E0499-E0503, etc that are supposed to be fixed by polonius - #[inline] - fn launder<'he2>(&mut self) -> &'he2 mut Self { - unsafe { core::mem::transmute(self) } - } -} - -pub struct Cursor { - prev: Shared>, - // For harris, this keeps the mark bit. Don't mix harris and harris-michael. - curr: Shared>, - // `anchor` is used for `find_harris` - // anchor and anchor_next are non-null iff exist - anchor: Shared>, - anchor_next: Shared>, -} - -impl Cursor { - pub fn new<'d, 'h>(head: &Atomic>, shields: &mut EraShields<'d, 'h>) -> Self { - Self { - prev: Shared::from(head as *const _ as usize), - curr: head.protect(&mut shields.curr_h), - anchor: Shared::null(), - anchor_next: Shared::null(), - } - } -} - -impl Cursor -where - K: Ord, -{ - /// Optimistically traverses while maintaining `anchor` and `anchor_next`. - /// It is used for both Harris and Harris-Herlihy-Shavit traversals. - #[inline] - fn traverse_with_anchor<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h>, - ) -> Result { - // Invariants: - // anchor, anchor_next: protected if they are not null. - // prev: always protected with prev_sh - // curr: not protected. - // curr: also has tag value when it is obtained from prev. - Ok(loop { - let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { - break false; - }; - - // Validation depending on the state of `self.curr`. - // - // - If it is marked, validate on anchor. - // - If it is not marked, it is already protected safely by the Crystalline. - if self.curr.tag() != 0 { - // Validate on anchor. - - debug_assert!(!self.anchor.is_null()); - debug_assert!(!self.anchor_next.is_null()); - let an_new = unsafe { self.anchor.deref() }.next.load(Ordering::Acquire); - - if an_new.tag() != 0 { - return Err(()); - } else if !an_new.ptr_eq(self.anchor_next) { - // Anchor is updated but clear, so can restart from anchor. - - self.prev = self.anchor; - self.curr = an_new; - self.anchor = Shared::null(); - - // Set prev HP as anchor HP, since prev should always be protected. - HazardEra::swap(&mut shields.prev_h, &mut shields.anchor_h); - continue; - } - } - - let next = curr_node.next.protect(&mut shields.next_h); - if next.tag() == 0 { - if curr_node.key < *key { - self.prev = self.curr; - self.curr = next; - self.anchor = Shared::null(); - HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - } else { - break curr_node.key == *key; - } - } else { - if self.anchor.is_null() { - self.anchor = self.prev; - self.anchor_next = self.curr; - HazardEra::swap(&mut shields.anchor_h, &mut shields.prev_h); - } else if self.anchor_next.ptr_eq(self.prev) { - HazardEra::swap(&mut shields.anchor_next_h, &mut shields.prev_h); - } - self.prev = self.curr; - self.curr = next; - HazardEra::swap(&mut shields.curr_h, &mut shields.prev_h); - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - } - }) - } - - #[inline] - fn find_harris<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Result { - // Finding phase - // - cursor.curr: first unmarked node w/ key >= search key (4) - // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) - // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) - - let found = self.traverse_with_anchor(key, shields)?; - - scopeguard::defer! { - shields.anchor_h.clear(); - shields.anchor_next_h.clear(); - } - - if self.anchor.is_null() { - self.prev = self.prev.with_tag(0); - self.curr = self.curr.with_tag(0); - Ok(found) - } else { - debug_assert_eq!(self.anchor_next.tag(), 0); - // TODO: on CAS failure, if anchor is not tagged, we can restart from anchor. - unsafe { &self.anchor.deref().next } - .compare_exchange( - self.anchor_next, - self.curr.with_tag(0), - Ordering::AcqRel, - Ordering::Relaxed, - ) - .map_err(|_| { - self.curr = self.curr.with_tag(0); - () - })?; - - let mut node = self.anchor_next; - while !node.with_tag(0).ptr_eq(self.curr.with_tag(0)) { - // NOTE: It may seem like this can be done with a NA load, but we do a `fetch_or` in remove, which always does an write. - // This can be a NA load if the `fetch_or` in delete is changed to a CAS, but it is not clear if it is worth it. - let next = unsafe { node.deref().next.load(Ordering::Relaxed) }; - debug_assert!(next.tag() != 0); - unsafe { handle.retire(node) }; - node = next; - } - self.prev = self.anchor.with_tag(0); - self.curr = self.curr.with_tag(0); - Ok(found) - } - } - - #[inline] - fn find_harris_michael<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Result { - loop { - debug_assert_eq!(self.curr.tag(), 0); - let Some(curr_node) = (unsafe { self.curr.as_ref() }) else { - return Ok(false); - }; - let mut next = curr_node.next.protect(&mut shields.next_h); - - if next.tag() != 0 { - next = next.with_tag(0); - unsafe { self.prev.deref() } - .next - .compare_exchange(self.curr, next, Ordering::AcqRel, Ordering::Acquire) - .map_err(|_| ())?; - unsafe { handle.retire(self.curr) }; - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - self.curr = next; - continue; - } - - match curr_node.key.cmp(key) { - Less => { - HazardEra::swap(&mut shields.prev_h, &mut shields.curr_h); - HazardEra::swap(&mut shields.curr_h, &mut shields.next_h); - self.prev = self.curr; - self.curr = next; - } - Equal => return Ok(true), - Greater => return Ok(false), - } - } - } - - #[inline] - fn find_harris_herlihy_shavit<'d, 'h>( - &mut self, - key: &K, - shields: &mut EraShields<'d, 'h>, - _handle: &'h Handle<'d>, - ) -> Result { - let found = self.traverse_with_anchor(key, shields)?; - // Return only the found `curr` node. - // Others are not necessary because we are not going to do insertion or deletion - // with this Harris-Herlihy-Shavit traversal. - self.curr = self.curr.with_tag(0); - shields.anchor_h.clear(); - shields.anchor_next_h.clear(); - Ok(found) - } -} - -impl List -where - K: Ord + 'static, -{ - /// Creates a new list. - pub fn new() -> Self { - List { - head: Atomic::null(), - } - } - - #[inline] - fn get<'d, 'h, 'he, F>( - &self, - key: &K, - find: F, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> - where - F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, - { - loop { - let mut cursor = Cursor::new(&self.head, shields); - match find(&mut cursor, key, shields, handle) { - Ok(true) => return unsafe { Some(&(cursor.curr.deref().value)) }, - Ok(false) => return None, - Err(_) => continue, - } - } - } - - fn insert_inner<'d, 'h, 'he, F>( - &self, - node: Shared>, - find: &F, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Result - where - F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, - { - loop { - let mut cursor = Cursor::new(&self.head, shields); - let found = find(&mut cursor, unsafe { &node.deref().key }, shields, handle)?; - if found { - drop(unsafe { node.into_owned() }); - return Ok(false); - } - - unsafe { node.deref() } - .next - .store(cursor.curr, Ordering::Relaxed); - if unsafe { cursor.prev.deref() } - .next - .compare_exchange(cursor.curr, node, Ordering::Release, Ordering::Relaxed) - .is_ok() - { - return Ok(true); - } - } - } - - #[inline] - fn insert<'d, 'h, 'he, F>( - &self, - key: K, - value: V, - find: F, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool - where - F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, - { - let node = Shared::new( - Node { - key, - value, - next: Atomic::null(), - }, - handle, - ); - - loop { - match self.insert_inner(node, &find, shields, handle) { - Ok(r) => return r, - Err(()) => continue, - } - } - } - - fn remove_inner<'d, 'h, 'he, F>( - &self, - key: &K, - find: &F, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Result, ()> - where - F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, - { - loop { - let mut cursor = Cursor::new(&self.head, shields); - let found = find(&mut cursor, key, shields, handle)?; - if !found { - return Ok(None); - } - - let curr_node = unsafe { cursor.curr.deref() }; - let next = curr_node.next.fetch_or(1, Ordering::AcqRel); - if next.tag() == 1 { - continue; - } - - if unsafe { &cursor.prev.deref().next } - .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) - .is_ok() - { - unsafe { handle.retire(cursor.curr) }; - } - - return Ok(Some(&curr_node.value)); - } - } - - #[inline] - fn remove<'d, 'h, 'he, F>( - &self, - key: &K, - find: F, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> - where - F: Fn(&mut Cursor, &K, &mut EraShields<'d, 'h>, &'h Handle<'d>) -> Result, - { - loop { - match self.remove_inner(key, &find, shields.launder(), handle) { - Ok(r) => return r, - Err(_) => continue, - } - } - } - - pub fn harris_get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.get(key, Cursor::find_harris, shields, handle) - } - - pub fn harris_insert<'d, 'h, 'he>( - &self, - key: K, - value: V, - shields: &mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool { - self.insert(key, value, Cursor::find_harris, shields, handle) - } - - pub fn harris_remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.remove(key, Cursor::find_harris, shields, handle) - } - - pub fn harris_michael_get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.get(key, Cursor::find_harris_michael, shields, handle) - } - - pub fn harris_michael_insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool { - self.insert(key, value, Cursor::find_harris_michael, shields, handle) - } - - pub fn harris_michael_remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.remove(key, Cursor::find_harris_michael, shields, handle) - } - - pub fn harris_herlihy_shavit_get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.get(key, Cursor::find_harris_herlihy_shavit, shields, handle) - } -} - -pub struct HList { - inner: List, -} - -impl ConcurrentMap for HList -where - K: Ord + 'static, - V: 'static, -{ - type Shields<'d, 'h> - = EraShields<'d, 'h> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(_: &'h Handle<'d>) -> Self { - Self { inner: List::new() } - } - - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.inner.harris_get(key, shields, handle) - } - - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool { - self.inner.harris_insert(key, value, shields, handle) - } - - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.inner.harris_remove(key, shields, handle) - } -} - -pub struct HMList { - inner: List, -} - -impl ConcurrentMap for HMList -where - K: Ord + 'static, - V: 'static, -{ - type Shields<'d, 'h> - = EraShields<'d, 'h> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(_: &'h Handle<'d>) -> Self { - Self { inner: List::new() } - } - - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.inner.harris_michael_get(key, shields, handle) - } - - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool { - self.inner - .harris_michael_insert(key, value, shields, handle) - } - - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.inner.harris_michael_remove(key, shields, handle) - } -} - -pub struct HHSList { - inner: List, -} - -impl ConcurrentMap for HHSList -where - K: Ord + 'static, - V: 'static, -{ - type Shields<'d, 'h> - = EraShields<'d, 'h> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(_: &'h Handle<'d>) -> Self { - Self { inner: List::new() } - } - - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.inner.harris_herlihy_shavit_get(key, shields, handle) - } - - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool { - self.inner.harris_insert(key, value, shields, handle) - } - - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.inner.harris_remove(key, shields, handle) - } -} - -#[cfg(test)] -mod tests { - use super::{HHSList, HList, HMList}; - use crate::ds_impl::he::concurrent_map; - - #[test] - fn smoke_h_list() { - concurrent_map::tests::smoke::>(); - } - - #[test] - fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); - } - - #[test] - fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); - } -} diff --git a/src/ds_impl/he/michael_hash_map.rs b/src/ds_impl/he/michael_hash_map.rs deleted file mode 100644 index c3e6a5d6..00000000 --- a/src/ds_impl/he/michael_hash_map.rs +++ /dev/null @@ -1,102 +0,0 @@ -use super::concurrent_map::ConcurrentMap; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; - -use super::list::HHSList; -pub use super::list::{Cursor, EraShields, Node}; - -use he::Handle; - -pub struct HashMap { - buckets: Vec>, -} - -impl HashMap -where - K: Ord + Hash + 'static, - V: 'static, -{ - pub fn with_capacity<'d, 'h>(n: usize, handle: &'h Handle<'d>) -> Self { - let mut buckets = Vec::with_capacity(n); - for _ in 0..n { - buckets.push(HHSList::new(handle)); - } - - HashMap { buckets } - } - - #[inline] - pub fn get_bucket(&self, index: usize) -> &HHSList { - unsafe { self.buckets.get_unchecked(index % self.buckets.len()) } - } - - #[inline] - fn hash(k: &K) -> usize { - let mut s = DefaultHasher::new(); - k.hash(&mut s); - s.finish() as usize - } -} - -impl ConcurrentMap for HashMap -where - K: Ord + Hash + Send + 'static, - V: Send + 'static, -{ - type Shields<'d, 'h> = EraShields<'d, 'h> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(handle: &'h Handle<'d>) -> Self { - Self::with_capacity(30000, handle) - } - - #[inline(always)] - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - let i = Self::hash(key); - self.get_bucket(i).get(key, shields, handle) - } - - #[inline(always)] - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool { - let i = Self::hash(&key); - self.get_bucket(i).insert(key, value, shields, handle) - } - - #[inline(always)] - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - let i = Self::hash(key); - self.get_bucket(i).remove(key, shields, handle) - } -} - -#[cfg(test)] -mod tests { - use super::HashMap; - use crate::ds_impl::he::concurrent_map; - - #[test] - fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); - } -} diff --git a/src/ds_impl/he/mod.rs b/src/ds_impl/he/mod.rs deleted file mode 100644 index f6d449ed..00000000 --- a/src/ds_impl/he/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub mod concurrent_map; - -pub mod list; -pub mod michael_hash_map; -pub mod natarajan_mittal_tree; - -pub use self::concurrent_map::ConcurrentMap; - -pub use self::list::{HHSList, HList, HMList}; -pub use self::michael_hash_map::HashMap; -pub use self::natarajan_mittal_tree::NMTreeMap; diff --git a/src/ds_impl/he/natarajan_mittal_tree.rs b/src/ds_impl/he/natarajan_mittal_tree.rs deleted file mode 100644 index 12d7d9a6..00000000 --- a/src/ds_impl/he/natarajan_mittal_tree.rs +++ /dev/null @@ -1,683 +0,0 @@ -use super::concurrent_map::ConcurrentMap; - -use std::cmp; -use std::sync::atomic::Ordering; - -use he::{Atomic, Handle, HazardEra, Shared}; - -bitflags! { - /// TODO - /// A remove operation is registered by marking the corresponding edges: the (parent, target) - /// edge is _flagged_ and the (parent, sibling) edge is _tagged_. - struct Marks: usize { - const FLAG = 1usize.wrapping_shl(1); - const TAG = 1usize.wrapping_shl(0); - } -} - -impl Marks { - fn new(flag: bool, tag: bool) -> Self { - (if flag { Marks::FLAG } else { Marks::empty() }) - | (if tag { Marks::TAG } else { Marks::empty() }) - } - - fn flag(self) -> bool { - !(self & Marks::FLAG).is_empty() - } - - fn tag(self) -> bool { - !(self & Marks::TAG).is_empty() - } - - fn marked(self) -> bool { - !(self & (Marks::TAG | Marks::FLAG)).is_empty() - } -} - -#[derive(Clone, PartialEq, Eq, Debug)] -enum Key { - Fin(K), - Inf, -} - -impl PartialOrd for Key -where - K: PartialOrd, -{ - fn partial_cmp(&self, other: &Self) -> Option { - match (self, other) { - (Key::Fin(k1), Key::Fin(k2)) => k1.partial_cmp(k2), - (Key::Fin(_), Key::Inf) => Some(std::cmp::Ordering::Less), - (Key::Inf, Key::Fin(_)) => Some(std::cmp::Ordering::Greater), - (Key::Inf, Key::Inf) => Some(std::cmp::Ordering::Equal), - } - } -} - -impl PartialEq for Key -where - K: PartialEq, -{ - fn eq(&self, rhs: &K) -> bool { - match self { - Key::Fin(k) => k == rhs, - _ => false, - } - } -} - -impl PartialOrd for Key -where - K: PartialOrd, -{ - fn partial_cmp(&self, rhs: &K) -> Option { - match self { - Key::Fin(k) => k.partial_cmp(rhs), - _ => Some(std::cmp::Ordering::Greater), - } - } -} - -impl Key -where - K: Ord, -{ - fn cmp(&self, rhs: &K) -> std::cmp::Ordering { - match self { - Key::Fin(k) => k.cmp(rhs), - _ => std::cmp::Ordering::Greater, - } - } -} - -pub struct Node { - key: Key, - value: Option, - left: Atomic>, - right: Atomic>, -} - -impl Node -where - K: Clone, - V: Clone, -{ - fn new_leaf(key: Key, value: Option) -> Node { - Node { - key, - value, - left: Atomic::null(), - right: Atomic::null(), - } - } - - /// Make a new internal node, consuming the given left and right nodes, - /// using the right node's key. - fn new_internal(left: Node, right: Node, handle: &Handle<'_>) -> Node { - Node { - key: right.key.clone(), - value: None, - left: Atomic::new(left, handle), - right: Atomic::new(right, handle), - } - } -} - -#[derive(Clone, Copy)] -enum Direction { - L, - R, -} - -pub struct EraShields<'d, 'h> { - ancestor_h: HazardEra<'d, 'h>, - successor_h: HazardEra<'d, 'h>, - parent_h: HazardEra<'d, 'h>, - leaf_h: HazardEra<'d, 'h>, - curr_h: HazardEra<'d, 'h>, -} - -impl<'d, 'h> EraShields<'d, 'h> { - pub fn new(handle: &'h Handle<'d>) -> Self { - Self { - ancestor_h: HazardEra::new(handle), - successor_h: HazardEra::new(handle), - parent_h: HazardEra::new(handle), - leaf_h: HazardEra::new(handle), - curr_h: HazardEra::new(handle), - } - } - - // bypass E0499-E0503, etc that are supposed to be fixed by polonius - #[inline] - fn launder<'he2>(&mut self) -> &'he2 mut Self { - unsafe { core::mem::transmute(self) } - } -} - -/// All Shared<_> are unmarked. -/// -/// All of the edges of path from `successor` to `parent` are in the process of removal. -pub struct SeekRecord { - /// Parent of `successor` - ancestor: Shared>, - /// The first internal node with a marked outgoing edge - successor: Shared>, - /// The direction of successor from ancestor. - successor_dir: Direction, - /// Parent of `leaf` - parent: Shared>, - /// The end of the access path. - leaf: Shared>, - /// The direction of leaf from parent. - leaf_dir: Direction, -} - -impl SeekRecord { - fn new() -> Self { - Self { - ancestor: Shared::null(), - successor: Shared::null(), - successor_dir: Direction::L, - parent: Shared::null(), - leaf: Shared::null(), - leaf_dir: Direction::L, - } - } - - fn successor_addr(&self) -> &Atomic> { - match self.successor_dir { - Direction::L => unsafe { &self.ancestor.deref().left }, - Direction::R => unsafe { &self.ancestor.deref().right }, - } - } - - fn leaf_addr(&self) -> &Atomic> { - match self.leaf_dir { - Direction::L => unsafe { &self.parent.deref().left }, - Direction::R => unsafe { &self.parent.deref().right }, - } - } - - fn leaf_sibling_addr(&self) -> &Atomic> { - match self.leaf_dir { - Direction::L => unsafe { &self.parent.deref().right }, - Direction::R => unsafe { &self.parent.deref().left }, - } - } -} - -pub struct NMTreeMap { - r: Node, -} - -impl Default for NMTreeMap -where - K: Ord + Clone, - V: Clone, -{ - fn default() -> Self { - todo!("new") - } -} - -impl Drop for NMTreeMap { - fn drop(&mut self) { - unsafe { - let mut stack = vec![ - self.r.left.load(Ordering::Relaxed), - self.r.right.load(Ordering::Relaxed), - ]; - assert!(self.r.value.is_none()); - - while let Some(node) = stack.pop() { - if node.is_null() { - continue; - } - - let node_ref = node.deref(); - - stack.push(node_ref.left.load(Ordering::Relaxed)); - stack.push(node_ref.right.load(Ordering::Relaxed)); - drop(node.into_owned()); - } - } - } -} - -impl NMTreeMap -where - K: Ord + Clone + 'static, - V: Clone + 'static, -{ - pub fn new<'d>(handle: &Handle<'d>) -> Self { - // An empty tree has 5 default nodes with infinite keys so that the SeekRecord is allways - // well-defined. - // r - // / \ - // s inf2 - // / \ - // inf0 inf1 - let inf0 = Node::new_leaf(Key::Inf, None); - let inf1 = Node::new_leaf(Key::Inf, None); - let inf2 = Node::new_leaf(Key::Inf, None); - let s = Node::new_internal(inf0, inf1, handle); - let r = Node::new_internal(s, inf2, handle); - NMTreeMap { r } - } - - fn seek<'d, 'h>( - &self, - key: &K, - record: &mut SeekRecord, - shields: &mut EraShields<'d, 'h>, - ) -> Result<(), ()> { - let s = self.r.left.load(Ordering::Relaxed).with_tag(0); - let s_node = unsafe { s.deref() }; - - // The root node is always alive; we do not have to protect it. - record.ancestor = Shared::from(&self.r as *const _ as usize); - record.successor = s; // TODO: should preserve tag? - - record.successor_dir = Direction::L; - // The `s` node is always alive; we do not have to protect it. - record.parent = s; - - record.leaf = s_node.left.protect(&mut shields.leaf_h); - record.leaf_dir = Direction::L; - - let mut prev_tag = Marks::from_bits_truncate(record.leaf.tag()).tag(); - let mut curr_dir = Direction::L; - let mut curr = unsafe { record.leaf.deref() } - .left - .protect(&mut shields.curr_h); - - // `ancestor` always points untagged node. - while !curr.is_null() { - if !prev_tag { - // untagged edge: advance ancestor and successor pointers - record.ancestor = record.parent; - record.successor = record.leaf; - record.successor_dir = record.leaf_dir; - // `ancestor` and `successor` are already protected by - // hazard pointers of `parent` and `leaf`. - - // Advance the parent and leaf pointers when the cursor looks like the following: - // (): protected by its dedicated shield. - // - // (parent), ancestor -> O (ancestor) -> O - // / \ / \ - // (leaf), successor -> O O => (parent), successor -> O O - // / \ / \ - // O O (leaf) -> O O - record.parent = record.leaf; - HazardEra::swap(&mut shields.ancestor_h, &mut shields.parent_h); - HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); - } else if record.successor.ptr_eq(record.parent) { - // Advance the parent and leaf pointer when the cursor looks like the following: - // (): protected by its dedicated shield. - // - // (ancestor) -> O (ancestor) -> O - // / \ / \ - // (parent), successor -> O O (successor) -> O O - // / \ => / \ - // (leaf) -> O O (parent) -> O O - // / \ / \ - // O O (leaf) -> O O - record.parent = record.leaf; - HazardEra::swap(&mut shields.successor_h, &mut shields.parent_h); - HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); - } else { - // Advance the parent and leaf pointer when the cursor looks like the following: - // (): protected by its dedicated shield. - // - // (ancestor) -> O - // / \ - // (successor) -> O O - // ... ... - // (parent) -> O - // / \ - // (leaf) -> O O - record.parent = record.leaf; - HazardEra::swap(&mut shields.parent_h, &mut shields.leaf_h); - } - debug_assert_eq!(record.successor.tag(), 0); - - if Marks::from_bits_truncate(curr.tag()).marked() { - // `curr` is marked. Validate by `ancestor`. - let succ_new = record.successor_addr().load(Ordering::Acquire); - if Marks::from_bits_truncate(succ_new.tag()).marked() - || !record.successor.ptr_eq(succ_new) - { - // Validation is failed. Let's restart from the root. - // TODO: Maybe it can be optimized (by restarting from the anchor), but - // it would require a serious reasoning (including shield swapping, etc). - return Err(()); - } - } - - record.leaf = curr; - record.leaf_dir = curr_dir; - - // update other variables - prev_tag = Marks::from_bits_truncate(curr.tag()).tag(); - let curr_node = unsafe { curr.deref() }; - if curr_node.key.cmp(key) == cmp::Ordering::Greater { - curr_dir = Direction::L; - curr = curr_node.left.load(Ordering::Acquire); - } else { - curr_dir = Direction::R; - curr = curr_node.right.load(Ordering::Acquire); - } - } - Ok(()) - } - - /// Physically removes node. - /// - /// Returns true if it successfully unlinks the flagged node in `record`. - fn cleanup<'d>(&self, record: &mut SeekRecord, handle: &Handle<'d>) -> bool { - // Identify the node(subtree) that will replace `successor`. - let leaf_marked = record.leaf_addr().load(Ordering::Acquire); - let leaf_flag = Marks::from_bits_truncate(leaf_marked.tag()).flag(); - let target_sibling_addr = if leaf_flag { - record.leaf_sibling_addr() - } else { - record.leaf_addr() - }; - - // NOTE: the ibr implementation uses CAS - // tag (parent, sibling) edge -> all of the parent's edges can't change now - // TODO: Is Release enough? - target_sibling_addr.fetch_or(Marks::TAG.bits(), Ordering::AcqRel); - - // Try to replace (ancestor, successor) w/ (ancestor, sibling). - // Since (parent, sibling) might have been concurrently flagged, copy - // the flag to the new edge (ancestor, sibling). - let target_sibling = target_sibling_addr.load(Ordering::Acquire); - let flag = Marks::from_bits_truncate(target_sibling.tag()).flag(); - let is_unlinked = record - .successor_addr() - .compare_exchange( - record.successor.with_tag(0), - target_sibling.with_tag(Marks::new(flag, false).bits()), - Ordering::AcqRel, - Ordering::Acquire, - ) - .is_ok(); - - if is_unlinked { - unsafe { - // destroy the subtree of successor except target_sibling - let mut stack = vec![record.successor]; - - while let Some(node) = stack.pop() { - if node.is_null() || node.with_tag(0).ptr_eq(target_sibling.with_tag(0)) { - continue; - } - - let node_ref = node.deref(); - - stack.push(node_ref.left.load(Ordering::Relaxed)); - stack.push(node_ref.right.load(Ordering::Relaxed)); - handle.retire(node); - } - } - } - - is_unlinked - } - - fn get_inner<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - ) -> Result, ()> { - let mut record = SeekRecord::new(); - - self.seek(key, &mut record, shields)?; - let leaf_node = unsafe { record.leaf.deref() }; - - if leaf_node.key.cmp(key) != cmp::Ordering::Equal { - return Ok(None); - } - - Ok(Some(leaf_node.value.as_ref().unwrap())) - } - - pub fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - ) -> Option<&'he V> { - loop { - if let Ok(r) = self.get_inner(key, shields.launder()) { - return r; - } - } - } - - fn insert_inner<'d, 'h, 'he>( - &self, - key: &K, - value: V, - record: &mut SeekRecord, - shields: &'he mut EraShields<'d, 'h>, - handle: &Handle<'d>, - ) -> Result<(), Result> { - let mut new_leaf = Shared::new(Node::new_leaf(Key::Fin(key.clone()), Some(value)), handle); - - let mut new_internal = Shared::new( - Node:: { - key: Key::Inf, // temporary placeholder - value: None, - left: Atomic::null(), - right: Atomic::null(), - }, - handle, - ); - - loop { - self.seek(key, record, shields).map_err(|_| unsafe { - let value = new_leaf.deref_mut().value.take().unwrap(); - drop(new_leaf.into_owned()); - drop(new_internal.into_owned()); - Err(value) - })?; - let leaf = record.leaf.with_tag(0); - - let (new_left, new_right) = match unsafe { leaf.deref() }.key.cmp(key) { - cmp::Ordering::Equal => { - // Newly created nodes that failed to be inserted are free'd here. - let value = unsafe { new_leaf.deref_mut() }.value.take().unwrap(); - unsafe { - drop(new_leaf.into_owned()); - drop(new_internal.into_owned()); - } - return Err(Ok(value)); - } - cmp::Ordering::Greater => (new_leaf, leaf), - cmp::Ordering::Less => (leaf, new_leaf), - }; - - let new_internal_node = unsafe { new_internal.deref_mut() }; - new_internal_node.key = unsafe { new_right.deref().key.clone() }; - new_internal_node.left.store(new_left, Ordering::Relaxed); - new_internal_node.right.store(new_right, Ordering::Relaxed); - - // NOTE: record.leaf_addr is called childAddr in the paper. - match record.leaf_addr().compare_exchange( - leaf, - new_internal, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => return Ok(()), - Err(e) => { - // Insertion failed. Help the conflicting remove operation if needed. - // NOTE: The paper version checks if any of the mark is set, which is redundant. - if e.current.with_tag(0).ptr_eq(leaf) { - self.cleanup(record, handle); - } - } - } - } - } - - pub fn insert<'d, 'h, 'he>( - &self, - key: K, - mut value: V, - shields: &'he mut EraShields<'d, 'h>, - handle: &Handle<'d>, - ) -> Result<(), (K, V)> { - loop { - let mut record = SeekRecord::new(); - match self.insert_inner(&key, value, &mut record, shields, handle) { - Ok(()) => return Ok(()), - Err(Ok(v)) => return Err((key, v)), - Err(Err(v)) => value = v, - } - } - } - - fn remove_inner<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - handle: &Handle<'d>, - ) -> Result, ()> { - // `leaf` and `value` are the snapshot of the node to be deleted. - // NOTE: The paper version uses one big loop for both phases. - // injection phase - let mut record = SeekRecord::new(); - let (leaf, value) = loop { - self.seek(key, &mut record, shields)?; - - // candidates - let leaf = record.leaf.with_tag(0); - let leaf_node = unsafe { record.leaf.deref() }; - - if leaf_node.key.cmp(key) != cmp::Ordering::Equal { - return Ok(None); - } - - let value = leaf_node.value.as_ref().unwrap(); - - // Try injecting the deletion flag. - match record.leaf_addr().compare_exchange( - leaf, - leaf.with_tag(Marks::new(true, false).bits()), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - // Finalize the node to be removed - if self.cleanup(&mut record, handle) { - return Ok(Some(value)); - } - // In-place cleanup failed. Enter the cleanup phase. - break (leaf, value); - } - Err(e) => { - // Flagging failed. - // case 1. record.leaf_addr(e.current) points to another node: restart. - // case 2. Another thread flagged/tagged the edge to leaf: help and restart - // NOTE: The paper version checks if any of the mark is set, which is redundant. - if leaf.ptr_eq(e.current.with_tag(Marks::empty().bits())) { - self.cleanup(&mut record, handle); - } - } - } - }; - - let leaf = leaf.with_tag(0); - - // cleanup phase - loop { - self.seek(key, &mut record, shields)?; - if !record.leaf.with_tag(0).ptr_eq(leaf) { - // The edge to leaf flagged for deletion was removed by a helping thread - return Ok(Some(value)); - } - - // leaf is still present in the tree. - if self.cleanup(&mut record, handle) { - return Ok(Some(value)); - } - } - } - - pub fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut EraShields<'d, 'h>, - handle: &Handle<'d>, - ) -> Option<&'he V> { - loop { - if let Ok(r) = self.remove_inner(key, shields.launder(), handle) { - return r; - } - } - } -} - -impl ConcurrentMap for NMTreeMap -where - K: Ord + Clone + 'static, - V: Clone + 'static, -{ - type Shields<'d, 'h> = EraShields<'d, 'h> - where - 'd: 'h; - - fn shields<'d, 'h>(handle: &'h Handle<'d>) -> Self::Shields<'d, 'h> { - EraShields::new(handle) - } - - fn new<'d, 'h>(handle: &'h Handle<'d>) -> Self { - Self::new(handle) - } - - #[inline(always)] - fn get<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - _handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.get(key, shields) - } - - #[inline(always)] - fn insert<'d, 'h>( - &self, - key: K, - value: V, - shields: &mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> bool { - self.insert(key, value, shields, handle).is_ok() - } - - #[inline(always)] - fn remove<'d, 'h, 'he>( - &self, - key: &K, - shields: &'he mut Self::Shields<'d, 'h>, - handle: &'h Handle<'d>, - ) -> Option<&'he V> { - self.remove(key, shields, handle) - } -} - -#[cfg(test)] -mod tests { - use super::NMTreeMap; - use crate::ds_impl::he::concurrent_map; - - #[test] - fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); - } -} diff --git a/src/ds_impl/mod.rs b/src/ds_impl/mod.rs index 320a67f6..d7917775 100644 --- a/src/ds_impl/mod.rs +++ b/src/ds_impl/mod.rs @@ -1,9 +1,7 @@ pub mod cdrc; pub mod circ_ebr; pub mod circ_hp; -pub mod crystalline_l; pub mod ebr; -pub mod he; pub mod hp; pub mod hp_brcu; pub mod hp_pp; diff --git a/test-scripts/sanitize-crystalline-l.sh b/test-scripts/sanitize-crystalline-l.sh deleted file mode 100755 index f0a158f2..00000000 --- a/test-scripts/sanitize-crystalline-l.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' - -he="cargo run --bin crystalline-l --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " - -set -e -for i in {1..5000}; do - $he -dh-list -i3 -t48 -r10 -g1 - $he -dhm-list -i3 -t48 -r10 -g1 - $he -dhhs-list -i3 -t48 -r10 -g1 -done diff --git a/test-scripts/sanitize-he.sh b/test-scripts/sanitize-he.sh deleted file mode 100755 index bc05db07..00000000 --- a/test-scripts/sanitize-he.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' - -he="cargo run --bin he --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " - -set -e -for i in {1..5000}; do - $he -dh-list -i3 -t48 -r10 -g1 - $he -dhm-list -i3 -t48 -r10 -g1 - $he -dhhs-list -i3 -t48 -r10 -g1 -done From 9c2dea27245385d0fb85c369caa26bd6bcfa117c Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 27 Nov 2024 10:44:32 +0000 Subject: [PATCH 83/84] Update README and add an experiment script --- README.md | 2 ++ bench-scripts/hp-revisited/experiment.sh | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 bench-scripts/hp-revisited/experiment.sh diff --git a/README.md b/README.md index 62711c23..5dff6fb0 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ where * `skip-list`: lock-free skiplist by Herlihy and Shavit, with wait-free get() for schemes other than HP \[3\] * `bonsai-tree`: A non-blocking variant of Bonsai tree \[5\] * `efrb-tree`: Ellen et al. ’s tree \[6\] + * `elim-ab-tree`: An (a,b) tree with elimination \[17\] * Reclamation scheme * `nr`: A baseline that does not reclaim memory * `ebr`: Epoch-based RCU \[1,7\] @@ -276,3 +277,4 @@ Note that sanitizer may report memory leaks when used against CIRC EBR. This is * \[14\] Jeonghyeon Kim, Jaehwang Jung, and Jeehoon Kang. 2024. Expediting Hazard Pointers with Bounded RCU Critical Sections. In Proceedings of the 36th ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2024), June 17–21, 2024, Nantes, France. ACM, New York, NY, USA, 34 pages. * \[15\] Jaehwang Jung, Jeonghyeon Kim, Matthew J. Parkinson, and Jeehoon Kang. 2024. Concurrent Immediate Reference Counting. Proc. ACM Program. Lang. 8, PLDI, Article 153 (June 2024), 24 pages. * \[16\] Gali Sheffi, Maurice Herlihy, and Erez Petrank. 2021. VBR: Version Based Reclamation. In Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures (Virtual Event, USA) (SPAA ’21). Association for Computing Machinery, New York, NY, USA, 443–445. +* \[17\] Anubhav Srivastava and Trevor Brown. 2022. Elimination (a,b)-trees with fast, durable updates. In Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (PPoPP '22). Association for Computing Machinery, New York, NY, USA, 416–430. diff --git a/bench-scripts/hp-revisited/experiment.sh b/bench-scripts/hp-revisited/experiment.sh new file mode 100644 index 00000000..75e50137 --- /dev/null +++ b/bench-scripts/hp-revisited/experiment.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +python3 ./bench-scripts/hp-revisited/bench.py +python3 ./bench-scripts/hp-revisited/bench-short-lists.py +python3 ./bench-scripts/hp-revisited/bench-hp-trees.py + +python3 ./bench-scripts/hp-revisited/plot.py +python3 ./bench-scripts/hp-revisited/plot-short-lists.py +python3 ./bench-scripts/hp-revisited/plot-hp-trees.py From 7401573149e67cf6846c5816e54e9d3c211d30b1 Mon Sep 17 00:00:00 2001 From: Jeonghyeon Kim Date: Wed, 27 Nov 2024 10:45:53 +0000 Subject: [PATCH 84/84] Add execution permission for HP experiment script --- bench-scripts/hp-revisited/experiment.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bench-scripts/hp-revisited/experiment.sh diff --git a/bench-scripts/hp-revisited/experiment.sh b/bench-scripts/hp-revisited/experiment.sh old mode 100644 new mode 100755