diff --git a/.gitignore b/.gitignore index d670581d..3e9a8c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ tags *.pdf *.csv .python-version +__pycache__ diff --git a/Cargo.lock b/Cargo.lock index 7eac0b38..bf342462 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "atomic" @@ -259,14 +259,14 @@ dependencies = [ [[package]] name = "crossbeam-pebr-epoch" version = "0.7.1" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#8c4b969c866a7d1b1c698f6b374e781615efb8c8" dependencies = [ "arrayvec 0.4.12", "bitflags 1.3.2", "cfg-if 0.1.10", "crossbeam-pebr-utils", "lazy_static", - "membarrier 0.2.3 (git+https://github.com/kaist-cp/crossbeam?branch=pebr)", + "membarrier 0.2.3 (git+https://github.com/jeehoonkang/membarrier-rs.git?rev=99254da47b6dab382c9860a024cab3ab9bac6504)", "memoffset", "murmur3", "scopeguard", @@ -276,7 +276,7 @@ dependencies = [ [[package]] name = "crossbeam-pebr-utils" version = "0.6.5" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#8c4b969c866a7d1b1c698f6b374e781615efb8c8" dependencies = [ "cfg-if 0.1.10", "lazy_static", @@ -369,7 +369,7 @@ dependencies = [ name = "hp-brcu" version = "0.1.0" dependencies = [ - "arrayvec 0.7.4", + "arrayvec 0.7.6", "atomic", "bitflags 2.5.0", "cfg-if 1.0.0", @@ -454,7 +454,7 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "membarrier" version = "0.2.3" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/jeehoonkang/membarrier-rs.git?branch=smr-benchmark#bc3cf47144212655ea3daf38b10bf646ba94fbe9" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -465,7 +465,7 @@ dependencies = [ [[package]] name = "membarrier" version = "0.2.3" -source = "git+https://github.com/jeehoonkang/membarrier-rs.git?branch=smr-benchmark#bc3cf47144212655ea3daf38b10bf646ba94fbe9" +source = "git+https://github.com/jeehoonkang/membarrier-rs.git?rev=99254da47b6dab382c9860a024cab3ab9bac6504#99254da47b6dab382c9860a024cab3ab9bac6504" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -497,7 +497,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "murmur3" version = "0.4.1" -source = "git+https://github.com/kaist-cp/crossbeam?branch=pebr#05d55f2e1683e91f2bcbbbf06ff8ae4666ec22e5" +source = "git+https://github.com/kaist-cp/murmur3.git?branch=upgrade#0843ec5d49329d24e7c646d681a55701dbf7cf52" dependencies = [ "byteorder", "static_assertions 0.3.4", @@ -807,6 +807,7 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" name = "smr-benchmark" version = "0.1.0" dependencies = [ + "arrayvec 0.7.6", "bitflags 2.5.0", "cdrc", "cfg-if 1.0.0", @@ -821,6 +822,7 @@ dependencies = [ "nbr", "num", "rand", + "scopeguard", "tikv-jemalloc-ctl", "tikv-jemallocator", "typenum", @@ -909,7 +911,6 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" name = "vbr" version = "0.1.0" dependencies = [ - "arrayvec 0.7.4", "atomic", "crossbeam-utils 0.8.20", "portable-atomic", diff --git a/Cargo.toml b/Cargo.toml index 47c77721..202b860e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ csv = "1.3.0" rand = "0.8" typenum = "1.17" num = "0.4.3" +arrayvec = "0.7.6" +scopeguard = "1" hp_pp = { path = "./smrs/hp-pp" } nbr = { path = "./smrs/nbr" } cdrc = { path = "./smrs/cdrc" } diff --git a/README.md b/README.md index 62711c23..5dff6fb0 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ where * `skip-list`: lock-free skiplist by Herlihy and Shavit, with wait-free get() for schemes other than HP \[3\] * `bonsai-tree`: A non-blocking variant of Bonsai tree \[5\] * `efrb-tree`: Ellen et al. ’s tree \[6\] + * `elim-ab-tree`: An (a,b) tree with elimination \[17\] * Reclamation scheme * `nr`: A baseline that does not reclaim memory * `ebr`: Epoch-based RCU \[1,7\] @@ -276,3 +277,4 @@ Note that sanitizer may report memory leaks when used against CIRC EBR. This is * \[14\] Jeonghyeon Kim, Jaehwang Jung, and Jeehoon Kang. 2024. Expediting Hazard Pointers with Bounded RCU Critical Sections. In Proceedings of the 36th ACM Symposium on Parallelism in Algorithms and Architectures (SPAA 2024), June 17–21, 2024, Nantes, France. ACM, New York, NY, USA, 34 pages. * \[15\] Jaehwang Jung, Jeonghyeon Kim, Matthew J. Parkinson, and Jeehoon Kang. 2024. Concurrent Immediate Reference Counting. Proc. ACM Program. Lang. 8, PLDI, Article 153 (June 2024), 24 pages. * \[16\] Gali Sheffi, Maurice Herlihy, and Erez Petrank. 2021. VBR: Version Based Reclamation. In Proceedings of the 33rd ACM Symposium on Parallelism in Algorithms and Architectures (Virtual Event, USA) (SPAA ’21). Association for Computing Machinery, New York, NY, USA, 443–445. +* \[17\] Anubhav Srivastava and Trevor Brown. 2022. Elimination (a,b)-trees with fast, durable updates. In Proceedings of the 27th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (PPoPP '22). Association for Computing Machinery, New York, NY, USA, 416–430. diff --git a/bench-scripts/hp-revisited/bench-hp-trees.py b/bench-scripts/hp-revisited/bench-hp-trees.py new file mode 100644 index 00000000..50be5525 --- /dev/null +++ b/bench-scripts/hp-revisited/bench-hp-trees.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import subprocess +import os + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") +BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") + +dss = ['nm-tree', 'efrb-tree'] +# "-large" suffix if it uses a large garbage bag. +mms = ['hp'] +i = 10 +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = list(map(str, [1] + list(range(4, 33, 4)))) +elif cpu_count <= 64: + ts = list(map(str, [1] + list(range(8, 129, 8)))) +else: + ts = list(map(str, [1] + list(range(12, 193, 12)))) +runs = 2 +gs = [0, 1, 2] + +subprocess.run(['cargo', 'build', '--release']) + +def key_ranges(ds): + return ["100000"] + +def is_suffix(orig, suf): + return len(suf) <= len(orig) and mm[-len(suf):] == suf + +def make_cmd(mm, i, ds, g, t, kr): + bag = "small" + if is_suffix(mm, "-large"): + mm = mm[:len(mm)-len("-large")] + bag = "large" + + return [os.path.join(BIN_PATH, mm), + '-i', str(i), + '-d', str(ds), + '-g', str(g), + '-t', str(t), + '-r', str(kr), + '-b', bag, + '-o', os.path.join(RESULTS_PATH, f'{ds}.csv')] + +def invalid(mm, ds, g): + is_invalid = False + if ds == 'hhs-list': + is_invalid |= g == 0 # HHSList is just HList with faster get() + if mm == 'nbr': + is_invalid |= ds in ["hm-list", "skip-list"] + if ds == 'elim-ab-tree': + is_invalid |= mm in ["pebr", "hp-pp", "vbr"] + return is_invalid + +cmds = [] + +for ds in dss: + for kr in key_ranges(ds): + for mm in mms: + for g in gs: + if invalid(mm, ds, g): + continue + for t in ts: + cmds.append(make_cmd(mm, i, ds, g, t, kr)) + +print('number of configurations: ', len(cmds)) +print('estimated time: ', (len(cmds) * i * 1.1) // 60, ' min *', runs, 'times\n') + +for i, cmd in enumerate(cmds): + try: + print(f"\rdry-running commands... ({i+1}/{len(cmds)})", end="") + subprocess.run(cmd + ['--dry-run']) + except: + print(f"A dry-run for the following command is failed:\n{' '.join(cmd)}") + exit(1) +print("\nAll dry-runs passed!\n") + +os.makedirs(RESULTS_PATH, exist_ok=True) +failed = [] +for run in range(runs): + for i, cmd in enumerate(cmds): + print("run {}/{}, bench {}/{}: '{}'".format(run + 1, runs, i + 1, len(cmds), ' '.join(cmd))) + try: + subprocess.run(cmd, timeout=i+30) + except subprocess.TimeoutExpired: + print("timeout") + failed.append(' '.join(cmd)) + except KeyboardInterrupt: + if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) + exit(0) + except: + failed.append(' '.join(cmd)) + +if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) diff --git a/bench-scripts/hp-revisited/bench-short-lists.py b/bench-scripts/hp-revisited/bench-short-lists.py new file mode 100644 index 00000000..b2f6b047 --- /dev/null +++ b/bench-scripts/hp-revisited/bench-short-lists.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import subprocess +import os + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") +BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") + +dss = ['hhs-list', 'hm-list'] +# "-large" suffix if it uses a large garbage bag. +mms = ['hp', 'hp-pp'] +i = 10 +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = list(map(str, [1] + list(range(4, 33, 4)))) +elif cpu_count <= 64: + ts = list(map(str, [1] + list(range(8, 129, 8)))) +else: + ts = list(map(str, [1] + list(range(12, 193, 12)))) +runs = 2 +gs = [0, 1, 2] + +subprocess.run(['cargo', 'build', '--release']) + +def key_ranges(ds): + return ["16"] + +def is_suffix(orig, suf): + return len(suf) <= len(orig) and mm[-len(suf):] == suf + +def make_cmd(mm, i, ds, g, t, kr): + bag = "small" + if is_suffix(mm, "-large"): + mm = mm[:len(mm)-len("-large")] + bag = "large" + + return [os.path.join(BIN_PATH, mm), + '-i', str(i), + '-d', str(ds), + '-g', str(g), + '-t', str(t), + '-r', str(kr), + '-b', bag, + '-o', os.path.join(RESULTS_PATH, f'{ds}.csv')] + +def invalid(mm, ds, g): + is_invalid = False + if ds == 'hhs-list': + is_invalid |= g == 0 # HHSList is just HList with faster get() + if mm == 'nbr': + is_invalid |= ds in ["hm-list", "skip-list"] + if ds == 'elim-ab-tree': + is_invalid |= mm in ["pebr", "hp-pp", "vbr"] + return is_invalid + +cmds = [] + +for ds in dss: + for kr in key_ranges(ds): + for mm in mms: + for g in gs: + if invalid(mm, ds, g): + continue + for t in ts: + cmds.append(make_cmd(mm, i, ds, g, t, kr)) + +print('number of configurations: ', len(cmds)) +print('estimated time: ', (len(cmds) * i * 1.1) // 60, ' min *', runs, 'times\n') + +for i, cmd in enumerate(cmds): + try: + print(f"\rdry-running commands... ({i+1}/{len(cmds)})", end="") + subprocess.run(cmd + ['--dry-run']) + except: + print(f"A dry-run for the following command is failed:\n{' '.join(cmd)}") + exit(1) +print("\nAll dry-runs passed!\n") + +os.makedirs(RESULTS_PATH, exist_ok=True) +failed = [] +for run in range(runs): + for i, cmd in enumerate(cmds): + print("run {}/{}, bench {}/{}: '{}'".format(run + 1, runs, i + 1, len(cmds), ' '.join(cmd))) + try: + subprocess.run(cmd, timeout=i+30) + except subprocess.TimeoutExpired: + print("timeout") + failed.append(' '.join(cmd)) + except KeyboardInterrupt: + if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) + exit(0) + except: + failed.append(' '.join(cmd)) + +if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) diff --git a/bench-scripts/hp-revisited/bench.py b/bench-scripts/hp-revisited/bench.py new file mode 100644 index 00000000..f164404a --- /dev/null +++ b/bench-scripts/hp-revisited/bench.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +import subprocess +import os + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") +BIN_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "target", "release") + +dss = ['h-list', 'hm-list', 'hhs-list', 'hash-map', 'nm-tree', 'skip-list', 'elim-ab-tree'] +# "-large" suffix if it uses a large garbage bag. +mms = ['nr', 'ebr', 'pebr', 'hp', 'hp-pp', 'hp-brcu', 'vbr'] +i = 10 +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = list(map(str, [1] + list(range(4, 33, 4)))) +elif cpu_count <= 64: + ts = list(map(str, [1] + list(range(8, 129, 8)))) +else: + ts = list(map(str, [1] + list(range(12, 193, 12)))) +runs = 2 +gs = [0, 1, 2] + +subprocess.run(['cargo', 'build', '--release']) + +def key_ranges(ds): + if ds in ["h-list", "hm-list", "hhs-list"]: + # 1K and 10K + return ["1000", "10000"] + else: + # 100K and 100M + return ["100000", "100000000"] + +def is_suffix(orig, suf): + return len(suf) <= len(orig) and mm[-len(suf):] == suf + +def make_cmd(mm, i, ds, g, t, kr): + bag = "small" + if is_suffix(mm, "-large"): + mm = mm[:len(mm)-len("-large")] + bag = "large" + + return [os.path.join(BIN_PATH, mm), + '-i', str(i), + '-d', str(ds), + '-g', str(g), + '-t', str(t), + '-r', str(kr), + '-b', bag, + '-o', os.path.join(RESULTS_PATH, f'{ds}.csv')] + +def invalid(mm, ds, g): + is_invalid = False + if ds == 'hhs-list': + is_invalid |= g == 0 # HHSList is just HList with faster get() + if mm == 'nbr': + is_invalid |= ds in ["hm-list", "skip-list"] + if ds == 'elim-ab-tree': + is_invalid |= mm in ["hp-pp"] + return is_invalid + +cmds = [] + +for ds in dss: + for kr in key_ranges(ds): + for mm in mms: + for g in gs: + if invalid(mm, ds, g): + continue + for t in ts: + cmds.append(make_cmd(mm, i, ds, g, t, kr)) + +print('number of configurations: ', len(cmds)) +print('estimated time: ', (len(cmds) * i * 1.1) // 60, ' min *', runs, 'times\n') + +for i, cmd in enumerate(cmds): + try: + print(f"\rdry-running commands... ({i+1}/{len(cmds)})", end="") + subprocess.run(cmd + ['--dry-run']) + except: + print(f"A dry-run for the following command is failed:\n{' '.join(cmd)}") + exit(1) +print("\nAll dry-runs passed!\n") + +os.makedirs(RESULTS_PATH, exist_ok=True) +failed = [] +for run in range(runs): + for i, cmd in enumerate(cmds): + print("run {}/{}, bench {}/{}: '{}'".format(run + 1, runs, i + 1, len(cmds), ' '.join(cmd))) + try: + subprocess.run(cmd, timeout=i+30) + except subprocess.TimeoutExpired: + print("timeout") + failed.append(' '.join(cmd)) + except KeyboardInterrupt: + if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) + exit(0) + except: + failed.append(' '.join(cmd)) + +if len(failed) > 0: + print("====failed====") + print("\n".join(failed)) diff --git a/bench-scripts/hp-revisited/experiment.sh b/bench-scripts/hp-revisited/experiment.sh new file mode 100755 index 00000000..75e50137 --- /dev/null +++ b/bench-scripts/hp-revisited/experiment.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +python3 ./bench-scripts/hp-revisited/bench.py +python3 ./bench-scripts/hp-revisited/bench-short-lists.py +python3 ./bench-scripts/hp-revisited/bench-hp-trees.py + +python3 ./bench-scripts/hp-revisited/plot.py +python3 ./bench-scripts/hp-revisited/plot-short-lists.py +python3 ./bench-scripts/hp-revisited/plot-hp-trees.py diff --git a/bench-scripts/hp-revisited/legends.py b/bench-scripts/hp-revisited/legends.py new file mode 100644 index 00000000..db77815b --- /dev/null +++ b/bench-scripts/hp-revisited/legends.py @@ -0,0 +1,94 @@ +import os +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from matplotlib.path import Path +from matplotlib.transforms import Affine2D + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +EBR = "ebr" +PEBR = "pebr" +NR = "nr" +HP = "hp" +HP_PP = "hp-pp" +HP_BRCU = "hp-brcu" +HP_RCU = "hp-rcu" +VBR = "vbr" + +SMRs = [NR, EBR, HP_PP, HP, PEBR, HP_BRCU, HP_RCU, VBR] + +FACE_ALPHA = 0.85 + +# https://matplotlib.org/stable/gallery/lines_bars_and_markers/marker_reference.html +line_shapes = { + NR: { + "marker": ".", + "color": "k", + "linestyle": "-", + }, + EBR: { + "marker": "o", + "color": "c", + "linestyle": "-", + }, + HP: { + "marker": "v", + "color": "hotpink", + "linestyle": "dashed", + }, + HP_PP: { + "marker": "^", + "color": "purple", + "linestyle": "dashdot", + }, + PEBR: { + # Diamond("D") shape, but smaller. + "marker": Path.unit_rectangle() + .transformed(Affine2D().translate(-0.5, -0.5) + .rotate_deg(45)), + "color": "y", + "linestyle": (5, (10, 3)), + }, + HP_BRCU: { + "marker": "X", + "color": "r", + "linestyle": (5, (10, 3)), + }, + HP_RCU: { + "marker": "P", + "color": "green", + "linestyle": (5, (10, 3)), + }, + VBR: { + "marker": "d", + "color": "orange", + "linestyle": (0, (2, 1)), + }, + # Used in `plot-short-lists` + "PESSIM_HP": { + "marker": "v", + "color": "#828282", + "linestyle": "dotted", + }, +} + +# Add some common or derivable properties. +line_shapes = dict(map( + lambda kv: kv if kv[0] == NR else + (kv[0], { **kv[1], + "markerfacecolor": (*colors.to_rgb(kv[1]["color"]), FACE_ALPHA), + "markeredgecolor": "k", + "markeredgewidth": 0.75 }), + line_shapes.items() +)) + +if __name__ == "__main__": + os.makedirs(f'{RESULTS_PATH}/legends', exist_ok=True) + for smr in SMRs: + fig, ax = plt.subplots(ncols=1) + ax.plot([0] * 3, linewidth=3, markersize=18, **line_shapes[smr]) + ax.set_xlim(2/3, 4/3) + ax.set_axis_off() + fig.set_size_inches(1.25, 0.75) + fig.tight_layout() + fig.savefig(f"{RESULTS_PATH}/legends/{smr}.pdf", bbox_inches="tight") diff --git a/bench-scripts/hp-revisited/plot-hp-trees.py b/bench-scripts/hp-revisited/plot-hp-trees.py new file mode 100644 index 00000000..29862194 --- /dev/null +++ b/bench-scripts/hp-revisited/plot-hp-trees.py @@ -0,0 +1,151 @@ +import pandas as pd +import warnings +import os, math +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from legends import * + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +warnings.filterwarnings("ignore") +pd.set_option('display.max_rows', None) + +# avoid Type 3 fonts +matplotlib.rcParams['pdf.fonttype'] = 42 +matplotlib.rcParams['ps.fonttype'] = 42 + +# raw column names +THREADS = "threads" +THROUGHPUT = "throughput" +PEAK_MEM = "peak_mem" +AVG_GARB = "avg_garb" +PEAK_GARB = "peak_garb" + +# legend +SMR_ONLY = "SMR\n" + +EFRBTREE = "efrb-tree" +NMTREE = "nm-tree" + +# DS with read-dominated bench & write-only bench +dss_all = [NMTREE, EFRBTREE] + +WRITE, HALF, READ = "write", "half", "read" + +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = [1] + list(range(4, 33, 4)) +elif cpu_count <= 64: + ts = [1] + list(range(8, 129, 8)) +else: + ts = [1] + list(range(12, 193, 12)) +n_map = {0: ''} + +(label_size, xtick_size, ytick_size, marker_size) = (24, 20, 18, 20) + +SMRs = [HP] +COMBs = [f"{HP}_{NMTREE}", f"{HP}_{EFRBTREE}"] + +NMTREE_SHAPE = line_shapes[HP] +EFRBTREE_SHAPE = line_shapes["PESSIM_HP"] + +def plot_title(bench): + return 'NMTree v.s. EFRBTree' + +def range_to_str(kr: int): + UNITS = ["K", "M", "B", "T"] + for i in range(len(UNITS)): + unit = pow(10, 3 * (i+1)) + div = kr // unit + if div < 1000 or i == len(UNITS) - 1: + return f"{div}{UNITS[i]}" + +def draw(title, name, data, y_value, y_label=None, y_max=None, y_from_zero=False): + print(name) + plt.figure(figsize=(10, 7)) + plt.title(title, fontsize=36, pad=15) + + d = data[data.mm_ds == f"{HP}_{NMTREE}"].sort_values(by=[THREADS], axis=0) + h1, = plt.plot(d[THREADS], d[y_value], label="HP; NMTREE", + linewidth=3, markersize=marker_size, **NMTREE_SHAPE, zorder=30) + + d = data[data.mm_ds == f"{HP}_{EFRBTREE}"].sort_values(by=[THREADS], axis=0) + h2, = plt.plot(d[THREADS], d[y_value], label="HP; EFRBTREE", + linewidth=3, markersize=marker_size, **EFRBTREE_SHAPE, zorder=30) + + plt.legend(handles=[h1, h2], fontsize=label_size, loc="lower right") + + plt.xlabel("Threads", fontsize=label_size) + plt.ylabel(y_label, fontsize=label_size) + plt.yticks(fontsize=ytick_size) + plt.xticks(ts, fontsize=xtick_size, rotation=90) + plt.grid(alpha=0.5) + + if data.threads.max() >= cpu_count: + left, right = plt.xlim() + plt.axvspan(cpu_count, right, facecolor="#FF00000A") + plt.xlim(left, right) + + y_max = min(y_max, data[y_value].max()) if y_max else data[y_value].max() + y_min = 0 if y_from_zero else data[y_value].min() + y_margin = (y_max - y_min) * 0.05 + plt.ylim(y_min, y_max+y_margin) + + plt.savefig(name, bbox_inches='tight') + + +def draw_throughput(data, bench): + data = data.copy() + y_label = 'Throughput (M op/s)' + y_max = data.throughput.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/hp-trees_{bench}_throughput.pdf', + data, THROUGHPUT, y_label, y_max, True) + + +def draw_peak_garb(data, bench): + data = data.copy() + y_label = 'Peak unreclaimed nodes (×10⁴)' + y_max = data.peak_garb.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/hp-trees_{bench}_peak_garb.pdf', + data, PEAK_GARB, y_label, y_max) + + +raw_data = {} +# averaged data for write:read = 100:0, 50:50, 10:90 +avg_data = { WRITE: {}, HALF: {}, READ: {} } + +# preprocess +csvs = [] +for ds in dss_all: + csvs.append(pd.read_csv(f'{RESULTS_PATH}/' + ds + '.csv')) + +data = pd.concat(csvs) +data.throughput = data.throughput.map(lambda x: x / 1000_000) +data.peak_mem = data.peak_mem.map(lambda x: x / (2 ** 20)) +data.avg_mem = data.avg_mem.map(lambda x: x / (2 ** 20)) +data.peak_garb = data.peak_garb.map(lambda x: x / 10000) +data.avg_garb = data.avg_garb.map(lambda x: x / 10000) +data["mm_ds"] = list(map(lambda p: p[0] + "_" + p[1], zip(data.mm, data.ds))) +data.mm = list(map(lambda tup: tup[0] if tup[1] == "small" else tup[0] + "-large", zip(data.mm, data.bag_size))) +data = data[data.mm.isin(SMRs)] +data = data[data.key_range == 100000] +data = data.drop(["bag_size", "ds", "mm"], axis=1) + +# take average of each runs +avg = data.groupby(['mm_ds', 'threads', 'non_coop', 'get_rate', 'key_range']).mean().reset_index() + +avg[SMR_ONLY] = pd.Categorical(avg.mm_ds.map(str), COMBs) +avg.sort_values(by=SMR_ONLY, inplace=True) +for i, bench in [(0, WRITE), (1, HALF), (2, READ)]: + avg_data[bench] = avg[avg.get_rate == i] + +# 1. throughput graphs, 3 lines (SMR_ONLY) each. +draw_throughput(avg_data[WRITE], WRITE) +draw_throughput(avg_data[HALF], HALF) +draw_throughput(avg_data[READ], READ) + +# 2. peak garbage graph +draw_peak_garb(avg_data[WRITE], WRITE) +draw_peak_garb(avg_data[HALF], HALF) +draw_peak_garb(avg_data[READ], READ) diff --git a/bench-scripts/hp-revisited/plot-short-lists.py b/bench-scripts/hp-revisited/plot-short-lists.py new file mode 100644 index 00000000..a4a54489 --- /dev/null +++ b/bench-scripts/hp-revisited/plot-short-lists.py @@ -0,0 +1,151 @@ +import pandas as pd +import warnings +import os, math +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from legends import * + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +warnings.filterwarnings("ignore") +pd.set_option('display.max_rows', None) + +# avoid Type 3 fonts +matplotlib.rcParams['pdf.fonttype'] = 42 +matplotlib.rcParams['ps.fonttype'] = 42 + +# raw column names +THREADS = "threads" +THROUGHPUT = "throughput" +PEAK_MEM = "peak_mem" +AVG_GARB = "avg_garb" +PEAK_GARB = "peak_garb" + +# legend +SMR_ONLY = "SMR\n" + +HMLIST = "hm-list" +HHSLIST = "hhs-list" + +# DS with read-dominated bench & write-only bench +dss_all = [HHSLIST, HMLIST] + +WRITE, HALF, READ = "write", "half", "read" + +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = [1] + list(range(4, 33, 4)) +elif cpu_count <= 64: + ts = [1] + list(range(8, 129, 8)) +else: + ts = [1] + list(range(12, 193, 12)) +n_map = {0: ''} + +(label_size, xtick_size, ytick_size, marker_size) = (24, 20, 18, 20) + +SMRs = [HP] +COMBs = [f"{HP}_{HHSLIST}", f"{HP}_{HMLIST}"] + +HHSLIST_SHAPE = line_shapes[HP] +HMLIST_SHAPE = line_shapes["PESSIM_HP"] + +def plot_title(bench): + return 'HHSList v.s. HMList' + +def range_to_str(kr: int): + UNITS = ["K", "M", "B", "T"] + for i in range(len(UNITS)): + unit = pow(10, 3 * (i+1)) + div = kr // unit + if div < 1000 or i == len(UNITS) - 1: + return f"{div}{UNITS[i]}" + +def draw(title, name, data, y_value, y_label=None, y_max=None, y_from_zero=False): + print(name) + plt.figure(figsize=(10, 7)) + plt.title(title, fontsize=36, pad=15) + + d = data[data.mm_ds == f"{HP}_{HHSLIST}"].sort_values(by=[THREADS], axis=0) + h1, = plt.plot(d[THREADS], d[y_value], label="HP; HHSList", + linewidth=3, markersize=marker_size, **HHSLIST_SHAPE, zorder=30) + + d = data[data.mm_ds == f"{HP}_{HMLIST}"].sort_values(by=[THREADS], axis=0) + h2, = plt.plot(d[THREADS], d[y_value], label="HP; HMList", + linewidth=3, markersize=marker_size, **HMLIST_SHAPE, zorder=30) + + plt.legend(handles=[h1, h2], fontsize=label_size, loc="lower right") + + plt.xlabel("Threads", fontsize=label_size) + plt.ylabel(y_label, fontsize=label_size) + plt.yticks(fontsize=ytick_size) + plt.xticks(ts, fontsize=xtick_size, rotation=90) + plt.grid(alpha=0.5) + + if data.threads.max() >= cpu_count: + left, right = plt.xlim() + plt.axvspan(cpu_count, right, facecolor="#FF00000A") + plt.xlim(left, right) + + y_max = min(y_max, data[y_value].max()) if y_max else data[y_value].max() + y_min = 0 if y_from_zero else data[y_value].min() + y_margin = (y_max - y_min) * 0.05 + plt.ylim(y_min, y_max+y_margin) + + plt.savefig(name, bbox_inches='tight') + + +def draw_throughput(data, bench): + data = data.copy() + y_label = 'Throughput (M op/s)' + y_max = data.throughput.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/short-lists_{bench}_throughput.pdf', + data, THROUGHPUT, y_label, y_max, True) + + +def draw_peak_garb(data, bench): + data = data.copy() + y_label = 'Peak unreclaimed nodes (×10⁴)' + y_max = data.peak_garb.max() * 1.05 + draw(plot_title(bench), f'{RESULTS_PATH}/short-lists_{bench}_peak_garb.pdf', + data, PEAK_GARB, y_label, y_max) + + +raw_data = {} +# averaged data for write:read = 100:0, 50:50, 10:90 +avg_data = { WRITE: {}, HALF: {}, READ: {} } + +# preprocess +csvs = [] +for ds in dss_all: + csvs.append(pd.read_csv(f'{RESULTS_PATH}/' + ds + '.csv')) + +data = pd.concat(csvs) +data.throughput = data.throughput.map(lambda x: x / 1000_000) +data.peak_mem = data.peak_mem.map(lambda x: x / (2 ** 20)) +data.avg_mem = data.avg_mem.map(lambda x: x / (2 ** 20)) +data.peak_garb = data.peak_garb.map(lambda x: x / 10000) +data.avg_garb = data.avg_garb.map(lambda x: x / 10000) +data["mm_ds"] = list(map(lambda p: p[0] + "_" + p[1], zip(data.mm, data.ds))) +data.mm = list(map(lambda tup: tup[0] if tup[1] == "small" else tup[0] + "-large", zip(data.mm, data.bag_size))) +data = data[data.mm.isin(SMRs)] +data = data[data.key_range == 16] +data = data.drop(["bag_size", "ds", "mm"], axis=1) + +# take average of each runs +avg = data.groupby(['mm_ds', 'threads', 'non_coop', 'get_rate', 'key_range']).mean().reset_index() + +avg[SMR_ONLY] = pd.Categorical(avg.mm_ds.map(str), COMBs) +avg.sort_values(by=SMR_ONLY, inplace=True) +for i, bench in [(0, WRITE), (1, HALF), (2, READ)]: + avg_data[bench] = avg[avg.get_rate == i] + +# 1. throughput graphs, 3 lines (SMR_ONLY) each. +draw_throughput(avg_data[WRITE], WRITE) +draw_throughput(avg_data[HALF], HALF) +draw_throughput(avg_data[READ], READ) + +# 2. peak garbage graph +draw_peak_garb(avg_data[WRITE], WRITE) +draw_peak_garb(avg_data[HALF], HALF) +draw_peak_garb(avg_data[READ], READ) diff --git a/bench-scripts/hp-revisited/plot.py b/bench-scripts/hp-revisited/plot.py new file mode 100644 index 00000000..550629f0 --- /dev/null +++ b/bench-scripts/hp-revisited/plot.py @@ -0,0 +1,189 @@ +import pandas as pd +import warnings +import os, math +import matplotlib +import matplotlib.pyplot as plt +from legends import * + +RESULTS_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "results") + +warnings.filterwarnings("ignore") +pd.set_option('display.max_rows', None) + +# avoid Type 3 fonts +matplotlib.rcParams['pdf.fonttype'] = 42 +matplotlib.rcParams['ps.fonttype'] = 42 + +# raw column names +THREADS = "threads" +THROUGHPUT = "throughput" +PEAK_MEM = "peak_mem" +AVG_GARB = "avg_garb" +PEAK_GARB = "peak_garb" + +# legend +SMR_ONLY = "SMR\n" + +HLIST = "h-list" +HMLIST = "hm-list" +HHSLIST = "hhs-list" +HASHMAP = "hash-map" +NMTREE = "nm-tree" +SKIPLIST = "skip-list" +ELIMABTREE = "elim-ab-tree" + +FORMAL_NAMES = { + HLIST: "HList", + HMLIST: "HMList", + HHSLIST: "HHSList", + HASHMAP: "HashMap", + NMTREE: "NMTree", + SKIPLIST: "SkipList", + ELIMABTREE: "ElimAbTree", +} + +# DS with read-dominated bench & write-only bench +dss_all = [HLIST, HMLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] +dss_read = [HLIST, HMLIST, HHSLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] +dss_write = [HLIST, HMLIST, HASHMAP, NMTREE, SKIPLIST, ELIMABTREE] + +WRITE, HALF, READ = "write", "half", "read" + +cpu_count = os.cpu_count() +if not cpu_count or cpu_count <= 24: + ts = [1] + list(range(4, 33, 4)) +elif cpu_count <= 64: + ts = [1] + list(range(8, 129, 8)) +else: + ts = [1] + list(range(12, 193, 12)) +n_map = {0: ''} + +(label_size, xtick_size, ytick_size, marker_size) = (24, 20, 18, 20) + +mm_order = [ + HP_BRCU, + EBR, + HP, + HP_PP, + PEBR, + VBR, + NR, +] + +def plot_title(ds, bench): + if bench == WRITE and ds == HLIST: + return 'HList/HHSList' + return FORMAL_NAMES[ds] + +def key_ranges(ds): + if ds in [HLIST, HMLIST, HHSLIST]: + return [1000, 10000] + else: + return [100000, 100000000] + +def range_to_str(kr: int): + UNITS = ["K", "M", "B", "T"] + for i in range(len(UNITS)): + unit = pow(10, 3 * (i+1)) + div = kr // unit + if div < 1000 or i == len(UNITS) - 1: + return f"{div}{UNITS[i]}" + +def draw(title, name, data, y_value, y_label=None, y_max=None, y_min=None): + print(name) + plt.figure(figsize=(10, 7)) + plt.title(title, fontsize=36, pad=15) + + for mm in sorted(list(set(data.mm)), key=lambda mm: len(mm_order) - mm_order.index(mm)): + d = data[data.mm == mm].sort_values(by=[THREADS], axis=0) + plt.plot(d[THREADS], d[y_value], + linewidth=3, markersize=marker_size, **line_shapes[mm], zorder=30) + + plt.xlabel("Threads", fontsize=label_size) + plt.ylabel(y_label, fontsize=label_size) + plt.yticks(fontsize=ytick_size) + plt.xticks(ts, fontsize=xtick_size, rotation=90) + plt.grid(alpha=0.5) + + if data.threads.max() >= cpu_count: + left, right = plt.xlim() + plt.axvspan(cpu_count, right, facecolor="#FF00000A") + plt.xlim(left, right) + + y_max = min(y_max, data[y_value].max()) if y_max else data[y_value].max() + y_min = max(y_min, data[y_value].min()) if y_min else data[y_value].min() + y_margin = (y_max - y_min) * 0.05 + plt.ylim(y_min-y_margin, y_max+y_margin) + + plt.savefig(name, bbox_inches='tight') + + +def draw_throughput(data, ds, bench, key_range): + data = data[ds].copy() + data = data[data.key_range == key_range] + data = data[data.non_coop == 0] + y_label = 'Throughput (M op/s)' + y_max = data.throughput.max() * 1.05 + draw(plot_title(ds, bench), f'{RESULTS_PATH}/{ds}_{bench}_{range_to_str(key_range)}_throughput.pdf', + data, THROUGHPUT, y_label, y_max) + + +def draw_peak_garb(data, ds, bench, key_range): + data = data[ds].copy() + data = data[data.key_range == key_range] + data = data[data.mm != NR] + data = data[data.mm != VBR] + y_label = 'Peak unreclaimed nodes (×10⁴)' + y_max = 0 + for cand in [HP_PP, HP_BRCU]: + max_garb = data[data.mm == cand].peak_garb.max() + if not math.isnan(max_garb): + y_max = max(y_max, max_garb * 2) + draw(plot_title(ds, bench), f'{RESULTS_PATH}/{ds}_{bench}_{range_to_str(key_range)}_peak_garb.pdf', + data, PEAK_GARB, y_label, y_max) + + +raw_data = {} +# averaged data for write:read = 100:0, 50:50, 10:90 +avg_data = { WRITE: {}, HALF: {}, READ: {} } + +# preprocess +for ds in dss_all: + data = pd.read_csv(f'{RESULTS_PATH}/' + ds + '.csv') + + data.throughput = data.throughput.map(lambda x: x / 1000_000) + data.peak_mem = data.peak_mem.map(lambda x: x / (2 ** 20)) + data.avg_mem = data.avg_mem.map(lambda x: x / (2 ** 20)) + data.peak_garb = data.peak_garb.map(lambda x: x / 10000) + data.avg_garb = data.avg_garb.map(lambda x: x / 10000) + data.mm = list(map(lambda tup: tup[0] if tup[1] == "small" else tup[0] + "-large", zip(data.mm, data.bag_size))) + data = data.drop("bag_size", axis=1) + data = data[data.mm.isin(SMRs)] + + raw_data[ds] = data.copy() + + # take average of each runs + avg = data.groupby(['ds', 'mm', 'threads', 'non_coop', 'get_rate', 'key_range']).mean().reset_index() + + avg[SMR_ONLY] = pd.Categorical(avg.mm.map(str), SMRs) + avg.sort_values(by=SMR_ONLY, inplace=True) + for i, bench in enumerate([WRITE, HALF, READ]): + avg_data[bench][ds] = avg[avg.get_rate == i] + +# 1. throughput graphs, 3 lines (SMR_ONLY) each. +for ds in dss_write: + for kr in key_ranges(ds): + draw_throughput(avg_data[WRITE], ds, WRITE, kr) +for ds in dss_read: + for kr in key_ranges(ds): + draw_throughput(avg_data[HALF], ds, HALF, kr) + draw_throughput(avg_data[READ], ds, READ, kr) + +# 2. peak garbage graph +for ds in dss_write: + for kr in key_ranges(ds): + draw_peak_garb(avg_data[WRITE], ds, WRITE, kr) +for ds in dss_read: + for kr in key_ranges(ds): + draw_peak_garb(avg_data[HALF], ds, HALF, kr) + draw_peak_garb(avg_data[READ], ds, READ, kr) diff --git a/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs b/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs index 726ebe8d..afc6b1c6 100644 --- a/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs +++ b/smrs/cdrc/src/internal/smr/ebr_impl/internal.rs @@ -58,19 +58,19 @@ pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); /// Maximum number of objects a bag can contain. #[cfg(not(any(crossbeam_sanitize, miri)))] -static mut MAX_OBJECTS: usize = 64; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(64); // Makes it more likely to trigger any potential data races. #[cfg(any(crossbeam_sanitize, miri))] -static mut MAX_OBJECTS: usize = 4; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(4); -static mut MANUAL_EVENTS_BETWEEN_COLLECT: usize = 128; +static MANUAL_EVENTS_BETWEEN_COLLECT: AtomicUsize = AtomicUsize::new(128); /// Sets the capacity of thread-local deferred bag. /// /// Note that an update on this capacity may not be reflected immediately to concurrent threads, /// because there is no synchronization between reads and writes on the capacity variable. pub fn set_bag_capacity(max_objects: usize) { - unsafe { MAX_OBJECTS = max_objects }; + MAX_OBJECTS.store(max_objects, Ordering::Relaxed); } /// Sets the manual collection interval. @@ -78,7 +78,7 @@ pub fn set_bag_capacity(max_objects: usize) { /// Note that an update on this interval may not be reflected immediately to concurrent threads, /// because there is no synchronization between reads and writes on the interval variable. pub fn set_manual_collection_interval(interval: usize) { - unsafe { MANUAL_EVENTS_BETWEEN_COLLECT = interval }; + MANUAL_EVENTS_BETWEEN_COLLECT.store(interval, Ordering::Relaxed); } /// A bag of deferred functions. @@ -123,7 +123,7 @@ impl Bag { impl Default for Bag { fn default() -> Self { - Bag(Vec::with_capacity(unsafe { MAX_OBJECTS })) + Bag(Vec::with_capacity(MAX_OBJECTS.load(Ordering::Relaxed))) } } @@ -342,12 +342,12 @@ fn local_size() { impl Local { #[inline] fn counts_between_collect() -> usize { - unsafe { MAX_OBJECTS } + MAX_OBJECTS.load(Ordering::Relaxed) } #[inline] fn counts_between_try_advance() -> usize { - unsafe { MAX_OBJECTS * 2 } + MAX_OBJECTS.load(Ordering::Relaxed) * 2 } /// Registers a new `Local` in the provided `Global`. @@ -606,7 +606,7 @@ impl Local { let manual_count = self.manual_count.get().wrapping_add(1); self.manual_count.set(manual_count); - if manual_count % unsafe { MANUAL_EVENTS_BETWEEN_COLLECT } == 0 { + if manual_count % MANUAL_EVENTS_BETWEEN_COLLECT.load(Ordering::Relaxed) == 0 { self.flush(guard); } } @@ -663,7 +663,7 @@ mod tests { let mut bag = Bag::new(); assert!(bag.is_empty()); - for _ in 0..unsafe { MAX_OBJECTS } { + for _ in 0..MAX_OBJECTS.load(Ordering::Relaxed) { assert!(unsafe { bag.try_push(Deferred::new(incr)).is_ok() }); assert!(!bag.is_empty()); assert_eq!(FLAG.load(Ordering::Relaxed), 0); @@ -675,6 +675,9 @@ mod tests { assert_eq!(FLAG.load(Ordering::Relaxed), 0); drop(bag); - assert_eq!(FLAG.load(Ordering::Relaxed), unsafe { MAX_OBJECTS }); + assert_eq!( + FLAG.load(Ordering::Relaxed), + MAX_OBJECTS.load(Ordering::Relaxed) + ); } } diff --git a/smrs/cdrc/src/internal/smr/hp_impl/thread.rs b/smrs/cdrc/src/internal/smr/hp_impl/thread.rs index abab3c85..7922f2b7 100644 --- a/smrs/cdrc/src/internal/smr/hp_impl/thread.rs +++ b/smrs/cdrc/src/internal/smr/hp_impl/thread.rs @@ -1,26 +1,26 @@ use core::ptr; -use core::sync::atomic::{AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use std::cell::{Cell, RefCell}; use super::domain::Domain; use super::hazard::ThreadRecord; use super::retire::Retired; -pub static mut COUNTS_BETWEEN_FLUSH: usize = 64; +pub static COUNTS_BETWEEN_FLUSH: AtomicUsize = AtomicUsize::new(64); #[inline] pub fn set_counts_between_flush(counts: usize) { - unsafe { COUNTS_BETWEEN_FLUSH = counts }; + COUNTS_BETWEEN_FLUSH.store(counts, Ordering::Relaxed); } #[inline] pub fn counts_between_flush() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) } #[inline] pub fn counts_between_collect() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH * 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) * 2 } pub struct Thread { diff --git a/smrs/circ/src/smr/ebr_impl/internal.rs b/smrs/circ/src/smr/ebr_impl/internal.rs index 3737f615..e69b2510 100644 --- a/smrs/circ/src/smr/ebr_impl/internal.rs +++ b/smrs/circ/src/smr/ebr_impl/internal.rs @@ -58,19 +58,19 @@ pub static GLOBAL_GARBAGE_COUNT: AtomicUsize = AtomicUsize::new(0); /// Maximum number of objects a bag can contain. #[cfg(not(any(crossbeam_sanitize, miri)))] -static mut MAX_OBJECTS: usize = 64; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(64); // Makes it more likely to trigger any potential data races. #[cfg(any(crossbeam_sanitize, miri))] -static mut MAX_OBJECTS: usize = 4; +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(4); -static mut MANUAL_EVENTS_BETWEEN_COLLECT: usize = 64; +static MANUAL_EVENTS_BETWEEN_COLLECT: AtomicUsize = AtomicUsize::new(64); /// Sets the capacity of thread-local deferred bag. /// /// Note that an update on this capacity may not be reflected immediately to concurrent threads, /// because there is no synchronization between reads and writes on the capacity variable. pub fn set_bag_capacity(max_objects: usize) { - unsafe { MAX_OBJECTS = max_objects }; + MAX_OBJECTS.store(max_objects, Ordering::Relaxed); } /// A bag of deferred functions. @@ -115,7 +115,7 @@ impl Bag { impl Default for Bag { fn default() -> Self { - Bag(Vec::with_capacity(unsafe { MAX_OBJECTS })) + Bag(Vec::with_capacity(MAX_OBJECTS.load(Ordering::Relaxed))) } } @@ -602,7 +602,7 @@ impl Local { let manual_count = self.manual_count.get().wrapping_add(1); self.manual_count.set(manual_count); - if manual_count % unsafe { MANUAL_EVENTS_BETWEEN_COLLECT } == 0 { + if manual_count % MANUAL_EVENTS_BETWEEN_COLLECT.load(Ordering::Relaxed) == 0 { self.flush(guard); } } @@ -659,7 +659,7 @@ mod tests { let mut bag = Bag::new(); assert!(bag.is_empty()); - for _ in 0..unsafe { MAX_OBJECTS } { + for _ in 0..MAX_OBJECTS.load(Ordering::Relaxed) { assert!(unsafe { bag.try_push(Deferred::new(incr)).is_ok() }); assert!(!bag.is_empty()); assert_eq!(FLAG.load(Ordering::Relaxed), 0); @@ -671,6 +671,9 @@ mod tests { assert_eq!(FLAG.load(Ordering::Relaxed), 0); drop(bag); - assert_eq!(FLAG.load(Ordering::Relaxed), unsafe { MAX_OBJECTS }); + assert_eq!( + FLAG.load(Ordering::Relaxed), + MAX_OBJECTS.load(Ordering::Relaxed) + ); } } diff --git a/smrs/circ/src/smr/hp_impl/thread.rs b/smrs/circ/src/smr/hp_impl/thread.rs index 04a6e8c1..c16a243a 100644 --- a/smrs/circ/src/smr/hp_impl/thread.rs +++ b/smrs/circ/src/smr/hp_impl/thread.rs @@ -1,26 +1,26 @@ use core::ptr::null_mut; -use core::sync::atomic::{AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use std::cell::{Cell, RefCell}; use super::domain::Domain; use super::hazard::ThreadRecord; use super::retire::{Pile, Retired}; -pub static mut COUNTS_BETWEEN_FLUSH: usize = 64; +pub static COUNTS_BETWEEN_FLUSH: AtomicUsize = AtomicUsize::new(64); #[inline] pub fn set_counts_between_flush(counts: usize) { - unsafe { COUNTS_BETWEEN_FLUSH = counts }; + COUNTS_BETWEEN_FLUSH.store(counts, Ordering::Relaxed); } #[inline] pub fn counts_between_flush() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) } #[inline] pub fn counts_between_collect() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH * 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) * 2 } pub struct Thread { diff --git a/smrs/hp-brcu/src/deferred.rs b/smrs/hp-brcu/src/deferred.rs index c42fe70b..ee43ae96 100644 --- a/smrs/hp-brcu/src/deferred.rs +++ b/smrs/hp-brcu/src/deferred.rs @@ -1,12 +1,27 @@ use std::mem::forget; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::epoch::Epoch; -/// Maximum number of objects a bag can contain. -#[cfg(not(sanitize = "address"))] -pub(crate) const MAX_OBJECTS: usize = 128; -#[cfg(sanitize = "address")] -pub(crate) const MAX_OBJECTS: usize = 4; +#[cfg(not(feature = "sanitize"))] +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(64); +#[cfg(feature = "sanitize")] +static MAX_OBJECTS: AtomicUsize = AtomicUsize::new(4); + +/// Sets the capacity of thread-local garbage bag. +/// +/// This value applies to all threads. +#[inline] +pub fn set_bag_capacity(cap: usize) { + assert!(cap > 1, "capacity must be greater than 1."); + MAX_OBJECTS.store(cap, Ordering::Relaxed); +} + +/// Returns the current capacity of thread-local garbage bag. +#[inline] +pub fn bag_capacity() -> usize { + MAX_OBJECTS.load(Ordering::Relaxed) +} /// A deferred task consisted of data and a callable function. /// @@ -67,7 +82,7 @@ impl Bag { #[inline] pub fn new() -> Self { Self { - defs: Vec::with_capacity(MAX_OBJECTS), + defs: Vec::with_capacity(bag_capacity()), } } @@ -77,7 +92,7 @@ impl Bag { /// full. #[inline] pub fn try_push(&mut self, def: Deferred) -> Result<(), Deferred> { - if self.len() == MAX_OBJECTS { + if self.len() == bag_capacity() { return Err(def); } self.defs.push(def); diff --git a/smrs/hp-brcu/src/handle.rs b/smrs/hp-brcu/src/handle.rs index 70b8ce58..dd8c6bbf 100644 --- a/smrs/hp-brcu/src/handle.rs +++ b/smrs/hp-brcu/src/handle.rs @@ -1,7 +1,7 @@ use std::mem::{transmute, ManuallyDrop}; use std::sync::atomic::{compiler_fence, fence, AtomicUsize, Ordering}; -use crate::deferred::{Deferred, MAX_OBJECTS}; +use crate::deferred::{bag_capacity, Deferred}; use crate::internal::{free, Local}; use crate::pointers::Shared; use crate::rollback::Rollbacker; @@ -259,7 +259,7 @@ impl Thread { pub(crate) unsafe fn do_reclamation(&mut self, mut deferred: Vec) -> Vec { let deferred_len = deferred.len(); - if deferred_len < MAX_OBJECTS * 2 { + if deferred_len < bag_capacity() * 2 { return deferred; } @@ -334,6 +334,10 @@ impl CsGuard { /// Starts a non-crashable section where we can conduct operations with global side-effects. /// + /// It issues a `SeqCst` fence at the beginning, so that a protected hazard pointers can be + /// dereferenced within this section. If you are not going to use such local pointers, + /// it would be better to use `mask_light` instead, which issues a compiler fence only. + /// /// In this section, we do not restart immediately when we receive signals from reclaimers. /// The whole critical section restarts after this `mask` section ends, if a reclaimer sent /// a signal, or we advanced our epoch to reclaim a full local garbage bag. @@ -354,6 +358,33 @@ impl CsGuard { compiler_fence(Ordering::SeqCst); result } + + /// Starts a non-crashable section where we can conduct operations with global side-effects. + /// + /// It issues a compiler fence at the beginning to prevent unexpected instruction reordering + /// across this region boundary. If you are going to use local pointers that are protected + /// with hazard pointers, see `mask` instead. + /// + /// In this section, we do not restart immediately when we receive signals from reclaimers. + /// The whole critical section restarts after this `mask` section ends, if a reclaimer sent + /// a signal, or we advanced our epoch to reclaim a full local garbage bag. + /// + /// The body may return an arbitrary value, and it will be returned without any modifications. + /// However, it is required to return a *rollback-safe* variable from the body. For example, + /// [`String`] or [`Box`] is dangerous to return as it will be leaked on a crash! On the other + /// hand, [`Copy`] types is likely to be safe as they are totally defined by their bit-wise + /// representations, and have no possibilities to be leaked after an unexpected crash. + #[inline(always)] + pub fn mask_light(&self, body: F) -> R + where + F: FnOnce(&mut RaGuard) -> R, + R: Copy, + { + compiler_fence(Ordering::SeqCst); + let result = self.rb.atomic(|_| body(&mut RaGuard { local: self.local })); + compiler_fence(Ordering::SeqCst); + result + } } impl Handle for CsGuard {} diff --git a/smrs/hp-brcu/src/lib.rs b/smrs/hp-brcu/src/lib.rs index e27fe539..2370d2fa 100644 --- a/smrs/hp-brcu/src/lib.rs +++ b/smrs/hp-brcu/src/lib.rs @@ -11,6 +11,7 @@ mod pointers; mod queue; mod rollback; +pub use deferred::{bag_capacity, set_bag_capacity}; pub use handle::*; pub use internal::*; pub use pointers::*; diff --git a/smrs/hp-brcu/src/pointers.rs b/smrs/hp-brcu/src/pointers.rs index 6504ef90..bdaf0a91 100644 --- a/smrs/hp-brcu/src/pointers.rs +++ b/smrs/hp-brcu/src/pointers.rs @@ -125,12 +125,30 @@ impl Atomic { } } +impl<'g, T> From> for Atomic { + fn from(value: Shared<'g, T>) -> Self { + Self { + link: AtomicUsize::new(value.inner), + _marker: PhantomData, + } + } +} + impl Default for Atomic { fn default() -> Self { Self::null() } } +impl Clone for Atomic { + fn clone(&self) -> Self { + Self { + link: AtomicUsize::new(self.link.load(Ordering::Relaxed)), + _marker: PhantomData, + } + } +} + /// A pointer to a shared object. /// /// This pointer is valid for use only during the lifetime `'r`. @@ -231,7 +249,7 @@ impl<'r, T> Shared<'r, T> { /// /// The `self` must be a valid memory location. #[inline] - pub unsafe fn deref(&self) -> &T { + pub unsafe fn deref(&self) -> &'r T { &*decompose_data::(self.inner).0 } @@ -241,7 +259,7 @@ impl<'r, T> Shared<'r, T> { /// /// The `self` must be a valid memory location. #[inline] - pub unsafe fn deref_mut(&mut self) -> &mut T { + pub unsafe fn deref_mut(&mut self) -> &'r mut T { &mut *decompose_data::(self.inner).0 } diff --git a/smrs/hp-pp/src/thread.rs b/smrs/hp-pp/src/thread.rs index f7c0d24b..fca788b8 100644 --- a/smrs/hp-pp/src/thread.rs +++ b/smrs/hp-pp/src/thread.rs @@ -1,4 +1,4 @@ -use core::sync::atomic::{AtomicPtr, Ordering}; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use core::{mem, ptr}; use std::collections::VecDeque; @@ -9,7 +9,7 @@ use crate::retire::{Retired, Unlinked}; use crate::HazardPointer; use crate::{Invalidate, Unlink}; -pub static mut COUNTS_BETWEEN_FLUSH: usize = 64; +pub static COUNTS_BETWEEN_FLUSH: AtomicUsize = AtomicUsize::new(64); #[inline] pub fn set_counts_between_flush(counts: usize) { @@ -17,22 +17,22 @@ pub fn set_counts_between_flush(counts: usize) { counts >= 2 && counts % 2 == 0, "counts must be even and greater than 1." ); - unsafe { COUNTS_BETWEEN_FLUSH = counts }; + COUNTS_BETWEEN_FLUSH.store(counts, Ordering::Relaxed); } #[inline] pub fn counts_between_invalidation() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH / 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) / 2 } #[inline] pub fn counts_between_flush() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) } #[inline] pub fn counts_between_collect() -> usize { - unsafe { COUNTS_BETWEEN_FLUSH * 2 } + COUNTS_BETWEEN_FLUSH.load(Ordering::Relaxed) * 2 } pub struct Thread<'domain> { diff --git a/smrs/vbr/Cargo.toml b/smrs/vbr/Cargo.toml index 2261a5ee..be3a5bc2 100644 --- a/smrs/vbr/Cargo.toml +++ b/smrs/vbr/Cargo.toml @@ -9,4 +9,3 @@ edition = "2021" crossbeam-utils = "0.8" atomic = "0.5" portable-atomic = "1" -arrayvec = "0.7.4" diff --git a/smrs/vbr/src/lib.rs b/smrs/vbr/src/lib.rs index 3fa9d796..ff5b787b 100644 --- a/smrs/vbr/src/lib.rs +++ b/smrs/vbr/src/lib.rs @@ -1,28 +1,46 @@ use std::{ - cell::RefCell, - collections::VecDeque, - fmt::Display, - marker::PhantomData, - mem::{align_of, zeroed}, - ptr::null_mut, - sync::atomic::AtomicU64, + cell::RefCell, collections::VecDeque, fmt::Display, marker::PhantomData, mem::align_of, + ops::Deref, ptr::null_mut, sync::atomic::AtomicU64, }; -use arrayvec::ArrayVec; use atomic::{Atomic, Ordering}; use crossbeam_utils::CachePadded; -use portable_atomic::{compiler_fence, AtomicU128}; +use portable_atomic::{compiler_fence, AtomicU128, AtomicUsize}; -pub const ENTRIES_PER_BAG: usize = 128; +static ENTRIES_PER_BAG: AtomicUsize = AtomicUsize::new(128); pub const INIT_BAGS_PER_LOCAL: usize = 32; pub const NOT_RETIRED: u64 = u64::MAX; +/// Sets the capacity of thread-local garbage bag. +/// +/// This value applies to all threads. +#[inline] +pub fn set_bag_capacity(cap: usize) { + assert!(cap > 1, "capacity must be greater than 1."); + ENTRIES_PER_BAG.store(cap, Ordering::Relaxed); +} + +/// Returns the current capacity of thread-local garbage bag. +#[inline] +pub fn bag_capacity() -> usize { + ENTRIES_PER_BAG.load(Ordering::Relaxed) +} + +#[derive(Default)] pub struct Inner { birth: AtomicU64, retire: AtomicU64, data: T, } +impl Deref for Inner { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.data + } +} + pub struct Global { epoch: CachePadded, avail: BagStack>, @@ -31,10 +49,10 @@ pub struct Global { unsafe impl Sync for Global {} unsafe impl Send for Global {} -impl Global { +impl Global { pub fn new(capacity: usize) -> Self { let avail = BagStack::new(); - let count = capacity / ENTRIES_PER_BAG + if capacity % ENTRIES_PER_BAG > 0 { 1 } else { 0 }; + let count = capacity / bag_capacity() + if capacity % bag_capacity() > 0 { 1 } else { 0 }; for _ in 0..count { avail.push(Box::into_raw(Box::new(Bag::new_with_alloc()))); } @@ -153,30 +171,34 @@ impl Drop for BagStack { pub struct Bag { /// NOTE: A timestamp is necessary to prevent ABA problems. next: AtomicU128, - entries: ArrayVec<*mut T, ENTRIES_PER_BAG>, + entries: Vec<*mut T>, } -impl Bag { +impl Bag { fn new() -> Self { Self { next: AtomicU128::new(0), - entries: ArrayVec::new(), + entries: Vec::with_capacity(bag_capacity()), } } fn new_with_alloc() -> Self { - let mut alloc = [null_mut(); ENTRIES_PER_BAG]; - for ptr in &mut alloc { - *ptr = unsafe { Box::into_raw(Box::new(zeroed())) }; + let mut entries = vec![null_mut(); bag_capacity()]; + for ptr in &mut entries { + *ptr = Box::into_raw(Box::new(T::default())); } Self { next: AtomicU128::new(0), - entries: ArrayVec::from(alloc), + entries, } } fn push(&mut self, obj: *mut T) -> bool { - self.entries.try_push(obj).is_ok() + if self.entries.len() < bag_capacity() { + self.entries.push(obj); + return true; + } + false } fn pop(&mut self) -> Option<*mut T> { @@ -190,7 +212,7 @@ pub struct Local { retired: RefCell>>>, } -impl Local { +impl Local { fn global(&self) -> &Global { unsafe { &*self.global } } @@ -272,7 +294,7 @@ pub struct Guard { epoch: u64, } -impl Guard { +impl Guard { fn global(&self) -> &Global { unsafe { &*self.local().global } } @@ -285,13 +307,15 @@ impl Guard { self.epoch = self.global().epoch(); } - pub fn allocate<'g, F>(&'g self, init: F) -> Result, ()> + pub fn allocate<'g, F>(&'g self, mut init: F) -> Result, ()> where - F: Fn(Shared<'g, T>), + F: FnMut(Shared<'g, T>), { let ptr = self.local().pop_avail(); debug_assert!(!ptr.is_null()); let slot_ref = unsafe { &*ptr }; + // If the retire epoch is (greater than or) equal to the current epoch, + // try advance the global epoch. if self.epoch <= slot_ref.retire.load(Ordering::SeqCst) { self.local().return_avail(ptr); let _ = self.global().advance(self.epoch); @@ -311,12 +335,27 @@ impl Guard { Ok(result) } + /// # Safety + /// + /// The current must conceptually have exclusive permission to retire this pointer + /// (e.g., after successful physical deletion in a lock-free data structure). pub unsafe fn retire(&self, ptr: Shared) { let inner = ptr.as_inner().expect("Attempted to retire a null pointer."); + if inner.birth.load(Ordering::SeqCst) > ptr.birth { + // It is already retired and reclaimed. + return; + } + self.retire_raw(inner as *const _ as *mut _); + } - if inner.birth.load(Ordering::SeqCst) > ptr.birth - || inner.retire.load(Ordering::SeqCst) != NOT_RETIRED - { + /// # Safety + /// + /// The pointee must not be relcaimed yet, and the current must conceptually have exclusive + /// permission to retire this pointer (e.g., after successful physical deletion in a lock-free + /// data structure). + pub unsafe fn retire_raw(&self, ptr: *mut Inner) { + let inner = unsafe { &*ptr }; + if inner.retire.load(Ordering::SeqCst) != NOT_RETIRED { return; } @@ -419,10 +458,19 @@ pub struct MutAtomic { _marker: PhantomData, } +impl Default for MutAtomic { + fn default() -> Self { + Self { + link: AtomicU128::new(0), + _marker: PhantomData, + } + } +} + unsafe impl Sync for MutAtomic {} unsafe impl Send for MutAtomic {} -impl MutAtomic { +impl MutAtomic { pub fn null() -> Self { Self { link: AtomicU128::new(0), @@ -506,7 +554,7 @@ pub struct Entry { unsafe impl Sync for Entry {} unsafe impl Send for Entry {} -impl Entry { +impl Entry { pub fn new(init: Shared) -> Self { Self { link: init.as_raw(), @@ -532,6 +580,10 @@ impl Entry { }) } } + + pub fn load_raw(&self) -> *mut Inner { + self.link + } } pub struct ImmAtomic { @@ -541,14 +593,14 @@ pub struct ImmAtomic { unsafe impl Sync for ImmAtomic {} unsafe impl Send for ImmAtomic {} -impl ImmAtomic { +impl ImmAtomic { pub fn new(v: T) -> Self { Self { data: Atomic::new(v), } } - pub fn get(&self, guard: &Guard) -> Result { + pub fn get(&self, guard: &Guard) -> Result { let value = unsafe { self.get_unchecked() }; compiler_fence(Ordering::SeqCst); guard.validate_epoch()?; diff --git a/src/bin/cdrc-ebr-flush.rs b/src/bin/cdrc-ebr-flush.rs index b2cb15c0..0aa4ff7e 100644 --- a/src/bin/cdrc-ebr-flush.rs +++ b/src/bin/cdrc-ebr-flush.rs @@ -41,7 +41,11 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::BonsaiTree => { - bench_map::>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) } _ => panic!("Unsupported(or unimplemented) data structure for CDRC"), }; @@ -51,7 +55,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -69,6 +75,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/cdrc-ebr.rs b/src/bin/cdrc-ebr.rs index 62d60f0c..5337a4d7 100644 --- a/src/bin/cdrc-ebr.rs +++ b/src/bin/cdrc-ebr.rs @@ -41,7 +41,11 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::BonsaiTree => { - bench_map::>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) } _ => panic!("Unsupported(or unimplemented) data structure for CDRC"), }; @@ -51,7 +55,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -69,6 +75,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/cdrc-hp.rs b/src/bin/cdrc-hp.rs index 1f96e803..fe7109d4 100644 --- a/src/bin/cdrc-hp.rs +++ b/src/bin/cdrc-hp.rs @@ -40,7 +40,11 @@ fn bench(config: &Config, output: BenchWriter) { bench_map::>(config, PrefillStrategy::Decreasing) } DS::BonsaiTree => { - bench_map::>(config, PrefillStrategy::Random) + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) } _ => panic!("Unsupported(or unimplemented) data structure for CDRC"), }; @@ -50,7 +54,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -64,6 +70,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/circ-ebr.rs b/src/bin/circ-ebr.rs index fd47e152..33aba25e 100644 --- a/src/bin/circ-ebr.rs +++ b/src/bin/circ-ebr.rs @@ -34,7 +34,13 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) + } _ => panic!("Unsupported(or unimplemented) data structure for CIRC"), }; output.write_record(config, &perf); @@ -43,7 +49,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -57,6 +65,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let rng = &mut rand::thread_rng(); let count = config.prefill / threads diff --git a/src/bin/circ-hp.rs b/src/bin/circ-hp.rs index cbbbb446..8ffb15a1 100644 --- a/src/bin/circ-hp.rs +++ b/src/bin/circ-hp.rs @@ -33,7 +33,13 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + // Note: Using the `Random` strategy with the Bonsai tree is unsafe + // because it involves multiple threads with unprotected guards. + // It is safe for many other data structures that don't retire elements + // during insertion, but this is not the case for the Bonsai tree. + bench_map::>(config, PrefillStrategy::Decreasing) + } _ => panic!("Unsupported(or unimplemented) data structure for CIRC"), }; output.write_record(config, &perf); @@ -42,7 +48,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -56,6 +64,9 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { + // Safety: We assume that the insert operation does not retire + // any elements. Note that this assumption may not hold for all + // data structures (e.g., Bonsai tree). let cs = unsafe { &Cs::unprotected() }; let output = &mut M::empty_output(); let rng = &mut rand::thread_rng(); diff --git a/src/bin/ebr.rs b/src/bin/ebr.rs index d67c8a7e..bd278083 100644 --- a/src/bin/ebr.rs +++ b/src/bin/ebr.rs @@ -12,7 +12,8 @@ use typenum::{Unsigned, U1, U4}; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, OpsPerCs, Perf, DS}; use smr_benchmark::ds_impl::ebr::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -38,10 +39,12 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::, N>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::, N>(config, PrefillStrategy::Random), DS::BonsaiTree => { - bench_map::, N>(config, PrefillStrategy::Random) + // For Bonsai Tree, it would be faster to use a single thread to prefill. + bench_map::, N>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::, N>(config, PrefillStrategy::Random), DS::SkipList => bench_map::, N>(config, PrefillStrategy::Decreasing), + DS::ElimAbTree => bench_map::, N>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); @@ -49,12 +52,17 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } impl PrefillStrategy { fn prefill + Send + Sync>(self, config: &Config, map: &M) { + // Some data structures (e.g., Bonsai tree, Elim AB-Tree) need SMR's retirement + // functionality even during insertions. + let collector = &crossbeam_ebr::Collector::new(); match self { PrefillStrategy::Random => { let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); @@ -63,14 +71,14 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { - let guard = unsafe { crossbeam_ebr::unprotected() }; + let handle = collector.register(); let rng = &mut rand::thread_rng(); let count = config.prefill / threads + if t < config.prefill % threads { 1 } else { 0 }; for _ in 0..count { let key = config.key_dist.sample(rng); let value = key; - map.insert(key, value, guard); + map.insert(key, value, &handle.pin()); } }); } @@ -79,7 +87,7 @@ impl PrefillStrategy { for _ in 0..config.prefill {} } PrefillStrategy::Decreasing => { - let guard = unsafe { crossbeam_ebr::unprotected() }; + let handle = collector.register(); let rng = &mut rand::thread_rng(); let mut keys = Vec::with_capacity(config.prefill); for _ in 0..config.prefill { @@ -88,7 +96,7 @@ impl PrefillStrategy { keys.sort_by(|a, b| b.cmp(a)); for key in keys.drain(..) { let value = key; - map.insert(key, value, guard); + map.insert(key, value, &handle.pin()); } } } @@ -102,7 +110,7 @@ fn bench_map + Send + Sync, N: Unsigned>( strategy: PrefillStrategy, ) -> Perf { match config.bag_size { - BagSize::Small => crossbeam_ebr::set_bag_capacity(64), + BagSize::Small => crossbeam_ebr::set_bag_capacity(512), BagSize::Large => crossbeam_ebr::set_bag_capacity(4096), } let map = &M::new(); diff --git a/src/bin/hp-brcu.rs b/src/bin/hp-brcu.rs index 862d2db9..d484888d 100644 --- a/src/bin/hp-brcu.rs +++ b/src/bin/hp-brcu.rs @@ -1,5 +1,5 @@ use crossbeam_utils::thread::scope; -use hp_brcu::{global, THREAD}; +use hp_brcu::{global, set_bag_capacity, THREAD}; use rand::prelude::*; use std::cmp::max; use std::io::{stdout, Write}; @@ -10,7 +10,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp_brcu::{ - ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -33,6 +33,10 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), _ => panic!("Unsupported(or unimplemented) data structure for HP-BRCU"), }; output.write_record(config, &perf); @@ -41,7 +45,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -98,8 +104,9 @@ fn bench_map + Send + Sync>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for HP-BRCU."); + match config.bag_size { + BagSize::Small => set_bag_capacity(512), + BagSize::Large => set_bag_capacity(4096), } let map = &M::new(); strategy.prefill(config, map); diff --git a/src/bin/hp-pp.rs b/src/bin/hp-pp.rs index c526cc12..e80608dd 100644 --- a/src/bin/hp-pp.rs +++ b/src/bin/hp-pp.rs @@ -36,7 +36,10 @@ fn bench(config: &Config, output: BenchWriter) { DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } + _ => panic!("Unsupported(or unimplemented) data structure for HP++"), }; output.write_record(config, &perf); println!("{}", perf); @@ -44,7 +47,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -96,7 +101,7 @@ fn bench_map + Send + Sync>( strategy: PrefillStrategy, ) -> Perf { match config.bag_size { - BagSize::Small => set_counts_between_flush(64), + BagSize::Small => set_counts_between_flush(512), BagSize::Large => set_counts_between_flush(4096), } let map = &M::new(); diff --git a/src/bin/hp-rcu.rs b/src/bin/hp-rcu.rs index cbdc94ff..95dd481d 100644 --- a/src/bin/hp-rcu.rs +++ b/src/bin/hp-rcu.rs @@ -1,5 +1,5 @@ use crossbeam_utils::thread::scope; -use hp_brcu::{global, THREAD}; +use hp_brcu::{global, set_bag_capacity, THREAD}; use rand::prelude::*; use std::cmp::max; use std::io::{stdout, Write}; @@ -10,7 +10,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp_brcu::{ - ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -34,6 +34,10 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), _ => panic!("Unsupported(or unimplemented) data structure for HP-BRCU"), }; output.write_record(config, &perf); @@ -42,7 +46,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -99,8 +105,9 @@ fn bench_map + Send + Sync>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for HP-BRCU."); + match config.bag_size { + BagSize::Small => set_bag_capacity(512), + BagSize::Large => set_bag_capacity(4096), } let map = &M::new(); strategy.prefill(config, map); diff --git a/src/bin/hp.rs b/src/bin/hp.rs index f2bf09cb..bef365ea 100644 --- a/src/bin/hp.rs +++ b/src/bin/hp.rs @@ -11,7 +11,8 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::hp::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HMList, HashMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -28,12 +29,17 @@ fn main() { fn bench(config: &Config, output: BenchWriter) { println!("{}", config); let perf = match config.ds { + DS::HList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::HHSList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HMList => bench_map::>(config, PrefillStrategy::Decreasing), DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), - _ => panic!("Unsupported(or unimplemented) data structure for HP"), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } + DS::NMTree => bench_map::>(config, PrefillStrategy::Random), + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); @@ -41,7 +47,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -93,7 +101,7 @@ fn bench_map + Send + Sync>( strategy: PrefillStrategy, ) -> Perf { match config.bag_size { - BagSize::Small => set_counts_between_flush(64), + BagSize::Small => set_counts_between_flush(512), BagSize::Large => set_counts_between_flush(4096), } let map = &M::new(); diff --git a/src/bin/nbr.rs b/src/bin/nbr.rs index 9f0bfafb..afa79743 100644 --- a/src/bin/nbr.rs +++ b/src/bin/nbr.rs @@ -36,14 +36,16 @@ fn bench(config: &Config, output: BenchWriter) { fn extract_nbr_params(config: &Config) -> (usize, usize) { match config.bag_size { - BagSize::Small => (256, 32), + BagSize::Small => (512, 64), BagSize::Large => (8192, 1024), } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/nr.rs b/src/bin/nr.rs index 2458f436..b594c9aa 100644 --- a/src/bin/nr.rs +++ b/src/bin/nr.rs @@ -9,7 +9,8 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::nr::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -32,8 +33,11 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), - DS::BonsaiTree => bench_map::>(config, PrefillStrategy::Random), + DS::BonsaiTree => { + bench_map::>(config, PrefillStrategy::Decreasing) + } DS::EFRBTree => bench_map::>(config, PrefillStrategy::Random), + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); @@ -41,7 +45,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } diff --git a/src/bin/pebr.rs b/src/bin/pebr.rs index 4d04c9df..9a4a1e20 100644 --- a/src/bin/pebr.rs +++ b/src/bin/pebr.rs @@ -12,7 +12,8 @@ use typenum::{Unsigned, U1, U4}; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, OpsPerCs, Perf, DS}; use smr_benchmark::ds_impl::pebr::{ - BonsaiTreeMap, ConcurrentMap, EFRBTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + BonsaiTreeMap, ConcurrentMap, EFRBTree, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, + SkipList, }; fn main() { @@ -38,10 +39,12 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::, N>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::, N>(config, PrefillStrategy::Random), DS::BonsaiTree => { - bench_map::, N>(config, PrefillStrategy::Random) + // For Bonsai Tree, it would be faster to use a single thread to prefill. + bench_map::, N>(config, PrefillStrategy::Decreasing) } DS::EFRBTree => bench_map::, N>(config, PrefillStrategy::Random), DS::SkipList => bench_map::, N>(config, PrefillStrategy::Decreasing), + DS::ElimAbTree => bench_map::, N>(config, PrefillStrategy::Random), }; output.write_record(config, &perf); println!("{}", perf); @@ -49,12 +52,17 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } impl PrefillStrategy { fn prefill + Send + Sync>(self, config: &Config, map: &M) { + // Some data structures (e.g., Bonsai tree, Elim AB-Tree) need SMR's retirement + // functionality even during insertions. + let collector = &crossbeam_pebr::Collector::new(); match self { PrefillStrategy::Random => { let threads = available_parallelism().map(|v| v.get()).unwrap_or(1); @@ -63,7 +71,8 @@ impl PrefillStrategy { scope(|s| { for t in 0..threads { s.spawn(move |_| { - let guard = unsafe { crossbeam_pebr::unprotected() }; + let handle = collector.register(); + let guard = &mut handle.pin(); let mut handle = M::handle(guard); let rng = &mut rand::thread_rng(); let count = config.prefill / threads @@ -79,7 +88,8 @@ impl PrefillStrategy { .unwrap(); } PrefillStrategy::Decreasing => { - let guard = unsafe { crossbeam_pebr::unprotected() }; + let handle = collector.register(); + let guard = &mut handle.pin(); let mut handle = M::handle(guard); let rng = &mut rand::thread_rng(); let mut keys = Vec::with_capacity(config.prefill); @@ -102,8 +112,9 @@ fn bench_map + Send + Sync, N: Unsigned>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for PEBR."); + match config.bag_size { + BagSize::Small => crossbeam_pebr::set_bag_capacity(512), + BagSize::Large => crossbeam_pebr::set_bag_capacity(4096), } let map = &M::new(); strategy.prefill(config, map); diff --git a/src/bin/vbr.rs b/src/bin/vbr.rs index 420f71e8..4aea8c1d 100644 --- a/src/bin/vbr.rs +++ b/src/bin/vbr.rs @@ -9,7 +9,7 @@ use std::time::Instant; use smr_benchmark::config::map::{setup, BagSize, BenchWriter, Config, Op, Perf, DS}; use smr_benchmark::ds_impl::vbr::{ - ConcurrentMap, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, + ConcurrentMap, ElimABTree, HHSList, HList, HMList, HashMap, NMTreeMap, SkipList, }; fn main() { @@ -32,6 +32,7 @@ fn bench(config: &Config, output: BenchWriter) { DS::HashMap => bench_map::>(config, PrefillStrategy::Decreasing), DS::NMTree => bench_map::>(config, PrefillStrategy::Random), DS::SkipList => bench_map::>(config, PrefillStrategy::Decreasing), + DS::ElimAbTree => bench_map::>(config, PrefillStrategy::Random), _ => panic!("Unsupported(or unimplemented) data structure for VBR"), }; output.write_record(config, &perf); @@ -40,7 +41,9 @@ fn bench(config: &Config, output: BenchWriter) { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PrefillStrategy { + /// Inserts keys in a random order, with multiple threads. Random, + /// Inserts keys in an increasing order, with a single thread. Decreasing, } @@ -96,8 +99,9 @@ fn bench_map + Send + Sync>( config: &Config, strategy: PrefillStrategy, ) -> Perf { - if config.bag_size == BagSize::Large { - println!("Warning: Large bag size is currently unavailable for VBR."); + match config.bag_size { + BagSize::Small => vbr::set_bag_capacity(512), + BagSize::Large => vbr::set_bag_capacity(4096), } let global = &M::global(config.prefill); let local = &M::local(global); diff --git a/src/config/map.rs b/src/config/map.rs index 34351cb3..2a9fc670 100644 --- a/src/config/map.rs +++ b/src/config/map.rs @@ -17,6 +17,7 @@ pub enum DS { BonsaiTree, EFRBTree, SkipList, + ElimAbTree, } pub enum OpsPerCs { diff --git a/src/ds_impl/ebr/bonsai_tree.rs b/src/ds_impl/ebr/bonsai_tree.rs index 7b44c8f0..cf3ac70f 100644 --- a/src/ds_impl/ebr/bonsai_tree.rs +++ b/src/ds_impl/ebr/bonsai_tree.rs @@ -1,6 +1,6 @@ use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::sync::atomic::Ordering; @@ -720,7 +720,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.get(key, guard) } #[inline(always)] @@ -728,7 +728,7 @@ where self.insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -740,6 +740,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/concurrent_map.rs b/src/ds_impl/ebr/concurrent_map.rs index aaefdd96..6ef687a3 100644 --- a/src/ds_impl/ebr/concurrent_map.rs +++ b/src/ds_impl/ebr/concurrent_map.rs @@ -1,24 +1,46 @@ use crossbeam_ebr::Guard; +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { fn new() -> Self; - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V>; + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option>; fn insert(&self, key: K, value: V, guard: &Guard) -> bool; - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V>; + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::pin; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -29,7 +51,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(i, i.to_string(), &pin())); + assert!(map.insert(i, to_value(&i), &pin())); } }); } @@ -44,7 +66,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.remove(&i, &pin()).unwrap()); + assert_eq!(to_value(&i), *map.remove(&i, &pin()).unwrap().output()); } }); } @@ -59,7 +81,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.get(&i, &pin()).unwrap()); + assert_eq!(to_value(&i), *map.get(&i, &pin()).unwrap().output()); } }); } diff --git a/src/ds_impl/ebr/elim_ab_tree.rs b/src/ds_impl/ebr/elim_ab_tree.rs new file mode 100644 index 00000000..02e0c05c --- /dev/null +++ b/src/ds_impl/ebr/elim_ab_tree.rs @@ -0,0 +1,1205 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use arrayvec::ArrayVec; +use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next<'g>(&self, index: usize, guard: &'g Guard) -> Shared<'g, Self> { + self.next()[index].load(Ordering::Acquire, guard) + } + + fn store_next<'g>(&'g self, index: usize, ptr: impl Pointer, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: impl Pointer) { + self.next_mut()[index] = Atomic::from(ptr.into_usize() as *const Self); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + guard: &'g Guard, + ) -> impl Iterator, Shared<'g, Self>)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i, guard))) + .chain(once((None, self.load_next(self.key_count(), guard)))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +struct Cursor<'g, K, V> { + l: Shared<'g, Node>, + p: Shared<'g, Node>, + gp: Shared<'g, Node>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Owned::new(left)); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, guard: &Guard) -> Option { + let mut node = unsafe { self.entry.load_next(0, guard).deref() }; + while let NodeKind::Internal { next } = &node.kind { + let next = next[node.child_index(key)].load(Ordering::Acquire, guard); + node = unsafe { next.deref() }; + } + node.read_consistent(key).1 + } + + fn search<'g>( + &self, + key: &K, + target: Option>>, + guard: &'g Guard, + ) -> (bool, Cursor<'g, K, V>) { + let mut cursor = Cursor { + l: self.entry.load_next(0, guard), + p: unsafe { Shared::from_usize(&self.entry as *const _ as usize) }, + gp: Shared::null(), + gp_p_idx: 0, + p_l_idx: 0, + l_key_idx: 0, + val: None, + }; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { cursor.l.deref() }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.l = l_node.load_next(cursor.p_l_idx, guard); + } + + if let Some(target) = target { + (cursor.l == target, cursor) + } else { + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); + cursor.val = value; + cursor.l_key_idx = index; + (value.is_some(), cursor) + } + } + + pub fn insert(&self, key: &K, value: &V, guard: &Guard) -> Option { + loop { + let (_, cursor) = self.search(key, None, guard); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, &cursor, guard) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner<'g>( + &self, + key: &K, + value: &V, + cursor: &Cursor<'g, K, V>, + guard: &'g Guard, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Owned::new(left)); + internal.init_next(1, Owned::new(right)); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Owned::new(internal).into_shared(guard); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { guard.defer_destroy(cursor.l) }; + self.fix_tag_violation(new_internal, guard); + + Ok(None) + } + } + + fn fix_tag_violation<'g>(&self, viol: Shared<'g, Node>, guard: &'g Guard) { + loop { + let viol_node = unsafe { viol.deref() }; + if viol_node.weight { + return; + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0, guard) != viol); + + let (found, cursor) = self.search(&viol_node.search_key, Some(viol), guard); + if !found { + return; + } + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + if !eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + self.fix_tag_violation(cursor.p, guard); + continue; + } + + try_acq_val_or!(node, node_lock, Operation::Balance, None, continue); + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next(cursor.gp_p_idx, Owned::new(absorber), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l) }; + unsafe { guard.defer_destroy(cursor.p) }; + return; + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Owned::new(left)); + new_internal.init_next(1, Owned::new(right)); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Owned::new(new_internal).into_shared(guard); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l) }; + unsafe { guard.defer_destroy(cursor.p) }; + + drop((node_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(new_internal, guard); + return; + } + } + } + + pub fn remove(&self, key: &K, guard: &Guard) -> Option { + loop { + let (_, cursor) = self.search(key, None, guard); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor, guard) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner<'g>( + &self, + key: &K, + cursor: &Cursor<'g, K, V>, + guard: &'g Guard, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation(cursor.l, guard); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation<'g>(&self, viol: Shared<'g, Node>, guard: &'g Guard) { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + let viol_node = unsafe { viol.deref() }; + loop { + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0, guard) + { + // No degree violation at `viol`. + return; + } + + // Search for `viol`. + let (_, cursor) = self.search(&viol_node.search_key, Some(viol), guard); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p != self.entry.load_next(0, guard) + { + self.fix_underfull_violation(cursor.p, guard); + continue; + } + + if !eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + let sibling_idx = if cursor.p_l_idx > 0 { + cursor.p_l_idx - 1 + } else { + 1 + }; + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling_sh = parent.load_next(sibling_idx, guard); + let sibling = unsafe { sibling_sh.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if sibling_idx < cursor.p_l_idx { + ((sibling, sibling_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, sibling_idx)) + }; + + try_acq_val_or!(left, left_lock, Operation::Balance, None, continue); + try_acq_val_or!(right, right_lock, Operation::Balance, None, continue); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return; + } + + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight || !node.weight || !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(cursor.p, guard); + self.fix_tag_violation(cursor.l, guard); + self.fix_tag_violation(sibling_sh, guard); + continue; + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + } + .into_shared(guard); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l); + guard.defer_destroy(cursor.p); + guard.defer_destroy(sibling_sh); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node, guard); + return; + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..sibling_idx { + new_parent.init_next(i, parent.load_next(i, guard)); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next(i, guard)); + } + + new_parent.init_next( + cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 }), + new_node, + ); + let new_parent = Owned::new(new_parent).into_shared(guard); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l); + guard.defer_destroy(cursor.p); + guard.defer_destroy(sibling_sh); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node, guard); + self.fix_underfull_violation(new_parent, guard); + return; + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = + Owned::new(Node::leaf(true, left_size, Default::default())); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = + Owned::new(Node::leaf(true, right_size, Default::default())); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)); + + let (new_left, pivot) = { + let mut new_internal = + Owned::new(Node::internal(true, left_size, Default::default())); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = + Owned::new(Node::internal(true, right_size, Default::default())); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = + Owned::new(Node::internal(parent.weight, psize, parent.search_key)); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, new_left); + new_parent.init_next(right_idx, new_right); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l); + guard.defer_destroy(cursor.p); + guard.defer_destroy(sibling_sh); + } + + return; + } + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let mut stack = vec![]; + let guard = unsafe { unprotected() }; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self::new() + } + + #[inline(always)] + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { + self.search_basic(key, guard) + } + + #[inline(always)] + fn insert(&self, key: K, value: V, guard: &Guard) -> bool { + self.insert(&key, &value, guard).is_none() + } + + #[inline(always)] + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { + self.remove(key, guard) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::ebr::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/ebr/ellen_tree.rs b/src/ds_impl/ebr/ellen_tree.rs index b08f97b7..dab607a5 100644 --- a/src/ds_impl/ebr/ellen_tree.rs +++ b/src/ds_impl/ebr/ellen_tree.rs @@ -2,7 +2,7 @@ use std::sync::atomic::Ordering; use crossbeam_ebr::{unprotected, Atomic, CompareExchangeError, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -535,7 +535,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { match self.find(key, guard) { Some(node) => Some(node.value.as_ref().unwrap()), None => None, @@ -548,7 +548,7 @@ where } #[inline(always)] - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.delete(key, guard) } } @@ -560,6 +560,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/list.rs b/src/ds_impl/ebr/list.rs index f93d4bd8..0cae89dd 100644 --- a/src/ds_impl/ebr/list.rs +++ b/src/ds_impl/ebr/list.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; use std::cmp::Ordering::{Equal, Greater, Less}; @@ -398,7 +398,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_get(key, guard) } #[inline(always)] @@ -406,7 +406,7 @@ where self.inner.harris_insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_remove(key, guard) } } @@ -425,7 +425,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_michael_get(key, guard) } #[inline(always)] @@ -433,7 +433,7 @@ where self.inner.harris_michael_insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_michael_remove(key, guard) } } @@ -464,7 +464,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_herlihy_shavit_get(key, guard) } #[inline(always)] @@ -472,7 +472,7 @@ where self.inner.harris_insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.inner.harris_remove(key, guard) } } @@ -484,17 +484,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/ebr/michael_hash_map.rs b/src/ds_impl/ebr/michael_hash_map.rs index 2063c5ac..85e31e97 100644 --- a/src/ds_impl/ebr/michael_hash_map.rs +++ b/src/ds_impl/ebr/michael_hash_map.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_ebr::Guard; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -35,7 +35,7 @@ where s.finish() as usize } - pub fn get<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option<&'g V> { + pub fn get<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option + 'g> { let i = Self::hash(k); self.get_bucket(i).get(k, guard) } @@ -45,7 +45,7 @@ where self.get_bucket(i).insert(k, v, guard) } - pub fn remove<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option<&'g V> { + pub fn remove<'g>(&'g self, k: &'g K, guard: &'g Guard) -> Option + 'g> { let i = Self::hash(k); self.get_bucket(i).remove(k, guard) } @@ -61,7 +61,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.get(key, guard) } #[inline(always)] @@ -69,7 +69,7 @@ where self.insert(key, value, guard) } #[inline(always)] - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -81,6 +81,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/mod.rs b/src/ds_impl/ebr/mod.rs index 78d97a1e..9497097f 100644 --- a/src/ds_impl/ebr/mod.rs +++ b/src/ds_impl/ebr/mod.rs @@ -2,6 +2,7 @@ pub mod concurrent_map; pub mod bonsai_tree; pub mod double_link; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; @@ -12,6 +13,7 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; diff --git a/src/ds_impl/ebr/natarajan_mittal_tree.rs b/src/ds_impl/ebr/natarajan_mittal_tree.rs index 14f62bd1..b433c5d8 100644 --- a/src/ds_impl/ebr/natarajan_mittal_tree.rs +++ b/src/ds_impl/ebr/natarajan_mittal_tree.rs @@ -1,6 +1,6 @@ use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::sync::atomic::Ordering; @@ -506,7 +506,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.get(key, guard) } #[inline(always)] @@ -514,7 +514,7 @@ where self.insert(key, value, guard).is_ok() } #[inline(always)] - fn remove<'g>(&'g self, key: &K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -526,6 +526,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/ebr/skip_list.rs b/src/ds_impl/ebr/skip_list.rs index 790b4440..c5db6193 100644 --- a/src/ds_impl/ebr/skip_list.rs +++ b/src/ds_impl/ebr/skip_list.rs @@ -2,7 +2,7 @@ use std::sync::atomic::{fence, AtomicUsize, Ordering}; use crossbeam_ebr::{unprotected, Atomic, Guard, Owned, Shared}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -401,7 +401,7 @@ where } #[inline(always)] - fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn get<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { let cursor = self.find_optimistic(key, guard); cursor.found.map(|node| &node.value) } @@ -412,7 +412,7 @@ where } #[inline(always)] - fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option<&'g V> { + fn remove<'g>(&'g self, key: &'g K, guard: &'g Guard) -> Option> { self.remove(key, guard) } } @@ -424,6 +424,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/bonsai_tree.rs b/src/ds_impl/hp/bonsai_tree.rs index b351a417..a318d867 100644 --- a/src/ds_impl/hp/bonsai_tree.rs +++ b/src/ds_impl/hp/bonsai_tree.rs @@ -1,7 +1,7 @@ use hp_pp::{light_membarrier, Thread}; use hp_pp::{tag, tagged, untagged, HazardPointer, DEFAULT_DOMAIN}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::ptr; @@ -113,20 +113,21 @@ pub struct State<'domain, K, V> { retired_nodes: Vec<*mut Node>, /// Nodes newly constructed by the op. Should be destroyed if CAS fails. (`destroy`) new_nodes: Vec<*mut Node>, - thread: Thread<'domain>, + thread: Box>, } impl Default for State<'static, K, V> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { root_link: ptr::null(), curr_root: ptr::null_mut(), - root_h: Default::default(), - succ_h: Default::default(), - removed_h: Default::default(), + root_h: HazardPointer::new(&mut thread), + succ_h: HazardPointer::new(&mut thread), + removed_h: HazardPointer::new(&mut thread), retired_nodes: vec![], new_nodes: vec![], - thread: Thread::new(&DEFAULT_DOMAIN), + thread, } } } @@ -787,7 +788,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } @@ -797,7 +802,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -809,6 +818,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/concurrent_map.rs b/src/ds_impl/hp/concurrent_map.rs index 48b650fb..6c41a2f1 100644 --- a/src/ds_impl/hp/concurrent_map.rs +++ b/src/ds_impl/hp/concurrent_map.rs @@ -1,3 +1,19 @@ +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { type Handle<'domain>; @@ -5,24 +21,38 @@ pub trait ConcurrentMap { fn handle() -> Self::Handle<'static>; - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V>; + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option>; fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool; - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V>; + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -34,7 +64,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(&mut handle, i, i.to_string())); + assert!(map.insert(&mut handle, i, to_value(&i))); } }); } @@ -50,7 +80,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.remove(&mut handle, &i).unwrap()); + assert_eq!(to_value(&i), *map.remove(&mut handle, &i).unwrap().output()); } }); } @@ -66,7 +96,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.get(&mut handle, &i).unwrap()); + assert_eq!(to_value(&i), *map.get(&mut handle, &i).unwrap().output()); } }); } diff --git a/src/ds_impl/hp/double_link.rs b/src/ds_impl/hp/double_link.rs index dba1ef35..0dd04958 100644 --- a/src/ds_impl/hp/double_link.rs +++ b/src/ds_impl/hp/double_link.rs @@ -41,15 +41,16 @@ pub struct DoubleLink { pub struct Handle<'domain> { pri: HazardPointer<'domain>, sub: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - pri: HazardPointer::default(), - sub: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + pri: HazardPointer::new(&mut thread), + sub: HazardPointer::new(&mut thread), + thread, } } } diff --git a/src/ds_impl/hp/elim_ab_tree.rs b/src/ds_impl/hp/elim_ab_tree.rs new file mode 100644 index 00000000..16a8c33f --- /dev/null +++ b/src/ds_impl/hp/elim_ab_tree.rs @@ -0,0 +1,1492 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use super::pointers::{Atomic, Pointer, Shared}; +use arrayvec::ArrayVec; +use hp_pp::{light_membarrier, HazardPointer, Thread, DEFAULT_DOMAIN}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next(&self, index: usize) -> Shared { + self.next()[index].load(Ordering::Acquire) + } + + fn protect_next_pess( + &self, + index: usize, + slot: &mut HazardPointer<'_>, + ) -> Result, ()> { + let atomic = &self.next()[index]; + let mut ptr = atomic.load(Ordering::Relaxed); + loop { + slot.protect_raw(ptr.into_raw()); + light_membarrier(); + let new = atomic.load(Ordering::Acquire); + if ptr == new { + break; + } + ptr = new; + } + if self.marked.load(Ordering::Acquire) { + return Err(()); + } + Ok(ptr) + } + + /// Stores the next pointer into the given slot. + /// It retries if the marked state or the pointer changes. + fn protect_next_consistent( + &self, + index: usize, + slot: &mut HazardPointer<'_>, + ) -> (Shared, bool) { + let mut marked = self.marked.load(Ordering::Acquire); + let mut ptr = self.load_next(index); + loop { + slot.protect_raw(ptr.into_raw()); + light_membarrier(); + let marked_new = self.marked.load(Ordering::Acquire); + let ptr_new = self.load_next(index); + if marked_new == marked && ptr_new == ptr { + return (ptr, marked); + } + (marked, ptr) = (marked_new, ptr_new); + } + } + + fn store_next<'g>(&'g self, index: usize, ptr: impl Pointer, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: impl Pointer) { + self.next_mut()[index] = Atomic::from(ptr.into_raw()); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } + + fn p_s_idx(p_l_idx: usize) -> usize { + if p_l_idx > 0 { + p_l_idx - 1 + } else { + 1 + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator, Shared)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i))) + .chain(once((None, self.load_next(self.key_count())))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +struct Cursor { + l: Shared>, + p: Shared>, + gp: Shared>, + // A pointer to a sibling node. + s: Shared>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + p_s_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +pub struct Handle<'domain> { + l_h: HazardPointer<'domain>, + p_h: HazardPointer<'domain>, + gp_h: HazardPointer<'domain>, + /// A protector for the sibling node. + s_h: HazardPointer<'domain>, + /// Used in `search_basic`, which does optimistic traversal. + c_h: HazardPointer<'domain>, + thread: Box>, +} + +impl Default for Handle<'static> { + fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); + Self { + l_h: HazardPointer::new(&mut thread), + p_h: HazardPointer::new(&mut thread), + gp_h: HazardPointer::new(&mut thread), + s_h: HazardPointer::new(&mut thread), + c_h: HazardPointer::new(&mut thread), + thread, + } + } +} + +impl<'domain> Handle<'domain> { + // bypass E0499-E0503, etc that are supposed to be fixed by polonius + #[inline] + fn launder<'hp2>(&mut self) -> &'hp2 mut Self { + unsafe { core::mem::transmute(self) } + } +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Shared::from_owned(left)); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option { + loop { + match self.search_basic_inner(key, handle) { + Ok(found) => return found, + Err(_) => continue, + } + } + } + + pub fn search_basic_inner<'hp>( + &self, + key: &K, + handle: &'hp mut Handle<'_>, + ) -> Result, ()> { + let (a_h, an_h, p_h, l_h, n_h) = ( + &mut handle.gp_h, + &mut handle.s_h, + &mut handle.p_h, + &mut handle.l_h, + &mut handle.c_h, + ); + + let mut p = Shared::>::from(&self.entry as *const _ as usize); + let mut p_l_idx = 0; + let (mut l, mut l_marked) = self.entry.protect_next_consistent(p_l_idx, l_h); + + let mut a = Shared::>::null(); + let mut a_an_idx = 0; + let mut an = Shared::>::null(); + + loop { + // Validation depending on the state of `p`. + // + // - If it is marked, validate on anchor. + // - If it is not marked, it is already protected safely. + if l_marked { + // Validate on anchor. + debug_assert!(!a.is_null()); + debug_assert!(!an.is_null()); + let an_new = unsafe { a.deref() }.load_next(a_an_idx); + + if an != an_new { + if unsafe { a.deref() }.marked.load(Ordering::Acquire) { + return Err(()); + } + // Anchor is updated but clear, so can restart from anchor. + p = a; + p_l_idx = a_an_idx; + l = an_new; + l_marked = false; + a = Shared::null(); + + HazardPointer::swap(p_h, a_h); + continue; + } + } + + let l_node = unsafe { l.deref() }; + if l_node.is_leaf() { + return Ok(l_node.read_consistent(key).1); + } + + let l_n_idx = l_node.child_index(key); + let (n, n_marked) = l_node.protect_next_consistent(l_n_idx, n_h); + if n_marked { + if a.is_null() { + a = p; + a_an_idx = p_l_idx; + an = l; + HazardPointer::swap(a_h, p_h); + } else if an == p { + HazardPointer::swap(an_h, p_h); + } + p = l; + p_l_idx = l_n_idx; + l = n; + l_marked = n_marked; + HazardPointer::swap(l_h, p_h); + HazardPointer::swap(l_h, n_h); + } else { + p = l; + p_l_idx = l_n_idx; + l = n; + l_marked = n_marked; + a = Shared::null(); + HazardPointer::swap(l_h, p_h); + HazardPointer::swap(l_h, n_h); + } + } + } + + fn search<'hp>( + &self, + key: &K, + target: Option>>, + handle: &'hp mut Handle<'_>, + ) -> (bool, Cursor) { + loop { + match self.search_inner(key, target, handle.launder()) { + Ok(found) => return found, + Err(_) => continue, + } + } + } + + fn search_inner<'hp>( + &self, + key: &K, + target: Option>>, + handle: &'hp mut Handle<'_>, + ) -> Result<(bool, Cursor), ()> { + let mut cursor = Cursor { + l: self.entry.protect_next_pess(0, &mut handle.l_h)?, + s: self.entry.protect_next_pess(1, &mut handle.s_h)?, + p: Shared::from(&self.entry as *const _ as usize), + gp: Shared::null(), + gp_p_idx: 0, + p_l_idx: 0, + p_s_idx: 1, + l_key_idx: 0, + val: None, + }; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { cursor.l.deref() }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + HazardPointer::swap(&mut handle.gp_h, &mut handle.p_h); + HazardPointer::swap(&mut handle.p_h, &mut handle.l_h); + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.p_s_idx = Node::::p_s_idx(cursor.p_l_idx); + cursor.l = l_node.protect_next_pess(cursor.p_l_idx, &mut handle.l_h)?; + cursor.s = l_node.protect_next_pess(cursor.p_s_idx, &mut handle.s_h)?; + } + + if let Some(target) = target { + Ok((cursor.l == target, cursor)) + } else { + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); + cursor.val = value; + cursor.l_key_idx = index; + Ok((value.is_some(), cursor)) + } + } + + pub fn insert<'hp>(&self, key: &K, value: &V, handle: &'hp mut Handle<'_>) -> Option { + loop { + let (_, cursor) = self.search(key, None, handle.launder()); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, &cursor, handle) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner<'hp>( + &self, + key: &K, + value: &V, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Shared::from_owned(left)); + internal.init_next(1, Shared::from_owned(right)); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Shared::from_owned(internal); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { handle.thread.retire(cursor.l.into_raw()) }; + self.fix_tag_violation(kv_pairs[left_size].0, new_internal, handle); + + Ok(None) + } + } + + fn fix_tag_violation<'hp>( + &self, + search_key: K, + viol: Shared>, + handle: &'hp mut Handle<'_>, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + let (found, cursor) = self.search(&search_key, Some(viol), handle); + if !found || cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_tag_violation_inner(&cursor, handle); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_tag_violation_inner<'hp>( + &self, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> (bool, Option<(K, Shared>)>) { + let viol = cursor.l; + let viol_node = unsafe { cursor.l.deref() }; + if viol_node.weight { + return (true, None); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0) != viol); + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + return (false, Some((parent.search_key, cursor.p))); + } + + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, None) + ); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next(cursor.gp_p_idx, Shared::from_owned(absorber), &gparent_lock); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + + unsafe { handle.thread.retire(cursor.l.into_raw()) }; + unsafe { handle.thread.retire(cursor.p.into_raw()) }; + return (true, None); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Shared::from_owned(left)); + new_internal.init_next(1, Shared::from_owned(right)); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Shared::from_owned(new_internal); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + + unsafe { handle.thread.retire(cursor.l.into_raw()) }; + unsafe { handle.thread.retire(cursor.p.into_raw()) }; + + drop((node_lock, parent_lock, gparent_lock)); + return ( + true, + Some((keys[left_size - 1].get().unwrap(), new_internal)), + ); + } + } + + pub fn remove<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option { + loop { + let (_, cursor) = self.search(key, None, handle.launder()); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor, handle) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner<'hp>( + &self, + key: &K, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation(node.search_key, cursor.l, handle); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation<'hp>( + &self, + search_key: K, + viol: Shared>, + handle: &'hp mut Handle<'_>, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + let (_, cursor) = self.search(&search_key, Some(viol), handle); + if cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_underfull_violation_inner(&cursor, handle); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_underfull_violation_inner<'hp>( + &self, + cursor: &Cursor, + handle: &'hp mut Handle<'_>, + ) -> (bool, ArrayVec<(K, Shared>), 2>) { + let viol = cursor.l; + let viol_node = unsafe { viol.deref() }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0) + { + // No degree violation at `viol`. + return (true, ArrayVec::<_, 2>::new()); + } + + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p != self.entry.load_next(0) + { + return ( + false, + ArrayVec::from_iter(once((parent.search_key, cursor.p))), + ); + } + + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling = unsafe { cursor.s.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if cursor.p_s_idx < cursor.p_l_idx { + ((sibling, cursor.p_s_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, cursor.p_s_idx)) + }; + + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(parent.search_key, cursor.p, handle); + return (false, ArrayVec::new()); + } + if !node.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(node.search_key, cursor.l, handle); + return (false, ArrayVec::new()); + } + if !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(sibling.search_key, cursor.s, handle); + return (false, ArrayVec::new()); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Node::leaf(true, size, node.search_key); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Node::internal(true, size, node.search_key); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + }; + let new_node = Shared::from_owned(new_node); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + sibling.marked.store(true, Ordering::Release); + + unsafe { + handle.thread.retire(cursor.l.into_raw()); + handle.thread.retire(cursor.p.into_raw()); + handle.thread.retire(cursor.s.into_raw()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return (true, ArrayVec::from_iter(once((node.search_key, new_node)))); + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..cursor.p_s_idx { + new_parent.init_next(i, parent.load_next(i)); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next(i)); + } + + new_parent.init_next( + cursor.p_l_idx + - (if cursor.p_l_idx > cursor.p_s_idx { + 1 + } else { + 0 + }), + new_node, + ); + let new_parent = Shared::from_owned(new_parent); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + sibling.marked.store(true, Ordering::Release); + + unsafe { + handle.thread.retire(cursor.l.into_raw()); + handle.thread.retire(cursor.p.into_raw()); + handle.thread.retire(cursor.s.into_raw()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter( + [(node.search_key, new_node), (parent.search_key, new_parent)].into_iter(), + ), + ); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Node::leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Node::leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let (new_left, pivot) = { + let mut new_internal = Node::internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = Node::internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = Node::internal(parent.weight, psize, parent.search_key); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, Shared::from_owned(new_left)); + new_parent.init_next(right_idx, Shared::from_owned(new_right)); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next( + cursor.gp_p_idx, + Shared::from_owned(new_parent), + &gparent_lock, + ); + node.marked.store(true, Ordering::Release); + parent.marked.store(true, Ordering::Release); + sibling.marked.store(true, Ordering::Release); + + unsafe { + handle.thread.retire(cursor.l.into_raw()); + handle.thread.retire(cursor.p.into_raw()); + handle.thread.retire(cursor.s.into_raw()); + } + + return (true, ArrayVec::new()); + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let mut stack = vec![]; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + type Handle<'domain> = Handle<'domain>; + + fn new() -> Self { + ElimABTree::new() + } + + fn handle() -> Self::Handle<'static> { + Handle::default() + } + + #[inline(always)] + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.search_basic(key, handle) + } + + #[inline(always)] + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + self.insert(&key, &value, handle).is_none() + } + + #[inline(always)] + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.remove(key, handle) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::hp::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/hp/ellen_tree.rs b/src/ds_impl/hp/ellen_tree.rs index fd8ce7ab..e8822017 100644 --- a/src/ds_impl/hp/ellen_tree.rs +++ b/src/ds_impl/hp/ellen_tree.rs @@ -27,7 +27,7 @@ use hp_pp::{ decompose_ptr, light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, }; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -230,22 +230,23 @@ pub struct Handle<'domain> { new_internal_h: HazardPointer<'domain>, // Protect an owner of update which is currently being helped. help_src_h: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - gp_h: HazardPointer::default(), - p_h: HazardPointer::default(), - l_h: HazardPointer::default(), - l_other_h: HazardPointer::default(), - pupdate_h: HazardPointer::default(), - gpupdate_h: HazardPointer::default(), - aux_update_h: HazardPointer::default(), - new_internal_h: HazardPointer::default(), - help_src_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + gp_h: HazardPointer::new(&mut thread), + p_h: HazardPointer::new(&mut thread), + l_h: HazardPointer::new(&mut thread), + l_other_h: HazardPointer::new(&mut thread), + pupdate_h: HazardPointer::new(&mut thread), + gpupdate_h: HazardPointer::new(&mut thread), + aux_update_h: HazardPointer::new(&mut thread), + new_internal_h: HazardPointer::new(&mut thread), + help_src_h: HazardPointer::new(&mut thread), + thread, } } } @@ -848,11 +849,12 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - match self.find(key, handle) { - Some(value) => Some(value), - None => None, - } + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.find(key, handle) } #[inline(always)] @@ -861,7 +863,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.delete(key, handle) } } @@ -873,6 +879,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/list.rs b/src/ds_impl/hp/list.rs index 90d89b89..2d9e1084 100644 --- a/src/ds_impl/hp/list.rs +++ b/src/ds_impl/hp/list.rs @@ -1,31 +1,29 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use super::pointers::{Atomic, Pointer, Shared}; +use core::mem; use std::cmp::Ordering::{Equal, Greater, Less}; -use std::ptr; -use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::atomic::Ordering; -use hp_pp::{ - decompose_ptr, light_membarrier, tag, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, -}; +use hp_pp::{light_membarrier, HazardPointer, Thread, DEFAULT_DOMAIN}; // `#[repr(C)]` is used to ensure the first field // is also the first data in the memory alignment. #[repr(C)] -#[derive(Debug)] pub struct Node { /// Mark: tag(), Tag: not needed - next: AtomicPtr>, + next: Atomic>, key: K, value: V, } pub struct List { - head: AtomicPtr>, + head: Atomic>, } impl Default for List where - K: Ord, + K: Ord + 'static, { fn default() -> Self { Self::new() @@ -34,10 +32,10 @@ where impl Drop for List { fn drop(&mut self) { - let mut curr = untagged(*self.head.get_mut()); + let mut o_curr = mem::take(&mut self.head); - while !curr.is_null() { - curr = untagged(*unsafe { Box::from_raw(curr) }.next.get_mut()); + while let Some(curr) = unsafe { o_curr.try_into_owned() } { + o_curr = curr.next; } } } @@ -45,15 +43,21 @@ impl Drop for List { pub struct Handle<'domain> { prev_h: HazardPointer<'domain>, curr_h: HazardPointer<'domain>, - thread: Thread<'domain>, + // `anchor_h` and `anchor_next_h` are used for `find_harris` + anchor_h: HazardPointer<'domain>, + anchor_next_h: HazardPointer<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - prev_h: HazardPointer::default(), - curr_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + prev_h: HazardPointer::new(&mut thread), + curr_h: HazardPointer::new(&mut thread), + anchor_h: HazardPointer::new(&mut thread), + anchor_next_h: HazardPointer::new(&mut thread), + thread, } } } @@ -67,16 +71,23 @@ impl<'domain> Handle<'domain> { } pub struct Cursor<'domain, 'hp, K, V> { - prev: *mut Node, // not &AtomicPtr because we can't construct the cursor out of thin air - curr: *mut Node, + prev: Shared>, // not &Atomic because we can't construct the cursor out of thin air + // For harris, this keeps the mark bit. Don't mix harris and harris-micheal. + curr: Shared>, + // `anchor` is used for `find_harris` + // anchor and anchor_next are non-null iff exist + anchor: Shared>, + anchor_next: Shared>, handle: &'hp mut Handle<'domain>, } impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> { - pub fn new(head: &AtomicPtr>, handle: &'hp mut Handle<'domain>) -> Self { + pub fn new(head: &Atomic>, handle: &'hp mut Handle<'domain>) -> Self { Self { - prev: head as *const _ as *mut _, + prev: unsafe { Shared::from_raw(head as *const _ as *mut _) }, curr: head.load(Ordering::Acquire), + anchor: Shared::null(), + anchor_next: Shared::null(), handle, } } @@ -86,34 +97,168 @@ impl<'domain, 'hp, K, V> Cursor<'domain, 'hp, K, V> where K: Ord, { + /// Optimistically traverses while maintaining `anchor` and `anchor_next`. + /// It is used for both Harris and Harris-Herlihy-Shavit traversals. + #[inline] + fn traverse_with_anchor(&mut self, key: &K) -> Result { + // Invariants: + // anchor, anchor_next: protected if they are not null. + // prev: always protected with prev_sh + // curr: not protected. + // curr: also has tag value when it is obtained from prev. + Ok(loop { + if self.curr.is_null() { + break false; + } + + let prev_next = unsafe { &self.prev.deref().next }; + self.handle + .curr_h + .protect_raw(self.curr.with_tag(0).into_raw()); + light_membarrier(); + + // Validation depending on the state of `self.curr`. + // + // - If it is marked, validate on anchor. + // - If it is not marked, validate on prev. + + if self.curr.tag() != 0 { + // Validate on anchor. + + debug_assert!(!self.anchor.is_null()); + debug_assert!(!self.anchor_next.is_null()); + let an_new = unsafe { &self.anchor.deref().next }.load(Ordering::Acquire); + + if an_new.tag() != 0 { + return Err(()); + } else if an_new != self.anchor_next { + // Anchor is updated but clear, so can restart from anchor. + + self.prev = self.anchor; + self.curr = an_new; + self.anchor = Shared::null(); + + // Set prev HP as anchor HP, since prev should always be protected. + HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.anchor_h); + continue; + } + } else { + // Validate on prev. + debug_assert!(self.anchor.is_null()); + + let curr_new = prev_next.load(Ordering::Acquire); + + if curr_new.tag() != 0 { + // If prev is marked, then restart from head. + return Err(()); + } else if curr_new != self.curr { + // self.curr's tag was 0, so the above comparison ignores tags. + + // In contrary to what HP04 paper does, it's fine to retry protecting the new node + // without restarting from head as long as prev is not logically deleted. + self.curr = curr_new; + continue; + } + } + + let curr_node = unsafe { self.curr.deref() }; + let next = curr_node.next.load(Ordering::Acquire); + if next.tag() == 0 { + if curr_node.key < *key { + self.prev = self.curr; + self.curr = next; + self.anchor = Shared::null(); + HazardPointer::swap(&mut self.handle.curr_h, &mut self.handle.prev_h); + } else { + break curr_node.key == *key; + } + } else { + if self.anchor.is_null() { + self.anchor = self.prev; + self.anchor_next = self.curr; + HazardPointer::swap(&mut self.handle.anchor_h, &mut self.handle.prev_h); + } else if self.anchor_next == self.prev { + HazardPointer::swap(&mut self.handle.anchor_next_h, &mut self.handle.prev_h); + } + self.prev = self.curr; + self.curr = next; + HazardPointer::swap(&mut self.handle.prev_h, &mut self.handle.curr_h); + } + }) + } + + #[inline] + fn find_harris(&mut self, key: &K) -> Result { + // Finding phase + // - cursor.curr: first unmarked node w/ key >= search key (4) + // - cursor.prev: the ref of .next in previous unmarked node (1 -> 2) + // 1 -> 2 -x-> 3 -x-> 4 -> 5 -> ∅ (search key: 4) + + let found = self.traverse_with_anchor(key)?; + + if self.anchor.is_null() { + self.prev = self.prev.with_tag(0); + self.curr = self.curr.with_tag(0); + Ok(found) + } else { + debug_assert_eq!(self.anchor_next.tag(), 0); + // TODO: on CAS failure, if anchor is not tagged, we can restart from anchor. + unsafe { &self.anchor.deref().next } + .compare_exchange( + self.anchor_next, + self.curr.with_tag(0), + Ordering::AcqRel, + Ordering::Relaxed, + ) + .map_err(|_| { + self.curr = self.curr.with_tag(0); + () + })?; + + let mut node = self.anchor_next; + while node.with_tag(0) != self.curr.with_tag(0) { + // NOTE: It may seem like this can be done with a NA load, but we do a `fetch_or` in remove, which always does an write. + // This can be a NA load if the `fetch_or` in delete is changed to a CAS, but it is not clear if it is worth it. + let next = unsafe { node.deref().next.load(Ordering::Relaxed) }; + debug_assert!(next.tag() != 0); + unsafe { self.handle.thread.retire(node.with_tag(0).into_raw()) }; + node = next; + } + self.prev = self.anchor.with_tag(0); + self.curr = self.curr.with_tag(0); + Ok(found) + } + } + #[inline] fn find_harris_michael(&mut self, key: &K) -> Result { loop { - debug_assert_eq!(tag(self.curr), 0); + debug_assert_eq!(self.curr.tag(), 0); if self.curr.is_null() { return Ok(false); } - let prev = unsafe { &(*self.prev).next }; + let prev = unsafe { &self.prev.deref().next }; - self.handle.curr_h.protect_raw(self.curr); + self.handle + .curr_h + .protect_raw(self.curr.with_tag(0).into_raw()); light_membarrier(); - let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); - if curr_new_tag != 0 { + let curr_new = prev.load(Ordering::Acquire); + if curr_new.tag() != 0 { return Err(()); - } else if curr_new_base != self.curr { + } else if curr_new.with_tag(0) != self.curr { // In contrary to what HP04 paper does, it's fine to retry protecting the new node // without restarting from head as long as prev is not logically deleted. - self.curr = curr_new_base; + self.curr = curr_new.with_tag(0); continue; } - let curr_node = unsafe { &*self.curr }; + let curr_node = unsafe { self.curr.deref() }; let next = curr_node.next.load(Ordering::Acquire); - let (next_base, next_tag) = decompose_ptr(next); - if next_tag == 0 { + if next.tag() == 0 { match curr_node.key.cmp(key) { Less => { self.prev = self.curr; @@ -123,26 +268,40 @@ where Greater => return Ok(false), } } else if prev - .compare_exchange(self.curr, next_base, Ordering::Release, Ordering::Relaxed) + .compare_exchange( + self.curr, + next.with_tag(0), + Ordering::Release, + Ordering::Relaxed, + ) .is_ok() { - unsafe { self.handle.thread.retire(self.curr) }; + unsafe { self.handle.thread.retire(self.curr.with_tag(0).into_raw()) }; } else { return Err(()); } - self.curr = next_base; + self.curr = next.with_tag(0); } } + + fn find_harris_herlihy_shavit(&mut self, key: &K) -> Result { + let found = self.traverse_with_anchor(key)?; + // Return only the found `curr` node. + // Others are not necessary because we are not going to do insertion or deletion + // with this Harris-Herlihy-Shavit traversal. + self.curr = self.curr.with_tag(0); + Ok(found) + } } impl List where - K: Ord, + K: Ord + 'static, { /// Creates a new list. pub fn new() -> Self { List { - head: AtomicPtr::new(ptr::null_mut()), + head: Atomic::null(), } } @@ -159,7 +318,7 @@ where loop { let mut cursor = Cursor::new(&self.head, handle.launder()); match find(&mut cursor, key) { - Ok(true) => return unsafe { Some(&((*cursor.curr).value)) }, + Ok(true) => return Some(&unsafe { cursor.curr.deref() }.value), Ok(false) => return None, Err(_) => continue, } @@ -168,28 +327,31 @@ where fn insert_inner<'domain, 'hp, F>( &self, - node: *mut Node, + mut node: Box>, find: &F, handle: &'hp mut Handle<'domain>, - ) -> Result + ) -> bool where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { loop { let mut cursor = Cursor::new(&self.head, handle.launder()); - let found = find(&mut cursor, unsafe { &(*node).key })?; + let Ok(found) = find(&mut cursor, &node.key) else { + continue; + }; if found { - drop(unsafe { Box::from_raw(node) }); - return Ok(false); + return false; } - unsafe { &*node }.next.store(cursor.curr, Ordering::Relaxed); - if unsafe { &*cursor.prev } - .next - .compare_exchange(cursor.curr, node, Ordering::Release, Ordering::Relaxed) - .is_ok() - { - return Ok(true); + node.next = cursor.curr.into(); + match unsafe { cursor.prev.deref() }.next.compare_exchange( + cursor.curr, + node, + Ordering::Release, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(e) => node = e.new, } } } @@ -205,18 +367,13 @@ where where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { - let node = Box::into_raw(Box::new(Node { + let node = Box::new(Node { key, value, - next: AtomicPtr::new(ptr::null_mut()), - })); + next: Atomic::null(), + }); - loop { - match self.insert_inner(node, &find, handle.launder()) { - Ok(r) => return r, - Err(()) => continue, - } - } + self.insert_inner(node, &find, handle.launder()) } fn remove_inner<'domain, 'hp, F>( @@ -224,34 +381,40 @@ where key: &K, find: &F, handle: &'hp mut Handle<'domain>, - ) -> Result, ()> + ) -> Option<&'hp V> where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { loop { let mut cursor = Cursor::new(&self.head, handle.launder()); - let found = find(&mut cursor, key)?; + let Ok(found) = find(&mut cursor, key) else { + continue; + }; if !found { - return Ok(None); + return None; } - let curr_node = unsafe { &*cursor.curr }; + let curr_node = unsafe { cursor.curr.deref() }; let next = curr_node.next.fetch_or(1, Ordering::AcqRel); - let next_tag = tag(next); - if next_tag == 1 { + if next.tag() == 1 { continue; } - let prev = unsafe { &(*cursor.prev).next }; + let prev = unsafe { &cursor.prev.deref().next }; if prev .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) .is_ok() { - unsafe { cursor.handle.thread.retire(cursor.curr) }; + unsafe { + cursor + .handle + .thread + .retire(cursor.curr.with_tag(0).into_raw()) + }; } - return Ok(Some(&curr_node.value)); + return Some(&curr_node.value); } } @@ -265,23 +428,20 @@ where where F: Fn(&mut Cursor<'domain, 'hp, K, V>, &K) -> Result, { - loop { - match self.remove_inner(key, &find, handle.launder()) { - Ok(r) => return r, - Err(_) => continue, - } - } + self.remove_inner(key, &find, handle.launder()) } #[inline] fn pop_inner<'hp>(&self, handle: &'hp mut Handle<'_>) -> Result, ()> { let cursor = Cursor::new(&self.head, handle.launder()); - let prev = unsafe { &(*cursor.prev).next }; + let prev = unsafe { &cursor.prev.deref().next }; - handle.curr_h.protect_raw(cursor.curr); + handle + .curr_h + .protect_raw(cursor.curr.with_tag(0).into_raw()); light_membarrier(); - let (curr_new_base, curr_new_tag) = decompose_ptr(prev.load(Ordering::Acquire)); - if curr_new_tag != 0 || curr_new_base != cursor.curr { + let curr_new = prev.load(Ordering::Acquire); + if curr_new.tag() != 0 || curr_new.with_tag(0) != cursor.curr { return Err(()); } @@ -289,11 +449,10 @@ where return Ok(None); } - let curr_node = unsafe { &*cursor.curr }; + let curr_node = unsafe { cursor.curr.deref() }; let next = curr_node.next.fetch_or(1, Ordering::AcqRel); - let next_tag = tag(next); - if next_tag == 1 { + if next.tag() == 1 { return Err(()); } @@ -301,7 +460,7 @@ where .compare_exchange(cursor.curr, next, Ordering::Release, Ordering::Relaxed) .is_ok() { - unsafe { handle.thread.retire(cursor.curr) }; + unsafe { handle.thread.retire(cursor.curr.with_tag(0).into_raw()) }; } Ok(Some((&curr_node.key, &curr_node.value))) @@ -317,6 +476,18 @@ where } } + pub fn harris_get<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option<&'hp V> { + self.get(key, Cursor::find_harris, handle) + } + + pub fn harris_insert(&self, key: K, value: V, handle: &mut Handle<'_>) -> bool { + self.insert(key, value, Cursor::find_harris, handle) + } + + pub fn harris_remove<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option<&'hp V> { + self.remove(key, Cursor::find_harris, handle) + } + pub fn harris_michael_get<'hp>(&self, key: &K, handle: &'hp mut Handle<'_>) -> Option<&'hp V> { self.get(key, Cursor::find_harris_michael, handle) } @@ -332,6 +503,54 @@ where ) -> Option<&'hp V> { self.remove(key, Cursor::find_harris_michael, handle) } + + pub fn harris_herlihy_shavit_get<'hp>( + &self, + key: &K, + handle: &'hp mut Handle<'_>, + ) -> Option<&'hp V> { + self.get(key, Cursor::find_harris_herlihy_shavit, handle) + } +} + +pub struct HList { + inner: List, +} + +impl ConcurrentMap for HList +where + K: Ord + 'static, +{ + type Handle<'domain> = Handle<'domain>; + + fn handle() -> Self::Handle<'static> { + Handle::default() + } + + fn new() -> Self { + HList { inner: List::new() } + } + + #[inline(always)] + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.inner.harris_get(key, handle) + } + #[inline(always)] + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + self.inner.harris_insert(key, value, handle) + } + #[inline(always)] + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.inner.harris_remove(key, handle) + } } pub struct HMList { @@ -340,7 +559,7 @@ pub struct HMList { impl HMList where - K: Ord, + K: Ord + 'static, { /// Pop the first element efficiently. /// This method is used for only the fine grained benchmark (src/bin/long_running). @@ -351,7 +570,7 @@ where impl ConcurrentMap for HMList where - K: Ord, + K: Ord + 'static, { type Handle<'domain> = Handle<'domain>; @@ -364,7 +583,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_get(key, handle) } #[inline(always)] @@ -372,19 +595,73 @@ where self.inner.harris_michael_insert(key, value, handle) } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_remove(key, handle) } } +pub struct HHSList { + inner: List, +} + +impl ConcurrentMap for HHSList +where + K: Ord + 'static, +{ + type Handle<'domain> = Handle<'domain>; + + fn handle() -> Self::Handle<'static> { + Handle::default() + } + + fn new() -> Self { + HHSList { inner: List::new() } + } + + #[inline(always)] + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.inner.harris_herlihy_shavit_get(key, handle) + } + #[inline(always)] + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + self.inner.harris_insert(key, value, handle) + } + #[inline(always)] + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.inner.harris_remove(key, handle) + } +} + #[cfg(test)] mod tests { - use super::HMList; + use super::{HHSList, HList, HMList}; use crate::ds_impl::hp::concurrent_map; + #[test] + fn smoke_h_list() { + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); + } + #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); + } + + #[test] + fn smoke_hhs_list() { + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/hp/michael_hash_map.rs b/src/ds_impl/hp/michael_hash_map.rs index a0bb9557..b48adf21 100644 --- a/src/ds_impl/hp/michael_hash_map.rs +++ b/src/ds_impl/hp/michael_hash_map.rs @@ -1,29 +1,29 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -use super::list::HMList; +use super::list::HHSList; pub use super::list::{Cursor, Handle}; pub struct HashMap { - buckets: Vec>, + buckets: Vec>, } impl HashMap where - K: Ord + Hash, + K: Ord + Hash + 'static, { pub fn with_capacity(n: usize) -> Self { let mut buckets = Vec::with_capacity(n); for _ in 0..n { - buckets.push(HMList::new()); + buckets.push(HHSList::new()); } HashMap { buckets } } #[inline] - pub fn get_bucket(&self, index: usize) -> &HMList { + pub fn get_bucket(&self, index: usize) -> &HHSList { unsafe { self.buckets.get_unchecked(index % self.buckets.len()) } } @@ -33,26 +33,11 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get<'hp>(&self, handle: &'hp mut Handle<'_>, k: &K) -> Option<&'hp V> { - let i = Self::hash(k); - self.get_bucket(i).get(handle, k) - } - - pub fn insert(&self, handle: &mut Handle<'_>, k: K, v: V) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(handle, k, v) - } - - pub fn remove<'hp>(&self, handle: &'hp mut Handle<'_>, k: &K) -> Option<&'hp V> { - let i = Self::hash(k); - self.get_bucket(i).remove(handle, k) - } } impl ConcurrentMap for HashMap where - K: Ord + Hash + Send, + K: Ord + Hash + Send + 'static, V: Send, { type Handle<'domain> = Handle<'domain>; @@ -66,16 +51,27 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - self.get(handle, key) + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(handle, key) } #[inline(always)] fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { - self.insert(handle, key, value) + let i = Self::hash(&key); + self.get_bucket(i).insert(handle, key, value) } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - self.remove(handle, key) + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).remove(handle, key) } } @@ -86,6 +82,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/mod.rs b/src/ds_impl/hp/mod.rs index 329e422d..2d5551d7 100644 --- a/src/ds_impl/hp/mod.rs +++ b/src/ds_impl/hp/mod.rs @@ -1,7 +1,9 @@ pub mod concurrent_map; +pub mod pointers; pub mod bonsai_tree; pub mod double_link; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; @@ -12,8 +14,9 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; -pub use self::list::HMList; +pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; pub use self::natarajan_mittal_tree::NMTreeMap; pub use self::skip_list::SkipList; diff --git a/src/ds_impl/hp/natarajan_mittal_tree.rs b/src/ds_impl/hp/natarajan_mittal_tree.rs index 43861a2d..cf2b8c3b 100644 --- a/src/ds_impl/hp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp/natarajan_mittal_tree.rs @@ -1,9 +1,8 @@ -use hp_pp::Thread; +use hp_pp::{light_membarrier, Thread}; use hp_pp::{tag, tagged, untagged, HazardPointer, DEFAULT_DOMAIN}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; -use std::mem; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; @@ -30,6 +29,10 @@ impl Marks { fn tag(self) -> bool { !(self & Marks::TAG).is_empty() } + + fn marked(self) -> bool { + !(self & (Marks::TAG | Marks::FLAG)).is_empty() + } } #[derive(Clone, PartialEq, Eq, Debug)] @@ -136,17 +139,18 @@ pub struct Handle<'domain> { successor_h: HazardPointer<'domain>, parent_h: HazardPointer<'domain>, leaf_h: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - ancestor_h: HazardPointer::default(), - successor_h: HazardPointer::default(), - parent_h: HazardPointer::default(), - leaf_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + ancestor_h: HazardPointer::new(&mut thread), + successor_h: HazardPointer::new(&mut thread), + parent_h: HazardPointer::new(&mut thread), + leaf_h: HazardPointer::new(&mut thread), + thread, } } } @@ -274,12 +278,11 @@ where NMTreeMap { r } } - // All `Shared<_>` fields are unmarked. fn seek(&self, key: &K, record: &mut SeekRecord<'_, '_, K, V>) -> Result<(), ()> { let s = untagged(self.r.left.load(Ordering::Relaxed)); let s_node = unsafe { &*s }; - // We doesn't have to defend with hazard pointers here + // The root node is always alive; we do not have to protect it. record.ancestor = &self.r as *const _ as *mut _; record.successor = s; // TODO: should preserve tag? @@ -287,11 +290,12 @@ where let leaf = tagged(s_node.left.load(Ordering::Relaxed), Marks::empty().bits()); - // We doesn't have to defend with hazard pointers here + // The `s` node is always alive; we do not have to protect it. record.parent = s; record.handle.leaf_h.protect_raw(leaf); - if leaf != tagged(s_node.left.load(Ordering::Relaxed), Marks::empty().bits()) { + light_membarrier(); + if leaf != tagged(s_node.left.load(Ordering::Acquire), Marks::empty().bits()) { return Err(()); } record.leaf = leaf; @@ -301,36 +305,90 @@ where let mut curr_dir = Direction::L; let mut curr = unsafe { &*record.leaf }.left.load(Ordering::Relaxed); + // `ancestor` always points untagged node. while !untagged(curr).is_null() { if !prev_tag { // untagged edge: advance ancestor and successor pointers - record - .handle - .ancestor_h - .protect_raw(untagged(record.parent)); record.ancestor = record.parent; - record.handle.successor_h.protect_raw(untagged(record.leaf)); record.successor = record.leaf; record.successor_dir = record.leaf_dir; + // `ancestor` and `successor` are already protected by + // hazard pointers of `parent` and `leaf`. + + // Advance the parent and leaf pointers when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (parent), ancestor -> O (ancestor) -> O + // / \ / \ + // (leaf), successor -> O O => (parent), successor -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardPointer::swap(&mut record.handle.ancestor_h, &mut record.handle.parent_h); + HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); + } else if record.successor == record.parent { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O (ancestor) -> O + // / \ / \ + // (parent), successor -> O O (successor) -> O O + // / \ => / \ + // (leaf) -> O O (parent) -> O O + // / \ / \ + // O O (leaf) -> O O + record.parent = record.leaf; + HazardPointer::swap(&mut record.handle.successor_h, &mut record.handle.parent_h); + HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); + } else { + // Advance the parent and leaf pointer when the cursor looks like the following: + // (): protected by its dedicated shield. + // + // (ancestor) -> O + // / \ + // (successor) -> O O + // ... ... + // (parent) -> O + // / \ + // (leaf) -> O O + record.parent = record.leaf; + HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); } - - // advance parent and leaf pointers - mem::swap(&mut record.parent, &mut record.leaf); - HazardPointer::swap(&mut record.handle.parent_h, &mut record.handle.leaf_h); - let mut curr_base = untagged(curr); - loop { - record.handle.leaf_h.protect_raw(curr_base); - let curr_base_new = untagged(match curr_dir { - Direction::L => unsafe { &*record.parent }.left.load(Ordering::Acquire), - Direction::R => unsafe { &*record.parent }.right.load(Ordering::Acquire), - }); - if curr_base_new == curr_base { - break; + debug_assert_eq!(tag(record.successor), 0); + + let curr_base = untagged(curr); + record.handle.leaf_h.protect_raw(curr_base); + light_membarrier(); + + if Marks::from_bits_truncate(tag(curr)).marked() { + // `curr` is marked. Validate by `ancestor`. + let succ_new = record.successor_addr().load(Ordering::Acquire); + if Marks::from_bits_truncate(tag(succ_new)).marked() || record.successor != succ_new + { + // Validation is failed. Let's restart from the root. + // TODO: Maybe it can be optimized (by restarting from the anchor), but + // it would require a serious reasoning (including shield swapping, etc). + return Err(()); + } + } else { + // `curr` is unmarked. Validate by `parent`. + let curr_new = match curr_dir { + Direction::L => unsafe { &*untagged(record.parent) } + .left + .load(Ordering::Acquire), + Direction::R => unsafe { &*untagged(record.parent) } + .right + .load(Ordering::Acquire), + }; + if curr_new != curr { + // Validation is failed. Let's restart from the root. + // TODO: Maybe it can be optimized (by restarting from the parent), but + // it would require a serious reasoning (including shield swapping, etc). + return Err(()); } - curr_base = curr_base_new; } - record.leaf = curr_base; + record.leaf = curr; record.leaf_dir = curr_dir; // update other variables @@ -373,7 +431,7 @@ where let is_unlinked = record .successor_addr() .compare_exchange( - record.successor, + untagged(record.successor), tagged(target_sibling, Marks::new(flag, false).bits()), Ordering::AcqRel, Ordering::Acquire, @@ -446,9 +504,9 @@ where drop(Box::from_raw(new_internal)); Err(value) })?; - let leaf = record.leaf; + let leaf = untagged(record.leaf); - let (new_left, new_right) = match unsafe { &*untagged(leaf) }.key.cmp(key) { + let (new_left, new_right) = match unsafe { &*leaf }.key.cmp(key) { cmp::Ordering::Equal => { // Newly created nodes that failed to be inserted are free'd here. let value = unsafe { &mut *new_leaf }.value.take().unwrap(); @@ -510,8 +568,8 @@ where self.seek(key, &mut record)?; // candidates - let leaf = record.leaf; - let leaf_node = unsafe { &*untagged(record.leaf) }; + let leaf = untagged(record.leaf); + let leaf_node = unsafe { &*record.leaf }; if leaf_node.key.cmp(key) != cmp::Ordering::Equal { return Ok(None); @@ -551,7 +609,7 @@ where // cleanup phase loop { self.seek(key, &mut record)?; - if record.leaf != leaf { + if untagged(record.leaf) != leaf { // The edge to leaf flagged for deletion was removed by a helping thread return Ok(Some(value)); } @@ -588,7 +646,11 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } @@ -598,7 +660,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -610,6 +676,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp/pointers.rs b/src/ds_impl/hp/pointers.rs new file mode 100644 index 00000000..2020c4ed --- /dev/null +++ b/src/ds_impl/hp/pointers.rs @@ -0,0 +1,265 @@ +use core::mem; +use std::{ + ptr::null_mut, + sync::atomic::{AtomicPtr, Ordering}, +}; + +pub struct CompareExchangeError> { + pub new: P, + pub current: Shared, +} + +pub struct Atomic { + link: AtomicPtr, +} + +unsafe impl Sync for Atomic {} +unsafe impl Send for Atomic {} + +impl Atomic { + #[inline] + pub fn new(init: T) -> Self { + let link = AtomicPtr::new(Box::into_raw(Box::new(init))); + Self { link } + } + + #[inline] + pub fn null() -> Self { + let link = AtomicPtr::new(null_mut()); + Self { link } + } + + #[inline] + pub fn load(&self, order: Ordering) -> Shared { + let ptr = self.link.load(order); + Shared { ptr } + } + + #[inline] + pub fn store>(&self, ptr: P, order: Ordering) { + self.link.store(ptr.into_raw(), order) + } + + #[inline] + pub fn fetch_or(&self, val: usize, order: Ordering) -> Shared { + let ptr = self.link.fetch_or(val, order); + Shared { ptr } + } + + #[inline] + pub fn compare_exchange>( + &self, + current: Shared, + new: P, + success: Ordering, + failure: Ordering, + ) -> Result, CompareExchangeError> { + let current = current.into_raw(); + let new = new.into_raw(); + + match self.link.compare_exchange(current, new, success, failure) { + Ok(current) => Ok(Shared { ptr: current }), + Err(current) => { + let new = unsafe { P::from_raw(new) }; + Err(CompareExchangeError { + new, + current: Shared { ptr: current }, + }) + } + } + } + + #[inline] + pub unsafe fn try_into_owned(self) -> Option> { + let ptr = base_ptr(self.link.into_inner()); + if ptr.is_null() { + return None; + } else { + Some(unsafe { Box::from_raw(ptr) }) + } + } + + #[inline] + pub unsafe fn into_owned(self) -> Box { + unsafe { Box::from_raw(self.link.into_inner()) } + } + + #[inline] + // TODO: best API? might be better to just wrap as_ptr, without the deref. + pub unsafe fn as_shared(&self) -> Shared { + Shared { + ptr: unsafe { *self.link.as_ptr() }, + } + } +} + +impl Default for Atomic { + #[inline] + fn default() -> Self { + Self { + link: AtomicPtr::default(), + } + } +} + +impl From> for Atomic { + #[inline] + fn from(value: Shared) -> Self { + let link = AtomicPtr::new(value.into_raw()); + Self { link } + } +} + +impl From<*mut T> for Atomic { + #[inline] + fn from(value: *mut T) -> Self { + let link = AtomicPtr::new(value); + Self { link } + } +} + +impl Clone for Atomic { + fn clone(&self) -> Self { + Self { + link: AtomicPtr::new(self.link.load(Ordering::Relaxed)), + } + } +} + +pub struct Shared { + ptr: *mut T, +} + +impl Shared { + #[inline] + pub fn from_owned(init: T) -> Shared { + let ptr = Box::into_raw(Box::new(init)); + Self { ptr } + } + + #[inline] + pub unsafe fn into_owned(self) -> T { + unsafe { *Box::from_raw(base_ptr(self.ptr)) } + } + + #[inline] + pub fn null() -> Self { + Self { ptr: null_mut() } + } + + #[inline] + pub fn tag(&self) -> usize { + tag(self.ptr) + } + + #[inline] + pub fn is_null(&self) -> bool { + base_ptr(self.ptr).is_null() + } + + #[inline] + pub fn with_tag(&self, tag: usize) -> Self { + let ptr = compose_tag(self.ptr, tag); + Self { ptr } + } + + #[inline] + pub unsafe fn as_ref<'g>(&self) -> Option<&'g T> { + base_ptr(self.ptr).as_ref() + } + + #[inline] + pub unsafe fn as_mut<'g>(&self) -> Option<&'g mut T> { + base_ptr(self.ptr).as_mut() + } + + #[inline] + pub unsafe fn deref<'g>(&self) -> &'g T { + &*base_ptr(self.ptr) + } + + #[inline] + pub unsafe fn deref_mut<'g>(&mut self) -> &'g mut T { + &mut *base_ptr(self.ptr) + } +} + +impl From for Shared { + #[inline] + fn from(val: usize) -> Self { + Self { + ptr: val as *const T as *mut T, + } + } +} + +impl Clone for Shared { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Shared {} + +impl PartialEq for Shared { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.ptr == other.ptr + } +} + +impl Eq for Shared {} + +pub trait Pointer { + fn into_raw(self) -> *mut T; + unsafe fn from_raw(val: *mut T) -> Self; +} + +impl Pointer for Shared { + #[inline] + fn into_raw(self) -> *mut T { + self.ptr + } + + #[inline] + unsafe fn from_raw(val: *mut T) -> Self { + Shared::from(val as usize) + } +} + +impl Pointer for Box { + #[inline] + fn into_raw(self) -> *mut T { + Box::into_raw(self) + } + + #[inline] + unsafe fn from_raw(val: *mut T) -> Self { + Box::from_raw(val) + } +} + +/// Returns a bitmask containing the unused least significant bits of an aligned pointer to `T`. +#[inline] +fn low_bits() -> usize { + (1 << mem::align_of::().trailing_zeros()) - 1 +} + +/// Given a tagged pointer `data`, returns the same pointer, but tagged with `tag`. +/// +/// `tag` is truncated to fit into the unused bits of the pointer to `T`. +#[inline] +pub(crate) fn compose_tag(ptr: *mut T, tag: usize) -> *mut T { + ((ptr as usize & !low_bits::()) | (tag & low_bits::())) as _ +} + +#[inline] +pub(crate) fn base_ptr(ptr: *mut T) -> *mut T { + (ptr as usize & !low_bits::()) as _ +} + +#[inline] +pub(crate) fn tag(ptr: *mut T) -> usize { + ptr as usize & low_bits::() +} diff --git a/src/ds_impl/hp/skip_list.rs b/src/ds_impl/hp/skip_list.rs index 0f43fe93..9dc0f5f2 100644 --- a/src/ds_impl/hp/skip_list.rs +++ b/src/ds_impl/hp/skip_list.rs @@ -2,10 +2,9 @@ use std::mem::transmute; use std::ptr; use std::sync::atomic::{fence, AtomicPtr, AtomicUsize, Ordering}; -use hp_pp::{light_membarrier, tagged, Thread}; -use hp_pp::{tag, untagged, HazardPointer, DEFAULT_DOMAIN}; +use hp_pp::{light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -89,16 +88,17 @@ pub struct Handle<'g> { preds_h: [HazardPointer<'g>; MAX_HEIGHT], succs_h: [HazardPointer<'g>; MAX_HEIGHT], removed_h: HazardPointer<'g>, - thread: Thread<'g>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - preds_h: Default::default(), - succs_h: Default::default(), - removed_h: Default::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + preds_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + succs_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + removed_h: HazardPointer::new(&mut thread), + thread, } } } @@ -159,6 +159,122 @@ where } } + fn find_optimistic(&self, key: &K, handle: &mut Handle<'_>) -> Option<*mut Node> { + 'search: loop { + // This optimistic traversal doesn't have to use all shields in the `handle`. + let (anchor_h, anchor_next_h) = unsafe { + let splited = handle.preds_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; + let (pred_h, curr_h) = unsafe { + let splited = handle.succs_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; + + let mut level = MAX_HEIGHT; + while level >= 1 && self.head[level - 1].load(Ordering::Relaxed).is_null() { + level -= 1; + } + + let mut pred = &self.head as *const _ as *mut Node; + let mut curr = ptr::null_mut::>(); + let mut anchor = pred; + let mut anchor_next = ptr::null_mut::>(); + + while level >= 1 { + level -= 1; + curr = unsafe { &*untagged(anchor) }.next[level].load(Ordering::Acquire); + + loop { + if untagged(curr).is_null() { + break; + } + + curr_h.protect_raw(untagged(curr)); + light_membarrier(); + + // Validation depending on the state of `curr`. + // + // - If it is marked, validate on anchor. + // - If it is not marked, validate on pred. + + if tag(curr) != 0 { + // Validate on anchor. + let an_new = + unsafe { &*untagged(anchor) }.next[level].load(Ordering::Acquire); + + if tag(an_new) != 0 { + continue 'search; + } else if an_new != anchor_next { + // Anchor is updated but clear, so can restart from anchor. + + pred = anchor; + curr = an_new; + + // Set prev HP as anchor HP, since prev should always be protected. + HazardPointer::swap(pred_h, anchor_h); + continue; + } + } else { + // Validate on prev. + let curr_new = + unsafe { &*untagged(pred) }.next[level].load(Ordering::Acquire); + + if tag(curr_new) != 0 { + // If prev is marked, then restart from head. + continue 'search; + } else if curr_new != curr { + // curr's tag was 0, so the above comparison ignores tags. + + // In contrary to what HP04 paper does, it's fine to retry protecting the new node + // without restarting from head as long as prev is not logically deleted. + curr = curr_new; + continue; + } + } + + let curr_node = unsafe { &*untagged(curr) }; + let next = curr_node.next[level].load(Ordering::Acquire); + if tag(next) == 0 { + if curr_node.key < *key { + pred = curr; + curr = next; + anchor = pred; + HazardPointer::swap(curr_h, pred_h); + } else { + break; + } + } else { + if untagged(anchor) == untagged(pred) { + anchor_next = curr; + HazardPointer::swap(anchor_h, pred_h); + } else if untagged(anchor_next) == untagged(pred) { + HazardPointer::swap(anchor_next_h, pred_h); + } + pred = curr; + curr = next; + HazardPointer::swap(pred_h, curr_h); + } + } + } + + if let Some(curr_node) = unsafe { untagged(curr).as_ref() } { + if curr_node.key == *key + && (curr_node.next[0].load(Ordering::Acquire) as usize & 1) == 0 + { + return Some(untagged(curr)); + } + } + return None; + } + } + fn find(&self, key: &K, handle: &mut Handle<'_>) -> Cursor { 'search: loop { let mut cursor = Cursor::new(&self.head); @@ -386,11 +502,14 @@ where } #[inline(always)] - fn get<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { - let cursor = self.find(key, handle); - let node = unsafe { cursor.found?.as_ref()? }; - if node.key.eq(key) { - Some(unsafe { transmute(&node.value) }) + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let node = unsafe { &*self.find_optimistic(key, handle)? }; + if node.key.eq(&key) { + Some(&node.value) } else { None } @@ -402,7 +521,11 @@ where } #[inline(always)] - fn remove<'hp>(&self, handle: &'hp mut Self::Handle<'_>, key: &K) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -414,6 +537,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_brcu/bonsai_tree.rs b/src/ds_impl/hp_brcu/bonsai_tree.rs new file mode 100644 index 00000000..266ef1a5 --- /dev/null +++ b/src/ds_impl/hp_brcu/bonsai_tree.rs @@ -0,0 +1,812 @@ +use std::{cmp, sync::atomic::Ordering}; + +use hp_brcu::{ + Atomic, CsGuard, Owned, RaGuard, RollbackProof, Shared, Shield, Thread, Unprotected, +}; + +use super::concurrent_map::{ConcurrentMap, OutputHolder}; + +static WEIGHT: usize = 2; + +// TODO: optimization from the paper? IBR paper doesn't do that + +bitflags! { + /// TODO + struct Retired: usize { + const RETIRED = 1usize; + } +} + +impl Retired { + fn new(retired: bool) -> Self { + if retired { + Retired::RETIRED + } else { + Retired::empty() + } + } + + fn retired(self) -> bool { + !(self & Retired::RETIRED).is_empty() + } +} + +/// a real node in tree or a wrapper of State node +/// Retired node if Shared ptr of Node has RETIRED tag. +struct Node { + key: K, + value: V, + size: usize, + left: Atomic>, + right: Atomic>, +} + +impl Node +where + K: Ord + Clone, + V: Clone, +{ + fn retired_node<'g>() -> Shared<'g, Self> { + Shared::null().with_tag(Retired::new(true).bits()) + } + + fn is_retired<'g>(node: Shared<'g, Self>) -> bool { + Retired::from_bits_truncate(node.tag()).retired() + } + + fn is_retired_spot<'g>(node: Shared<'g, Self>, guard: &'g CsGuard) -> bool { + if Self::is_retired(node) { + return true; + } + + if let Some(node_ref) = unsafe { node.as_ref() } { + Self::is_retired(node_ref.left.load(Ordering::Acquire, guard)) + || Self::is_retired(node_ref.right.load(Ordering::Acquire, guard)) + } else { + false + } + } + + fn node_size<'g>(node: Shared<'g, Self>, _: &'g CsGuard) -> usize { + debug_assert!(!Self::is_retired(node)); + if let Some(node_ref) = unsafe { node.as_ref() } { + node_ref.size + } else { + 0 + } + } +} + +/// Rollback-proof buffer to memorize retired nodes and newly allocated nodes. +struct RBProofBuf { + /// Nodes that current op wants to remove from the tree. Should be retired if CAS succeeds. + /// (`retire`). If not, ignore. + retired_nodes: Vec>>, + /// Nodes newly constructed by the op. Should be destroyed if CAS fails. (`destroy`) + new_nodes: Vec>>, +} + +impl RBProofBuf +where + K: Ord + Clone, + V: Clone, +{ + fn new() -> Self { + Self { + retired_nodes: Vec::new(), + new_nodes: Vec::new(), + } + } + + /// Destroy the newly created state (self) that lost the race (reclaim_state) + fn abort(&mut self, _: &mut G) { + self.retired_nodes.clear(); + + for node in self.new_nodes.drain(..) { + drop(unsafe { node.into_owned() }); + } + } + + /// Retire the old state replaced by the new_state and the new_state.retired_nodes + fn commit(&mut self, guard: &mut G) { + self.new_nodes.clear(); + + for node in self.retired_nodes.drain(..) { + let node = node.load(Ordering::Relaxed, guard); + unsafe { + node.deref() + .left + .store(Node::retired_node(), Ordering::Release, guard); + node.deref() + .right + .store(Node::retired_node(), Ordering::Release, guard); + guard.retire(node); + } + } + } + + fn retire_node(&mut self, node: Shared>, _: &mut RaGuard) { + self.retired_nodes.push(Atomic::from(node)); + } + + fn add_new_node(&mut self, node: Shared>, _: &mut RaGuard) { + self.new_nodes.push(Atomic::from(node)); + } +} + +pub struct Protectors { + old_root: Shield>, + new_root: Shield>, + found_node: Shield>, +} + +impl OutputHolder for Protectors { + fn default(thread: &mut Thread) -> Self { + Self { + old_root: Shield::null(thread), + new_root: Shield::null(thread), + found_node: Shield::null(thread), + } + } + + fn output(&self) -> &V { + &self.found_node.as_ref().unwrap().value + } +} + +/// Each op creates a new local state and tries to update (CAS) the tree with it. +struct State<'g, K, V> { + root_link: &'g Atomic>, + curr_root: Shared<'g, Node>, + buf: &'g mut RBProofBuf, +} + +impl<'g, K, V> State<'g, K, V> +where + K: Ord + Clone, + V: Clone, +{ + fn new( + root_link: &'g Atomic>, + buf: &'g mut RBProofBuf, + guard: &'g CsGuard, + ) -> Self { + Self { + root_link, + curr_root: root_link.load(Ordering::Acquire, guard), + buf, + } + } + + // TODO get ref of K, V and clone here + fn mk_node( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + if Node::is_retired_spot(left, guard) || Node::is_retired_spot(right, guard) { + return Node::retired_node(); + } + + let left_size = Node::node_size(left, guard); + let right_size = Node::node_size(right, guard); + guard.mask_light(|guard| { + let new_node = Owned::new(Node { + key: key.clone(), + value: value.clone(), + size: left_size + right_size + 1, + left: Atomic::from(left), + right: Atomic::from(right), + }) + .into_shared(); + self.buf.add_new_node(new_node, guard); + new_node + }) + } + + /// Make a new balanced tree from cur (the root of a subtree) and newly constructed left and right subtree + fn mk_balanced( + &mut self, + cur: Shared<'g, Node>, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + if Node::is_retired_spot(cur, guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return Node::retired_node(); + } + + let cur_ref = unsafe { cur.deref() }; + let key = &cur_ref.key; + let value = &cur_ref.value; + + let l_size = Node::node_size(left, guard); + let r_size = Node::node_size(right, guard); + let res = if r_size > 0 + && ((l_size > 0 && r_size > WEIGHT * l_size) || (l_size == 0 && r_size > WEIGHT)) + { + self.mk_balanced_left(left, right, key, value, guard) + } else if l_size > 0 + && ((r_size > 0 && l_size > WEIGHT * r_size) || (r_size == 0 && l_size > WEIGHT)) + { + self.mk_balanced_right(left, right, key, value, guard) + } else { + self.mk_node(left, right, key, value, guard) + }; + guard.mask_light(|guard| self.buf.retire_node(cur, guard)); + res + } + + #[inline] + fn mk_balanced_left( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let right_ref = unsafe { right.deref() }; + let right_left = right_ref.left.load(Ordering::Acquire, guard); + let right_right = right_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(right_left, guard) + || Node::is_retired_spot(right_right, guard) + { + return Node::retired_node(); + } + + if Node::node_size(right_left, guard) < Node::node_size(right_right, guard) { + // single left rotation + return self.single_left(left, right, right_left, right_right, key, value, guard); + } + + // double left rotation + return self.double_left(left, right, right_left, right_right, key, value, guard); + } + + #[inline] + fn single_left( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + right_left: Shared<'g, Node>, + right_right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let right_ref = unsafe { right.deref() }; + let new_left = self.mk_node(left, right_left, key, value, guard); + let res = self.mk_node( + new_left, + right_right, + &right_ref.key, + &right_ref.value, + guard, + ); + guard.mask_light(|guard| self.buf.retire_node(right, guard)); + res + } + + #[inline] + fn double_left( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + right_left: Shared<'g, Node>, + right_right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let right_ref = unsafe { right.deref() }; + let right_left_ref = unsafe { right_left.deref() }; + let right_left_left = right_left_ref.left.load(Ordering::Acquire, guard); + let right_left_right = right_left_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(right_left_left, guard) + || Node::is_retired_spot(right_left_right, guard) + { + return Node::retired_node(); + } + + let new_left = self.mk_node(left, right_left_left, key, value, guard); + let new_right = self.mk_node( + right_left_right, + right_right, + &right_ref.key, + &right_ref.value, + guard, + ); + let res = self.mk_node( + new_left, + new_right, + &right_left_ref.key, + &right_left_ref.value, + guard, + ); + guard.mask_light(|guard| { + self.buf.retire_node(right_left, guard); + self.buf.retire_node(right, guard); + }); + res + } + + #[inline] + fn mk_balanced_right( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let left_ref = unsafe { left.deref() }; + let left_right = left_ref.right.load(Ordering::Acquire, guard); + let left_left = left_ref.left.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left_right, guard) + || Node::is_retired_spot(left_left, guard) + { + return Node::retired_node(); + } + + if Node::node_size(left_right, guard) < Node::node_size(left_left, guard) { + // single right rotation (fig 3) + return self.single_right(left, right, left_right, left_left, key, value, guard); + } + // double right rotation + return self.double_right(left, right, left_right, left_left, key, value, guard); + } + + #[inline] + fn single_right( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + left_right: Shared<'g, Node>, + left_left: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let left_ref = unsafe { left.deref() }; + let new_right = self.mk_node(left_right, right, key, value, guard); + let res = self.mk_node(left_left, new_right, &left_ref.key, &left_ref.value, guard); + guard.mask_light(|guard| self.buf.retire_node(left, guard)); + res + } + + #[inline] + fn double_right( + &mut self, + left: Shared<'g, Node>, + right: Shared<'g, Node>, + left_right: Shared<'g, Node>, + left_left: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> Shared<'g, Node> { + let left_ref = unsafe { left.deref() }; + let left_right_ref = unsafe { left_right.deref() }; + let left_right_left = left_right_ref.left.load(Ordering::Acquire, guard); + let left_right_right = left_right_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left_right_left, guard) + || Node::is_retired_spot(left_right_right, guard) + { + return Node::retired_node(); + } + + let new_left = self.mk_node( + left_left, + left_right_left, + &left_ref.key, + &left_ref.value, + guard, + ); + let new_right = self.mk_node(left_right_right, right, key, value, guard); + let res = self.mk_node( + new_left, + new_right, + &left_right_ref.key, + &left_right_ref.value, + guard, + ); + guard.mask_light(|guard| { + self.buf.retire_node(left_right, guard); + self.buf.retire_node(left, guard); + }); + res + } + + #[inline] + fn do_insert( + &mut self, + node: Shared<'g, Node>, + key: &K, + value: &V, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, bool) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), false); + } + + if node.is_null() { + return ( + self.mk_node(Shared::null(), Shared::null(), key, value, guard), + true, + ); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), false); + } + + match node_ref.key.cmp(key) { + cmp::Ordering::Equal => (node, false), + cmp::Ordering::Less => { + let (new_right, inserted) = self.do_insert(right, key, value, guard); + (self.mk_balanced(node, left, new_right, guard), inserted) + } + cmp::Ordering::Greater => { + let (new_left, inserted) = self.do_insert(left, key, value, guard); + (self.mk_balanced(node, new_left, right, guard), inserted) + } + } + } + + /// Hint: Returns a tuple of (new node, removed node if exists). + #[inline] + fn do_remove( + &mut self, + node: Shared<'g, Node>, + key: &K, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, Option>>) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), None); + } + + if node.is_null() { + return (Shared::null(), None); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), None); + } + + match node_ref.key.cmp(key) { + cmp::Ordering::Equal => { + guard.mask_light(|guard| self.buf.retire_node(node, guard)); + if node_ref.size == 1 { + return (Shared::null(), Some(node)); + } + + if !left.is_null() { + let (new_left, succ) = self.pull_rightmost(left, guard); + return (self.mk_balanced(succ, new_left, right, guard), Some(node)); + } + let (new_right, succ) = self.pull_leftmost(right, guard); + (self.mk_balanced(succ, left, new_right, guard), Some(node)) + } + cmp::Ordering::Less => { + let (new_right, removed) = self.do_remove(right, key, guard); + (self.mk_balanced(node, left, new_right, guard), removed) + } + cmp::Ordering::Greater => { + let (new_left, removed) = self.do_remove(left, key, guard); + (self.mk_balanced(node, new_left, right, guard), removed) + } + } + } + + fn pull_leftmost( + &mut self, + node: Shared<'g, Node>, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, Shared<'g, Node>) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), Node::retired_node()); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), Node::retired_node()); + } + + if !left.is_null() { + let (new_left, succ) = self.pull_leftmost(left, guard); + return (self.mk_balanced(node, new_left, right, guard), succ); + } + // node is the leftmost + let succ = self.mk_node( + Shared::null(), + Shared::null(), + &node_ref.key, + &node_ref.value, + guard, + ); + guard.mask_light(|guard| self.buf.retire_node(node, guard)); + (right, succ) + } + + fn pull_rightmost( + &mut self, + node: Shared<'g, Node>, + guard: &'g CsGuard, + ) -> (Shared<'g, Node>, Shared<'g, Node>) { + if Node::is_retired_spot(node, guard) { + return (Node::retired_node(), Node::retired_node()); + } + + let node_ref = unsafe { node.deref() }; + let left = node_ref.left.load(Ordering::Acquire, guard); + let right = node_ref.right.load(Ordering::Acquire, guard); + + if !self.check_root(guard) + || Node::is_retired_spot(left, guard) + || Node::is_retired_spot(right, guard) + { + return (Node::retired_node(), Node::retired_node()); + } + + if !right.is_null() { + let (new_right, succ) = self.pull_rightmost(right, guard); + return (self.mk_balanced(node, left, new_right, guard), succ); + } + // node is the rightmost + let succ = self.mk_node( + Shared::null(), + Shared::null(), + &node_ref.key, + &node_ref.value, + guard, + ); + guard.mask_light(|guard| self.buf.retire_node(node, guard)); + (left, succ) + } + + pub fn check_root(&self, guard: &CsGuard) -> bool { + self.curr_root == self.root_link.load(Ordering::Acquire, guard) + } +} + +pub struct BonsaiTreeMap { + root: Atomic>, +} + +impl Default for BonsaiTreeMap +where + K: Ord + Clone, + V: Clone, +{ + fn default() -> Self { + Self::new() + } +} + +impl BonsaiTreeMap +where + K: Ord + Clone, + V: Clone, +{ + pub fn new() -> Self { + Self { + root: Atomic::null(), + } + } + + pub fn get_inner( + &self, + key: &K, + output: &mut Protectors, + guard: &CsGuard, + ) -> Result { + let mut node = self.root.load(Ordering::Acquire, guard); + while !node.is_null() && !Node::is_retired(node) { + let node_ref = unsafe { node.deref() }; + match key.cmp(&node_ref.key) { + cmp::Ordering::Equal => break, + cmp::Ordering::Less => node = node_ref.left.load(Ordering::Acquire, guard), + cmp::Ordering::Greater => node = node_ref.right.load(Ordering::Acquire, guard), + } + } + if Node::is_retired_spot(node, guard) { + return Err(()); + } else if node.is_null() { + return Ok(false); + } + output.found_node.protect(node); + Ok(true) + } + + pub fn get(&self, key: &K, output: &mut Protectors, handle: &mut Thread) -> bool { + loop { + let result = + unsafe { handle.critical_section(|guard| self.get_inner(key, output, guard)) }; + if let Ok(found) = result { + return found; + } + } + } + + pub fn insert( + &self, + key: K, + value: V, + output: &mut Protectors, + handle: &mut Thread, + ) -> bool { + let mut buf = RBProofBuf::new(); + loop { + let inserted = unsafe { + handle.critical_section(|guard| { + output.found_node.release(); + guard.mask_light(|guard| buf.abort(guard)); + let mut state = State::new(&self.root, &mut buf, guard); + let old_root = state.curr_root; + let (new_node, inserted) = state.do_insert(old_root, &key, &value, guard); + output.old_root.protect(old_root); + output.new_root.protect(new_node); + inserted + }) + }; + + if Node::is_retired(output.new_root.shared()) { + buf.abort(handle); + continue; + } + + if self + .root + .compare_exchange( + output.old_root.shared(), + output.new_root.shared(), + Ordering::AcqRel, + Ordering::Acquire, + handle, + ) + .is_ok() + { + buf.commit(handle); + return inserted; + } + buf.abort(handle); + } + } + + pub fn remove(&self, key: &K, output: &mut Protectors, handle: &mut Thread) -> bool { + let mut buf = RBProofBuf::new(); + loop { + unsafe { + handle.critical_section(|guard| { + output.found_node.release(); + guard.mask_light(|guard| buf.abort(guard)); + let mut state = State::new(&self.root, &mut buf, guard); + let old_root = state.curr_root; + let (new_root, removed) = state.do_remove(old_root, key, guard); + if let Some(removed) = removed { + output.found_node.protect(removed); + } + output.old_root.protect(old_root); + output.new_root.protect(new_root); + }) + } + + if Node::is_retired(output.new_root.shared()) { + buf.abort(handle); + continue; + } + + if self + .root + .compare_exchange( + output.old_root.shared(), + output.new_root.shared(), + Ordering::AcqRel, + Ordering::Acquire, + handle, + ) + .is_ok() + { + buf.commit(handle); + return !output.found_node.is_null(); + } + buf.abort(handle); + } + } +} + +impl Drop for BonsaiTreeMap { + fn drop(&mut self) { + unsafe { + let guard = &Unprotected::new(); + let mut stack = vec![self.root.load(Ordering::Relaxed, guard)]; + + while let Some(mut node) = stack.pop() { + if node.is_null() { + continue; + } + + let node_ref = node.deref_mut(); + + stack.push(node_ref.left.load(Ordering::Relaxed, guard)); + stack.push(node_ref.right.load(Ordering::Relaxed, guard)); + drop(node.into_owned()); + } + } + } +} + +impl ConcurrentMap for BonsaiTreeMap +where + K: Ord + Clone, + V: Clone, +{ + type Output = Protectors; + + fn new() -> Self { + Self::new() + } + + fn get(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.get(key, output, thread) + } + + fn insert(&self, key: K, value: V, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.insert(key, value, output, thread) + } + + fn remove<'domain, 'hp>( + &self, + key: &K, + output: &mut Self::Output, + thread: &mut Thread, + ) -> bool { + self.remove(key, output, thread) + } +} + +#[cfg(test)] +mod tests { + use super::BonsaiTreeMap; + use crate::ds_impl::hp_brcu::concurrent_map; + + #[test] + fn smoke_bonsai_tree() { + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); + } +} diff --git a/src/ds_impl/hp_brcu/concurrent_map.rs b/src/ds_impl/hp_brcu/concurrent_map.rs index 2c5a4466..2c38024a 100644 --- a/src/ds_impl/hp_brcu/concurrent_map.rs +++ b/src/ds_impl/hp_brcu/concurrent_map.rs @@ -15,8 +15,7 @@ pub trait ConcurrentMap { fn new() -> Self; fn get(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool; fn insert(&self, key: K, value: V, output: &mut Self::Output, thread: &mut Thread) -> bool; - fn remove<'domain, 'hp>(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) - -> bool; + fn remove(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool; } #[cfg(test)] @@ -26,11 +25,17 @@ pub mod tests { use crossbeam_utils::thread; use hp_brcu::THREAD; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -44,7 +49,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(i, i.to_string(), output, thread)); + assert!(map.insert(i, to_value(&i), output, thread)); } }); }); @@ -83,7 +88,7 @@ pub mod tests { keys.shuffle(&mut rng); for i in keys { assert!(map.get(&i, output, thread)); - assert_eq!(i.to_string(), *output.output()); + assert_eq!(to_value(&i), *output.output()); } }); }); diff --git a/src/ds_impl/hp_brcu/elim_ab_tree.rs b/src/ds_impl/hp_brcu/elim_ab_tree.rs new file mode 100644 index 00000000..8d018e29 --- /dev/null +++ b/src/ds_impl/hp_brcu/elim_ab_tree.rs @@ -0,0 +1,1388 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use arrayvec::ArrayVec; +use hp_brcu::{Atomic, Handle, Owned, Pointer, RollbackProof, Shared, Shield, Thread, Unprotected}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next<'g, G: Handle>(&self, index: usize, guard: &'g G) -> Shared<'g, Self> { + self.next()[index].load(Ordering::Acquire, guard) + } + + fn load_next_locked<'g>(&'g self, index: usize, _: &MCSLockGuard<'g, K, V>) -> Shared { + // Safety: the node is locked by this thread. + self.next()[index].load(Ordering::Acquire, &unsafe { Unprotected::new() }) + } + + fn store_next<'g>(&'g self, index: usize, ptr: Shared, _: &MCSLockGuard<'g, K, V>) { + // Safety: the node is locked by this thread. + self.next()[index].store(ptr, Ordering::Release, &unsafe { Unprotected::new() }); + } + + fn init_next<'g>(&mut self, index: usize, ptr: Shared) { + self.next_mut()[index] = Atomic::from(ptr); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } + + fn p_s_idx(p_l_idx: usize) -> usize { + if p_l_idx > 0 { + p_l_idx - 1 + } else { + 1 + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &'g MCSLockGuard<'g, K, V>, + ) -> impl Iterator, Shared)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next_locked(i, lock))) + .chain(once((None, self.load_next_locked(self.key_count(), lock)))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +pub struct Cursor { + l: Shield>, + p: Shield>, + gp: Shield>, + // A pointer to a sibling node. + s: Shield>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + p_s_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +impl Cursor { + fn empty(thread: &mut Thread) -> Self { + Self { + l: Shield::null(thread), + p: Shield::null(thread), + gp: Shield::null(thread), + s: Shield::null(thread), + gp_p_idx: 0, + p_l_idx: 0, + p_s_idx: 0, + l_key_idx: 0, + val: None, + } + } +} + +impl OutputHolder for Cursor { + fn default(thread: &mut Thread) -> Self { + Self::empty(thread) + } + + fn output(&self) -> &V { + self.val.as_ref().unwrap() + } +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Owned::new(left).into_shared()); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, cursor: &mut Cursor, thread: &mut Thread) { + unsafe { + cursor.val = thread.critical_section(|guard| { + let mut node = self.entry.load_next(0, guard).deref(); + while let NodeKind::Internal { next } = &node.kind { + let next = next[node.child_index(key)].load(Ordering::Acquire, guard); + node = next.deref(); + } + node.read_consistent(key).1 + }); + } + } + + fn search( + &self, + key: &K, + target: Option>>, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> bool { + unsafe { + thread.critical_section(|guard| { + let mut l = self.entry.load_next(0, guard); + let mut s = self.entry.load_next(1, guard); + let mut p = Shared::from_usize(&self.entry as *const _ as usize); + let mut gp = Shared::null(); + let mut gp_p_idx = 0; + let mut p_l_idx = 0; + let mut p_s_idx = 1; + let mut l_key_idx = 0; + + while !l.deref().is_leaf() && target.map(|target| target != l).unwrap_or(true) { + let l_node = l.deref(); + gp = p; + p = l; + gp_p_idx = p_l_idx; + p_l_idx = l_node.child_index(key); + p_s_idx = Node::::p_s_idx(p_l_idx); + l = l_node.load_next(p_l_idx, guard); + s = l_node.load_next(p_s_idx, guard); + } + + let found = if let Some(target) = target { + l == target + } else { + let (index, value) = l.deref().read_consistent(key); + cursor.val = value; + l_key_idx = index; + value.is_some() + }; + cursor.l.protect(l); + cursor.s.protect(s); + cursor.p.protect(p); + cursor.gp.protect(gp); + cursor.gp_p_idx = gp_p_idx; + cursor.p_l_idx = p_l_idx; + cursor.p_s_idx = p_s_idx; + cursor.l_key_idx = l_key_idx; + found + }) + } + } + + /// TODO: start from here... + pub fn insert( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> bool { + loop { + self.search(key, None, cursor, thread); + if cursor.val.is_some() { + return false; + } + if let Ok(result) = self.insert_inner(key, value, cursor, thread) { + cursor.val = result; + return result.is_none(); + } + } + } + + fn insert_inner( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Owned::new(left).into_shared()); + internal.init_next(1, Owned::new(right).into_shared()); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Owned::new(internal).into_shared(); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { thread.retire(cursor.l.shared()) }; + self.fix_tag_violation(kv_pairs[left_size].0, new_internal.as_raw(), cursor, thread); + + Ok(None) + } + } + + fn fix_tag_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + thread: &mut Thread, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + let found = self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + thread, + ); + if !found || cursor.l.as_raw() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_tag_violation_inner(cursor, thread); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_tag_violation_inner<'c>( + &self, + cursor: &Cursor, + thread: &mut Thread, + ) -> (bool, Option<(K, usize)>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { cursor.l.deref() }; + if viol_node.weight { + return (true, None); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!( + !eq(viol_node, &self.entry) + && self.entry.load_next(0, unsafe { &Unprotected::new() }) != viol + ); + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + return (false, Some((parent.search_key, cursor.p.as_raw()))); + } + + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, None) + ); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next( + cursor.gp_p_idx, + Owned::new(absorber).into_shared(), + &gparent_lock, + ); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { thread.retire(cursor.l.shared()) }; + unsafe { thread.retire(cursor.p.shared()) }; + return (true, None); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Owned::new(left).into_shared()); + new_internal.init_next(1, Owned::new(right).into_shared()); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Owned::new(new_internal).into_shared(); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { thread.retire(cursor.l.shared()) }; + unsafe { thread.retire(cursor.p.shared()) }; + + drop((node_lock, parent_lock, gparent_lock)); + return ( + true, + Some((keys[left_size - 1].get().unwrap(), new_internal.as_raw())), + ); + } + } + + pub fn remove(&self, key: &K, cursor: &mut Cursor, thread: &mut Thread) -> bool { + loop { + self.search(key, None, cursor, thread); + if cursor.val.is_none() { + return false; + } + if let Ok(result) = self.remove_inner(key, cursor, thread) { + cursor.val = result; + return result.is_some(); + } + } + } + + fn remove_inner( + &self, + key: &K, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = cursor.gp.as_ref(); + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation( + node.search_key, + cursor.l.as_raw(), + cursor, + thread, + ); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + thread: &mut Thread, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + thread, + ); + if cursor.l.as_raw() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_underfull_violation_inner(cursor, thread); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_underfull_violation_inner( + &self, + cursor: &mut Cursor, + thread: &mut Thread, + ) -> (bool, ArrayVec<(K, usize), 2>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { viol.deref() }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0, unsafe { &Unprotected::new() }) + { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p.shared() != self.entry.load_next(0, unsafe { &Unprotected::new() }) + { + return ( + false, + ArrayVec::from_iter(once((parent.search_key, cursor.p.as_raw()))), + ); + } + + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling = unsafe { cursor.s.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if cursor.p_s_idx < cursor.p_l_idx { + ((sibling, cursor.p_s_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, cursor.p_s_idx)) + }; + + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(parent.search_key, cursor.p.as_raw(), cursor, thread); + return (false, ArrayVec::new()); + } + if !node.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(node.search_key, cursor.l.as_raw(), cursor, thread); + return (false, ArrayVec::new()); + } + if !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(sibling.search_key, cursor.s.as_raw(), cursor, thread); + return (false, ArrayVec::new()); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Node::leaf(true, size, node.search_key); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Node::internal(true, size, node.search_key); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + }; + let new_node = Owned::new(new_node).into_shared(); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + thread.retire(cursor.l.shared()); + thread.retire(cursor.p.shared()); + thread.retire(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter(once((node.search_key, new_node.as_raw()))), + ); + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..cursor.p_s_idx { + new_parent.init_next(i, parent.load_next(i, unsafe { &Unprotected::new() })); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent + .init_next(i - 1, parent.load_next(i, unsafe { &Unprotected::new() })); + } + + new_parent.init_next( + cursor.p_l_idx + - (if cursor.p_l_idx > cursor.p_s_idx { + 1 + } else { + 0 + }), + new_node, + ); + let new_parent = Owned::new(new_parent).into_shared(); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + thread.retire(cursor.l.shared()); + thread.retire(cursor.p.shared()); + thread.retire(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter( + [ + (node.search_key, new_node.as_raw()), + (parent.search_key, new_parent.as_raw()), + ] + .into_iter(), + ), + ); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Node::leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Node::leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let (new_left, pivot) = { + let mut new_internal = Node::internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = Node::internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = Node::internal(parent.weight, psize, parent.search_key); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, Owned::new(new_left).into_shared()); + new_parent.init_next(right_idx, Owned::new(new_right).into_shared()); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next( + cursor.gp_p_idx, + Owned::new(new_parent).into_shared(), + &gparent_lock, + ); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + thread.retire(cursor.l.shared()); + thread.retire(cursor.p.shared()); + thread.retire(cursor.s.shared()); + } + + return (true, ArrayVec::new()); + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let guard = unsafe { &Unprotected::new() }; + let mut stack = vec![]; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + type Output = Cursor; + + fn new() -> Self { + Self::new() + } + + fn get(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.search_basic(key, output, thread); + output.val.is_some() + } + + fn insert(&self, key: K, value: V, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.insert(&key, &value, output, thread) + } + + fn remove(&self, key: &K, output: &mut Self::Output, thread: &mut Thread) -> bool { + self.remove(key, output, thread) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::hp_brcu::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/hp_brcu/list.rs b/src/ds_impl/hp_brcu/list.rs index c222e4e6..da9065cc 100644 --- a/src/ds_impl/hp_brcu/list.rs +++ b/src/ds_impl/hp_brcu/list.rs @@ -525,15 +525,15 @@ where #[test] fn smoke_h_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } diff --git a/src/ds_impl/hp_brcu/list_alter.rs b/src/ds_impl/hp_brcu/list_alter.rs index 9524b5f9..57321cd8 100644 --- a/src/ds_impl/hp_brcu/list_alter.rs +++ b/src/ds_impl/hp_brcu/list_alter.rs @@ -646,15 +646,15 @@ where #[test] fn smoke_h_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - super::concurrent_map::tests::smoke::>(); + super::concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } diff --git a/src/ds_impl/hp_brcu/michael_hash_map.rs b/src/ds_impl/hp_brcu/michael_hash_map.rs index 788f8422..97f5ec55 100644 --- a/src/ds_impl/hp_brcu/michael_hash_map.rs +++ b/src/ds_impl/hp_brcu/michael_hash_map.rs @@ -101,6 +101,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_brcu/mod.rs b/src/ds_impl/hp_brcu/mod.rs index a3668daa..72f79678 100644 --- a/src/ds_impl/hp_brcu/mod.rs +++ b/src/ds_impl/hp_brcu/mod.rs @@ -1,5 +1,7 @@ pub mod concurrent_map; +mod bonsai_tree; +mod elim_ab_tree; mod list; pub mod list_alter; mod michael_hash_map; @@ -7,6 +9,8 @@ mod natarajan_mittal_tree; mod skip_list; pub use self::concurrent_map::ConcurrentMap; +pub use bonsai_tree::BonsaiTreeMap; +pub use elim_ab_tree::ElimABTree; pub use list::{HHSList, HList, HMList}; pub use michael_hash_map::HashMap; pub use natarajan_mittal_tree::NMTreeMap; diff --git a/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs b/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs index f9179665..5e036700 100644 --- a/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp_brcu/natarajan_mittal_tree.rs @@ -562,6 +562,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_brcu/skip_list.rs b/src/ds_impl/hp_brcu/skip_list.rs index e1ec2887..e16a3b02 100644 --- a/src/ds_impl/hp_brcu/skip_list.rs +++ b/src/ds_impl/hp_brcu/skip_list.rs @@ -454,6 +454,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/bonsai_tree.rs b/src/ds_impl/hp_pp/bonsai_tree.rs index 252a1784..dabf0de0 100644 --- a/src/ds_impl/hp_pp/bonsai_tree.rs +++ b/src/ds_impl/hp_pp/bonsai_tree.rs @@ -1,7 +1,7 @@ use hp_pp::{light_membarrier, Invalidate, Thread, Unlink}; use hp_pp::{tag, tagged, untagged, HazardPointer, ProtectError, DEFAULT_DOMAIN}; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::ptr; @@ -119,20 +119,21 @@ pub struct State<'domain, K, V> { retired_nodes: Vec<*mut Node>, /// Nodes newly constructed by the op. Should be destroyed if CAS fails. (`destroy`) new_nodes: Vec<*mut Node>, - thread: Thread<'domain>, + thread: Box>, } impl Default for State<'static, K, V> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { root_link: ptr::null(), curr_root: ptr::null_mut(), - root_h: Default::default(), - succ_h: Default::default(), - removed_h: Default::default(), + root_h: HazardPointer::new(&mut thread), + succ_h: HazardPointer::new(&mut thread), + removed_h: HazardPointer::new(&mut thread), retired_nodes: vec![], new_nodes: vec![], - thread: Thread::new(&DEFAULT_DOMAIN), + thread, } } } @@ -853,26 +854,25 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -884,6 +884,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/ellen_tree.rs b/src/ds_impl/hp_pp/ellen_tree.rs index cdfd5306..5f569622 100644 --- a/src/ds_impl/hp_pp/ellen_tree.rs +++ b/src/ds_impl/hp_pp/ellen_tree.rs @@ -28,7 +28,7 @@ use hp_pp::{ DEFAULT_DOMAIN, }; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -240,22 +240,23 @@ pub struct Handle<'domain> { new_internal_h: HazardPointer<'domain>, // Protect an owner of update which is currently being helped. help_src_h: HazardPointer<'domain>, - thread: Thread<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - gp_h: HazardPointer::default(), - p_h: HazardPointer::default(), - l_h: HazardPointer::default(), - l_other_h: HazardPointer::default(), - pupdate_h: HazardPointer::default(), - gpupdate_h: HazardPointer::default(), - aux_update_h: HazardPointer::default(), - new_internal_h: HazardPointer::default(), - help_src_h: HazardPointer::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + gp_h: HazardPointer::new(&mut thread), + p_h: HazardPointer::new(&mut thread), + l_h: HazardPointer::new(&mut thread), + l_other_h: HazardPointer::new(&mut thread), + pupdate_h: HazardPointer::new(&mut thread), + gpupdate_h: HazardPointer::new(&mut thread), + aux_update_h: HazardPointer::new(&mut thread), + new_internal_h: HazardPointer::new(&mut thread), + help_src_h: HazardPointer::new(&mut thread), + thread, } } } @@ -949,29 +950,25 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { - match self.find(key, handle) { - Some(value) => Some(value), - None => None, - } + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + self.find(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(&key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.delete(key, handle) } } @@ -983,6 +980,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/list.rs b/src/ds_impl/hp_pp/list.rs index 8c8c21c5..a0e568ce 100644 --- a/src/ds_impl/hp_pp/list.rs +++ b/src/ds_impl/hp_pp/list.rs @@ -1,10 +1,12 @@ -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp::Ordering::{Equal, Greater, Less}; use std::sync::atomic::{AtomicPtr, Ordering}; use std::{mem, ptr, slice}; -use hp_pp::{decompose_ptr, light_membarrier, tag, tagged, try_unlink, untagged, HazardPointer}; +use hp_pp::{ + decompose_ptr, light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, +}; // `#[repr(C)]` is used to ensure the first field // is also the first data in the memory alignment. @@ -46,15 +48,18 @@ pub struct Handle<'domain> { // `anchor_h` and `anchor_next_h` are used for `find_harris` anchor_h: HazardPointer<'domain>, anchor_next_h: HazardPointer<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - prev_h: HazardPointer::default(), - curr_h: HazardPointer::default(), - anchor_h: HazardPointer::default(), - anchor_next_h: HazardPointer::default(), + prev_h: HazardPointer::new(&mut thread), + curr_h: HazardPointer::new(&mut thread), + anchor_h: HazardPointer::new(&mut thread), + anchor_next_h: HazardPointer::new(&mut thread), + thread, } } } @@ -96,26 +101,28 @@ impl hp_pp::Invalidate for Node { } } -struct HarrisUnlink<'c, 'domain, 'hp, K, V> { - cursor: &'c Cursor<'domain, 'hp, K, V>, +struct HarrisUnlink { + anchor: *mut Node, + anchor_next: *mut Node, + curr: *mut Node, } -impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for HarrisUnlink<'r, 'domain, 'hp, K, V> { +impl hp_pp::Unlink> for HarrisUnlink { fn do_unlink(&self) -> Result>, ()> { - if unsafe { &*self.cursor.anchor } + if unsafe { &*self.anchor } .next .compare_exchange( - self.cursor.anchor_next, - self.cursor.curr, + self.anchor_next, + self.curr, Ordering::AcqRel, Ordering::Relaxed, ) .is_ok() { let mut collected = Vec::with_capacity(16); - let mut node = self.cursor.anchor_next; + let mut node = self.anchor_next; loop { - if untagged(node) == self.cursor.curr { + if untagged(node) == self.curr { break; } let node_ref = unsafe { node.as_ref() }.unwrap(); @@ -130,24 +137,25 @@ impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for HarrisUnlink<'r, 'dom } } -struct Unlink<'c, 'domain, 'hp, K, V> { - cursor: &'c Cursor<'domain, 'hp, K, V>, +struct Unlink { + prev: *mut Node, + curr: *mut Node, next_base: *mut Node, } -impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for Unlink<'r, 'domain, 'hp, K, V> { +impl hp_pp::Unlink> for Unlink { fn do_unlink(&self) -> Result>, ()> { - let prev = unsafe { &(*self.cursor.prev).next }; + let prev = unsafe { &(*self.prev).next }; if prev .compare_exchange( - self.cursor.curr, + self.curr, self.next_base, Ordering::Release, Ordering::Relaxed, ) .is_ok() { - Ok(vec![self.cursor.curr]) + Ok(vec![self.curr]) } else { Err(()) } @@ -212,8 +220,16 @@ where if self.anchor.is_null() { Ok(found) } else { - let unlink = HarrisUnlink { cursor: &self }; - if unsafe { try_unlink(unlink, slice::from_ref(&self.curr)) } { + let unlink = HarrisUnlink { + anchor: self.anchor, + anchor_next: self.anchor_next, + curr: self.curr, + }; + if unsafe { + self.handle + .thread + .try_unlink(unlink, slice::from_ref(&self.curr)) + } { self.prev = self.anchor; Ok(found) } else { @@ -262,10 +278,11 @@ where } else { let links = slice::from_ref(&next_base); let unlink = Unlink { - cursor: self, + prev: self.prev, + curr: self.curr, next_base, }; - if unsafe { !try_unlink(unlink, links) } { + if unsafe { !self.handle.thread.try_unlink(unlink, links) } { return Err(()); } } @@ -421,10 +438,11 @@ where let links = slice::from_ref(&next); let unlink = Unlink { - cursor: &cursor, + prev: cursor.prev, + curr: cursor.curr, next_base: next, }; - unsafe { try_unlink(unlink, links) }; + unsafe { handle.thread.try_unlink(unlink, links) }; return Ok(Some(&curr_node.value)); } @@ -476,10 +494,11 @@ where let links = slice::from_ref(&next); let unlink = Unlink { - cursor: &cursor, + prev: cursor.prev, + curr: cursor.curr, next_base: next, }; - unsafe { try_unlink(unlink, links) }; + unsafe { handle.thread.try_unlink(unlink, links) }; return Some((&curr_node.key, &curr_node.value)); } @@ -563,24 +582,23 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.inner.harris_insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_remove(key, handle) } } @@ -619,24 +637,23 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.inner.harris_michael_insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_michael_remove(key, handle) } } @@ -671,24 +688,23 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_herlihy_shavit_get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.inner.harris_insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.inner.harris_remove(key, handle) } } @@ -700,17 +716,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/hp_pp/michael_hash_map.rs b/src/ds_impl/hp_pp/michael_hash_map.rs index e964e2a7..2d6f70b5 100644 --- a/src/ds_impl/hp_pp/michael_hash_map.rs +++ b/src/ds_impl/hp_pp/michael_hash_map.rs @@ -1,4 +1,4 @@ -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -34,21 +34,6 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get<'domain, 'hp>(&self, handle: &'hp mut Handle<'domain>, k: &K) -> Option<&'hp V> { - let i = Self::hash(k); - self.get_bucket(i).get(handle, k) - } - - pub fn insert<'domain, 'hp>(&self, handle: &'hp mut Handle<'domain>, k: K, v: V) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(handle, k, v) - } - - pub fn remove<'domain, 'hp>(&self, handle: &'hp mut Handle<'domain>, k: &K) -> Option<&'hp V> { - let i = Self::hash(&k); - self.get_bucket(i).remove(handle, k) - } } impl ConcurrentMap for HashMap @@ -67,25 +52,27 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { - self.get(handle, key) + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(handle, key) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { - self.insert(handle, key, value) + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { + let i = Self::hash(&key); + self.get_bucket(i).insert(handle, key, value) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { - self.remove(handle, key) + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).remove(handle, key) } } @@ -96,6 +83,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/natarajan_mittal_tree.rs b/src/ds_impl/hp_pp/natarajan_mittal_tree.rs index 8ed1faa5..6cd1f008 100644 --- a/src/ds_impl/hp_pp/natarajan_mittal_tree.rs +++ b/src/ds_impl/hp_pp/natarajan_mittal_tree.rs @@ -1,7 +1,8 @@ -use hp_pp::{light_membarrier, tag, tagged, untagged, HazardPointer}; -use hp_pp::{try_unlink, ProtectError}; +use hp_pp::{ + light_membarrier, tag, tagged, untagged, HazardPointer, ProtectError, Thread, DEFAULT_DOMAIN, +}; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; use std::mem; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; @@ -144,15 +145,18 @@ pub struct Handle<'domain> { successor_h: HazardPointer<'domain>, parent_h: HazardPointer<'domain>, leaf_h: HazardPointer<'domain>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - ancestor_h: HazardPointer::default(), - successor_h: HazardPointer::default(), - parent_h: HazardPointer::default(), - leaf_h: HazardPointer::default(), + ancestor_h: HazardPointer::new(&mut thread), + successor_h: HazardPointer::new(&mut thread), + parent_h: HazardPointer::new(&mut thread), + leaf_h: HazardPointer::new(&mut thread), + thread, } } } @@ -276,31 +280,25 @@ impl hp_pp::Invalidate for Node { } } -struct Unlink<'r, 'domain, 'hp, K, V> { - record: &'r SeekRecord<'domain, 'hp, K, V>, +struct Unlink { + successor_addr: *mut AtomicPtr>, + successor: *mut Node, target_sibling: *mut Node, flag: bool, } -impl<'r, 'domain, 'hp, K, V> hp_pp::Unlink> for Unlink<'r, 'domain, 'hp, K, V> { +impl hp_pp::Unlink> for Unlink { fn do_unlink(&self) -> Result>, ()> { let link = tagged( self.target_sibling, Marks::new(false, self.flag, false).bits(), ); - if self - .record - .successor_addr() - .compare_exchange( - self.record.successor, - link, - Ordering::AcqRel, - Ordering::Acquire, - ) + if unsafe { &*self.successor_addr } + .compare_exchange(self.successor, link, Ordering::AcqRel, Ordering::Acquire) .is_ok() { // destroy the subtree of successor except target_sibling - let mut stack = vec![self.record.successor]; + let mut stack = vec![self.successor]; let mut collected = Vec::with_capacity(32); while let Some(node) = stack.pop() { @@ -518,7 +516,7 @@ where /// Physically removes node. /// /// Returns true if it successfully unlinks the flagged node in `record`. - fn cleanup(&self, record: &SeekRecord) -> bool { + fn cleanup(&self, record: &mut SeekRecord) -> bool { // Identify the node(subtree) that will replace `successor`. let leaf_marked = record.leaf_addr().load(Ordering::Acquire); let leaf_flag = Marks::from_bits_truncate(tag(leaf_marked)).flag(); @@ -540,13 +538,19 @@ where let flag = Marks::from_bits_truncate(tag(target_sibling)).flag(); let link = tagged(target_sibling, Marks::new(false, flag, false).bits()); let unlink = Unlink { - record, + successor_addr: record.successor_addr() as *const _ as *mut _, + successor: record.successor, target_sibling, flag, }; let link = untagged(link); - unsafe { try_unlink(unlink, slice::from_ref(&link)) } + unsafe { + record + .handle + .thread + .try_unlink(unlink, slice::from_ref(&link)) + } } fn get_inner<'domain, 'hp>( @@ -629,7 +633,7 @@ where // Insertion failed. Help the conflicting remove operation if needed. // NOTE: The paper version checks if any of the mark is set, which is redundant. if untagged(current) == leaf { - self.cleanup(&record); + self.cleanup(record); } } } @@ -683,7 +687,7 @@ where ) { Ok(_) => { // Finalize the node to be removed - if self.cleanup(&record) { + if self.cleanup(&mut record) { return Ok(Some(value)); } // In-place cleanup failed. Enter the cleanup phase. @@ -695,7 +699,7 @@ where // case 2. Another thread flagged/tagged the edge to leaf: help and restart // NOTE: The paper version checks if any of the mark is set, which is redundant. if leaf == tagged(current, Marks::empty().bits()) { - self.cleanup(&record); + self.cleanup(&mut record); } } } @@ -712,7 +716,7 @@ where } // leaf is still present in the tree. - if self.cleanup(&record) { + if self.cleanup(&mut record) { return Ok(Some(value)); } } @@ -747,26 +751,25 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.get(key, handle) } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(key, value, handle).is_ok() } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -778,6 +781,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/hp_pp/skip_list.rs b/src/ds_impl/hp_pp/skip_list.rs index cc71abe3..7ae00040 100644 --- a/src/ds_impl/hp_pp/skip_list.rs +++ b/src/ds_impl/hp_pp/skip_list.rs @@ -6,7 +6,7 @@ use hp_pp::{ decompose_ptr, light_membarrier, tag, tagged, untagged, HazardPointer, Thread, DEFAULT_DOMAIN, }; -use crate::ds_impl::hp::concurrent_map::ConcurrentMap; +use crate::ds_impl::hp::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -100,16 +100,17 @@ pub struct Handle<'g> { preds_h: [HazardPointer<'g>; MAX_HEIGHT], succs_h: [HazardPointer<'g>; MAX_HEIGHT], removed_h: HazardPointer<'g>, - thread: Thread<'g>, + thread: Box>, } impl Default for Handle<'static> { fn default() -> Self { + let mut thread = Box::new(Thread::new(&DEFAULT_DOMAIN)); Self { - preds_h: Default::default(), - succs_h: Default::default(), - removed_h: Default::default(), - thread: Thread::new(&DEFAULT_DOMAIN), + preds_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + succs_h: [(); MAX_HEIGHT].map(|_| HazardPointer::new(&mut thread)), + removed_h: HazardPointer::new(&mut thread), + thread, } } } @@ -164,9 +165,23 @@ where &self, key: &K, handle: &'hp mut Handle<'domain>, - ) -> Option> { + ) -> Option<*mut Node> { 'search: loop { - let mut cursor = Cursor::new(&self.head); + // This optimistic traversal doesn't have to use all shields in the `handle`. + let (anchor_h, anchor_next_h) = unsafe { + let splited = handle.preds_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; + let (pred_h, curr_h) = unsafe { + let splited = handle.succs_h.split_at_mut_unchecked(1); + ( + splited.0.get_unchecked_mut(0), + splited.1.get_unchecked_mut(0), + ) + }; let mut level = MAX_HEIGHT; while level >= 1 && self.head[level - 1].load(Ordering::Relaxed).is_null() { @@ -174,55 +189,64 @@ where } let mut pred = &self.head as *const _ as *mut Node; + let mut curr = ptr::null_mut(); + let mut anchor = pred; + let mut anchor_next = ptr::null_mut(); + while level >= 1 { level -= 1; // untagged - let mut curr = untagged(unsafe { &*pred }.next[level].load(Ordering::Acquire)); + curr = untagged(unsafe { &*anchor }.next[level].load(Ordering::Acquire)); loop { if curr.is_null() { break; } - - let pred_ref = unsafe { &*pred }; - - // Inlined version of hp++ protection, without duplicate load - handle.succs_h[level].protect_raw(curr); - light_membarrier(); - let (curr_new_base, curr_new_tag) = - decompose_ptr(pred_ref.next[level].load(Ordering::Acquire)); - if curr_new_tag == 3 { - // Invalidated. Restart from head. + if curr_h + .try_protect_pp( + curr, + unsafe { &*pred }, + unsafe { &(*pred).next[level] }, + &|node| node.next[level].load(Ordering::Acquire) as usize & 3 == 3, + ) + .is_err() + { continue 'search; - } else if curr_new_base != curr { - // If link changed but not invalidated, retry protecting the new node. - curr = curr_new_base; - continue; } let curr_node = unsafe { &*curr }; - - match curr_node.key.cmp(key) { - std::cmp::Ordering::Less => { + let (next_base, next_tag) = + decompose_ptr(curr_node.next[level].load(Ordering::Acquire)); + if next_tag == 0 { + if curr_node.key < *key { pred = curr; - curr = untagged(curr_node.next[level].load(Ordering::Acquire)); - HazardPointer::swap( - &mut handle.preds_h[level], - &mut handle.succs_h[level], - ); + curr = next_base; + anchor = pred; + HazardPointer::swap(curr_h, pred_h); + } else { + break; } - std::cmp::Ordering::Equal => { - if curr_new_tag == 0 { - cursor.found = Some(curr); - return Some(cursor); - } else { - return None; - } + } else { + if anchor == pred { + anchor_next = curr; + HazardPointer::swap(anchor_h, pred_h); + } else if anchor_next == pred { + HazardPointer::swap(anchor_next_h, pred_h); } - std::cmp::Ordering::Greater => break, + pred = curr; + curr = next_base; + HazardPointer::swap(pred_h, curr_h); } } } + + if let Some(curr_node) = unsafe { curr.as_ref() } { + if curr_node.key == *key + && (curr_node.next[0].load(Ordering::Acquire) as usize & 1) == 0 + { + return Some(curr); + } + } return None; } } @@ -502,9 +526,12 @@ where } #[inline(always)] - fn get<'domain, 'hp>(&self, handle: &'hp mut Self::Handle<'domain>, key: &K) -> Option<&'hp V> { - let cursor = self.find_optimistic(key, handle)?; - let node = unsafe { &*cursor.found? }; + fn get<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { + let node = unsafe { &*self.find_optimistic(key, handle)? }; if node.key.eq(&key) { Some(&node.value) } else { @@ -513,21 +540,16 @@ where } #[inline(always)] - fn insert<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: K, - value: V, - ) -> bool { + fn insert(&self, handle: &mut Self::Handle<'_>, key: K, value: V) -> bool { self.insert(key, value, handle) } #[inline(always)] - fn remove<'domain, 'hp>( - &self, - handle: &'hp mut Self::Handle<'domain>, - key: &K, - ) -> Option<&'hp V> { + fn remove<'hp>( + &'hp self, + handle: &'hp mut Self::Handle<'_>, + key: &'hp K, + ) -> Option> { self.remove(key, handle) } } @@ -539,6 +561,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/bonsai_tree.rs b/src/ds_impl/nr/bonsai_tree.rs index 8e88e243..b06a5985 100644 --- a/src/ds_impl/nr/bonsai_tree.rs +++ b/src/ds_impl/nr/bonsai_tree.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; use std::cmp; @@ -610,7 +610,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.get(key) } #[inline(always)] @@ -618,7 +618,7 @@ where self.insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.remove(key) } } @@ -630,6 +630,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/concurrent_map.rs b/src/ds_impl/nr/concurrent_map.rs index 277e927c..3457618a 100644 --- a/src/ds_impl/nr/concurrent_map.rs +++ b/src/ds_impl/nr/concurrent_map.rs @@ -1,21 +1,43 @@ +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { fn new() -> Self; - fn get(&self, key: &K) -> Option<&'static V>; + fn get(&self, key: &K) -> Option>; fn insert(&self, key: K, value: V) -> bool; - fn remove(&self, key: &K) -> Option<&'static V>; + fn remove(&self, key: &K) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -26,7 +48,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(i, i.to_string())); + assert!(map.insert(i, to_value(&i))); } }); } @@ -41,7 +63,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.remove(&i).unwrap()); + assert_eq!(to_value(&i), *map.remove(&i).unwrap().output()); } }); } @@ -56,7 +78,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert_eq!(i.to_string(), *map.get(&i).unwrap()); + assert_eq!(to_value(&i), *map.get(&i).unwrap().output()); } }); } diff --git a/src/ds_impl/nr/elim_ab_tree.rs b/src/ds_impl/nr/elim_ab_tree.rs new file mode 100644 index 00000000..8404d303 --- /dev/null +++ b/src/ds_impl/nr/elim_ab_tree.rs @@ -0,0 +1,1159 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use super::pointers::{Atomic, Shared}; +use arrayvec::ArrayVec; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next<'g>(&self, index: usize) -> Shared { + self.next()[index].load(Ordering::Acquire) + } + + fn store_next<'g>(&'g self, index: usize, ptr: Shared, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: Shared) { + self.next_mut()[index] = Atomic::from(ptr); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator, Shared)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i))) + .chain(once((None, self.load_next(self.key_count())))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +struct Cursor { + l: Shared>, + p: Shared>, + gp: Shared>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Shared::from_owned(left)); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K) -> Option { + let mut node = unsafe { self.entry.load_next(0).deref() }; + while let NodeKind::Internal { next } = &node.kind { + let next = next[node.child_index(key)].load(Ordering::Acquire); + node = unsafe { next.deref() }; + } + node.read_consistent(key).1 + } + + fn search(&self, key: &K, target: Option>>) -> (bool, Cursor) { + let mut cursor = Cursor { + l: self.entry.load_next(0), + p: Shared::from(&self.entry as *const _ as usize), + gp: Shared::null(), + gp_p_idx: 0, + p_l_idx: 0, + l_key_idx: 0, + val: None, + }; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { cursor.l.deref() }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.l = l_node.load_next(cursor.p_l_idx); + } + + if let Some(target) = target { + (cursor.l == target, cursor) + } else { + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); + cursor.val = value; + cursor.l_key_idx = index; + (value.is_some(), cursor) + } + } + + pub fn insert(&self, key: &K, value: &V) -> Option { + loop { + let (_, cursor) = self.search(key, None); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, &cursor) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner(&self, key: &K, value: &V, cursor: &Cursor) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Shared::from_owned(left)); + internal.init_next(1, Shared::from_owned(right)); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Shared::from_owned(internal); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + self.fix_tag_violation(new_internal); + + Ok(None) + } + } + + fn fix_tag_violation(&self, viol: Shared>) { + loop { + let viol_node = unsafe { viol.deref() }; + if viol_node.weight { + return; + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0) != viol); + + let (found, cursor) = self.search(&viol_node.search_key, Some(viol)); + if !found { + return; + } + + debug_assert!(!cursor.gp.is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + if !eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + self.fix_tag_violation(cursor.p); + continue; + } + + try_acq_val_or!(node, node_lock, Operation::Balance, None, continue); + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next(cursor.gp_p_idx, Shared::from_owned(absorber), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + return; + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Shared::from_owned(left)); + new_internal.init_next(1, Shared::from_owned(right)); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Shared::from_owned(new_internal); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + drop((node_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(new_internal); + return; + } + } + } + + pub fn remove(&self, key: &K) -> Option { + loop { + let (_, cursor) = self.search(key, None); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner(&self, key: &K, cursor: &Cursor) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation(cursor.l); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation(&self, viol: Shared>) { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + let viol_node = unsafe { viol.deref() }; + loop { + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0) + { + // No degree violation at `viol`. + return; + } + + // Search for `viol`. + let (_, cursor) = self.search(&viol_node.search_key, Some(viol)); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p != self.entry.load_next(0) + { + self.fix_underfull_violation(cursor.p); + continue; + } + + if !eq(node, viol_node) { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + return; + } + + let sibling_idx = if cursor.p_l_idx > 0 { + cursor.p_l_idx - 1 + } else { + 1 + }; + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling_sh = parent.load_next(sibling_idx); + let sibling = unsafe { sibling_sh.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if sibling_idx < cursor.p_l_idx { + ((sibling, sibling_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, sibling_idx)) + }; + + try_acq_val_or!(left, left_lock, Operation::Balance, None, continue); + try_acq_val_or!(right, right_lock, Operation::Balance, None, continue); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return; + } + + try_acq_val_or!(parent, parent_lock, Operation::Balance, None, continue); + try_acq_val_or!(gparent, gparent_lock, Operation::Balance, None, continue); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight || !node.weight || !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(cursor.p); + self.fix_tag_violation(cursor.l); + self.fix_tag_violation(sibling_sh); + continue; + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Node::leaf(true, size, node.search_key); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Node::internal(true, size, node.search_key); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + }; + let new_node = Shared::from_owned(new_node); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node); + return; + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..sibling_idx { + new_parent.init_next(i, parent.load_next(i)); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next(i)); + } + + new_parent.init_next( + cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 }), + new_node, + ); + let new_parent = Shared::from_owned(new_parent); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_underfull_violation(new_node); + self.fix_underfull_violation(new_parent); + return; + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Node::leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Node::leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let (new_left, pivot) = { + let mut new_internal = Node::internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = Node::internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = Node::internal(parent.weight, psize, parent.search_key); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, Shared::from_owned(new_left)); + new_parent.init_next(right_idx, Shared::from_owned(new_right)); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next( + cursor.gp_p_idx, + Shared::from_owned(new_parent), + &gparent_lock, + ); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + return; + } + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let mut stack = vec![]; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self::new() + } + + fn get(&self, key: &K) -> Option> { + self.search_basic(key) + } + + fn insert(&self, key: K, value: V) -> bool { + self.insert(&key, &value).is_none() + } + + fn remove(&self, key: &K) -> Option> { + self.remove(key) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::nr::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/nr/ellen_tree.rs b/src/ds_impl/nr/ellen_tree.rs index 05755c4a..6f86733d 100644 --- a/src/ds_impl/nr/ellen_tree.rs +++ b/src/ds_impl/nr/ellen_tree.rs @@ -1,6 +1,6 @@ use std::sync::atomic::Ordering; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; bitflags! { @@ -468,7 +468,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { match self.find(key) { Some(node) => Some(node.value.as_ref().unwrap()), None => None, @@ -481,7 +481,7 @@ where } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.delete(key) } } @@ -493,6 +493,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/list.rs b/src/ds_impl/nr/list.rs index 3c06bd94..41abfbaa 100644 --- a/src/ds_impl/nr/list.rs +++ b/src/ds_impl/nr/list.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; use std::cmp::Ordering::{Equal, Greater, Less}; @@ -316,7 +316,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.inner.harris_get(key) } #[inline(always)] @@ -324,7 +324,7 @@ where self.inner.harris_insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.inner.harris_remove(key) } } @@ -343,7 +343,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.inner.harris_michael_get(key) } #[inline(always)] @@ -351,7 +351,7 @@ where self.inner.harris_michael_insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.inner.harris_michael_remove(key) } } @@ -382,7 +382,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.inner.harris_herlihy_shavit_get(key) } #[inline(always)] @@ -390,7 +390,7 @@ where self.inner.harris_insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.inner.harris_remove(key) } } @@ -402,17 +402,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/nr/michael_hash_map.rs b/src/ds_impl/nr/michael_hash_map.rs index 602565f9..63009c2e 100644 --- a/src/ds_impl/nr/michael_hash_map.rs +++ b/src/ds_impl/nr/michael_hash_map.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -33,21 +33,6 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get(&self, k: &K) -> Option<&'static V> { - let i = Self::hash(k); - self.get_bucket(i).get(k) - } - - pub fn insert(&self, k: K, v: V) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(k, v) - } - - pub fn remove(&self, k: &K) -> Option<&'static V> { - let i = Self::hash(k); - self.get_bucket(i).remove(k) - } } impl ConcurrentMap for HashMap @@ -60,16 +45,19 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { - self.get(key) + fn get(&self, key: &K) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(key) } #[inline(always)] fn insert(&self, key: K, value: V) -> bool { - self.insert(key, value) + let i = Self::hash(&key); + self.get_bucket(i).insert(key, value) } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { - self.remove(key) + fn remove(&self, key: &K) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).remove(key) } } @@ -80,6 +68,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/mod.rs b/src/ds_impl/nr/mod.rs index bd66b816..2d5551d7 100644 --- a/src/ds_impl/nr/mod.rs +++ b/src/ds_impl/nr/mod.rs @@ -3,6 +3,7 @@ pub mod pointers; pub mod bonsai_tree; pub mod double_link; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; @@ -13,6 +14,7 @@ pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; pub use self::double_link::DoubleLink; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; diff --git a/src/ds_impl/nr/natarajan_mittal_tree.rs b/src/ds_impl/nr/natarajan_mittal_tree.rs index bf63c898..c4759ed9 100644 --- a/src/ds_impl/nr/natarajan_mittal_tree.rs +++ b/src/ds_impl/nr/natarajan_mittal_tree.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; use std::cmp; use std::sync::atomic::Ordering; @@ -445,7 +445,7 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { self.get(key) } #[inline(always)] @@ -453,7 +453,7 @@ where self.insert(key, value).is_ok() } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { + fn remove(&self, key: &K) -> Option> { self.remove(key) } } @@ -465,6 +465,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/nr/pointers.rs b/src/ds_impl/nr/pointers.rs index 36055b6b..5abbee52 100644 --- a/src/ds_impl/nr/pointers.rs +++ b/src/ds_impl/nr/pointers.rs @@ -83,6 +83,14 @@ impl From> for Atomic { } } +impl Clone for Atomic { + fn clone(&self) -> Self { + Self { + link: AtomicPtr::new(self.link.load(Ordering::Relaxed)), + } + } +} + pub struct Shared { ptr: *mut T, } diff --git a/src/ds_impl/nr/skip_list.rs b/src/ds_impl/nr/skip_list.rs index 082507eb..8df399ff 100644 --- a/src/ds_impl/nr/skip_list.rs +++ b/src/ds_impl/nr/skip_list.rs @@ -1,7 +1,7 @@ use std::mem::transmute; use std::sync::atomic::Ordering; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::pointers::{Atomic, Shared}; const MAX_HEIGHT: usize = 32; @@ -331,9 +331,9 @@ where } #[inline(always)] - fn get(&self, key: &K) -> Option<&'static V> { + fn get(&self, key: &K) -> Option> { let cursor = self.find_optimistic(key); - unsafe { transmute(cursor.found.map(|node| &node.value)) } + unsafe { transmute::<_, Option<&'static V>>(cursor.found.map(|node| &node.value)) } } #[inline(always)] @@ -342,8 +342,8 @@ where } #[inline(always)] - fn remove(&self, key: &K) -> Option<&'static V> { - unsafe { transmute(self.remove(key)) } + fn remove(&self, key: &K) -> Option> { + unsafe { transmute::<_, Option<&'static V>>(self.remove(key)) } } } @@ -354,6 +354,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/bonsai_tree.rs b/src/ds_impl/pebr/bonsai_tree.rs index 4353810f..c45148ff 100644 --- a/src/ds_impl/pebr/bonsai_tree.rs +++ b/src/ds_impl/pebr/bonsai_tree.rs @@ -1,6 +1,6 @@ use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use super::shield_pool::ShieldPool; use std::cmp; @@ -783,7 +783,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.get(key, handle, guard) } @@ -793,7 +793,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.remove(key, handle, guard) } } @@ -805,6 +810,6 @@ mod tests { #[test] fn smoke_bonsai_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, BonsaiTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/concurrent_map.rs b/src/ds_impl/pebr/concurrent_map.rs index 72a0e176..388ee3b6 100644 --- a/src/ds_impl/pebr/concurrent_map.rs +++ b/src/ds_impl/pebr/concurrent_map.rs @@ -1,5 +1,21 @@ use crossbeam_pebr::Guard; +pub trait OutputHolder { + fn output(&self) -> &V; +} + +impl<'g, V> OutputHolder for &'g V { + fn output(&self) -> &V { + self + } +} + +impl OutputHolder for V { + fn output(&self) -> &V { + self + } +} + pub trait ConcurrentMap { type Handle; @@ -12,23 +28,34 @@ pub trait ConcurrentMap { handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V>; + ) -> Option>; fn insert(&self, handle: &mut Self::Handle, key: K, value: V, guard: &mut Guard) -> bool; - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option; + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option>; } #[cfg(test)] pub mod tests { extern crate rand; - use super::ConcurrentMap; + use super::{ConcurrentMap, OutputHolder}; use crossbeam_pebr::pin; use crossbeam_utils::thread; use rand::prelude::*; + use std::fmt::Debug; const THREADS: i32 = 30; const ELEMENTS_PER_THREADS: i32 = 1000; - pub fn smoke + Send + Sync>() { + pub fn smoke(to_value: &F) + where + V: Eq + Debug, + M: ConcurrentMap + Send + Sync, + F: Sync + Fn(&i32) -> V, + { let map = &M::new(); thread::scope(|s| { @@ -40,7 +67,7 @@ pub mod tests { (0..ELEMENTS_PER_THREADS).map(|k| k * THREADS + t).collect(); keys.shuffle(&mut rng); for i in keys { - assert!(map.insert(&mut handle, i, i.to_string(), &mut pin())); + assert!(map.insert(&mut handle, i, to_value(&i), &mut pin())); } }); } @@ -57,8 +84,8 @@ pub mod tests { keys.shuffle(&mut rng); for i in keys { assert_eq!( - i.to_string(), - map.remove(&mut handle, &i, &mut pin()).unwrap() + to_value(&i), + *map.remove(&mut handle, &i, &mut pin()).unwrap().output() ); } }); @@ -76,8 +103,8 @@ pub mod tests { keys.shuffle(&mut rng); for i in keys { assert_eq!( - i.to_string(), - *map.get(&mut handle, &i, &mut pin()).unwrap() + to_value(&i), + *map.get(&mut handle, &i, &mut pin()).unwrap().output() ); } }); diff --git a/src/ds_impl/pebr/elim_ab_tree.rs b/src/ds_impl/pebr/elim_ab_tree.rs new file mode 100644 index 00000000..a8abcecf --- /dev/null +++ b/src/ds_impl/pebr/elim_ab_tree.rs @@ -0,0 +1,1446 @@ +use super::concurrent_map::{ConcurrentMap, OutputHolder}; +use arrayvec::ArrayVec; +use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::mem::swap; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $acq_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node.marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + }; +} + +struct MCSLockSlot { + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> { + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> { + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> { + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> { + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +struct Node { + keys: [Cell>; DEGREE], + search_key: K, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + size: AtomicUsize, + weight: bool, + marked: AtomicBool, + kind: NodeKind, +} + +// Leaf or Internal node specific data. +enum NodeKind { + Leaf { + values: [Cell>; DEGREE], + write_version: AtomicUsize, + }, + Internal { + next: [Atomic>; DEGREE], + }, +} + +impl Node { + fn is_leaf(&self) -> bool { + match &self.kind { + NodeKind::Leaf { .. } => true, + NodeKind::Internal { .. } => false, + } + } + + fn next(&self) -> &[Atomic; DEGREE] { + match &self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn next_mut(&mut self) -> &mut [Atomic; DEGREE] { + match &mut self.kind { + NodeKind::Internal { next } => next, + _ => panic!("No next pointers for a leaf node."), + } + } + + fn load_next<'g>(&self, index: usize, guard: &'g Guard) -> Shared<'g, Self> { + self.next()[index].load(Ordering::Acquire, guard) + } + + fn protect_next<'g>( + &self, + index: usize, + slot: &mut Shield, + guard: &'g Guard, + ) -> Result, ShieldError> { + let ptr = self.next()[index].load(Ordering::Acquire, guard); + slot.defend(ptr, guard).map(|_| ptr) + } + + fn store_next<'g>(&'g self, index: usize, ptr: impl Pointer, _: &MCSLockGuard<'g, K, V>) { + self.next()[index].store(ptr, Ordering::Release); + } + + fn init_next<'g>(&mut self, index: usize, ptr: impl Pointer) { + self.next_mut()[index] = Atomic::from(ptr.into_usize() as *const Self); + } + + /// # Safety + /// + /// The write version record must be accessed by `start_write` and `WriteGuard`. + unsafe fn write_version(&self) -> &AtomicUsize { + match &self.kind { + NodeKind::Leaf { write_version, .. } => write_version, + _ => panic!("No write version for an internal node."), + } + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let version = unsafe { self.write_version() }; + let init_version = version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + version.store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn key_count(&self) -> usize { + match &self.kind { + NodeKind::Leaf { .. } => self.size.load(Ordering::Acquire), + NodeKind::Internal { .. } => self.size.load(Ordering::Acquire) - 1, + } + } + + fn p_s_idx(p_l_idx: usize) -> usize { + if p_l_idx > 0 { + p_l_idx - 1 + } else { + 1 + } + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn get_value(&self, index: usize) -> Option { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].get(), + _ => panic!("No values for an internal node."), + } + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + match &self.kind { + NodeKind::Leaf { values, .. } => values[index].set(Some(val)), + _ => panic!("No values for an internal node."), + } + } + + fn init_value(&mut self, index: usize, val: V) { + match &mut self.kind { + NodeKind::Leaf { values, .. } => *values[index].get_mut() = Some(val), + _ => panic!("No values for an internal node."), + } + } + + fn get_key(&self, index: usize) -> Option { + self.keys[index].get() + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self.keys[index].set(key); + } + + fn init_key(&mut self, index: usize, key: Option) { + *self.keys[index].get_mut() = key; + } + + fn internal(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Internal { + next: Default::default(), + }, + } + } + + fn leaf(weight: bool, size: usize, search_key: K) -> Self { + Self { + keys: Default::default(), + search_key, + lock: Default::default(), + size: AtomicUsize::new(size), + weight, + marked: AtomicBool::new(false), + kind: NodeKind::Leaf { + values: Default::default(), + write_version: AtomicUsize::new(0), + }, + } + } + + fn child_index(&self, key: &K) -> usize { + let key_count = self.key_count(); + let mut index = 0; + while index < key_count && !(key < &self.keys[index].get().unwrap()) { + index += 1; + } + index + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K) -> (usize, Option) { + let NodeKind::Leaf { + values, + write_version, + } = &self.kind + else { + panic!("Attempted to read value from an internal node."); + }; + loop { + let mut version = write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.keys[key_index].get() != Some(*key) { + key_index += 1; + } + let value = values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == write_version.load(Ordering::Acquire) { + return (key_index, value); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(self.is_leaf()); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child( + &self, + child: &Self, + child_idx: usize, + ) -> ( + [Atomic>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let mut next: [Atomic>; DEGREE * 2] = Default::default(); + let mut keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.size.load(Ordering::Relaxed); + let nsize = child.size.load(Ordering::Relaxed); + + slice_clone(&self.next()[0..], &mut next[0..], child_idx); + slice_clone(&child.next()[0..], &mut next[child_idx..], nsize); + slice_clone( + &self.next()[child_idx + 1..], + &mut next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + slice_clone(&self.keys[0..], &mut keys[0..], child_idx); + slice_clone(&child.keys[0..], &mut keys[child_idx..], child.key_count()); + slice_clone( + &self.keys[child_idx..], + &mut keys[child_idx + child.key_count()..], + self.key_count() - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + _: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.keys + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value(i).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + guard: &'g Guard, + ) -> impl Iterator, Shared<'g, Self>)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next(i, guard))) + .chain(once((None, self.load_next(self.key_count(), guard)))) + } +} + +struct WriteGuard<'g, K, V> { + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> { + fn drop(&mut self) { + unsafe { self.node.write_version() }.store(self.init_version + 2, Ordering::Release); + } +} + +pub struct Cursor { + l: Shield>, + p: Shield>, + gp: Shield>, + /// A protector for the sibling node. + s: Shield>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + p_s_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +impl Cursor { + fn new(guard: &Guard) -> Self { + Self { + l: Shield::null(guard), + p: Shield::null(guard), + gp: Shield::null(guard), + s: Shield::null(guard), + gp_p_idx: 0, + p_l_idx: 0, + p_s_idx: 0, + l_key_idx: 0, + val: None, + } + } + + fn release(&mut self) { + self.l.release(); + self.p.release(); + self.gp.release(); + self.s.release(); + } +} + +pub struct ElimABTree { + entry: Node, +} + +unsafe impl Sync for ElimABTree {} +unsafe impl Send for ElimABTree {} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new() -> Self { + let left = Node::leaf(true, 0, K::default()); + let mut entry = Node::internal(true, 1, K::default()); + entry.init_next(0, Owned::new(left)); + Self { entry } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, cursor: &mut Cursor, guard: &mut Guard) -> Option { + loop { + match self.search_basic_inner(key, cursor, guard) { + Ok(found) => return found, + Err(_) => guard.repin(), + } + } + } + + fn search_basic_inner( + &self, + key: &K, + cursor: &mut Cursor, + guard: &Guard, + ) -> Result, ShieldError> { + let mut node = unsafe { self.entry.protect_next(0, &mut cursor.p, guard)?.deref() }; + while !node.is_leaf() { + let next = node.protect_next(node.child_index(key), &mut cursor.l, guard)?; + swap(&mut cursor.l, &mut cursor.p); + node = unsafe { next.deref() }; + } + Ok(node.read_consistent(key).1) + } + + fn search<'g>( + &self, + key: &K, + target: Option>>, + cursor: &mut Cursor, + guard: &'g mut Guard, + ) -> bool { + loop { + match self.search_inner(key, target, cursor, guard) { + Ok(found) => return found, + Err(_) => guard.repin(), + } + } + } + + fn search_inner<'g>( + &self, + key: &K, + target: Option>>, + cursor: &mut Cursor, + guard: &'g Guard, + ) -> Result { + self.entry.protect_next(0, &mut cursor.l, guard)?; + self.entry.protect_next(1, &mut cursor.s, guard)?; + unsafe { + cursor + .p + .defend_fake(Shared::from_usize(&self.entry as *const _ as usize)) + }; + cursor.gp.release(); + cursor.gp_p_idx = 0; + cursor.p_l_idx = 0; + cursor.p_s_idx = 1; + cursor.l_key_idx = 0; + cursor.val = None; + + while !unsafe { cursor.l.deref() }.is_leaf() + && target + .map(|target| target != cursor.l.shared()) + .unwrap_or(true) + { + swap(&mut cursor.gp, &mut cursor.p); + swap(&mut cursor.p, &mut cursor.l); + let l_node = unsafe { cursor.p.deref() }; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key); + cursor.p_s_idx = Node::::p_s_idx(cursor.p_l_idx); + l_node.protect_next(cursor.p_l_idx, &mut cursor.l, guard)?; + l_node.protect_next(cursor.p_s_idx, &mut cursor.s, guard)?; + } + + if let Some(target) = target { + Ok(cursor.l.shared() == target) + } else { + let (index, value) = unsafe { cursor.l.deref() }.read_consistent(key); + cursor.val = value; + cursor.l_key_idx = index; + Ok(value.is_some()) + } + } + + pub fn insert( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> Option { + loop { + self.search(key, None, cursor, guard); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, cursor, guard) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner( + &self, + key: &K, + value: &V, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + if node.marked.load(Ordering::SeqCst) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + return Ok(Some(node.get_value(i).unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.size.load(Ordering::Acquire) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key(i).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.size + .store(node.size.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!(parent, parent_lock, Operation::Insert, None, return Err(())); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let mut left = Node::leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.init_key(i, Some(kv_pairs[i].0)); + left.init_value(i, kv_pairs[i].1); + } + + let mut right = Node::leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.init_key(i, Some(kv_pairs[i + left_size].0)); + right.init_value(i, kv_pairs[i + left_size].1); + } + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let mut internal = Node::internal(eq(parent, &self.entry), 2, kv_pairs[left_size].0); + internal.init_key(0, Some(kv_pairs[left_size].0)); + internal.init_next(0, Owned::new(left)); + internal.init_next(1, Owned::new(right)); + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + let new_internal = Owned::new(internal).into_shared(guard); + parent.store_next(cursor.p_l_idx, new_internal, &parent_lock); + node.marked.store(true, Ordering::Release); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { guard.defer_destroy(cursor.l.shared()) }; + self.fix_tag_violation( + kv_pairs[left_size].0, + new_internal.into_usize(), + cursor, + guard, + ); + + Ok(None) + } + } + + fn fix_tag_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + guard: &mut Guard, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + let found = self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + guard, + ); + if !found || cursor.l.shared().into_usize() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_tag_violation_inner(cursor, guard); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_tag_violation_inner<'g>( + &self, + cursor: &mut Cursor, + guard: &'g Guard, + ) -> (bool, Option<(K, usize)>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { viol.deref() }; + if viol_node.weight { + return (true, None); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf()); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!(!eq(viol_node, &self.entry) && self.entry.load_next(0, guard) != viol); + + debug_assert!(!cursor.gp.shared().is_null()); + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.deref() }; + debug_assert!(!node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(!gparent.is_leaf()); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight { + return ( + false, + Some((parent.search_key, cursor.p.shared().into_usize())), + ); + } + + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, None) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, None) + ); + + let psize = parent.size.load(Ordering::Relaxed); + let nsize = viol_node.size.load(Ordering::Relaxed); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let mut absorber = Node::internal(true, size, parent.get_key(0).unwrap()); + slice_clone(&next, absorber.next_mut(), DEGREE); + slice_clone(&keys, &mut absorber.keys, DEGREE); + + gparent.store_next(cursor.gp_p_idx, Owned::new(absorber), &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l.shared()) }; + unsafe { guard.defer_destroy(cursor.p.shared()) }; + return (true, None); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let mut left = Node::internal(true, left_size, keys[0].get().unwrap()); + slice_clone(&keys[0..], &mut left.keys[0..], left_size - 1); + slice_clone(&next[0..], &mut left.next_mut()[0..], left_size); + + let right_size = size - left_size; + let mut right = Node::internal(true, right_size, keys[left_size].get().unwrap()); + slice_clone(&keys[left_size..], &mut right.keys[0..], right_size - 1); + slice_clone(&next[left_size..], &mut right.next_mut()[0..], right_size); + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let mut new_internal = Node::internal( + eq(gparent, &self.entry), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.init_key(0, keys[left_size - 1].get()); + new_internal.init_next(0, Owned::new(left)); + new_internal.init_next(1, Owned::new(right)); + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + let new_internal = Owned::new(new_internal).into_shared(guard); + gparent.store_next(cursor.gp_p_idx, new_internal, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + + unsafe { guard.defer_destroy(cursor.l.shared()) }; + unsafe { guard.defer_destroy(cursor.p.shared()) }; + + drop((node_lock, parent_lock, gparent_lock)); + return ( + true, + Some(( + keys[left_size - 1].get().unwrap(), + new_internal.into_usize(), + )), + ); + } + } + + pub fn remove(&self, key: &K, cursor: &mut Cursor, guard: &mut Guard) -> Option { + loop { + self.search(key, None, cursor, guard); + if cursor.val.is_none() { + return None; + } + if let Ok(result) = self.remove_inner(key, cursor, guard) { + cursor.val = result; + return result; + } + } + } + + fn remove_inner( + &self, + key: &K, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> Result, ()> { + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf()); + debug_assert!(!parent.is_leaf()); + debug_assert!(gparent.map(|gp| !gp.is_leaf()).unwrap_or(true)); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.size.load(Ordering::Acquire) == 0 { + return Err(()); + } + + let new_size = node.size.load(Ordering::Relaxed) - 1; + for i in 0..DEGREE { + if node.get_key(i) == Some(*key) { + let val = node.get_value(i).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.size.store(new_size, Ordering::Relaxed); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + drop(node_lock); + self.fix_underfull_violation( + node.search_key, + cursor.l.shared().into_usize(), + cursor, + guard, + ); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation( + &self, + search_key: K, + viol: usize, + cursor: &mut Cursor, + guard: &mut Guard, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + self.search( + &search_key, + Some(unsafe { Shared::from_usize(viol) }), + cursor, + guard, + ); + if cursor.l.shared().into_usize() != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let (success, recur) = self.fix_underfull_violation_inner(cursor, guard); + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_underfull_violation_inner( + &self, + cursor: &mut Cursor, + guard: &mut Guard, + ) -> (bool, ArrayVec<(K, usize), 2>) { + let viol = cursor.l.shared(); + let viol_node = unsafe { viol.deref() }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, &self.entry) + || viol == self.entry.load_next(0, guard) + { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + let node = unsafe { cursor.l.deref() }; + let parent = unsafe { cursor.p.deref() }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.shared().is_null()); + let gparent = unsafe { cursor.gp.deref() }; + + if parent.size.load(Ordering::Relaxed) < Self::UNDERFULL_THRESHOLD + && !eq(parent, &self.entry) + && cursor.p.shared() != self.entry.load_next(0, guard) + { + return ( + false, + ArrayVec::from_iter(once((parent.search_key, cursor.p.shared().into_usize()))), + ); + } + + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling = unsafe { cursor.s.deref() }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if cursor.p_s_idx < cursor.p_l_idx { + ((sibling, cursor.p_s_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, cursor.p_s_idx)) + }; + + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // Repeat this check, this might have changed while we locked `viol`. + if viol_node.size.load(Ordering::Relaxed) >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return (true, ArrayVec::new()); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + return (false, ArrayVec::new()) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation( + parent.search_key, + cursor.p.shared().into_usize(), + cursor, + guard, + ); + return (false, ArrayVec::new()); + } + if !node.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation( + node.search_key, + cursor.l.shared().into_usize(), + cursor, + guard, + ); + return (false, ArrayVec::new()); + } + if !sibling.weight { + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation( + sibling.search_key, + cursor.s.shared().into_usize(), + cursor, + guard, + ); + return (false, ArrayVec::new()); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!(parent.weight && node.weight && sibling.weight); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (node.is_leaf() && sibling.is_leaf()) || (!node.is_leaf() && !sibling.is_leaf()) + ); + + let lsize = left.size.load(Ordering::Relaxed); + let rsize = right.size.load(Ordering::Relaxed); + let psize = parent.size.load(Ordering::Relaxed); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = if left.is_leaf() { + debug_assert!(right.is_leaf()); + let mut new_leaf = Owned::new(Node::leaf(true, size, node.search_key)); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + new_leaf + } else { + debug_assert!(!right.is_leaf()); + let mut new_internal = Owned::new(Node::internal(true, size, node.search_key)); + let key_btw = parent.get_key(left_idx).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + new_internal + } + .into_shared(guard); + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, &self.entry) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l.shared()); + guard.defer_destroy(cursor.p.shared()); + guard.defer_destroy(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter(once((node.search_key, new_node.into_usize()))), + ); + } else { + debug_assert!(!eq(gparent, &self.entry) || psize > 2); + let mut new_parent = Node::internal(true, psize - 1, parent.search_key); + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key(i)); + } + for i in 0..cursor.p_s_idx { + new_parent.init_next(i, parent.load_next(i, guard)); + } + for i in left_idx + 1..parent.key_count() { + new_parent.init_key(i - 1, parent.get_key(i)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next(i, guard)); + } + + new_parent.init_next( + cursor.p_l_idx + - (if cursor.p_l_idx > cursor.p_s_idx { + 1 + } else { + 0 + }), + new_node, + ); + let new_parent = Owned::new(new_parent).into_shared(guard); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l.shared()); + guard.defer_destroy(cursor.p.shared()); + guard.defer_destroy(cursor.s.shared()); + } + + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return ( + true, + ArrayVec::from_iter( + [ + (node.search_key, new_node.into_usize()), + (parent.search_key, new_parent.into_usize()), + ] + .into_iter(), + ), + ); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf() == right.is_leaf()); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf() { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = { + let mut new_leaf = Owned::new(Node::leaf(true, left_size, Default::default())); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.search_key = new_leaf.get_key(0).unwrap(); + new_leaf + }; + + let (new_right, pivot) = { + debug_assert!(left.is_leaf()); + let mut new_leaf = Owned::new(Node::leaf(true, right_size, Default::default())); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + let pivot = new_leaf.get_key(0).unwrap(); + new_leaf.search_key = pivot; + (new_leaf, pivot) + }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key(left_idx).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock, guard) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock, guard)); + + let (new_left, pivot) = { + let mut new_internal = + Owned::new(Node::internal(true, left_size, Default::default())); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + let pivot = new_internal.keys[left_size - 1].take().unwrap(); + new_internal.search_key = new_internal.get_key(0).unwrap(); + (new_internal, pivot) + }; + + let new_right = { + let mut new_internal = + Owned::new(Node::internal(true, right_size, Default::default())); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.search_key = new_internal.get_key(0).unwrap(); + new_internal + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let mut new_parent = + Owned::new(Node::internal(parent.weight, psize, parent.search_key)); + slice_clone( + &parent.keys[0..], + &mut new_parent.keys[0..], + parent.key_count(), + ); + slice_clone(&parent.next()[0..], &mut new_parent.next_mut()[0..], psize); + new_parent.init_next(left_idx, new_left); + new_parent.init_next(right_idx, new_right); + new_parent.init_key(left_idx, Some(pivot)); + + gparent.store_next(cursor.gp_p_idx, new_parent, &gparent_lock); + node.marked.store(true, Ordering::Relaxed); + parent.marked.store(true, Ordering::Relaxed); + sibling.marked.store(true, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(cursor.l.shared()); + guard.defer_destroy(cursor.p.shared()); + guard.defer_destroy(cursor.s.shared()); + } + + return (true, ArrayVec::new()); + } + } +} + +impl Drop for ElimABTree { + fn drop(&mut self) { + let mut stack = vec![]; + let guard = unsafe { unprotected() }; + for next in &self.entry.next()[0..self.entry.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + + while let Some(node) = stack.pop() { + let node_ref = unsafe { node.deref() }; + if !node_ref.is_leaf() { + for next in &node_ref.next()[0..node_ref.size.load(Ordering::Relaxed)] { + stack.push(next.load(Ordering::Relaxed, guard)); + } + } + drop(unsafe { node.into_owned() }); + } + } +} + +/// Similar to `memcpy`, but for `Clone` types. +#[inline] +fn slice_clone(src: &[T], dst: &mut [T], len: usize) { + dst[0..len].clone_from_slice(&src[0..len]); +} + +impl ConcurrentMap for ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + type Handle = Cursor; + + fn new() -> Self { + Self::new() + } + + fn handle<'g>(guard: &'g Guard) -> Self::Handle { + Cursor::new(guard) + } + + fn clear(handle: &mut Self::Handle) { + handle.release(); + } + + fn get<'g>( + &'g self, + handle: &'g mut Self::Handle, + key: &'g K, + guard: &'g mut Guard, + ) -> Option> { + self.search_basic(key, handle, guard) + } + + fn insert(&self, handle: &mut Self::Handle, key: K, value: V, guard: &mut Guard) -> bool { + self.insert(&key, &value, handle, guard).is_none() + } + + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { + self.remove(key, handle, guard) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::pebr::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::<_, ElimABTree, _>(&|a| *a); + } +} diff --git a/src/ds_impl/pebr/ellen_tree.rs b/src/ds_impl/pebr/ellen_tree.rs index 24669a91..358427a0 100644 --- a/src/ds_impl/pebr/ellen_tree.rs +++ b/src/ds_impl/pebr/ellen_tree.rs @@ -3,7 +3,7 @@ use std::sync::atomic::Ordering; use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -808,7 +808,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { match self.find(key, handle, guard) { Some(node) => Some(node), None => None, @@ -821,7 +821,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.delete(key, handle, guard) } } @@ -833,6 +838,6 @@ mod tests { #[test] fn smoke_efrb_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, EFRBTree, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/list.rs b/src/ds_impl/pebr/list.rs index 5d2c8967..ee366e4e 100644 --- a/src/ds_impl/pebr/list.rs +++ b/src/ds_impl/pebr/list.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; use std::cmp::Ordering::{Equal, Greater, Less}; @@ -549,9 +549,9 @@ where fn get<'g>( &'g self, handle: &'g mut Self::Handle, - key: &K, + key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.inner.harris_get(key, handle, guard) } #[inline(always)] @@ -559,7 +559,12 @@ where self.inner.harris_insert(key, value, handle, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.inner.harris_remove(key, handle, guard) } } @@ -606,9 +611,9 @@ where fn get<'g>( &'g self, handle: &'g mut Self::Handle, - key: &K, + key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.inner.harris_michael_get(key, handle, guard) } #[inline(always)] @@ -616,7 +621,12 @@ where self.inner.harris_michael_insert(key, value, handle, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.inner.harris_michael_remove(key, handle, guard) } } @@ -658,9 +668,9 @@ where fn get<'g>( &'g self, handle: &'g mut Self::Handle, - key: &K, + key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.inner.harris_herlihy_shavit_get(key, handle, guard) } #[inline(always)] @@ -668,7 +678,12 @@ where self.inner.harris_insert(key, value, handle, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.inner.harris_remove(key, handle, guard) } } @@ -680,17 +695,17 @@ mod tests { #[test] fn smoke_h_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HList, _>(&i32::to_string); } #[test] fn smoke_hm_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HMList, _>(&i32::to_string); } #[test] fn smoke_hhs_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HHSList, _>(&i32::to_string); } #[test] diff --git a/src/ds_impl/pebr/michael_hash_map.rs b/src/ds_impl/pebr/michael_hash_map.rs index 66ef494f..40dcde06 100644 --- a/src/ds_impl/pebr/michael_hash_map.rs +++ b/src/ds_impl/pebr/michael_hash_map.rs @@ -1,4 +1,4 @@ -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use crossbeam_pebr::Guard; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -35,26 +35,6 @@ where k.hash(&mut s); s.finish() as usize } - - pub fn get<'g>( - &'g self, - cursor: &'g mut Cursor, - k: &'g K, - guard: &'g mut Guard, - ) -> Option<&'g V> { - let i = Self::hash(k); - self.get_bucket(i).get(cursor, k, guard) - } - - pub fn insert(&self, cursor: &mut Cursor, k: K, v: V, guard: &mut Guard) -> bool { - let i = Self::hash(&k); - self.get_bucket(i).insert(cursor, k, v, guard) - } - - pub fn remove(&self, cursor: &mut Cursor, k: &K, guard: &mut Guard) -> Option { - let i = Self::hash(&k); - self.get_bucket(i).remove(cursor, k, guard) - } } impl ConcurrentMap for HashMap @@ -81,16 +61,24 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { - self.get(handle, key, guard) + ) -> Option> { + let i = Self::hash(key); + self.get_bucket(i).get(handle, key, guard) } #[inline(always)] fn insert(&self, handle: &mut Self::Handle, key: K, value: V, guard: &mut Guard) -> bool { - self.insert(handle, key, value, guard) + let i = Self::hash(&key); + self.get_bucket(i).insert(handle, key, value, guard) } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { - self.remove(handle, key, guard) + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { + let i = Self::hash(&key); + self.get_bucket(i).remove(handle, key, guard) } } @@ -101,6 +89,6 @@ mod tests { #[test] fn smoke_hashmap() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, HashMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/mod.rs b/src/ds_impl/pebr/mod.rs index 33b22bef..b12f32a2 100644 --- a/src/ds_impl/pebr/mod.rs +++ b/src/ds_impl/pebr/mod.rs @@ -3,6 +3,7 @@ pub mod shield_pool; pub mod concurrent_map; pub mod bonsai_tree; +pub mod elim_ab_tree; pub mod ellen_tree; pub mod list; pub mod michael_hash_map; @@ -12,6 +13,7 @@ pub mod skip_list; pub use self::concurrent_map::ConcurrentMap; pub use self::bonsai_tree::BonsaiTreeMap; +pub use self::elim_ab_tree::ElimABTree; pub use self::ellen_tree::EFRBTree; pub use self::list::{HHSList, HList, HMList}; pub use self::michael_hash_map::HashMap; diff --git a/src/ds_impl/pebr/natarajan_mittal_tree.rs b/src/ds_impl/pebr/natarajan_mittal_tree.rs index d4a5199d..c7982ad7 100644 --- a/src/ds_impl/pebr/natarajan_mittal_tree.rs +++ b/src/ds_impl/pebr/natarajan_mittal_tree.rs @@ -1,6 +1,6 @@ use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; use std::cmp; use std::mem; use std::sync::atomic::Ordering; @@ -601,7 +601,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { self.get(key, handle, guard) } @@ -611,7 +611,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.remove(key, handle, guard) } } @@ -623,6 +628,6 @@ mod tests { #[test] fn smoke_nm_tree() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, NMTreeMap, _>(&i32::to_string); } } diff --git a/src/ds_impl/pebr/skip_list.rs b/src/ds_impl/pebr/skip_list.rs index 6f233263..ccabf7b4 100644 --- a/src/ds_impl/pebr/skip_list.rs +++ b/src/ds_impl/pebr/skip_list.rs @@ -5,7 +5,7 @@ use std::{ use crossbeam_pebr::{unprotected, Atomic, Guard, Owned, Pointer, Shared, Shield, ShieldError}; -use super::concurrent_map::ConcurrentMap; +use super::concurrent_map::{ConcurrentMap, OutputHolder}; const MAX_HEIGHT: usize = 32; @@ -474,7 +474,7 @@ where handle: &'g mut Self::Handle, key: &'g K, guard: &'g mut Guard, - ) -> Option<&'g V> { + ) -> Option> { let cursor = self.find_optimistic(key, handle, guard)?; let node = unsafe { cursor.found?.deref() }; if node.key.eq(&key) { @@ -490,7 +490,12 @@ where } #[inline(always)] - fn remove(&self, handle: &mut Self::Handle, key: &K, guard: &mut Guard) -> Option { + fn remove( + &self, + handle: &mut Self::Handle, + key: &K, + guard: &mut Guard, + ) -> Option> { self.remove(key, handle, guard) } } @@ -502,6 +507,6 @@ mod tests { #[test] fn smoke_skip_list() { - concurrent_map::tests::smoke::>(); + concurrent_map::tests::smoke::<_, SkipList, _>(&i32::to_string); } } diff --git a/src/ds_impl/vbr/elim_ab_tree.rs b/src/ds_impl/vbr/elim_ab_tree.rs new file mode 100644 index 00000000..99b6ac26 --- /dev/null +++ b/src/ds_impl/vbr/elim_ab_tree.rs @@ -0,0 +1,1638 @@ +use super::concurrent_map::ConcurrentMap; +use arrayvec::ArrayVec; +use vbr::{Entry, Global, Guard, Inner, Local}; + +use std::cell::{Cell, UnsafeCell}; +use std::hint::spin_loop; +use std::iter::once; +use std::ptr::{eq, null, null_mut}; +use std::sync::atomic::{compiler_fence, AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +// Copied from the original author's code: +// https://gitlab.com/trbot86/setbench/-/blob/f4711af3ace28d8b4fa871559db74fb4e0e62cc0/ds/srivastava_abtree_mcs/adapter.h#L17 +const DEGREE: usize = 11; + +macro_rules! try_acq_val_or { + ($node:ident, $lock:ident, $op:expr, $key:expr, $guard:ident, $acq_val_err:expr, $epoch_val_err:expr) => { + let __slot = UnsafeCell::new(MCSLockSlot::new()); + let $lock = match ( + $node.acquire($op, $key, &__slot), + $node._marked.load(Ordering::Acquire), + ) { + (AcqResult::Acquired(lock), false) => lock, + _ => $acq_val_err, + }; + if $guard.validate_epoch().is_err() { + $epoch_val_err; + } + }; +} + +struct MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + node: *const Node, + op: Operation, + key: Option, + next: AtomicPtr, + owned: AtomicBool, + short_circuit: AtomicBool, + ret: Cell>, +} + +impl MCSLockSlot +where + K: Default + Copy, + V: Default + Copy, +{ + fn new() -> Self { + Self { + node: null(), + op: Operation::Insert, + key: Default::default(), + next: Default::default(), + owned: AtomicBool::new(false), + short_circuit: AtomicBool::new(false), + ret: Cell::new(None), + } + } + + fn init(&mut self, node: &Node, op: Operation, key: Option) { + self.node = node; + self.op = op; + self.key = key; + } +} + +struct MCSLockGuard<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + slot: &'l UnsafeCell>, +} + +impl<'l, K, V> MCSLockGuard<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + fn new(slot: &'l UnsafeCell>) -> Self { + Self { slot } + } + + unsafe fn owner_node(&self) -> &Node { + &*(&*self.slot.get()).node + } +} + +impl<'l, K, V> Drop for MCSLockGuard<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + fn drop(&mut self) { + let slot = unsafe { &*self.slot.get() }; + let node = unsafe { &*slot.node }; + debug_assert!(slot.owned.load(Ordering::Acquire)); + + if let Some(next) = unsafe { slot.next.load(Ordering::Acquire).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + + if node + .lock + .compare_exchange( + self.slot.get(), + null_mut(), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + slot.owned.store(false, Ordering::Release); + return; + } + loop { + if let Some(next) = unsafe { slot.next.load(Ordering::Relaxed).as_ref() } { + next.owned.store(true, Ordering::Release); + slot.owned.store(false, Ordering::Release); + return; + } + spin_loop(); + } + } +} + +enum AcqResult<'l, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + Acquired(MCSLockGuard<'l, K, V>), + Eliminated(V), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Operation { + Insert, + Delete, + Balance, +} + +#[derive(Default)] +pub struct Node +where + K: Copy + Default, + V: Copy + Default, +{ + _keys: [Cell>; DEGREE], + _search_key: Cell, + lock: AtomicPtr>, + /// The number of next pointers (for an internal node) or values (for a leaf node). + /// Note that it may not be equal to the number of keys, because the last next pointer + /// is mapped by a bottom key (i.e., `None`). + _size: AtomicUsize, + _weight: Cell, + _marked: AtomicBool, + // Leaf node data + _values: [Cell>; DEGREE], + _write_version: AtomicUsize, + // Internal node data + _next: [AtomicPtr>>; DEGREE], + _is_leaf: Cell, +} + +impl Node +where + K: Default + Copy, + V: Default + Copy, +{ + fn get_marked_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> bool { + self._marked.load(Ordering::Acquire) + } + + fn set_marked_locked<'l>(&'l self, mark: bool, _: &MCSLockGuard<'l, K, V>) { + self._marked.store(mark, Ordering::Release); + } + + fn weight(&self, guard: &Guard) -> Result { + let result = self._weight.get(); + guard.validate_epoch().map(|_| result) + } + + fn weight_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> bool { + self._weight.get() + } + + fn search_key(&self, guard: &Guard) -> Result { + let result = self._search_key.get(); + guard.validate_epoch().map(|_| result) + } + + fn search_key_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> K { + self._search_key.get() + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn search_key_unchecked(&self) -> K { + self._search_key.get() + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_search_key(&self, key: K) { + self._search_key.set(key); + } + + fn is_leaf(&self, guard: &Guard) -> Result { + let result = self._is_leaf.get(); + guard.validate_epoch().map(|_| result) + } + + fn is_leaf_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> bool { + self._is_leaf.get() + } + + fn load_next(&self, index: usize, guard: &Guard) -> Result<*mut Inner, ()> { + let result = self._next[index].load(Ordering::Acquire); + guard.validate_epoch().map(|_| result) + } + + fn load_next_locked<'l>( + &'l self, + index: usize, + _: &MCSLockGuard<'l, K, V>, + ) -> *mut Inner { + self._next[index].load(Ordering::Acquire) + } + + fn store_next<'l>(&'l self, index: usize, ptr: *mut Inner, _: &MCSLockGuard<'l, K, V>) { + self._next[index].store(ptr, Ordering::Release); + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_next(&self, index: usize, ptr: *mut Inner) { + self._next[index].store(ptr, Ordering::Release); + } + + fn next_slice<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> &[AtomicPtr>>] { + &self._next + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn next_slice_unchecked(&self) -> &[AtomicPtr>>] { + &self._next + } + + fn start_write<'g>(&'g self, lock: &MCSLockGuard<'g, K, V>) -> WriteGuard<'g, K, V> { + debug_assert!(eq(unsafe { lock.owner_node() }, self)); + let init_version = self._write_version.load(Ordering::Acquire); + debug_assert!(init_version % 2 == 0); + // It is safe to skip the epoch validation, as we are grabbing the lock. + self._write_version + .store(init_version + 1, Ordering::Release); + compiler_fence(Ordering::SeqCst); + + return WriteGuard { + init_version, + node: self, + }; + } + + fn get_size(&self, guard: &Guard) -> Result { + let result = self._size.load(Ordering::Acquire); + guard.validate_epoch().map(|_| result) + } + + fn get_size_locked<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> usize { + self._size.load(Ordering::Acquire) + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn get_size_unchecked(&self) -> usize { + self._size.load(Ordering::Acquire) + } + + fn set_size_locked<'l>(&'l self, size: usize, _: &MCSLockGuard<'l, K, V>) { + self._size.store(size, Ordering::Release); + } + + fn key_count(&self, guard: &Guard) -> Result { + let result = if self.is_leaf(guard)? { + self.get_size(guard)? + } else { + self.get_size(guard)? - 1 + }; + guard.validate_epoch().map(|_| result) + } + + fn key_count_locked<'l>(&'l self, lock: &MCSLockGuard<'l, K, V>) -> usize { + if self._is_leaf.get() { + self.get_size_locked(lock) + } else { + self.get_size_locked(lock) - 1 + } + } + + fn get_value(&self, index: usize, guard: &Guard) -> Result, ()> { + let value = self._values[index].get(); + guard.validate_epoch().map(|_| value) + } + + fn get_value_locked<'l>(&'l self, index: usize, _: &MCSLockGuard<'l, K, V>) -> Option { + self._values[index].get() + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_value(&self, index: usize, val: V) { + self._values[index].set(Some(val)); + } + + fn set_value<'g>(&'g self, index: usize, val: V, _: &WriteGuard<'g, K, V>) { + self._values[index].set(Some(val)); + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn get_key_unchecked(&self, index: usize) -> Option { + self._keys[index].get() + } + + fn get_key(&self, index: usize, guard: &Guard) -> Result, ()> { + let result = self._keys[index].get(); + guard.validate_epoch().map(|_| result) + } + + fn get_key_locked<'l>(&'l self, index: usize, _: &MCSLockGuard<'l, K, V>) -> Option { + self._keys[index].get() + } + + fn key_slice<'l>(&'l self, _: &MCSLockGuard<'l, K, V>) -> &[Cell>] { + &self._keys + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn key_slice_unchecked(&self) -> &[Cell>] { + &self._keys + } + + /// # Safety + /// + /// The current thread has an exclusive ownership and it is not exposed to the shared memory. + /// (e.g., in the `init` closure of `allocate` function) + unsafe fn init_key(&self, index: usize, key: Option) { + self._keys[index].set(key); + } + + fn set_key<'g>(&'g self, index: usize, key: Option, _: &WriteGuard<'g, K, V>) { + self._keys[index].set(key); + } + + unsafe fn init_on_allocate(&self, weight: bool, size: usize, search_key: K) { + for key in &self._keys { + key.set(Default::default()); + } + self._search_key.set(search_key); + self._size.store(size, Ordering::Release); + self._weight.set(weight); + self._marked.store(false, Ordering::Release); + for next in &self._next { + next.store(null_mut(), Ordering::Release); + } + for value in &self._values { + value.set(Default::default()); + } + self._write_version.store(0, Ordering::Release); + } + + unsafe fn init_for_internal(&self, weight: bool, size: usize, search_key: K) { + self.init_on_allocate(weight, size, search_key); + self._is_leaf.set(false); + } + + unsafe fn init_for_leaf(&self, weight: bool, size: usize, search_key: K) { + self.init_on_allocate(weight, size, search_key); + self._is_leaf.set(true); + } +} + +impl Node +where + K: PartialOrd + Eq + Default + Copy, + V: Default + Copy, +{ + fn child_index(&self, key: &K, guard: &Guard) -> Result { + let key_count = self.key_count(guard)?; + let mut index = 0; + while index < key_count && !(key < &some_or!(self.get_key(index, guard)?, return Err(()))) { + index += 1; + } + guard.validate_epoch().map(|_| index) + } + + // Search a node for a key repeatedly until we successfully read a consistent version. + fn read_consistent(&self, key: &K, guard: &Guard) -> Result<(usize, Option), ()> { + loop { + guard.validate_epoch()?; + let mut version = self._write_version.load(Ordering::Acquire); + while version & 1 > 0 { + version = self._write_version.load(Ordering::Acquire); + } + let mut key_index = 0; + while key_index < DEGREE && self.get_key(key_index, guard)? != Some(*key) { + key_index += 1; + } + let value = self._values.get(key_index).and_then(|value| value.get()); + compiler_fence(Ordering::SeqCst); + + if version == self._write_version.load(Ordering::Acquire) { + return guard.validate_epoch().map(|_| (key_index, value)); + } + } + } + + fn acquire<'l>( + &'l self, + op: Operation, + key: Option, + slot: &'l UnsafeCell>, + ) -> AcqResult<'l, K, V> { + unsafe { &mut *slot.get() }.init(self, op, key); + let old_tail = self.lock.swap(slot.get(), Ordering::AcqRel); + let curr = unsafe { &*slot.get() }; + + if let Some(old_tail) = unsafe { old_tail.as_ref() } { + old_tail.next.store(slot.get(), Ordering::Release); + while !curr.owned.load(Ordering::Acquire) && !curr.short_circuit.load(Ordering::Acquire) + { + spin_loop(); + } + debug_assert!( + !curr.owned.load(Ordering::Relaxed) || !curr.short_circuit.load(Ordering::Relaxed) + ); + if curr.short_circuit.load(Ordering::Relaxed) { + return AcqResult::Eliminated(curr.ret.get().unwrap()); + } + debug_assert!(curr.owned.load(Ordering::Relaxed)); + } else { + curr.owned.store(true, Ordering::Release); + } + return AcqResult::Acquired(MCSLockGuard::new(slot)); + } + + fn elim_key_ops<'l>( + &'l self, + value: V, + wguard: WriteGuard<'l, K, V>, + guard: &MCSLockGuard<'l, K, V>, + ) { + let slot = unsafe { &*guard.slot.get() }; + debug_assert!(slot.owned.load(Ordering::Relaxed)); + debug_assert!(slot.op != Operation::Balance); + + let stop_node = self.lock.load(Ordering::Acquire); + drop(wguard); + + if eq(stop_node.cast(), slot) { + return; + } + + let mut prev_alive = guard.slot.get(); + let mut curr = slot.next.load(Ordering::Acquire); + while curr.is_null() { + curr = slot.next.load(Ordering::Acquire); + } + + while curr != stop_node { + let curr_node = unsafe { &*curr }; + let mut next = curr_node.next.load(Ordering::Acquire); + while next.is_null() { + next = curr_node.next.load(Ordering::Acquire); + } + + if curr_node.key != slot.key || curr_node.op == Operation::Balance { + unsafe { &*prev_alive }.next.store(curr, Ordering::Release); + prev_alive = curr; + } else { + // Shortcircuit curr. + curr_node.ret.set(Some(value)); + curr_node.short_circuit.store(true, Ordering::Release); + } + curr = next; + } + + unsafe { &*prev_alive } + .next + .store(stop_node, Ordering::Release); + } + + /// Merge keys of p and l into one big array (and similarly for nexts). + /// We essentially replace the pointer to l with the contents of l. + fn absorb_child<'l1, 'l2>( + &'l1 self, + child: &'l2 Self, + child_idx: usize, + self_lock: &MCSLockGuard<'l1, K, V>, + child_lock: &MCSLockGuard<'l2, K, V>, + ) -> ( + [AtomicPtr>>; DEGREE * 2], + [Cell>; DEGREE * 2], + ) { + let next: [AtomicPtr>>; DEGREE * 2] = Default::default(); + let keys: [Cell>; DEGREE * 2] = Default::default(); + let psize = self.get_size_locked(self_lock); + let nsize = child.get_size_locked(child_lock); + + atomic_clone(&self.next_slice(self_lock)[0..], &next[0..], child_idx); + atomic_clone( + &child.next_slice(child_lock)[0..], + &next[child_idx..], + nsize, + ); + atomic_clone( + &self.next_slice(self_lock)[child_idx + 1..], + &next[child_idx + nsize..], + psize - (child_idx + 1), + ); + + cell_clone(&self.key_slice(self_lock)[0..], &keys[0..], child_idx); + cell_clone( + &child.key_slice(child_lock)[0..], + &keys[child_idx..], + child.key_count_locked(child_lock), + ); + // Safety: Both `parent` and `child` is locked, so they cannot be reclaimed. + cell_clone( + &self.key_slice(self_lock)[child_idx..], + &keys[child_idx + child.key_count_locked(child_lock)..], + self.key_count_locked(self_lock) - child_idx, + ); + + (next, keys) + } + + /// It requires a lock to guarantee the consistency. + /// Its length is equal to `key_count`. + fn enumerate_key<'g>( + &'g self, + lock: &MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.key_slice(lock) + .iter() + .enumerate() + .filter_map(|(i, k)| k.get().map(|k| (i, k))) + } + + /// Iterates key-value pairs in this **leaf** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node. + fn iter_key_value<'g>( + &'g self, + lock: &'g MCSLockGuard<'g, K, V>, + ) -> impl Iterator + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (k, self.get_value_locked(i, lock).unwrap())) + } + + /// Iterates key-next pairs in this **internal** node. + /// It requires a lock to guarantee the consistency. + /// Its length is equal to the size of this node, and only the last key is `None`. + fn iter_key_next<'g>( + &'g self, + lock: &'g MCSLockGuard<'g, K, V>, + ) -> impl Iterator, *mut Inner)> + 'g { + self.enumerate_key(lock) + .map(|(i, k)| (Some(k), self.load_next_locked(i, lock))) + .chain(once(( + None, + self.load_next_locked(self.key_count_locked(lock), lock), + ))) + } +} + +struct WriteGuard<'g, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + init_version: usize, + node: &'g Node, +} + +impl<'g, K, V> Drop for WriteGuard<'g, K, V> +where + K: Default + Copy, + V: Default + Copy, +{ + fn drop(&mut self) { + self.node + ._write_version + .store(self.init_version + 2, Ordering::Release); + } +} + +struct Cursor +where + K: Default + Copy, + V: Default + Copy, +{ + l: *mut Inner>, + p: *mut Inner>, + gp: *mut Inner>, + /// Index of `p` in `gp`. + gp_p_idx: usize, + /// Index of `l` in `p`. + p_l_idx: usize, + /// Index of the key in `l`. + l_key_idx: usize, + val: Option, +} + +pub struct ElimABTree +where + K: Default + Copy, + V: Default + Copy, +{ + entry: Entry>, +} + +unsafe impl Sync for ElimABTree +where + K: Sync + Default + Copy, + V: Sync + Default + Copy, +{ +} +unsafe impl Send for ElimABTree +where + K: Send + Default + Copy, + V: Send + Default + Copy, +{ +} + +impl ElimABTree +where + K: Ord + Eq + Default + Copy, + V: Default + Copy, +{ + const ABSORB_THRESHOLD: usize = DEGREE; + const UNDERFULL_THRESHOLD: usize = if DEGREE / 4 < 2 { 2 } else { DEGREE / 4 }; + + pub fn new(local: &Local>) -> Self { + loop { + let guard = &local.guard(); + let left = ok_or!( + guard.allocate(|node| unsafe { node.deref().init_for_leaf(true, 0, K::default()) }), + continue + ); + let entry = ok_or!( + guard.allocate(|node| unsafe { + node.deref().init_for_internal(true, 1, K::default()); + node.deref().init_next(0, left.as_raw()); + }), + unsafe { + guard.retire(left); + continue; + } + ); + return Self { + entry: Entry::new(entry), + }; + } + } + + /// Performs a basic search and returns the value associated with the key, + /// or `None` if nothing is found. Unlike other search methods, it does not return + /// any path information, making it slightly faster. + pub fn search_basic(&self, key: &K, local: &Local>) -> Option { + loop { + let guard = &local.guard(); + return ok_or!(self.search_basic_inner(key, guard), continue); + } + } + + fn search_basic_inner(&self, key: &K, guard: &Guard>) -> Result, ()> { + let entry = unsafe { self.entry.load(guard)?.deref() }; + let mut node = unsafe { &*entry.load_next(0, guard)? }; + while !node.is_leaf(guard)? { + let next = node.load_next(node.child_index(key, guard)?, guard)?; + node = unsafe { &*next }; + } + Ok(node.read_consistent(key, guard)?.1) + } + + fn search( + &self, + key: &K, + target: Option<*mut Inner>>, + guard: &mut Guard>, + ) -> (bool, Cursor) { + loop { + if let Ok(result) = self.search_inner(key, target, guard) { + return result; + } + guard.refresh(); + } + } + + fn search_inner( + &self, + key: &K, + target: Option<*mut Inner>>, + guard: &Guard>, + ) -> Result<(bool, Cursor), ()> { + let entry_sh = self.entry.load(guard)?; + let entry = unsafe { entry_sh.deref() }; + let mut cursor = Cursor { + l: entry.load_next(0, guard)?, + p: entry_sh.as_raw(), + gp: null_mut(), + gp_p_idx: 0, + p_l_idx: 0, + l_key_idx: 0, + val: None, + }; + + while !unsafe { &*cursor.l }.is_leaf(guard)? + && target.map(|target| target != cursor.l).unwrap_or(true) + { + let l_node = unsafe { &*cursor.l }; + cursor.gp = cursor.p; + cursor.p = cursor.l; + cursor.gp_p_idx = cursor.p_l_idx; + cursor.p_l_idx = l_node.child_index(key, guard)?; + cursor.l = l_node.load_next(cursor.p_l_idx, guard)?; + } + + if let Some(target) = target { + Ok((cursor.l == target, cursor)) + } else { + let (index, value) = unsafe { &*cursor.l }.read_consistent(key, guard)?; + cursor.val = value; + cursor.l_key_idx = index; + Ok((value.is_some(), cursor)) + } + } + + pub fn insert(&self, key: &K, value: &V, local: &Local>) -> Option { + loop { + let guard = &mut local.guard(); + let (_, cursor) = self.search(key, None, guard); + if let Some(value) = cursor.val { + return Some(value); + } + match self.insert_inner(key, value, &cursor, guard) { + Ok(result) => return result, + Err(_) => continue, + } + } + } + + fn insert_inner<'g>( + &self, + key: &K, + value: &V, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result, ()> { + let node = unsafe { &*cursor.l }; + let parent = unsafe { &*cursor.p }; + + debug_assert!(node.is_leaf(guard)?); + debug_assert!(!parent.is_leaf(guard)?); + + let node_lock_slot = UnsafeCell::new(MCSLockSlot::new()); + let node_lock = match node.acquire(Operation::Insert, Some(*key), &node_lock_slot) { + AcqResult::Acquired(lock) => lock, + AcqResult::Eliminated(value) => return Ok(Some(value)), + }; + guard.validate_epoch()?; + if node.get_marked_locked(&node_lock) { + return Err(()); + } + for i in 0..DEGREE { + if node.get_key_locked(i, &node_lock) == Some(*key) { + return Ok(Some(node.get_value(i, guard)?.unwrap())); + } + } + // At this point, we are guaranteed key is not in the node. + + if node.get_size_locked(&node_lock) < Self::ABSORB_THRESHOLD { + // We have the capacity to fit this new key. So let's just find an empty slot. + for i in 0..DEGREE { + if node.get_key_locked(i, &node_lock).is_some() { + continue; + } + let wguard = node.start_write(&node_lock); + node.set_key(i, Some(*key), &wguard); + node.set_value(i, *value, &wguard); + node.set_size_locked(node.get_size_locked(&node_lock) + 1, &node_lock); + + node.elim_key_ops(*value, wguard, &node_lock); + + drop(node_lock); + return Ok(None); + } + unreachable!("Should never happen"); + } else { + // We do not have a room for this key. We need to make new nodes. + try_acq_val_or!( + parent, + parent_lock, + Operation::Insert, + None, + guard, + return Err(()), + return Err(()) + ); + + let mut kv_pairs = node + .iter_key_value(&node_lock) + .chain(once((*key, *value))) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + + // Create new node(s). + // Since the new arrays are too big to fit in a single node, + // we replace `l` by a new subtree containing three new nodes: a parent, and two leaves. + // The array contents are then split between the two new leaves. + + let left_size = kv_pairs.len() / 2; + let right_size = DEGREE + 1 - left_size; + + let left = guard.allocate(|left| unsafe { + left.deref().init_for_leaf(true, left_size, kv_pairs[0].0); + for i in 0..left_size { + left.deref().init_key(i, Some(kv_pairs[i].0)); + left.deref().init_value(i, kv_pairs[i].1); + } + })?; + + let Ok(right) = guard.allocate(|right| unsafe { + right + .deref() + .init_for_leaf(true, right_size, kv_pairs[left_size].0); + for i in 0..right_size { + right.deref().init_key(i, Some(kv_pairs[i + left_size].0)); + right.deref().init_value(i, kv_pairs[i + left_size].1); + } + }) else { + unsafe { guard.retire(left) }; + return Err(()); + }; + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time as this Overflow if `n` will become the root. + let Ok(internal) = guard.allocate(|internal| unsafe { + internal.deref().init_for_internal( + eq(parent, self.entry.load_raw()), + 2, + kv_pairs[left_size].0, + ); + internal.deref().init_key(0, Some(kv_pairs[left_size].0)); + internal.deref().init_next(0, left.as_raw()); + internal.deref().init_next(1, right.as_raw()); + }) else { + unsafe { guard.retire(left) }; + unsafe { guard.retire(right) }; + return Err(()); + }; + + // If the parent is not marked, `parent.next[cursor.p_l_idx]` is guaranteed to contain + // a node since any update to parent would have deleted node (and hence we would have + // returned at the `node.marked` check). + parent.store_next(cursor.p_l_idx, internal.as_raw(), &parent_lock); + node.set_marked_locked(true, &node_lock); + + // Manually unlock and fix the tag. + drop((parent_lock, node_lock)); + unsafe { guard.retire_raw(cursor.l) }; + self.fix_tag_violation(kv_pairs[left_size].0, internal.as_raw(), guard); + + Ok(None) + } + } + + fn fix_tag_violation( + &self, + search_key: K, + viol: *mut Inner>, + guard: &mut Guard>, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + guard.refresh(); + let (found, cursor) = self.search(&search_key, Some(viol), guard); + if !found || cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let Ok((success, recur)) = self.fix_tag_violation_inner(&cursor, guard) else { + stack.push((search_key, viol)); + continue; + }; + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_tag_violation_inner( + &self, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result<(bool, Option<(K, *mut Inner>)>), ()> { + let viol = cursor.l; + let viol_node = unsafe { &*viol }; + if viol_node.weight(guard)? { + return Ok((true, None)); + } + + // `viol` should be internal because leaves always have weight = 1. + debug_assert!(!viol_node.is_leaf(guard)?); + // `viol` is not the entry or root node because both should always have weight = 1. + debug_assert!( + !eq(viol_node, self.entry.load_raw()) + && unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? != viol + ); + + debug_assert!(!cursor.gp.is_null()); + let node = viol_node; + let parent = unsafe { &*cursor.p }; + let gparent = unsafe { &*cursor.gp }; + debug_assert!(!node.is_leaf(guard)?); + debug_assert!(!parent.is_leaf(guard)?); + debug_assert!(!gparent.is_leaf(guard)?); + + // We cannot apply this update if p has a weight violation. + // So, we check if this is the case, and, if so, try to fix it. + if !parent.weight(guard)? { + return Ok((false, Some((parent.search_key(guard)?, cursor.p)))); + } + + try_acq_val_or!( + node, + node_lock, + Operation::Balance, + None, + guard, + return Ok((false, None)), + return Err(()) + ); + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + guard, + return Ok((false, None)), + return Err(()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + guard, + return Ok((false, None)), + return Err(()) + ); + + let psize = parent.get_size_locked(&parent_lock); + let nsize = viol_node.get_size_locked(&node_lock); + // We don't ever change the size of a tag node, so its size should always be 2. + debug_assert_eq!(nsize, 2); + let c = psize + nsize; + let size = c - 1; + let (next, keys) = parent.absorb_child(node, cursor.p_l_idx, &parent_lock, &node_lock); + + if size <= Self::ABSORB_THRESHOLD { + // Absorb case. + + // Create new node(s). + // The new arrays are small enough to fit in a single node, + // so we replace p by a new internal node. + let absorber = guard.allocate(|absorber| unsafe { + absorber.deref().init_for_internal( + true, + size, + parent.get_key_locked(0, &parent_lock).unwrap(), + ); + atomic_clone(&next, &absorber.deref().next_slice_unchecked(), DEGREE); + cell_clone(&keys, &absorber.deref().key_slice_unchecked(), DEGREE); + })?; + + gparent.store_next(cursor.gp_p_idx, absorber.as_raw(), &gparent_lock); + node.set_marked_locked(true, &node_lock); + parent.set_marked_locked(true, &parent_lock); + + unsafe { guard.retire_raw(cursor.l) }; + unsafe { guard.retire_raw(cursor.p) }; + return Ok((true, None)); + } else { + // Split case. + + // The new arrays are too big to fit in a single node, + // so we replace p by a new internal node and two new children. + // + // We take the big merged array and split it into two arrays, + // which are used to create two new children u and v. + // we then create a new internal node (whose weight will be zero + // if it is not the root), with u and v as its children. + + // Create new node(s). + let left_size = size / 2; + let left = guard.allocate(|left| unsafe { + left.deref() + .init_for_internal(true, left_size, keys[0].get().unwrap()); + cell_clone(&keys, &left.deref().key_slice_unchecked(), left_size - 1); + atomic_clone(&next, &left.deref().next_slice_unchecked(), left_size); + })?; + + let right_size = size - left_size; + let Ok(right) = guard.allocate(|right| unsafe { + right + .deref() + .init_for_internal(true, right_size, keys[left_size].get().unwrap()); + cell_clone( + &keys[left_size..], + &right.deref().key_slice_unchecked()[0..], + right_size - 1, + ); + atomic_clone( + &next[left_size..], + &right.deref().next_slice_unchecked()[0..], + right_size, + ); + }) else { + unsafe { guard.retire(left) }; + return Err(()); + }; + + // Note: keys[left_size - 1] should be the same as new_internal.keys[0]. + let Ok(new_internal) = guard.allocate(|new_internal| unsafe { + new_internal.deref().init_for_internal( + eq(gparent, self.entry.load_raw()), + 2, + keys[left_size - 1].get().unwrap(), + ); + new_internal.deref().init_key(0, keys[left_size - 1].get()); + new_internal.deref().init_next(0, left.as_raw()); + new_internal.deref().init_next(1, right.as_raw()); + }) else { + unsafe { guard.retire(left) }; + unsafe { guard.retire(right) }; + return Err(()); + }; + + // The weight of new internal node `n` will be zero, unless it is the root. + // This is because we test `p == entry`, above; in doing this, we are actually + // performing Root-Zero at the same time + // as this Overflow if `n` will become the root. + + gparent.store_next(cursor.gp_p_idx, new_internal.as_raw(), &gparent_lock); + node.set_marked_locked(true, &node_lock); + parent.set_marked_locked(true, &parent_lock); + + unsafe { guard.retire_raw(cursor.l) }; + unsafe { guard.retire_raw(cursor.p) }; + + drop((node_lock, parent_lock, gparent_lock)); + return Ok(( + true, + Some((keys[left_size - 1].get().unwrap(), new_internal.as_raw())), + )); + } + } + + pub fn remove(&self, key: &K, local: &Local>) -> Option { + loop { + let guard = &mut local.guard(); + let (_, cursor) = self.search(key, None, guard); + if cursor.val.is_none() { + return None; + } + match self.remove_inner(key, &cursor, guard) { + Ok(result) => return result, + Err(()) => continue, + } + } + } + + fn remove_inner( + &self, + key: &K, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result, ()> { + let node = unsafe { &*cursor.l }; + let parent = unsafe { &*cursor.p }; + let gparent = unsafe { cursor.gp.as_ref() }; + + debug_assert!(node.is_leaf(guard)?); + debug_assert!(!parent.is_leaf(guard)?); + debug_assert!(if let Some(gp) = gparent { + !gp.is_leaf(guard)? + } else { + true + }); + + try_acq_val_or!( + node, + node_lock, + Operation::Delete, + Some(*key), + guard, + return Err(()), + return Err(()) + ); + // Bug Fix: Added a check to ensure the node size is greater than 0. + // This prevents underflows caused by decrementing the size value. + // This check is not present in the original code. + if node.get_size_locked(&node_lock) == 0 { + return Err(()); + } + + let new_size = node.get_size_locked(&node_lock) - 1; + for i in 0..DEGREE { + if node.get_key_locked(i, &node_lock) == Some(*key) { + let val = node.get_value_locked(i, &node_lock).unwrap(); + let wguard = node.start_write(&node_lock); + node.set_key(i, None, &wguard); + node.set_size_locked(new_size, &node_lock); + + node.elim_key_ops(val, wguard, &node_lock); + + if new_size == Self::UNDERFULL_THRESHOLD - 1 { + let search_key = node.search_key_locked(&node_lock); + drop(node_lock); + self.fix_underfull_violation(search_key, cursor.l, guard); + } + return Ok(Some(val)); + } + } + Err(()) + } + + fn fix_underfull_violation( + &self, + search_key: K, + viol: *mut Inner>, + guard: &mut Guard>, + ) { + let mut stack = vec![(search_key, viol)]; + while let Some((search_key, viol)) = stack.pop() { + // We search for `viol` and try to fix any violation we find there. + // This entails performing AbsorbSibling or Distribute. + guard.refresh(); + let (_, cursor) = self.search(&search_key, Some(viol), guard); + if cursor.l != viol { + // `viol` was replaced by another update. + // We hand over responsibility for `viol` to that update. + continue; + } + let Ok((success, recur)) = self.fix_underfull_violation_inner(&cursor, guard) else { + stack.push((search_key, viol)); + continue; + }; + if !success { + stack.push((search_key, viol)); + } + stack.extend(recur); + } + } + + fn fix_underfull_violation_inner( + &self, + cursor: &Cursor, + guard: &mut Guard>, + ) -> Result<(bool, ArrayVec<(K, *mut Inner>), 2>), ()> { + let viol = cursor.l; + let viol_node = unsafe { &*viol }; + + // We do not need a lock for the `viol == entry.ptrs[0]` check since since we cannot + // "be turned into" the root. The root is only created by the root absorb + // operation below, so a node that is not the root will never become the root. + if viol_node.get_size(guard)? >= Self::UNDERFULL_THRESHOLD + || eq(viol_node, self.entry.load_raw()) + || viol == unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? + { + // No degree violation at `viol`. + return Ok((true, ArrayVec::<_, 2>::new())); + } + + let node = viol_node; + let parent = unsafe { &*cursor.p }; + // `gp` cannot be null, because if AbsorbSibling or Distribute can be applied, + // then `p` is not the root. + debug_assert!(!cursor.gp.is_null()); + let gparent = unsafe { &*cursor.gp }; + + if parent.get_size(guard)? < Self::UNDERFULL_THRESHOLD + && !eq(parent, self.entry.load_raw()) + && cursor.p != unsafe { self.entry.load(guard)?.deref() }.load_next(0, guard)? + { + return Ok(( + false, + ArrayVec::from_iter(once((parent.search_key(guard)?, cursor.p))), + )); + } + + let sibling_idx = if cursor.p_l_idx > 0 { + cursor.p_l_idx - 1 + } else { + 1 + }; + // Don't need a lock on parent here because if the pointer to sibling changes + // to a different node after this, sibling will be marked + // (Invariant: when a pointer switches away from a node, the node is marked) + let sibling_sh = parent.load_next(sibling_idx, guard)?; + let sibling = unsafe { &*sibling_sh }; + + // Prevent deadlocks by acquiring left node first. + let ((left, left_idx), (right, right_idx)) = if sibling_idx < cursor.p_l_idx { + ((sibling, sibling_idx), (node, cursor.p_l_idx)) + } else { + ((node, cursor.p_l_idx), (sibling, sibling_idx)) + }; + + try_acq_val_or!( + left, + left_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + try_acq_val_or!( + right, + right_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + + // Repeat this check, this might have changed while we locked `viol`. + // Safety: `viol_node` is locked by either `left_lock` or `right_lock`. + if unsafe { viol_node.get_size_unchecked() } >= Self::UNDERFULL_THRESHOLD { + // No degree violation at `viol`. + return Ok((true, ArrayVec::new())); + } + + try_acq_val_or!( + parent, + parent_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + try_acq_val_or!( + gparent, + gparent_lock, + Operation::Balance, + None, + guard, + return Ok((false, ArrayVec::new())), + return Err(()) + ); + + // We can only apply AbsorbSibling or Distribute if there are no + // weight violations at `parent`, `node`, or `sibling`. + // So, we first check for any weight violations and fix any that we see. + if !parent.weight_locked(&parent_lock) { + let search_key = parent.search_key_locked(&parent_lock); + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(search_key, cursor.p, guard); + return Ok((false, ArrayVec::new())); + } + if !left.weight_locked(&left_lock) { + let search_key = left.search_key_locked(&left_lock); + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(search_key, left as *const _ as *mut _, guard); + return Ok((false, ArrayVec::new())); + } + if !right.weight_locked(&right_lock) { + let search_key = right.search_key_locked(&right_lock); + drop((left_lock, right_lock, parent_lock, gparent_lock)); + self.fix_tag_violation(search_key, right as *const _ as *mut _, guard); + return Ok((false, ArrayVec::new())); + } + + // There are no weight violations at `parent`, `node` or `sibling`. + debug_assert!( + parent.weight_locked(&parent_lock) + && left.weight_locked(&left_lock) + && right.weight_locked(&right_lock) + ); + // l and s are either both leaves or both internal nodes, + // because there are no weight violations at these nodes. + debug_assert!( + (left.is_leaf_locked(&left_lock) && right.is_leaf_locked(&right_lock)) + || (!left.is_leaf_locked(&left_lock) && !right.is_leaf_locked(&right_lock)) + ); + + let lsize = left.get_size_locked(&left_lock); + let rsize = right.get_size_locked(&right_lock); + let psize = parent.get_size_locked(&parent_lock); + let size = lsize + rsize; + + if size < 2 * Self::UNDERFULL_THRESHOLD { + // AbsorbSibling + let new_node = guard.allocate(|new_node| unsafe { + if left.is_leaf_locked(&left_lock) { + debug_assert!(right.is_leaf_locked(&right_lock)); + let new_leaf = new_node.deref(); + // Safety: `node` is locked. + new_leaf.init_for_leaf(true, size, node.search_key_unchecked()); + let kv_iter = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .enumerate(); + for (i, (key, value)) in kv_iter { + new_leaf.init_key(i, Some(key)); + new_leaf.init_value(i, value); + } + } else { + debug_assert!(!right.is_leaf_locked(&right_lock)); + let new_internal = new_node.deref(); + // Safety: `node` is locked. + new_internal.init_for_internal(true, size, node.search_key_unchecked()); + let key_btw = parent.get_key_locked(left_idx, &parent_lock).unwrap(); + let kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)) + .enumerate(); + for (i, (key, next)) in kn_iter { + new_internal.init_key(i, key); + new_internal.init_next(i, next); + } + } + })?; + + // Now, we atomically replace `p` and its children with the new nodes. + // If appropriate, we perform RootAbsorb at the same time. + if eq(gparent, self.entry.load_raw()) && psize == 2 { + debug_assert!(cursor.gp_p_idx == 0); + gparent.store_next(cursor.gp_p_idx, new_node.as_raw(), &gparent_lock); + left.set_marked_locked(true, &left_lock); + parent.set_marked_locked(true, &parent_lock); + right.set_marked_locked(true, &right_lock); + + unsafe { + guard.retire_raw(cursor.l); + guard.retire_raw(cursor.p); + guard.retire_raw(sibling_sh); + } + + // Safety: `node` is locked. + let search_key = unsafe { node.search_key_unchecked() }; + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return Ok(( + true, + ArrayVec::from_iter(once((search_key, new_node.as_raw()))), + )); + } else { + debug_assert!(!eq(gparent, self.entry.load_raw()) || psize > 2); + let Ok(new_parent) = guard.allocate(|new_parent| unsafe { + let new_parent = new_parent.deref(); + new_parent.init_for_internal( + true, + psize - 1, + parent.search_key_locked(&parent_lock), + ); + + for i in 0..left_idx { + new_parent.init_key(i, parent.get_key_locked(i, &parent_lock)); + } + for i in 0..sibling_idx { + new_parent.init_next(i, parent.load_next_locked(i, &parent_lock)); + } + for i in left_idx + 1..parent.key_count_locked(&parent_lock) { + new_parent.init_key(i - 1, parent.get_key_locked(i, &parent_lock)); + } + for i in cursor.p_l_idx + 1..psize { + new_parent.init_next(i - 1, parent.load_next_locked(i, &parent_lock)); + } + + new_parent.init_next( + cursor.p_l_idx - (if cursor.p_l_idx > sibling_idx { 1 } else { 0 }), + new_node.as_raw(), + ); + }) else { + unsafe { guard.retire(new_node) }; + return Err(()); + }; + + gparent.store_next(cursor.gp_p_idx, new_parent.as_raw(), &gparent_lock); + left.set_marked_locked(true, &left_lock); + parent.set_marked_locked(true, &parent_lock); + right.set_marked_locked(true, &right_lock); + + unsafe { + guard.retire_raw(cursor.l); + guard.retire_raw(cursor.p); + guard.retire_raw(sibling_sh); + } + + // Safety: `node` is locked. + let node_key = unsafe { node.search_key_unchecked() }; + let parent_key = parent.search_key_locked(&parent_lock); + drop((left_lock, right_lock, parent_lock, gparent_lock)); + return Ok(( + true, + ArrayVec::from_iter( + [ + (node_key, new_node.as_raw()), + (parent_key, new_parent.as_raw()), + ] + .into_iter(), + ), + )); + } + } else { + // Distribute + let left_size = size / 2; + let right_size = size - left_size; + + assert!(left.is_leaf_locked(&left_lock) == right.is_leaf_locked(&right_lock)); + + // `pivot`: Reserve one key for the parent + // (to go between `new_left` and `new_right`). + let (new_left, new_right, pivot) = if left.is_leaf_locked(&left_lock) { + // Combine the contents of `l` and `s`. + let mut kv_pairs = left + .iter_key_value(&left_lock) + .chain(right.iter_key_value(&right_lock)) + .collect::>(); + kv_pairs.sort_by_key(|(k, _)| *k); + let mut kv_iter = kv_pairs.iter().copied(); + + let new_left = guard.allocate(|new_leaf| unsafe { + let new_leaf = new_leaf.deref(); + new_leaf.init_for_leaf(true, left_size, Default::default()); + for i in 0..left_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.init_search_key(new_leaf.get_key_unchecked(0).unwrap()); + })?; + + let Ok(new_right) = guard.allocate(|new_leaf| unsafe { + debug_assert!(left.is_leaf_locked(&left_lock)); + let new_leaf = new_leaf.deref(); + new_leaf.init_for_leaf(true, right_size, Default::default()); + for i in 0..right_size { + let (k, v) = kv_iter.next().unwrap(); + new_leaf.init_key(i, Some(k)); + new_leaf.init_value(i, v); + } + new_leaf.init_search_key(new_leaf.get_key_unchecked(0).unwrap()); + }) else { + unsafe { guard.retire(new_left) }; + return Err(()); + }; + let pivot = unsafe { new_right.deref().search_key_unchecked() }; + + debug_assert!(kv_iter.next().is_none()); + (new_left, new_right, pivot) + } else { + // Combine the contents of `l` and `s` + // (and one key from `p` if `l` and `s` are internal). + let key_btw = parent.get_key_locked(left_idx, &parent_lock).unwrap(); + let mut kn_iter = left + .iter_key_next(&left_lock) + .map(|(k, n)| (Some(k.unwrap_or(key_btw)), n)) + .chain(right.iter_key_next(&right_lock)); + + let new_left = guard.allocate(|new_internal| unsafe { + let new_internal = new_internal.deref(); + new_internal.init_for_internal(true, left_size, Default::default()); + for i in 0..left_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.init_search_key(new_internal.get_key_unchecked(0).unwrap()); + })?; + let pivot = unsafe { new_left.deref().get_key_unchecked(left_size - 1) } + .take() + .unwrap(); + + let Ok(new_right) = guard.allocate(|new_internal| unsafe { + let new_internal = new_internal.deref(); + new_internal.init_for_internal(true, right_size, Default::default()); + for i in 0..right_size { + let (k, n) = kn_iter.next().unwrap(); + new_internal.init_key(i, k); + new_internal.init_next(i, n); + } + new_internal.init_search_key(new_internal.get_key_unchecked(0).unwrap()); + }) else { + unsafe { guard.retire(new_left) }; + return Err(()); + }; + + debug_assert!(kn_iter.next().is_none()); + (new_left, new_right, pivot) + }; + + let Ok(new_parent) = guard.allocate(|new_parent| unsafe { + let new_parent = new_parent.deref(); + new_parent.init_for_internal( + parent.weight_locked(&parent_lock), + psize, + parent.search_key_locked(&parent_lock), + ); + cell_clone( + &parent.key_slice(&parent_lock)[0..], + &new_parent.key_slice_unchecked()[0..], + parent.key_count_locked(&parent_lock), + ); + atomic_clone( + &parent.next_slice(&parent_lock)[0..], + &new_parent.next_slice_unchecked()[0..], + psize, + ); + new_parent.init_next(left_idx, new_left.as_raw()); + new_parent.init_next(right_idx, new_right.as_raw()); + new_parent.init_key(left_idx, Some(pivot)); + }) else { + unsafe { guard.retire(new_left) }; + unsafe { guard.retire(new_right) }; + return Err(()); + }; + + gparent.store_next(cursor.gp_p_idx, new_parent.as_raw(), &gparent_lock); + left.set_marked_locked(true, &left_lock); + parent.set_marked_locked(true, &parent_lock); + right.set_marked_locked(true, &right_lock); + + unsafe { + guard.retire_raw(cursor.l); + guard.retire_raw(cursor.p); + guard.retire_raw(sibling_sh); + } + + return Ok((true, ArrayVec::new())); + } + } +} + +/// Similar to `memcpy`, but for `Cell` types. +#[inline] +fn cell_clone(src: &[Cell], dst: &[Cell], len: usize) { + for i in 0..len { + dst[i].set(src[i].get()); + } +} + +/// Similar to `memcpy`, but for `AtomicPtr` types. +#[inline] +fn atomic_clone(src: &[AtomicPtr], dst: &[AtomicPtr], len: usize) { + for i in 0..len { + dst[i].store(src[i].load(Ordering::Relaxed), Ordering::Relaxed); + } +} + +impl ConcurrentMap for ElimABTree +where + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, +{ + type Global = Global>; + type Local = Local>; + + fn global(key_range_hint: usize) -> Self::Global { + Global::new(key_range_hint) + } + + fn local(global: &Self::Global) -> Self::Local { + Local::new(global) + } + + fn new(local: &Self::Local) -> Self { + ElimABTree::new(local) + } + + #[inline(always)] + fn get(&self, key: &K, local: &Self::Local) -> Option { + self.search_basic(key, local) + } + + #[inline(always)] + fn insert(&self, key: K, value: V, local: &Self::Local) -> bool { + self.insert(&key, &value, local).is_none() + } + + #[inline(always)] + fn remove(&self, key: &K, local: &Self::Local) -> Option { + self.remove(key, local) + } +} + +#[cfg(test)] +mod tests { + use super::ElimABTree; + use crate::ds_impl::vbr::concurrent_map; + + #[test] + fn smoke_elim_ab_tree() { + concurrent_map::tests::smoke::>(); + } +} diff --git a/src/ds_impl/vbr/list.rs b/src/ds_impl/vbr/list.rs index d7447d68..199a8864 100644 --- a/src/ds_impl/vbr/list.rs +++ b/src/ds_impl/vbr/list.rs @@ -7,8 +7,8 @@ use std::sync::atomic::Ordering; pub struct Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { /// Mark: tag(), Tag: not needed next: MutAtomic>, @@ -16,18 +16,32 @@ where value: ImmAtomic, } +impl Default for Node +where + K: 'static + Copy + Default + Default, + V: 'static + Copy + Default + Default, +{ + fn default() -> Self { + Self { + next: MutAtomic::null(), + key: ImmAtomic::new(Default::default()), + value: ImmAtomic::new(Default::default()), + } + } +} + struct List where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { head: Entry>, } struct Cursor<'g, K, V> where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { prev: Shared<'g, Node>, // Tag of `curr` should always be zero so when `curr` is stored in a `prev`, we don't store a @@ -37,8 +51,8 @@ where impl<'g, K, V> Cursor<'g, K, V> where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { /// Creates the head cursor. #[inline] @@ -57,8 +71,8 @@ where impl List where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { /// Creates a new list. #[inline] @@ -417,16 +431,16 @@ where pub struct HList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { inner: List, } impl ConcurrentMap for HList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; @@ -461,16 +475,16 @@ where pub struct HMList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { inner: List, } impl ConcurrentMap for HMList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; @@ -505,16 +519,16 @@ where pub struct HHSList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { inner: List, } impl HHSList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { /// Pop the first element efficiently. /// This method is used for only the fine grained benchmark (src/bin/long_running). @@ -525,8 +539,8 @@ where impl ConcurrentMap for HHSList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; diff --git a/src/ds_impl/vbr/michael_hash_map.rs b/src/ds_impl/vbr/michael_hash_map.rs index b984d860..e3918b21 100644 --- a/src/ds_impl/vbr/michael_hash_map.rs +++ b/src/ds_impl/vbr/michael_hash_map.rs @@ -8,16 +8,16 @@ use super::list::{HHSList, Node}; pub struct HashMap where - K: 'static + Ord + Hash + Copy, - V: 'static + Copy, + K: 'static + Ord + Hash + Copy + Default, + V: 'static + Copy + Default, { buckets: Vec>, } impl HashMap where - K: 'static + Ord + Hash + Copy, - V: 'static + Copy, + K: 'static + Ord + Hash + Copy + Default, + V: 'static + Copy + Default, { pub fn with_capacity(n: usize, local: &Local>) -> Self { let mut buckets = Vec::with_capacity(n); @@ -59,8 +59,8 @@ where impl ConcurrentMap for HashMap where - K: 'static + Ord + Hash + Copy, - V: 'static + Copy, + K: 'static + Ord + Hash + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; diff --git a/src/ds_impl/vbr/mod.rs b/src/ds_impl/vbr/mod.rs index 37f66d63..7bfded52 100644 --- a/src/ds_impl/vbr/mod.rs +++ b/src/ds_impl/vbr/mod.rs @@ -1,5 +1,6 @@ pub mod concurrent_map; +pub mod elim_ab_tree; pub mod list; pub mod michael_hash_map; pub mod natarajan_mittal_tree; @@ -7,6 +8,7 @@ pub mod skip_list; pub use self::concurrent_map::ConcurrentMap; +pub use elim_ab_tree::ElimABTree; pub use list::{HHSList, HList, HMList}; pub use michael_hash_map::HashMap; pub use natarajan_mittal_tree::NMTreeMap; diff --git a/src/ds_impl/vbr/natarajan_mittal_tree.rs b/src/ds_impl/vbr/natarajan_mittal_tree.rs index 863d6db8..03774415 100644 --- a/src/ds_impl/vbr/natarajan_mittal_tree.rs +++ b/src/ds_impl/vbr/natarajan_mittal_tree.rs @@ -34,8 +34,8 @@ impl Marks { pub struct Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { key: ImmAtomic, value: ImmAtomic, @@ -43,10 +43,25 @@ where right: MutAtomic>, } +impl Default for Node +where + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, +{ + fn default() -> Self { + Self { + key: ImmAtomic::new(Default::default()), + value: ImmAtomic::new(Default::default()), + left: MutAtomic::null(), + right: MutAtomic::null(), + } + } +} + impl Node where - K: 'static + Copy + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, { fn new_leaf<'g>( key: K, @@ -90,8 +105,8 @@ enum Direction { /// All of the edges of path from `successor` to `parent` are in the process of removal. struct SeekRecord<'g, K, V> where - K: 'static + Copy + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, { /// Parent of `successor` ancestor: Shared<'g, Node>, @@ -109,8 +124,8 @@ where impl<'g, K, V> SeekRecord<'g, K, V> where - K: 'static + Copy + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Bounded, + V: 'static + Copy + Default, { fn successor_addr(&self) -> &MutAtomic> { match self.successor_dir { @@ -136,16 +151,16 @@ where pub struct NMTreeMap where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { r: Entry>, } impl NMTreeMap where - K: 'static + Copy + Ord + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Ord + Bounded, + V: 'static + Copy + Default, { pub fn new(local: &Local>) -> Self { // An empty tree has 5 default nodes with infinite keys so that the SeekRecord is allways @@ -498,8 +513,8 @@ where impl ConcurrentMap for NMTreeMap where - K: 'static + Copy + Ord + Bounded, - V: 'static + Copy, + K: 'static + Copy + Default + Ord + Bounded, + V: 'static + Copy + Default, { type Global = Global>; diff --git a/src/ds_impl/vbr/skip_list.rs b/src/ds_impl/vbr/skip_list.rs index 32a1ac34..cfca91a1 100644 --- a/src/ds_impl/vbr/skip_list.rs +++ b/src/ds_impl/vbr/skip_list.rs @@ -14,8 +14,8 @@ type Tower = [MutAtomic>; MAX_HEIGHT]; pub struct Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { key: ImmAtomic, value: ImmAtomic, @@ -24,6 +24,22 @@ where refs: AtomicUsize, } +impl Default for Node +where + K: 'static + Copy + Default, + V: 'static + Copy + Default, +{ + fn default() -> Self { + Self { + key: ImmAtomic::new(Default::default()), + value: ImmAtomic::new(Default::default()), + next: unsafe { zeroed() }, + height: ImmAtomic::new(0), + refs: AtomicUsize::new(0), + } + } +} + fn generate_height() -> usize { // returns 1 with probability 3/4 if rand::random::() % 4 < 3 { @@ -39,8 +55,8 @@ fn generate_height() -> usize { impl Node where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { pub fn decrement(ptr: Shared>, guard: &Guard>) { let prev = unsafe { ptr.deref() }.refs.fetch_sub(1, Ordering::SeqCst); @@ -92,8 +108,8 @@ where struct Cursor<'g, K, V> where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { found: Option>>, preds: [Shared<'g, Node>; MAX_HEIGHT], @@ -102,8 +118,8 @@ where impl<'g, K, V> Cursor<'g, K, V> where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { fn new(head: Shared<'g, Node>) -> Self { Self { @@ -116,16 +132,16 @@ where pub struct SkipList where - K: 'static + Copy, - V: 'static + Copy, + K: 'static + Copy + Default, + V: 'static + Copy + Default, { head: Entry>, } impl SkipList where - K: 'static + Copy + Ord, - V: 'static + Copy, + K: 'static + Copy + Default + Ord, + V: 'static + Copy + Default, { pub fn new(local: &Local>) -> Self { let guard = &local.guard(); @@ -392,8 +408,8 @@ where impl ConcurrentMap for SkipList where - K: 'static + Ord + Copy, - V: 'static + Copy, + K: 'static + Ord + Copy + Default, + V: 'static + Copy + Default, { type Global = Global>; type Local = Local>; diff --git a/test-scripts/sanitize-elim.sh b/test-scripts/sanitize-elim.sh new file mode 100755 index 00000000..d5ec5153 --- /dev/null +++ b/test-scripts/sanitize-elim.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' + +run="cargo run --bin hp --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " + +set -e +for i in {1..5000}; do + $run -delim-ab-tree -i3 -t256 -r100000 -g1 +done diff --git a/test-scripts/sanitize-hp.sh b/test-scripts/sanitize-hp.sh new file mode 100755 index 00000000..6f70fc75 --- /dev/null +++ b/test-scripts/sanitize-hp.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +export RUST_BACKTRACE=1 RUSTFLAGS='-Z sanitizer=address' + +hps="cargo run --bin hp --profile=release-simple --target x86_64-unknown-linux-gnu --features sanitize -- " + +set -e +for i in {1..5000}; do + $hps -dh-list -i3 -t256 -r10 -g1 + $hps -dnm-tree -i3 -t256 -r10 -g1 +done