perf: keep a skewed pointer in HP (#4095)

Keeping the `HP` as _always skewed_ allows us to eliminate the subtraction on each individual allocation, at the expense of (un)skewing in a few seldom operations (which possibly can also be transitioned to skewed pointers sometime). Removing an instruction from an already compact _hot path_ has profound effects, as already has been demonstrated by #4064. This PR realises one of the suggestions from there.

The runtime savings amount to 1.5% on allocation-heavy benchmarks, and 0.3% on average, which is rather nice.

_Note_: Also included are a few `u32` -> `usize` changes, which may come handy in the 64-bit world.

rts/motoko-rts-tests/src/gc.rs

@@ -444,7 +444,7 @@ fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx],
 impl GC {
     #[non_incremental_gc]
     fn run(&self, heap: &mut MotokoHeap, _round: usize) -> bool {
-        let heap_base = heap.heap_base_address() as u32;
+        let heap_base = heap.heap_base_address();
         let static_roots = Value::from_ptr(heap.static_root_array_address());
         let continuation_table_ptr_address = heap.continuation_table_ptr_address() as *mut Value;
 
@@ -505,7 +505,7 @@ impl GC {
                 };
                 unsafe {
                     REMEMBERED_SET = Some(RememberedSet::new(heap));
-                    LAST_HP = heap_1.last_ptr_address() as u32;
+                    LAST_HP = heap_1.last_ptr_address();
 
                     let limits = motoko_rts::gc::generational::Limits {
                         base: heap_base as usize,

rts/motoko-rts/src/barriers.rs

@@ -9,7 +9,7 @@ pub unsafe fn init_with_barrier<M: Memory>(_mem: &mut M, location: *mut Value, v
 #[non_incremental_gc]
 pub unsafe fn init_with_barrier<M: Memory>(mem: &mut M, location: *mut Value, value: Value) {
     *location = value;
-    crate::gc::generational::write_barrier::post_write_barrier(mem, location as u32);
+    crate::gc::generational::write_barrier::post_write_barrier(mem, location as usize);
 }
 
 #[incremental_gc]
@@ -20,7 +20,7 @@ pub unsafe fn write_with_barrier<M: Memory>(mem: &mut M, location: *mut Value, v
 #[non_incremental_gc]
 pub unsafe fn write_with_barrier<M: Memory>(mem: &mut M, location: *mut Value, value: Value) {
     *location = value;
-    crate::gc::generational::write_barrier::post_write_barrier(mem, location as u32);
+    crate::gc::generational::write_barrier::post_write_barrier(mem, location as usize);
 }
 
 #[incremental_gc]

rts/motoko-rts/src/gc.rs

@@ -12,15 +12,15 @@ use motoko_rts_macros::*;
 #[cfg(feature = "ic")]
 #[non_incremental_gc]
 unsafe fn should_do_gc(max_live: crate::types::Bytes<u64>) -> bool {
-    use crate::memory::ic::linear_memory::{getHP, LAST_HP};
+    use crate::memory::ic::linear_memory::{get_hp_unskewed, LAST_HP};
 
     // A factor of last heap size. We allow at most this much allocation before doing GC.
     const HEAP_GROWTH_FACTOR: f64 = 1.5;
 
     let heap_limit = core::cmp::min(
-        (f64::from(LAST_HP) * HEAP_GROWTH_FACTOR) as u64,
-        (u64::from(LAST_HP) + max_live.0) / 2,
+        (f64::from(LAST_HP as u32) * HEAP_GROWTH_FACTOR) as u64,
+        (u64::from(LAST_HP as u32) + max_live.0) / 2,
     );
 
-    u64::from(getHP()) >= heap_limit
+    u64::from(get_hp_unskewed() as u32) >= heap_limit
 }

rts/motoko-rts/src/gc/copying.rs

@@ -31,9 +31,9 @@ unsafe fn copying_gc<M: Memory>(mem: &mut M) {
         mem,
         ic::get_aligned_heap_base(),
         // get_hp
-        || linear_memory::getHP() as usize,
+        || linear_memory::get_hp_unskewed(),
         // set_hp
-        |hp| linear_memory::setHP(hp),
+        |hp| linear_memory::set_hp_unskewed(hp),
         ic::get_static_roots(),
         crate::continuation_table::continuation_table_loc(),
         // note_live_size
@@ -42,18 +42,18 @@ unsafe fn copying_gc<M: Memory>(mem: &mut M) {
         |reclaimed| linear_memory::RECLAIMED += Bytes(u64::from(reclaimed.as_u32())),
     );
 
-    linear_memory::LAST_HP = linear_memory::getHP();
+    linear_memory::LAST_HP = linear_memory::get_hp_unskewed();
 }
 
 pub unsafe fn copying_gc_internal<
     M: Memory,
     GetHp: Fn() -> usize,
-    SetHp: FnMut(u32),
+    SetHp: FnMut(usize),
     NoteLiveSize: Fn(Bytes<u32>),
     NoteReclaimed: Fn(Bytes<u32>),
 >(
     mem: &mut M,
-    heap_base: u32,
+    heap_base: usize,
     get_hp: GetHp,
     mut set_hp: SetHp,
     static_roots: Value,
@@ -105,7 +105,7 @@ pub unsafe fn copying_gc_internal<
 
     // Reset the heap pointer
     let new_hp = begin_from_space + (end_to_space - begin_to_space);
-    set_hp(new_hp as u32);
+    set_hp(new_hp);
 }
 
 /// Evacuate (copy) an object in from-space to to-space.

rts/motoko-rts/src/gc/generational.rs

@@ -83,17 +83,17 @@ unsafe fn get_limits() -> Limits {
     use crate::memory::ic::{self, linear_memory};
     assert!(linear_memory::LAST_HP >= ic::get_aligned_heap_base());
     Limits {
-        base: ic::get_aligned_heap_base() as usize,
-        last_free: linear_memory::LAST_HP as usize,
-        free: linear_memory::getHP() as usize,
+        base: ic::get_aligned_heap_base(),
+        last_free: linear_memory::LAST_HP,
+        free: (linear_memory::get_hp_unskewed()),
     }
 }
 
 #[cfg(feature = "ic")]
 unsafe fn set_limits(limits: &Limits) {
     use crate::memory::ic::linear_memory;
-    linear_memory::setHP(limits.free as u32);
-    linear_memory::LAST_HP = limits.free as u32;
+    linear_memory::set_hp_unskewed(limits.free);
+    linear_memory::LAST_HP = limits.free;
 }
 
 #[cfg(feature = "ic")]

rts/motoko-rts/src/gc/generational/write_barrier.rs

@@ -6,8 +6,8 @@ use crate::types::Value;
 use motoko_rts_macros::ic_mem_fn;
 
 pub static mut REMEMBERED_SET: Option<RememberedSet> = None;
-pub static mut HEAP_BASE: u32 = 0;
-pub static mut LAST_HP: u32 = 0;
+pub static mut HEAP_BASE: usize = 0;
+pub static mut LAST_HP: usize = 0;
 
 #[cfg(feature = "ic")]
 /// (Re-)initialize the write barrier for generational GC.
@@ -24,22 +24,22 @@ pub(crate) unsafe fn init_generational_write_barrier<M: Memory>(mem: &mut M) {
 /// As the barrier is called after the write, `*location` refers to the NEW value.
 /// No effect if the write barrier is deactivated.
 #[ic_mem_fn]
-pub unsafe fn post_write_barrier<M: Memory>(mem: &mut M, location: u32) {
+pub unsafe fn post_write_barrier<M: Memory>(mem: &mut M, location: usize) {
     // Must be an unskewed address.
     debug_assert_eq!(location & 0b1, 0);
     // Checks have been optimized according to the frequency of occurrence.
     // Only record locations inside old generation. Static roots are anyway marked by GC.
     if location < LAST_HP {
         // Nested ifs are more efficient when counting instructions on IC (explicit return counts as an instruction).
         let value = *(location as *mut Value);
-        if value.points_to_or_beyond(LAST_HP as usize) {
+        if value.points_to_or_beyond(LAST_HP) {
             #[allow(clippy::collapsible_if)]
             if location >= HEAP_BASE {
                 // Trap pointers that lead from old generation (or static roots) to young generation.
                 REMEMBERED_SET
                     .as_mut()
                     .unwrap()
-                    .insert(mem, Value::from_raw(location));
+                    .insert(mem, Value::from_raw(location as u32));
             }
         }
     }

rts/motoko-rts/src/gc/incremental.rs

@@ -38,7 +38,7 @@ pub mod time;
 #[ic_mem_fn(ic_only)]
 unsafe fn initialize_incremental_gc<M: Memory>(mem: &mut M) {
     use crate::memory::ic;
-    IncrementalGC::<M>::initialize(mem, ic::get_aligned_heap_base() as usize);
+    IncrementalGC::<M>::initialize(mem, ic::get_aligned_heap_base());
 }
 
 #[ic_mem_fn(ic_only)]
@@ -101,7 +101,7 @@ unsafe fn record_gc_stop<M: Memory>() {
     use crate::memory::ic::{self, partitioned_memory};
 
     let heap_size = partitioned_memory::get_heap_size();
-    let static_size = Bytes(ic::get_aligned_heap_base());
+    let static_size = Bytes(ic::get_aligned_heap_base() as u32);
     debug_assert!(heap_size >= static_size);
     let dynamic_size = heap_size - static_size;
     ic::MAX_LIVE = ::core::cmp::max(ic::MAX_LIVE, dynamic_size);

rts/motoko-rts/src/gc/mark_compact.rs

@@ -46,9 +46,9 @@ unsafe fn compacting_gc<M: Memory>(mem: &mut M) {
         mem,
         ic::get_aligned_heap_base(),
         // get_hp
-        || linear_memory::getHP() as usize,
+        || linear_memory::get_hp_unskewed(),
         // set_hp
-        |hp| linear_memory::setHP(hp),
+        |hp| linear_memory::set_hp_unskewed(hp),
         ic::get_static_roots(),
         crate::continuation_table::continuation_table_loc(),
         // note_live_size
@@ -57,18 +57,18 @@ unsafe fn compacting_gc<M: Memory>(mem: &mut M) {
         |reclaimed| linear_memory::RECLAIMED += Bytes(u64::from(reclaimed.as_u32())),
     );
 
-    linear_memory::LAST_HP = linear_memory::getHP();
+    linear_memory::LAST_HP = linear_memory::get_hp_unskewed();
 }
 
 pub unsafe fn compacting_gc_internal<
     M: Memory,
     GetHp: Fn() -> usize,
-    SetHp: Fn(u32),
+    SetHp: Fn(usize),
     NoteLiveSize: Fn(Bytes<u32>),
     NoteReclaimed: Fn(Bytes<u32>),
 >(
     mem: &mut M,
-    heap_base: u32,
+    heap_base: usize,
     get_hp: GetHp,
     set_hp: SetHp,
     static_roots: Value,
@@ -92,21 +92,21 @@ pub unsafe fn compacting_gc_internal<
     let reclaimed = old_hp - (get_hp() as u32);
     note_reclaimed(Bytes(reclaimed));
 
-    let live = get_hp() as u32 - heap_base;
-    note_live_size(Bytes(live));
+    let live = get_hp() - heap_base;
+    note_live_size(Bytes(live as u32));
 }
 
-unsafe fn mark_compact<M: Memory, SetHp: Fn(u32)>(
+unsafe fn mark_compact<M: Memory, SetHp: Fn(usize)>(
     mem: &mut M,
     set_hp: SetHp,
-    heap_base: u32,
+    heap_base: usize,
     heap_end: u32,
     static_roots: Value,
     continuation_table_ptr_loc: *mut Value,
 ) {
-    let mem_size = Bytes(heap_end - heap_base);
+    let mem_size = Bytes(heap_end - heap_base as u32);
 
-    alloc_bitmap(mem, mem_size, heap_base / WORD_SIZE);
+    alloc_bitmap(mem, mem_size, heap_base as u32 / WORD_SIZE);
     alloc_mark_stack(mem);
 
     mark_static_roots(mem, static_roots, heap_base);
@@ -126,15 +126,15 @@ unsafe fn mark_compact<M: Memory, SetHp: Fn(u32)>(
     free_bitmap();
 }
 
-unsafe fn mark_static_roots<M: Memory>(mem: &mut M, static_roots: Value, heap_base: u32) {
+unsafe fn mark_static_roots<M: Memory>(mem: &mut M, static_roots: Value, heap_base: usize) {
     let root_array = static_roots.as_array();
 
     // Static objects are not in the dynamic heap so don't need marking.
     for i in 0..root_array.len() {
         let obj = root_array.get(i).as_obj();
         // Root array should only have pointers to other static MutBoxes
         debug_assert_eq!(obj.tag(), TAG_MUTBOX); // check tag
-        debug_assert!((obj as u32) < heap_base); // check that MutBox is static
+        debug_assert!((obj as usize) < heap_base); // check that MutBox is static
         mark_root_mutbox_fields(mem, obj as *mut MutBox, heap_base);
     }
 }
@@ -157,18 +157,18 @@ unsafe fn mark_object<M: Memory>(mem: &mut M, obj: Value) {
     push_mark_stack(mem, obj as usize, obj_tag);
 }
 
-unsafe fn mark_stack<M: Memory>(mem: &mut M, heap_base: u32) {
+unsafe fn mark_stack<M: Memory>(mem: &mut M, heap_base: usize) {
     while let Some((obj, tag)) = pop_mark_stack() {
         mark_fields(mem, obj as *mut Obj, tag, heap_base)
     }
 }
 
-unsafe fn mark_fields<M: Memory>(mem: &mut M, obj: *mut Obj, obj_tag: Tag, heap_base: u32) {
+unsafe fn mark_fields<M: Memory>(mem: &mut M, obj: *mut Obj, obj_tag: Tag, heap_base: usize) {
     visit_pointer_fields(
         mem,
         obj,
         obj_tag,
-        heap_base as usize,
+        heap_base,
         |mem, field_addr| {
             let field_value = *field_addr;
             mark_object(mem, field_value);
@@ -194,9 +194,9 @@ unsafe fn mark_fields<M: Memory>(mem: &mut M, obj: *mut Obj, obj_tag: Tag, heap_
 }
 
 /// Specialized version of `mark_fields` for root `MutBox`es.
-unsafe fn mark_root_mutbox_fields<M: Memory>(mem: &mut M, mutbox: *mut MutBox, heap_base: u32) {
+unsafe fn mark_root_mutbox_fields<M: Memory>(mem: &mut M, mutbox: *mut MutBox, heap_base: usize) {
     let field_addr = &mut (*mutbox).field;
-    if pointer_to_dynamic_heap(field_addr, heap_base as usize) {
+    if pointer_to_dynamic_heap(field_addr, heap_base) {
         mark_object(mem, *field_addr);
         // It's OK to thread forward pointers here as the static objects won't be moved, so we will
         // be able to unthread objects pointed by these fields later.
@@ -213,7 +213,7 @@ unsafe fn mark_root_mutbox_fields<M: Memory>(mem: &mut M, mutbox: *mut MutBox, h
 ///
 /// - Thread forward pointers of the object
 ///
-unsafe fn update_refs<SetHp: Fn(u32)>(set_hp: SetHp, heap_base: u32) {
+unsafe fn update_refs<SetHp: Fn(usize)>(set_hp: SetHp, heap_base: usize) {
     let mut free = heap_base;
 
     let mut bitmap_iter = iter_bits();
@@ -236,7 +236,7 @@ unsafe fn update_refs<SetHp: Fn(u32)>(set_hp: SetHp, heap_base: u32) {
             debug_assert!(new_obj.tag() >= TAG_OBJECT && new_obj.tag() <= TAG_NULL);
         }
 
-        free += p_size_words.to_bytes().as_u32();
+        free += p_size_words.to_bytes().as_usize();
 
         // Thread forward pointers of the object
         thread_fwd_pointers(p_new as *mut Obj, heap_base);
@@ -248,12 +248,12 @@ unsafe fn update_refs<SetHp: Fn(u32)>(set_hp: SetHp, heap_base: u32) {
 }
 
 /// Thread forward pointers in object
-unsafe fn thread_fwd_pointers(obj: *mut Obj, heap_base: u32) {
+unsafe fn thread_fwd_pointers(obj: *mut Obj, heap_base: usize) {
     visit_pointer_fields(
         &mut (),
         obj,
         obj.tag(),
-        heap_base as usize,
+        heap_base,
         |_, field_addr| {
             if (*field_addr).get_ptr() > obj as usize {
                 thread(field_addr)
@@ -273,14 +273,14 @@ unsafe fn thread(field: *mut Value) {
 }
 
 /// Unthread all references at given header, replacing with `new_loc`. Restores object header.
-unsafe fn unthread(obj: *mut Obj, new_loc: u32) {
+unsafe fn unthread(obj: *mut Obj, new_loc: usize) {
     let mut header = obj.tag();
 
     // All objects and fields are word-aligned, and tags have the lowest bit set, so use the lowest
     // bit to distinguish a header (tag) from a field address.
     while header & 0b1 == 0 {
         let tmp = (header as *const Obj).tag();
-        (*(header as *mut Value)) = Value::from_ptr(new_loc as usize);
+        (*(header as *mut Value)) = Value::from_ptr(new_loc);
         header = tmp;
     }
 

rts/motoko-rts/src/memory/ic.rs

@@ -14,11 +14,11 @@ use motoko_rts_macros::*;
 
 // Provided by generated code
 extern "C" {
-    fn get_heap_base() -> u32;
+    fn get_heap_base() -> usize;
     pub(crate) fn get_static_roots() -> Value;
 }
 
-pub(crate) unsafe fn get_aligned_heap_base() -> u32 {
+pub(crate) unsafe fn get_aligned_heap_base() -> usize {
     // align to 32 bytes
     ((get_heap_base() + 31) / 32) * 32
 }