feat: add percpu mode

.gitignore

@@ -1 +1,3 @@
-target
+target
+output.log
+qemu.log

Cargo.toml

@@ -20,7 +20,6 @@ log = "0.4"
 fdt = "0.1.5"
 bitflags = "2.0.2"
 cfg-if = "1.0.0"
-percpu = { git = "https://github.com/Byte-OS/percpu.git" }
 polyhal-macro = { path = "polyhal-macro" }
 
 [target.'cfg(target_arch = "riscv64")'.dependencies]

example/Makefile

@@ -1,7 +1,10 @@
 # Building
 ARCH := riscv64
-QEMU_EXEC := timeout 40
+TEST := false
 
+ifeq ($(TEST), true)
+QEMU_EXEC := timeout 40
+endif
 
 ifeq ($(ARCH), x86_64)
   TARGET := x86_64-unknown-none
@@ -62,11 +65,17 @@ clean:
 
 run: run-inner
 
-QEMU_EXEC += -nographic -smp 1 >output.log 2>&1
+QEMU_EXEC += -nographic -smp 1 
+QEMU_EXEC += -D qemu.log -d in_asm,int,pcall,cpu_reset,guest_errors
+ifeq ($(TEST), true)
+QEMU_EXEC += >output.log 2>&1 || echo "QEMU exited"
+endif
 
 run-inner: build
-	$(QEMU_EXEC) || echo "QEMU exited"
+	$(QEMU_EXEC)
+ifeq ($(TEST), true)
 	grep "\[kernel\] Hello, world!" output.log
+endif
 
 test:
 	make ARCH=aarch64 run

example/linker/linker-riscv64.ld

@@ -48,20 +48,6 @@ SECTIONS
         _ebss = .;
     }
 
-    . = ALIGN(4K);
-    _percpu_start = .;
-    .percpu 0x0 : AT(_percpu_start) {
-        _percpu_load_start = .;
-        *(.percpu .percpu.*)
-        _percpu_load_end = .;
-        . = ALIGN(64);
-        _percpu_size_aligned = .;
-
-        . = _percpu_load_start + _percpu_size_aligned * 1;
-    }
-    . = _percpu_start + SIZEOF(.percpu);
-    _percpu_end = .;
-
     PROVIDE(end = .);
     /DISCARD/ : {
         *(.comment) *(.gnu*) *(.note*) *(.eh_frame*)

polyhal-macro/src/lib.rs

@@ -1,6 +1,9 @@
+mod percpu;
+
 use proc_macro::TokenStream;
-use quote::quote;
-use syn::{parse_macro_input, ItemFn};
+use proc_macro2::Span;
+use quote::{format_ident, quote};
+use syn::{parse_macro_input, Error, ItemFn, ItemStatic};
 
 #[proc_macro_attribute]
 pub fn arch_entry(_input: TokenStream, annotated_item: TokenStream) -> TokenStream {
@@ -19,3 +22,155 @@ pub fn arch_interrupt(_input: TokenStream, annotated_item: TokenStream) -> Token
         #annotated_item
     })
 }
+
+/// Defines a per-CPU data structure.
+///
+/// It should be used on a `static` variable.
+///
+/// See the [crate-level documentation](../percpu/index.html) for more details.
+#[proc_macro_attribute]
+pub fn def_percpu(attr: TokenStream, item: TokenStream) -> TokenStream {
+    if !attr.is_empty() {
+        return compiler_error(Error::new(
+            Span::call_site(),
+            "expect an empty attribute: `#[def_percpu]`",
+        ));
+    }
+
+    let ast = syn::parse_macro_input!(item as ItemStatic);
+
+    let attrs = &ast.attrs;
+    let vis = &ast.vis;
+    let name = &ast.ident;
+    let ty = &ast.ty;
+    let init_expr = &ast.expr;
+
+    let inner_symbol_name = &format_ident!("__PERCPU_{}", name);
+    let struct_name = &format_ident!("{}_WRAPPER", name);
+
+    let ty_str = quote!(#ty).to_string();
+    let is_primitive_int = ["bool", "u8", "u16", "u32", "u64", "usize"].contains(&ty_str.as_str());
+
+    // Do not generate `fn read_current()`, `fn write_current()`, etc for non primitive types.
+    let read_write_methods = if is_primitive_int {
+        quote! {
+            /// Returns the value of the per-CPU data on the current CPU.
+            ///
+            /// # Safety
+            ///
+            /// Caller must ensure that preemption is disabled on the current CPU.
+            #[inline]
+            pub unsafe fn read_current_raw(&self) -> #ty {
+                unsafe { *self.current_ptr() }
+            }
+
+            /// Set the value of the per-CPU data on the current CPU.
+            ///
+            /// # Safety
+            ///
+            /// Caller must ensure that preemption is disabled on the current CPU.
+            #[inline]
+            pub unsafe fn write_current_raw(&self, val: #ty) {
+                unsafe { *self.current_ref_mut_raw() = val };
+            }
+
+            /// Returns the value of the per-CPU data on the current CPU. Preemption will
+            /// be disabled during the call.
+            pub fn read_current(&self) -> #ty {
+                unsafe { self.read_current_raw() }
+            }
+
+            /// Set the value of the per-CPU data on the current CPU. Preemption will
+            /// be disabled during the call.
+            pub fn write_current(&self, val: #ty) {
+                unsafe { self.write_current_raw(val) }
+            }
+        }
+    } else {
+        quote! {}
+    };
+
+    let current_ptr = percpu::gen_current_ptr(inner_symbol_name, ty);
+    quote! {
+        #[cfg_attr(not(target_os = "macos"), link_section = "percpu")] // unimplemented on macos
+        #[used(linker)]
+        #(#attrs)*
+        static mut #inner_symbol_name: #ty = #init_expr;
+
+        #[doc = concat!("Wrapper struct for the per-CPU data [`", stringify!(#name), "`]")]
+        #[allow(non_camel_case_types)]
+        #vis struct #struct_name {}
+
+        #(#attrs)*
+        #vis static #name: #struct_name = #struct_name {};
+
+        impl #struct_name {
+            /// Returns the offset relative to the per-CPU data area base on the current CPU.
+            #[inline]
+            pub fn offset(&self) -> usize {
+                extern "Rust" {
+                    fn __start_percpu();
+                }
+                unsafe {
+                    &#inner_symbol_name as *const _ as usize - __start_percpu as usize
+                }
+            }
+
+            /// Returns the raw pointer of this per-CPU data on the current CPU.
+            ///
+            /// # Safety
+            ///
+            /// Caller must ensure that preemption is disabled on the current CPU.
+            #[inline]
+            pub unsafe fn current_ptr(&self) -> *const #ty {
+                #current_ptr
+            }
+
+            /// Returns the reference of the per-CPU data on the current CPU.
+            ///
+            /// # Safety
+            ///
+            /// Caller must ensure that preemption is disabled on the current CPU.
+            #[inline]
+            pub unsafe fn current_ref_raw(&self) -> &#ty {
+                &*self.current_ptr()
+            }
+
+            /// Returns the mutable reference of the per-CPU data on the current CPU.
+            ///
+            /// # Safety
+            ///
+            /// Caller must ensure that preemption is disabled on the current CPU.
+            #[inline]
+            #[allow(clippy::mut_from_ref)]
+            pub unsafe fn current_ref_mut_raw(&self) -> &mut #ty {
+                &mut *(self.current_ptr() as *mut #ty)
+            }
+
+            /// Manipulate the per-CPU data on the current CPU in the given closure.
+            /// Preemption will be disabled during the call.
+            pub fn with_current<F, T>(&self, f: F) -> T
+            where
+                F: FnOnce(&mut #ty) -> T,
+            {
+                f(unsafe { self.current_ref_mut_raw() })
+            }
+
+            #read_write_methods
+        }
+    }
+    .into()
+}
+
+#[doc(hidden)]
+#[proc_macro]
+pub fn percpu_symbol_offset(item: TokenStream) -> TokenStream {
+    let symbol = &format_ident!("{}", item.to_string());
+    let offset = percpu::gen_offset(symbol);
+    quote!({ #offset }).into()
+}
+
+
+fn compiler_error(err: Error) -> TokenStream {
+    err.to_compile_error().into()
+}

polyhal-macro/src/percpu.rs

@@ -0,0 +1,69 @@
+use quote::quote;
+use syn::{Ident, Type};
+
+pub fn gen_offset(symbol: &Ident) -> proc_macro2::TokenStream {
+    quote! {
+        let value: usize;
+        unsafe {
+            cfg_match! {
+                cfg(target_arch = "x86_64") => {
+                    ::core::arch::asm!(
+                        "movabs {0}, offset {VAR}",
+                        out(reg) value,
+                        VAR = sym #symbol,
+                    );
+                }
+                cfg(target_arch = "aarch64") => {
+                    ::core::arch::asm!(
+                        "movz {0}, #:abs_g0_nc:{VAR}",
+                        out(reg) value,
+                        VAR = sym #symbol,
+                    );
+                }
+                cfg(any(target_arch = "riscv32", target_arch = "riscv64")) => {
+                    ::core::arch::asm!(
+                        "lui {0}, %hi({VAR})",
+                        "addi {0}, {0}, %lo({VAR})",
+                        out(reg) value,
+                        VAR = sym #symbol,
+                    );
+                }
+                cfg(target_arch = "loongarch64") => {
+                    ::core::arch::asm!(
+                        "la.abs {0}, {VAR}",
+                        out(reg) value,
+                        VAR = sym #symbol,
+                    );
+                }
+            }
+        }
+        value
+    }
+}
+
+pub fn gen_current_ptr(symbol: &Ident, ty: &Type) -> proc_macro2::TokenStream {
+    quote! {
+        let base: usize;
+        #[cfg(target_arch = "x86_64")]
+        {
+            // `__PERCPU_SELF_PTR` stores GS_BASE, which is defined in crate `percpu`.
+            ::core::arch::asm!(
+                "mov {0}, gs:[offset __PERCPU_SELF_PTR]",
+                "add {0}, offset {VAR}",
+                out(reg) base,
+                VAR = sym #symbol,
+            );
+            base as *const #ty
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            #[cfg(target_arch = "aarch64")]
+            ::core::arch::asm!("mrs {}, TPIDR_EL1", out(reg) base);
+            #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
+            ::core::arch::asm!("mv {}, gp", out(reg) base);
+            #[cfg(target_arch = "loongarch64")]
+            ::core::arch::asm!("move {}, $r21", out(reg) base);
+            (base + self.offset()) as *const #ty
+        }
+    }
+}