try to add percpu

Cargo.toml

@@ -12,6 +12,8 @@ members = [
     "crates/cv1811-sd",
     "crates/memory_addr",
     "crates/crate_interface",
+    "crates/percpu",
+    "crates/percpu_macros",
 
     # modules
     "modules/logging",

arch/Cargo.toml

@@ -27,4 +27,5 @@ riscv = { git = "https://github.com/rcore-os/riscv", features = ["inline-asm"] }
 x86 = "0.52"
 x86_64 = "0.14"
 spin = { version = "0.9.8", features = ["mutex"] }
-multiboot = "0.8.0"
+multiboot = "0.8.0"
+percpu = { path = "../crates/percpu" }

arch/src/lib.rs

@@ -5,6 +5,7 @@
 #![feature(stdsimd)]
 #![feature(const_mut_refs)]
 #![feature(const_slice_from_raw_parts_mut)]
+#![feature(result_option_inspect)]
 
 extern crate alloc;
 

arch/src/x86_64/mod.rs

@@ -22,6 +22,10 @@ use x86_64::instructions::port::PortWriteOnly;
 
 use crate::x86_64::multiboot::use_multiboot;
 
+
+#[percpu::def_percpu]
+static CPU_ID: usize = 1;
+
 pub fn shutdown() -> ! {
     unsafe { PortWriteOnly::new(0x604).write(0x2000u16) };
 
@@ -31,27 +35,26 @@ pub fn shutdown() -> ! {
 fn rust_tmp_main(magic: usize, mboot_ptr: usize) {
     crate::clear_bss();
     crate::prepare_init();
+    percpu::init(1);
+    percpu::set_local_thread_pointer(0);
+
+    info!("TEST CPU ID: {}  ptr: {:#x}", CPU_ID.read_current(), unsafe { CPU_ID.current_ptr() } as usize);
     idt::init();
     pic::init();
 
     info!("magic: {:#x}, mboot_ptr: {:#x}", magic, mboot_ptr);
 
     if let Some(mboot) = use_multiboot(mboot_ptr as _) {
+        mboot.boot_loader_name().inspect(|x| info!("bootloader: {}", x));
+        mboot.command_line().inspect(|x| info!("command_line: {}", x));
         if mboot.has_memory_map() {
-            info!("has memory map");
             mboot
                 .memory_regions()
                 .unwrap()
                 .filter(|x| x.memory_type() == MemoryType::Available)
                 .for_each(|x| {
                     let start = x.base_address() as usize | VIRT_ADDR_START;
                     let end = x.length() as usize | VIRT_ADDR_START;
-                    info!(
-                        "memory region: {:#x} length: {:#x}, type: {:#x?}",
-                        start,
-                        end,
-                        x.memory_type()
-                    );
                     crate::ArchInterface::add_memory_region(start, end);
                 });
         }

crates/percpu/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "percpu"
+version = "0.1.0"
+edition = "2021"
+authors = ["Yuekai Jia <[email protected]>"]
+description = "Define and access per-CPU data structures"
+license = "GPL-3.0-or-later OR Apache-2.0"
+homepage = "https://github.com/rcore-os/arceos"
+repository = "https://github.com/rcore-os/arceos/tree/main/crates/percpu"
+documentation = "https://rcore-os.github.io/arceos/percpu/index.html"
+
+[features]
+# For single CPU use, just make the per-CPU data a global variable.
+sp-naive = ["percpu_macros/sp-naive"]
+
+# Whether the system enables preemption.
+preempt = ["percpu_macros/preempt"]
+
+default = []
+
+[dependencies]
+cfg-if = "1.0"
+percpu_macros = { path = "../percpu_macros" }
+
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+x86 = "0.52"
+
+[target.'cfg(not(target_os = "none"))'.dependencies]
+spin = "0.9"

crates/percpu/build.rs

@@ -0,0 +1,9 @@
+use std::path::Path;
+
+fn main() {
+    if cfg!(target_os = "linux") && cfg!(not(feature = "sp-naive")) {
+        let ld_script_path = Path::new(std::env!("CARGO_MANIFEST_DIR")).join("test_percpu.x");
+        println!("cargo:rustc-link-arg-tests=-no-pie");
+        println!("cargo:rustc-link-arg-tests=-T{}", ld_script_path.display());
+    }
+}

crates/percpu/src/imp.rs

@@ -0,0 +1,125 @@
+const fn align_up(val: usize) -> usize {
+    const PAGE_SIZE: usize = 0x1000;
+    (val + PAGE_SIZE - 1) & !(PAGE_SIZE - 1)
+}
+
+#[cfg(not(target_os = "none"))]
+static PERCPU_AREA_BASE: spin::once::Once<usize> = spin::once::Once::new();
+
+/// Returns the per-CPU data area size for one CPU.
+#[doc(cfg(not(feature = "sp-naive")))]
+pub fn percpu_area_size() -> usize {
+    extern "C" {
+        fn _percpu_load_start();
+        fn _percpu_load_end();
+    }
+    use percpu_macros::percpu_symbol_offset;
+    percpu_symbol_offset!(_percpu_load_end) - percpu_symbol_offset!(_percpu_load_start)
+}
+
+/// Returns the base address of the per-CPU data area on the given CPU.
+///
+/// if `cpu_id` is 0, it returns the base address of all per-CPU data areas.
+#[doc(cfg(not(feature = "sp-naive")))]
+pub fn percpu_area_base(cpu_id: usize) -> usize {
+    cfg_if::cfg_if! {
+        if #[cfg(target_os = "none")] {
+            extern "C" {
+                fn _percpu_start();
+            }
+            let base = _percpu_start as usize;
+        } else {
+            let base = *PERCPU_AREA_BASE.get().unwrap();
+        }
+    }
+    base + cpu_id * align_up(percpu_area_size())
+}
+
+/// Initialize the per-CPU data area for `max_cpu_num` CPUs.
+pub fn init(max_cpu_num: usize) {
+    let size = percpu_area_size();
+
+    #[cfg(target_os = "linux")]
+    {
+        // we not load the percpu section in ELF, allocate them here.
+        let total_size = align_up(size) * max_cpu_num;
+        let layout = std::alloc::Layout::from_size_align(total_size, 0x1000).unwrap();
+        PERCPU_AREA_BASE.call_once(|| unsafe { std::alloc::alloc(layout) as usize });
+    }
+
+    let base = percpu_area_base(0);
+    for i in 1..max_cpu_num {
+        let secondary_base = percpu_area_base(i);
+        // copy per-cpu data of the primary CPU to other CPUs.
+        unsafe {
+            core::ptr::copy_nonoverlapping(base as *const u8, secondary_base as *mut u8, size);
+        }
+    }
+}
+
+/// Read the architecture-specific thread pointer register on the current CPU.
+pub fn get_local_thread_pointer() -> usize {
+    let tp;
+    unsafe {
+        cfg_if::cfg_if! {
+            if #[cfg(target_arch = "x86_64")] {
+                tp = if cfg!(target_os = "linux") {
+                    SELF_PTR.read_current_raw()
+                } else if cfg!(target_os = "none") {
+                    x86::msr::rdmsr(x86::msr::IA32_GS_BASE) as usize
+                } else {
+                    unimplemented!()
+                };
+            } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] {
+                core::arch::asm!("mv {}, gp", out(reg) tp)
+            } else if #[cfg(target_arch = "aarch64")] {
+                core::arch::asm!("mrs {}, TPIDR_EL1", out(reg) tp)
+            }
+        }
+    }
+    tp
+}
+
+/// Set the architecture-specific thread pointer register to the per-CPU data
+/// area base on the current CPU.
+///
+/// `cpu_id` indicates which per-CPU data area to use.
+pub fn set_local_thread_pointer(cpu_id: usize) {
+    let tp = percpu_area_base(cpu_id);
+    unsafe {
+        cfg_if::cfg_if! {
+            if #[cfg(target_arch = "x86_64")] {
+                if cfg!(target_os = "linux") {
+                    const ARCH_SET_GS: u32 = 0x1001;
+                    const SYS_ARCH_PRCTL: u32 = 158;
+                    core::arch::asm!(
+                        "syscall",
+                        in("eax") SYS_ARCH_PRCTL,
+                        in("edi") ARCH_SET_GS,
+                        in("rsi") tp,
+                    );
+                } else if cfg!(target_os = "none") {
+                    x86::msr::wrmsr(x86::msr::IA32_GS_BASE, tp as u64);
+                } else {
+                    unimplemented!()
+                }
+                SELF_PTR.write_current_raw(tp);
+            } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] {
+                core::arch::asm!("mv gp, {}", in(reg) tp)
+            } else if #[cfg(target_arch = "aarch64")] {
+                core::arch::asm!("msr TPIDR_EL1, {}", in(reg) tp)
+            }
+        }
+    }
+}
+
+/// To use `percpu::__priv::NoPreemptGuard::new()` in macro expansion.
+#[allow(unused_imports)]
+#[cfg(feature = "preempt")]
+use crate as percpu;
+
+/// On x86, we use `gs:SELF_PTR` to store the address of the per-CPU data area base.
+#[cfg(target_arch = "x86_64")]
+#[no_mangle]
+#[percpu_macros::def_percpu]
+static SELF_PTR: usize = 0;

crates/percpu/src/lib.rs

@@ -0,0 +1,69 @@
+//! Define and access per-CPU data structures.
+//!
+//! All per-CPU data is placed into several contiguous memory regions called
+//! **per-CPU data areas**, the number of which is the number of CPUs. Each CPU
+//! has its own per-CPU data area. The architecture-specific thread pointer
+//! register (e.g., `GS_BASE` on x86_64) is set to the base address of the area
+//! on initialization.
+//!
+//! When accessing the per-CPU data on the current CPU, it first use the thread
+//! pointer register to obtain the corresponding per-CPU data area, and then add
+//! an offset to access the corresponding field.
+//!
+//! # Notes
+//!
+//! Since RISC-V does not provide separate thread pointer registers for user and
+//! kernel mode, we temporarily use the `gp` register to point to the per-CPU data
+//! area, while the `tp` register is used for thread-local storage.
+//!
+//! # Examples
+//!
+//! ```no_run
+//! #[percpu::def_percpu]
+//! static CPU_ID: usize = 0;
+//!
+//! // initialize per-CPU data for 4 CPUs.
+//! percpu::init(4);
+//! // set the thread pointer register to the per-CPU data area 0.
+//! percpu::set_local_thread_pointer(0);
+//!
+//! // access the per-CPU data `CPU_ID` on the current CPU.
+//! println!("{}", CPU_ID.read_current()); // prints "0"
+//! CPU_ID.write_current(1);
+//! println!("{}", CPU_ID.read_current()); // prints "1"
+//! ```
+//!
+//! # Cargo Features
+//!
+//! - `sp-naive`: For **single-core** use. In this case, each per-CPU data is
+//!    just a global variable, architecture-specific thread pointer register is
+//!    not used.
+//! - `preempt`: For **preemptible** system use. In this case, we need to disable
+//!    preemption when accessing per-CPU data. Otherwise, the data may be corrupted
+//!    when it's being accessing and the current thread happens to be preempted.
+
+#![cfg_attr(target_os = "none", no_std)]
+#![feature(doc_cfg)]
+
+extern crate percpu_macros;
+
+#[cfg_attr(feature = "sp-naive", path = "naive.rs")]
+mod imp;
+
+pub use self::imp::*;
+pub use percpu_macros::def_percpu;
+
+#[doc(hidden)]
+pub mod __priv {
+    #[cfg(feature = "preempt")]
+    pub use kernel_guard::NoPreempt as NoPreemptGuard;
+}
+
+cfg_if::cfg_if! {
+    if #[cfg(doc)] {
+        /// Example per-CPU data for documentation only.
+        #[doc(cfg(doc))]
+        #[def_percpu]
+        pub static EXAMPLE_PERCPU_DATA: usize = 0;
+    }
+}

crates/percpu/src/naive.rs

@@ -0,0 +1,10 @@
+/// No effect for "sp-naive" use.
+pub fn init(_max_cpu_num: usize) {}
+
+/// Always returns `0` for "sp-naive" use.
+pub fn get_local_thread_pointer() -> usize {
+    0
+}
+
+/// No effect for "sp-naive" use.
+pub fn set_local_thread_pointer(_cpu_id: usize) {}

crates/percpu/test_percpu.x

@@ -0,0 +1,19 @@
+CPU_NUM = 4;
+
+SECTIONS
+{
+    . = ALIGN(4K);
+    _percpu_start = .;
+    .percpu 0x0 (NOLOAD) : AT(_percpu_start) {
+        _percpu_load_start = .;
+        *(.percpu .percpu.*)
+        _percpu_load_end = .;
+        . = ALIGN(64);
+        _percpu_size_aligned = .;
+
+        . = _percpu_load_start + _percpu_size_aligned * CPU_NUM;
+    }
+    . = _percpu_start + SIZEOF(.percpu);
+    _percpu_end = .;
+}
+INSERT BEFORE .bss;