From 11a59cf65eb3a04a4eae2cfb94f7e60bdfef154d Mon Sep 17 00:00:00 2001 From: Mohanson Date: Wed, 8 Nov 2023 10:38:37 +0800 Subject: [PATCH] Refactored flexible memory implementation (#378) * Refactor dymnaic memory * Refactor memory flags * Refactor frames flags * Small optimization * Use less instructions for _CHECK_READ_FRAMES on aarch64 * Small optimization * Small optimization * Use alloc instead of vec * Add cast_ptr_to_slice and cast_ptr_to_slice_mut methods * Fix clippy * Remove memory size limit * Let AsmCoreMachine be responsible for memory allocation and release --- Makefile | 3 +- definitions/src/asm.rs | 58 +++-- definitions/src/generate_asm_constants.rs | 32 +-- definitions/src/lib.rs | 11 +- fuzz/fuzz_targets/isa_a.rs | 1 - src/lib.rs | 7 +- src/machine/asm/cdefinitions_generated.h | 12 +- src/machine/asm/execute_aarch64.S | 213 +++++++---------- src/machine/asm/execute_x64.S | 278 +++++++++++++--------- src/machine/asm/mod.rs | 138 ++++++----- src/memory/flat.rs | 25 +- src/memory/mod.rs | 21 +- src/memory/sparse.rs | 5 +- src/memory/wxorx.rs | 28 ++- tests/test_a_extension.rs | 4 +- tests/test_asm.rs | 7 +- tests/test_misc.rs | 12 +- tests/test_reset.rs | 6 +- tests/test_resume2.rs | 8 +- 19 files changed, 454 insertions(+), 415 deletions(-) diff --git a/Makefile b/Makefile index 36ad8fad..998f6e56 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,8 @@ clippy_rule = -D warnings \ -A clippy::inconsistent_digit_grouping \ -A clippy::large_digit_groups \ -A clippy::suspicious_operation_groupings \ - -A clippy::unnecessary_cast + -A clippy::unnecessary_cast \ + -A clippy::mut_from_ref clippy: cargo clippy --all --features=asm -- $(clippy_rule) cd definitions && cargo clippy --all -- $(clippy_rule) diff --git a/definitions/src/asm.rs b/definitions/src/asm.rs index 140757d1..ed5402e7 100644 --- a/definitions/src/asm.rs +++ b/definitions/src/asm.rs @@ -1,8 +1,8 @@ use crate::{ - instructions::Instruction, MEMORY_FRAMES, MEMORY_FRAMESIZE, MEMORY_FRAME_SHIFTS, - RISCV_GENERAL_REGISTER_NUMBER, RISCV_MAX_MEMORY, RISCV_PAGES, RISCV_PAGESIZE, + instructions::Instruction, DEFAULT_MEMORY_SIZE, MEMORY_FRAMESIZE, MEMORY_FRAME_SHIFTS, + RISCV_GENERAL_REGISTER_NUMBER, RISCV_PAGESIZE, }; -use std::alloc::{alloc, Layout}; +use std::alloc::{alloc, alloc_zeroed, dealloc, Layout}; // The number of trace items to keep pub const TRACE_SIZE: usize = 8192; @@ -101,15 +101,25 @@ pub struct AsmCoreMachine { pub last_read_frame: u64, pub last_write_page: u64, - pub flags: [u8; RISCV_PAGES], - pub frames: [u8; MEMORY_FRAMES], + pub memory_ptr: u64, + pub flags_ptr: u64, + pub frames_ptr: u64, +} - pub memory: [u8; RISCV_MAX_MEMORY], +impl Drop for AsmCoreMachine { + fn drop(&mut self) { + let memory_layout = Layout::array::(self.memory_size as usize).unwrap(); + unsafe { dealloc(self.memory_ptr as *mut u8, memory_layout) }; + let flags_layout = Layout::array::(self.flags_size as usize).unwrap(); + unsafe { dealloc(self.flags_ptr as *mut u8, flags_layout) }; + let frames_layout = Layout::array::(self.frames_size as usize).unwrap(); + unsafe { dealloc(self.frames_ptr as *mut u8, frames_layout) }; + } } impl AsmCoreMachine { pub fn new(isa: u8, version: u32, max_cycles: u64) -> Box { - Self::new_with_memory(isa, version, max_cycles, RISCV_MAX_MEMORY) + Self::new_with_memory(isa, version, max_cycles, DEFAULT_MEMORY_SIZE) } pub fn new_with_memory( @@ -119,15 +129,10 @@ impl AsmCoreMachine { memory_size: usize, ) -> Box { assert_ne!(memory_size, 0); - assert!(memory_size <= RISCV_MAX_MEMORY); assert_eq!(memory_size % RISCV_PAGESIZE, 0); assert_eq!(memory_size % (1 << MEMORY_FRAME_SHIFTS), 0); - let mut machine = unsafe { - let machine_size = - std::mem::size_of::() - RISCV_MAX_MEMORY + memory_size; - - let layout = Layout::array::(machine_size).unwrap(); + let layout = Layout::new::(); let raw_allocation = alloc(layout) as *mut AsmCoreMachine; Box::from_raw(raw_allocation) }; @@ -146,8 +151,6 @@ impl AsmCoreMachine { machine.reset_signal = 0; machine.version = version; machine.isa = isa; - machine.flags = [0; RISCV_PAGES]; - machine.frames = [0; MEMORY_FRAMES]; machine.memory_size = memory_size as u64; machine.frames_size = (memory_size / MEMORY_FRAMESIZE) as u64; @@ -156,6 +159,13 @@ impl AsmCoreMachine { machine.last_read_frame = u64::max_value(); machine.last_write_page = u64::max_value(); + let memory_layout = Layout::array::(machine.memory_size as usize).unwrap(); + machine.memory_ptr = unsafe { alloc(memory_layout) } as u64; + let flags_layout = Layout::array::(machine.flags_size as usize).unwrap(); + machine.flags_ptr = unsafe { alloc_zeroed(flags_layout) } as u64; + let frames_layout = Layout::array::(machine.frames_size as usize).unwrap(); + machine.frames_ptr = unsafe { alloc_zeroed(frames_layout) } as u64; + machine } @@ -163,3 +173,21 @@ impl AsmCoreMachine { self.max_cycles = cycles; } } + +impl AsmCoreMachine { + pub fn cast_ptr_to_slice(&self, ptr: u64, offset: usize, size: usize) -> &[u8] { + unsafe { + let ptr = ptr as *mut u8; + let ptr = ptr.add(offset); + std::slice::from_raw_parts(ptr, size) + } + } + + pub fn cast_ptr_to_slice_mut(&self, ptr: u64, offset: usize, size: usize) -> &mut [u8] { + unsafe { + let ptr = ptr as *mut u8; + let ptr = ptr.add(offset); + std::slice::from_raw_parts_mut(ptr, size) + } + } +} diff --git a/definitions/src/generate_asm_constants.rs b/definitions/src/generate_asm_constants.rs index d3f08b63..eefe822f 100644 --- a/definitions/src/generate_asm_constants.rs +++ b/definitions/src/generate_asm_constants.rs @@ -8,8 +8,8 @@ use ckb_vm_definitions::{ instructions::{instruction_opcode_name, MAXIMUM_OPCODE, MINIMAL_OPCODE}, memory::{FLAG_DIRTY, FLAG_EXECUTABLE, FLAG_FREEZED, FLAG_WRITABLE, FLAG_WXORX_BIT}, registers::{RA, SP}, - MEMORY_FRAMES, MEMORY_FRAMESIZE, MEMORY_FRAME_PAGE_SHIFTS, MEMORY_FRAME_SHIFTS, - RISCV_MAX_MEMORY, RISCV_PAGES, RISCV_PAGESIZE, RISCV_PAGE_SHIFTS, + MEMORY_FRAMESIZE, MEMORY_FRAME_PAGE_SHIFTS, MEMORY_FRAME_SHIFTS, RISCV_PAGESIZE, + RISCV_PAGE_SHIFTS, }; use std::mem::{size_of, zeroed}; @@ -30,17 +30,14 @@ macro_rules! print_inst_label { // of this as a workaround to the problem that build.rs cannot depend on any // of its crate contents. fn main() { - println!("#define CKB_VM_ASM_RISCV_MAX_MEMORY {}", RISCV_MAX_MEMORY); println!("#define CKB_VM_ASM_RISCV_PAGE_SHIFTS {}", RISCV_PAGE_SHIFTS); println!("#define CKB_VM_ASM_RISCV_PAGE_SIZE {}", RISCV_PAGESIZE); println!("#define CKB_VM_ASM_RISCV_PAGE_MASK {}", RISCV_PAGESIZE - 1); - println!("#define CKB_VM_ASM_RISCV_PAGES {}", RISCV_PAGES); println!( "#define CKB_VM_ASM_MEMORY_FRAME_SHIFTS {}", MEMORY_FRAME_SHIFTS ); println!("#define CKB_VM_ASM_MEMORY_FRAMESIZE {}", MEMORY_FRAMESIZE); - println!("#define CKB_VM_ASM_MEMORY_FRAMES {}", MEMORY_FRAMES); println!( "#define CKB_VM_ASM_MEMORY_FRAME_PAGE_SHIFTS {}", MEMORY_FRAME_PAGE_SHIFTS @@ -183,31 +180,18 @@ fn main() { "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_LAST_WRITE_PAGE {}", (&m.last_write_page as *const u64 as usize) - m_address ); - - println!( - "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS {}", - (&m.flags as *const u8 as usize) - m_address - ); - let memory_offset_address = (&m.memory as *const u8 as usize) - m_address; - println!( - "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY {}", - memory_offset_address - ); println!( - "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES {}", - (&m.frames as *const u8 as usize) - m_address + "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR {}", + (&m.memory_ptr as *const u64 as usize) - m_address ); - println!(); - println!( - "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_H {}", - memory_offset_address.wrapping_shr(12).wrapping_shl(12) + "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_PTR {}", + (&m.flags_ptr as *const u64 as usize) - m_address ); println!( - "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_L {}", - memory_offset_address & 0xFFF + "#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR {}", + (&m.frames_ptr as *const u64 as usize) - m_address ); - println!(); for op in MINIMAL_OPCODE..MAXIMUM_OPCODE { diff --git a/definitions/src/lib.rs b/definitions/src/lib.rs index 72e8f066..5a4e5b05 100644 --- a/definitions/src/lib.rs +++ b/definitions/src/lib.rs @@ -6,17 +6,12 @@ pub mod registers; pub const RISCV_PAGE_SHIFTS: usize = 12; pub const RISCV_PAGESIZE: usize = 1 << RISCV_PAGE_SHIFTS; pub const RISCV_GENERAL_REGISTER_NUMBER: usize = 32; -// 4 MB -pub const RISCV_MAX_MEMORY: usize = 4 << 20; -// 1 MB -pub const DEFAULT_STACK_SIZE: usize = 1 << 20; -pub const RISCV_PAGES: usize = RISCV_MAX_MEMORY / RISCV_PAGESIZE; -// 256 KB pub const MEMORY_FRAME_SHIFTS: usize = 18; -pub const MEMORY_FRAMESIZE: usize = 1 << MEMORY_FRAME_SHIFTS; -pub const MEMORY_FRAMES: usize = RISCV_MAX_MEMORY / MEMORY_FRAMESIZE; +pub const MEMORY_FRAMESIZE: usize = 1 << MEMORY_FRAME_SHIFTS; // 256 KB pub const MEMORY_FRAME_PAGE_SHIFTS: usize = MEMORY_FRAME_SHIFTS - RISCV_PAGE_SHIFTS; +pub const DEFAULT_MEMORY_SIZE: usize = 4 << 20; // 4 MB + pub const ISA_IMC: u8 = 0b0000_0000; pub const ISA_B: u8 = 0b0000_0001; pub const ISA_MOP: u8 = 0b0000_0010; diff --git a/fuzz/fuzz_targets/isa_a.rs b/fuzz/fuzz_targets/isa_a.rs index 8e95b3f2..25e4688c 100644 --- a/fuzz/fuzz_targets/isa_a.rs +++ b/fuzz/fuzz_targets/isa_a.rs @@ -43,7 +43,6 @@ fuzz_target!(|data: [u8; 512]| { ckb_vm::machine::asm::AsmCoreMachine::new(ckb_vm_isa, ckb_vm_version, u64::MAX), ) .build(); - let insts: [u32; 18] = [ 0b00001_00_00000_00000_010_00000_0101111, // AMOSWAP.W 0b00000_00_00000_00000_010_00000_0101111, // AMOADD.W diff --git a/src/lib.rs b/src/lib.rs index e0b6d784..d6563a72 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,9 +30,8 @@ pub use crate::{ pub use bytes::Bytes; pub use ckb_vm_definitions::{ - registers, DEFAULT_STACK_SIZE, ISA_A, ISA_B, ISA_IMC, ISA_MOP, MEMORY_FRAMES, MEMORY_FRAMESIZE, - MEMORY_FRAME_SHIFTS, RISCV_GENERAL_REGISTER_NUMBER, RISCV_MAX_MEMORY, RISCV_PAGES, - RISCV_PAGESIZE, RISCV_PAGE_SHIFTS, + registers, DEFAULT_MEMORY_SIZE, ISA_A, ISA_B, ISA_IMC, ISA_MOP, MEMORY_FRAMESIZE, + MEMORY_FRAME_SHIFTS, RISCV_GENERAL_REGISTER_NUMBER, RISCV_PAGESIZE, RISCV_PAGE_SHIFTS, }; pub use error::Error; @@ -74,7 +73,7 @@ mod tests { #[test] fn test_max_memory_must_be_multiple_of_pages() { - assert_eq!(RISCV_MAX_MEMORY % RISCV_PAGESIZE, 0); + assert_eq!(DEFAULT_MEMORY_SIZE % RISCV_PAGESIZE, 0); } #[test] diff --git a/src/machine/asm/cdefinitions_generated.h b/src/machine/asm/cdefinitions_generated.h index 13b326f8..ab57330c 100644 --- a/src/machine/asm/cdefinitions_generated.h +++ b/src/machine/asm/cdefinitions_generated.h @@ -1,11 +1,8 @@ -#define CKB_VM_ASM_RISCV_MAX_MEMORY 4194304 #define CKB_VM_ASM_RISCV_PAGE_SHIFTS 12 #define CKB_VM_ASM_RISCV_PAGE_SIZE 4096 #define CKB_VM_ASM_RISCV_PAGE_MASK 4095 -#define CKB_VM_ASM_RISCV_PAGES 1024 #define CKB_VM_ASM_MEMORY_FRAME_SHIFTS 18 #define CKB_VM_ASM_MEMORY_FRAMESIZE 262144 -#define CKB_VM_ASM_MEMORY_FRAMES 16 #define CKB_VM_ASM_MEMORY_FRAME_PAGE_SHIFTS 6 #define CKB_VM_ASM_MAXIMUM_TRACE_ADDRESS_LENGTH 64 @@ -52,12 +49,9 @@ #define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_SIZE 336 #define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_LAST_READ_FRAME 344 #define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_LAST_WRITE_PAGE 352 -#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS 360 -#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY 1400 -#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES 1384 - -#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_H 0 -#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_L 1400 +#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR 360 +#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_PTR 368 +#define CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR 376 #define CKB_VM_ASM_OP_UNLOADED 16 #define CKB_VM_ASM_OP_ADD 17 diff --git a/src/machine/asm/execute_aarch64.S b/src/machine/asm/execute_aarch64.S index 97e1c44b..50952edf 100644 --- a/src/machine/asm/execute_aarch64.S +++ b/src/machine/asm/execute_aarch64.S @@ -44,7 +44,7 @@ #define REGISTER_BASE x28 #define ZERO_VALUE xzr -#define MEMORY_OFFSET_ADDRESS x29 +#define MEMORY_PTR x29 #ifdef __APPLE__ #define SEP %% @@ -139,13 +139,13 @@ mov TEMP1, address_reg SEP \ lsr TEMP1, TEMP1, CKB_VM_ASM_MEMORY_FRAME_SHIFTS SEP \ str TEMP1, [MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_LAST_READ_FRAME] SEP \ - ldr TEMP4, =CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES SEP \ - add TEMP5, TEMP1, TEMP4 SEP \ - ldrb TEMP2w, [MACHINE, TEMP5] SEP \ + ldr TEMP4, =CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR SEP \ + ldr TEMP4, [MACHINE, TEMP4] SEP \ + ldrb TEMP2w, [TEMP4, TEMP1] SEP \ cmp TEMP2, 0 SEP \ bne 1f SEP \ mov TEMP3, 1 SEP \ - strb TEMP3w, [MACHINE, TEMP5] SEP \ + strb TEMP3w, [TEMP4, TEMP1] SEP \ PREPCALL SEP \ mov x1, MACHINE SEP \ mov x0, TEMP1 SEP \ @@ -156,11 +156,10 @@ add TEMP1, TEMP1, length SEP \ sub TEMP1, TEMP1, 1 SEP \ lsr TEMP1, TEMP1, CKB_VM_ASM_MEMORY_FRAME_SHIFTS SEP \ - add TEMP5, TEMP1, TEMP4 SEP \ - ldrb TEMP2w, [MACHINE, TEMP5] SEP \ + ldrb TEMP2w, [TEMP4, TEMP1] SEP \ cmp TEMP2, 0 SEP \ bne 2f SEP \ - strb TEMP3w, [MACHINE, TEMP5] SEP \ + strb TEMP3w, [TEMP4, TEMP1] SEP \ PREPCALL SEP \ mov x1, MACHINE SEP \ mov x0, TEMP1 SEP \ @@ -226,23 +225,23 @@ ldr TEMP2, [MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_SIZE] SEP \ cmp TEMP1, TEMP2 SEP \ bhs .exit_out_of_bound SEP \ - add TEMP5, TEMP1, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS SEP \ - ldrb TEMP3w, [MACHINE, TEMP5] SEP \ + ldr TEMP5, [MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_PTR] SEP \ + ldrb TEMP3w, [TEMP5, TEMP1] SEP \ mov TEMP2, TEMP3 SEP \ and TEMP3, TEMP3, CKB_VM_ASM_MEMORY_FLAG_WXORX_BIT SEP \ cmp TEMP3, CKB_VM_ASM_MEMORY_FLAG_WRITABLE SEP \ bne .exit_invalid_permission SEP \ orr TEMP2, TEMP2, CKB_VM_ASM_MEMORY_FLAG_DIRTY SEP \ - strb TEMP2w, [MACHINE, TEMP5] SEP \ + strb TEMP2w, [TEMP5, TEMP1] SEP \ mov TEMP2, TEMP1 SEP \ lsr TEMP1, TEMP1, CKB_VM_ASM_MEMORY_FRAME_PAGE_SHIFTS SEP \ - ldr TEMP5, =CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES SEP \ - add TEMP5, TEMP1, TEMP5 SEP \ - ldrb TEMP3w, [MACHINE, TEMP5] SEP \ + ldr TEMP4, =CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR SEP \ + ldr TEMP5, [MACHINE, TEMP4] SEP \ + ldrb TEMP3w, [TEMP5, TEMP1] SEP \ cmp TEMP3, 0 SEP \ bne 1f SEP \ mov TEMP4, 1 SEP \ - strb TEMP4w, [MACHINE, TEMP5] SEP \ + strb TEMP4w, [TEMP5, TEMP1] SEP \ PREPCALL SEP \ mov x1, MACHINE SEP \ mov x0, TEMP1 SEP \ @@ -260,21 +259,22 @@ ldr TEMP2, [MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_SIZE] SEP \ cmp TEMP1, TEMP2 SEP \ bhs .exit_out_of_bound SEP \ - add TEMP5, TEMP1, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS SEP \ - ldrb TEMP3w, [MACHINE, TEMP5] SEP \ + ldr TEMP5, [MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_PTR] SEP \ + ldrb TEMP3w, [TEMP5, TEMP1] SEP \ mov TEMP2, TEMP3 SEP \ and TEMP3, TEMP3, CKB_VM_ASM_MEMORY_FLAG_WXORX_BIT SEP \ cmp TEMP3, CKB_VM_ASM_MEMORY_FLAG_WRITABLE SEP \ bne .exit_invalid_permission SEP \ orr TEMP2, TEMP2, CKB_VM_ASM_MEMORY_FLAG_DIRTY SEP \ - strb TEMP2w, [MACHINE, TEMP5] SEP \ - lsr TEMP1, TEMP1, CKB_VM_ASM_MEMORY_FRAME_PAGE_SHIFTS SEP \ - ldr TEMP5, =CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES SEP \ - add TEMP5, TEMP1, TEMP5 SEP \ - ldrb TEMP3w, [MACHINE, TEMP5] SEP \ + strb TEMP2w, [TEMP5, TEMP1] SEP \ + lsr TEMP1, TEMP1, CKB_VM_ASM_MEMORY_FRAME_PAGE_SHIFTS SEP \ + ldr TEMP4, =CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR SEP \ + ldr TEMP5, [MACHINE, TEMP4] SEP \ + ldrb TEMP3w, [TEMP5, TEMP1] SEP \ cmp TEMP3, 0 SEP \ bne 2f SEP \ - strb TEMP4w, [MACHINE, TEMP5] SEP \ + mov TEMP4, 1 SEP \ + strb TEMP4w, [TEMP5, TEMP1] SEP \ PREPCALL SEP \ mov x1, MACHINE SEP \ mov x0, TEMP1 SEP \ @@ -297,8 +297,7 @@ ckb_vm_x64_execute: stp x27, x28, [sp, 64] stp x29, x30, [sp, 80] add REGISTER_BASE, MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_REGISTERS - mov MEMORY_OFFSET_ADDRESS, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_H - add MEMORY_OFFSET_ADDRESS, MEMORY_OFFSET_ADDRESS, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_L + ldr MEMORY_PTR, [MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR] .CKB_VM_ASM_LABEL_OP_CUSTOM_TRACE_END: ldr TEMP2, [MACHINE, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_PC] @@ -602,8 +601,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION0(RS1, 1) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsb RS1, [MACHINE, RS1] + ldrsb RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LB_VERSION1: @@ -611,8 +609,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION1(RS1, 1) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsb RS1, [MACHINE, RS1] + ldrsb RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LBU_VERSION0: @@ -620,8 +617,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION0(RS1, 1) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrb RS1w, [MACHINE, RS1] + ldrb RS1w, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LBU_VERSION1: @@ -629,8 +625,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION1(RS1, 1) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrb RS1w, [MACHINE, RS1] + ldrb RS1w, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LD_VERSION0: @@ -638,8 +633,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION0(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr RS1, [MACHINE, RS1] + ldr RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LD_VERSION1: @@ -647,8 +641,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION1(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr RS1, [MACHINE, RS1] + ldr RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LH_VERSION0: @@ -656,8 +649,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION0(RS1, 2) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsh RS1, [MACHINE, RS1] + ldrsh RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LH_VERSION1: @@ -665,8 +657,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION1(RS1, 2) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsh RS1, [MACHINE, RS1] + ldrsh RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LHU_VERSION0: @@ -674,8 +665,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION0(RS1, 2) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrh RS1w, [MACHINE, RS1] + ldrh RS1w, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LHU_VERSION1: @@ -683,8 +673,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION1(RS1, 2) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrh RS1w, [MACHINE, RS1] + ldrh RS1w, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LUI: @@ -702,8 +691,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION0(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw RS1, [MACHINE, RS1] + ldrsw RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LW_VERSION1: @@ -711,8 +699,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION1(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw RS1, [MACHINE, RS1] + ldrsw RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LWU_VERSION0: @@ -720,8 +707,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION0(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr RS1w, [MACHINE, RS1] + ldr RS1w, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_LWU_VERSION1: @@ -729,8 +715,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_READ_VERSION1(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr RS1w, [MACHINE, RS1] + ldr RS1w, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_MUL: @@ -861,27 +846,24 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_WRITE(RS1, 1) - add RS1, RS1, MEMORY_OFFSET_ADDRESS ldr RS2, REGISTER_ADDRESS(RS2) - strb RS2w, [MACHINE, RS1] + strb RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_SD: DECODE_S ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS ldr RS2, REGISTER_ADDRESS(RS2) - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_SH: DECODE_S ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_WRITE(RS1, 2) - add RS1, RS1, MEMORY_OFFSET_ADDRESS ldr RS2, REGISTER_ADDRESS(RS2) - strh RS2w, [MACHINE, RS1] + strh RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_SLL: DECODE_R @@ -1020,9 +1002,8 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) add RS1, RS1, IMMEDIATE CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS ldr RS2, REGISTER_ADDRESS(RS2) - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_XOR: DECODE_R @@ -1042,8 +1023,7 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) CHECK_READ_VERSION1(RS1, 4) str RS1, LOAD_RESERVATION_ADDRESS - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw RS1, [MACHINE, RS1] + ldrsw RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_SC_W: @@ -1054,11 +1034,10 @@ ckb_vm_x64_execute: ldr TEMP1, LOAD_RESERVATION_ADDRESS mov TEMP2, UINT64_MAX str TEMP2, LOAD_RESERVATION_ADDRESS - add TEMP2, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP3w, [MACHINE, TEMP2] + ldr TEMP2w, [MEMORY_PTR, RS1] cmp RS1, TEMP1 - csel RS2w, RS2w, TEMP3w, eq - str RS2w, [MACHINE, TEMP2] + csel RS2w, RS2w, TEMP2w, eq + str RS2w, [MEMORY_PTR, RS1] mov TEMP2, #1 csel TEMP1, XZR, TEMP2, eq WRITE_RD(TEMP1) @@ -1068,110 +1047,100 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOADD_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) add RS2, RS2, TEMP1 - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOXOR_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) eor RS2, RS2, TEMP1 - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOAND_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) and RS2, RS2, TEMP1 - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOOR_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) orr RS2, RS2, TEMP1 - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMIN_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2w, TEMP1w csel RS2w, RS2w, TEMP1w, le - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMAX_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2w, TEMP1w csel RS2w, RS2w, TEMP1w, ge - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMINU_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2w, TEMP1w csel RS2w, RS2w, TEMP1w, ls - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMAXU_W: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 4) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldrsw TEMP1, [MACHINE, RS1] + ldrsw TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2w, TEMP1w csel RS2w, RS2w, TEMP1w, hs - str RS2w, [MACHINE, RS1] + str RS2w, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_LR_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) CHECK_READ_VERSION1(RS1, 8) str RS1, LOAD_RESERVATION_ADDRESS - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr RS1, [MACHINE, RS1] + ldr RS1, [MEMORY_PTR, RS1] WRITE_RD(RS1) NEXT_INST .CKB_VM_ASM_LABEL_OP_SC_D: @@ -1182,11 +1151,10 @@ ckb_vm_x64_execute: ldr TEMP1, LOAD_RESERVATION_ADDRESS mov TEMP2, UINT64_MAX str TEMP2, LOAD_RESERVATION_ADDRESS - add TEMP2, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP3, [MACHINE, TEMP2] + ldr TEMP2, [MEMORY_PTR, RS1] cmp RS1, TEMP1 - csel RS2, RS2, TEMP3, eq - str RS2, [MACHINE, TEMP2] + csel RS2, RS2, TEMP2, eq + str RS2, [MEMORY_PTR, RS1] mov TEMP2, #1 csel TEMP1, XZR, TEMP2, eq WRITE_RD(TEMP1) @@ -1196,102 +1164,93 @@ ckb_vm_x64_execute: ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOADD_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) add RS2, RS2, TEMP1 - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOXOR_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) eor RS2, RS2, TEMP1 - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOAND_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) and RS2, RS2, TEMP1 - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOOR_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) orr RS2, RS2, TEMP1 - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMIN_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2, TEMP1 csel RS2, RS2, TEMP1, le - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMAX_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2, TEMP1 csel RS2, RS2, TEMP1, ge - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMINU_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2, TEMP1 csel RS2, RS2, TEMP1, ls - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_AMOMAXU_D: DECODE_R ldr RS1, REGISTER_ADDRESS(RS1) ldr RS2, REGISTER_ADDRESS(RS2) CHECK_WRITE(RS1, 8) - add RS1, RS1, MEMORY_OFFSET_ADDRESS - ldr TEMP1, [MACHINE, RS1] + ldr TEMP1, [MEMORY_PTR, RS1] WRITE_RD(TEMP1) cmp RS2, TEMP1 csel RS2, RS2, TEMP1, hs - str RS2, [MACHINE, RS1] + str RS2, [MEMORY_PTR, RS1] NEXT_INST .CKB_VM_ASM_LABEL_OP_ADDUW: DECODE_R diff --git a/src/machine/asm/execute_x64.S b/src/machine/asm/execute_x64.S index 2f336df1..429b911d 100644 --- a/src/machine/asm/execute_x64.S +++ b/src/machine/asm/execute_x64.S @@ -192,10 +192,11 @@ movq address_reg, TEMP1; \ shr $CKB_VM_ASM_MEMORY_FRAME_SHIFTS, TEMP1; \ movq TEMP1, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_LAST_READ_FRAME(MACHINE); \ - movzbl CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1), TEMP2d; \ + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR(MACHINE), TEMP3; \ + movzbl (TEMP3, TEMP1), TEMP2d; \ cmp $0, TEMP2d; \ jne 1f; \ - movb $1, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1); \ + movb $1, (TEMP3, TEMP1); \ PREPCALL; \ MOV_TEMP1_TO_ARG1; \ MOV_MACHINE_TO_ARG2; \ @@ -206,10 +207,11 @@ addq $length, TEMP1; \ subq $1, TEMP1; \ shr $CKB_VM_ASM_MEMORY_FRAME_SHIFTS, TEMP1; \ - movzbl CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1), TEMP2d; \ + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR(MACHINE), TEMP3; \ + movzbl (TEMP3, TEMP1), TEMP2d; \ cmp $0, TEMP2d; \ jne 2f; \ - movb $1, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1);\ + movb $1, (TEMP3, TEMP1);\ PREPCALL; \ MOV_TEMP1_TO_ARG1; \ MOV_MACHINE_TO_ARG2; \ @@ -273,19 +275,21 @@ movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_SIZE(MACHINE), TEMP2; \ cmp TEMP2, TEMP1; \ jae .exit_out_of_bound; \ - movzbl CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS(MACHINE, TEMP1), temp_regd; \ + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_PTR(MACHINE), TEMP3; \ + movzbl (TEMP3, TEMP1), temp_regd; \ mov temp_regd, TEMP2d; \ and $CKB_VM_ASM_MEMORY_FLAG_WXORX_BIT, temp_regd; \ cmp $CKB_VM_ASM_MEMORY_FLAG_WRITABLE, temp_regd; \ jne .exit_invalid_permission; \ or $CKB_VM_ASM_MEMORY_FLAG_DIRTY, TEMP2b; \ - movb TEMP2b, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS(MACHINE, TEMP1); \ + movb TEMP2b, (TEMP3, TEMP1); \ movq TEMP1, TEMP2; \ shr $CKB_VM_ASM_MEMORY_FRAME_PAGE_SHIFTS, TEMP1; \ - movzbl CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1), temp_regd; \ + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR(MACHINE), TEMP3; \ + movzbl (TEMP3, TEMP1), temp_regd; \ cmp $0, temp_regd; \ jne 1f; \ - movb $1, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1); \ + movb $1, (TEMP3, TEMP1); \ PREPCALL; \ MOV_TEMP1_TO_ARG1; \ MOV_MACHINE_TO_ARG2; \ @@ -303,18 +307,20 @@ movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_SIZE(MACHINE), TEMP2; \ cmp TEMP2, TEMP1; \ jae .exit_out_of_bound; \ - movzbl CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS(MACHINE, TEMP1), temp_regd; \ + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS_PTR(MACHINE), TEMP3; \ + movzbl (TEMP3, TEMP1), temp_regd; \ mov temp_regd, TEMP2d; \ and $CKB_VM_ASM_MEMORY_FLAG_WXORX_BIT, temp_regd; \ cmp $CKB_VM_ASM_MEMORY_FLAG_WRITABLE, temp_regd; \ jne .exit_invalid_permission; \ or $CKB_VM_ASM_MEMORY_FLAG_DIRTY, TEMP2b; \ - movb TEMP2b, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FLAGS(MACHINE, TEMP1); \ + movb TEMP2b, (TEMP3, TEMP1); \ shr $CKB_VM_ASM_MEMORY_FRAME_PAGE_SHIFTS, TEMP1; \ - movzbl CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1), temp_regd; \ + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES_PTR(MACHINE), TEMP3; \ + movzbl (TEMP3, TEMP1), temp_regd; \ cmp $0, temp_regd; \ jne 2f; \ - movb $1, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_FRAMES(MACHINE, TEMP1); \ + movb $1, (TEMP3, TEMP1); \ PREPCALL; \ MOV_TEMP1_TO_ARG1; \ MOV_MACHINE_TO_ARG2; \ @@ -749,7 +755,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION0(RS1, 1) - movsbq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movsbq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -758,7 +765,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION1(RS1, 1) - movsbq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movsbq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -767,7 +775,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION0(RS1, 1) - movzbq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movzbq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -776,7 +785,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION1(RS1, 1) - movzbq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movzbq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -785,7 +795,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION0(RS1, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -794,7 +805,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION1(RS1, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -803,7 +815,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION0(RS1, 2) - movswq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movswq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -812,7 +825,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION1(RS1, 2) - movswq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movswq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -821,7 +835,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION0(RS1, 2) - movzwq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movzwq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -830,7 +845,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION1(RS1, 2) - movzwq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movzwq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -851,7 +867,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION0(RS1, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -860,7 +877,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION1(RS1, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), RS1 WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -869,7 +887,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION0(RS1, 4) - mov CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1d + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + mov (TEMP1, RS1), RS1d WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -878,7 +897,8 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 addq IMMEDIATE, RS1 CHECK_READ_VERSION1(RS1, 4) - mov CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS1d + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + mov (TEMP1, RS1), RS1d WRITE_RD(RS1) NEXT_INST .p2align 3 @@ -1076,7 +1096,8 @@ ckb_vm_x64_execute: addq IMMEDIATE, RS1 CHECK_WRITE(RS1, RS2rd, 1) movq REGISTER_ADDRESS(RS2s), RS2s - mov RS2sb, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + mov RS2sb, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_SD: @@ -1085,7 +1106,8 @@ ckb_vm_x64_execute: addq IMMEDIATE, RS1 CHECK_WRITE(RS1, RS2rd, 8) movq REGISTER_ADDRESS(RS2s), RS2s - movq RS2s, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq RS2s, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_SH: @@ -1094,7 +1116,8 @@ ckb_vm_x64_execute: addq IMMEDIATE, RS1 CHECK_WRITE(RS1, RS2rd, 2) movq REGISTER_ADDRESS(RS2s), RS2s - mov RS2sh, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + mov RS2sh, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_SLL: @@ -1261,7 +1284,8 @@ ckb_vm_x64_execute: addq IMMEDIATE, RS1 CHECK_WRITE(RS1, RS2rd, 4) movq REGISTER_ADDRESS(RS2s), RS2s - mov RS2sd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + mov RS2sd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_XOR: @@ -1282,8 +1306,9 @@ ckb_vm_x64_execute: DECODE_R movq REGISTER_ADDRESS(RS1), RS1 CHECK_READ_VERSION1(RS1, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) movq RS1, LOAD_RESERVATION_ADDRESS NEXT_INST .p2align 3 @@ -1294,9 +1319,10 @@ ckb_vm_x64_execute: CHECK_WRITE(RS1, RS3d, 4) movq LOAD_RESERVATION_ADDRESS, TEMP1 movq $UINT64_MAX, LOAD_RESERVATION_ADDRESS + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP2 cmp RS1, TEMP1 - cmovne CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS2rd - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + cmovne (TEMP2, RS1), RS2rd + mov RS2rd, (TEMP2, RS1) setnz TEMP1b movzx TEMP1b, TEMP1 WRITE_RD(TEMP1) @@ -1307,9 +1333,10 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOADD_W: @@ -1317,10 +1344,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - add TEMP1, RS2r - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + add TEMP2, RS2r + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOXOR_W: @@ -1328,10 +1356,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - xor TEMP1, RS2r - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + xor TEMP2, RS2r + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOAND_W: @@ -1339,10 +1368,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - and TEMP1, RS2r - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + and TEMP2, RS2r + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOOR_W: @@ -1350,10 +1380,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - or TEMP1, RS2r - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + or TEMP2, RS2r + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMIN_W: @@ -1361,11 +1392,12 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2rd, TEMP1d - cmovle TEMP1d, RS2rd - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2rd, TEMP2d + cmovle TEMP2d, RS2rd + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMAX_W: @@ -1373,11 +1405,12 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2rd, TEMP1d - cmovge TEMP1d, RS2rd - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2rd, TEMP2d + cmovge TEMP2d, RS2rd + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMINU_W: @@ -1385,11 +1418,12 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2rd, TEMP1d - cmovbe TEMP1d, RS2rd - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2rd, TEMP2d + cmovbe TEMP2d, RS2rd + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMAXU_W: @@ -1397,20 +1431,22 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 4) - movslq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2rd, TEMP1d - cmovae TEMP1d, RS2rd - mov RS2rd, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movslq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2rd, TEMP2d + cmovae TEMP2d, RS2rd + mov RS2rd, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_LR_D: DECODE_R movq REGISTER_ADDRESS(RS1), RS1 CHECK_READ_VERSION1(RS1, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 movq RS1, LOAD_RESERVATION_ADDRESS - WRITE_RD(TEMP1) + WRITE_RD(TEMP2) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_SC_D: @@ -1420,9 +1456,10 @@ ckb_vm_x64_execute: CHECK_WRITE(RS1, RS3d, 8) movq LOAD_RESERVATION_ADDRESS, TEMP1 movq $UINT64_MAX, LOAD_RESERVATION_ADDRESS + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP2 cmp RS1, TEMP1 - cmovne CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + cmovne (TEMP2, RS1), RS2r + movq RS2r, (TEMP2, RS1) setnz TEMP1b movzx TEMP1b, TEMP1 WRITE_RD(TEMP1) @@ -1433,9 +1470,10 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOADD_D: @@ -1443,10 +1481,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - add TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + add TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOXOR_D: @@ -1454,10 +1493,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - xor TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + xor TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOAND_D: @@ -1465,10 +1505,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - and TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + and TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOOR_D: @@ -1476,10 +1517,11 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - or TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + or TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMIN_D: @@ -1487,11 +1529,12 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2r, TEMP1 - cmovle TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2r, TEMP2 + cmovle TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMAX_D: @@ -1499,11 +1542,12 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2r, TEMP1 - cmovge TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2r, TEMP2 + cmovge TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMINU_D: @@ -1511,11 +1555,12 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2r, TEMP1 - cmovbe TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2r, TEMP2 + cmovbe TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_AMOMAXU_D: @@ -1523,11 +1568,12 @@ ckb_vm_x64_execute: movq REGISTER_ADDRESS(RS1), RS1 movq REGISTER_ADDRESS(RS2r), RS2r CHECK_WRITE(RS1, RS3d, 8) - movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1), TEMP1 - WRITE_RD(TEMP1) - cmp RS2r, TEMP1 - cmovae TEMP1, RS2r - movq RS2r, CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY(MACHINE, RS1) + movq CKB_VM_ASM_ASM_CORE_MACHINE_OFFSET_MEMORY_PTR(MACHINE), TEMP1 + movq (TEMP1, RS1), TEMP2 + WRITE_RD(TEMP2) + cmp RS2r, TEMP2 + cmovae TEMP2, RS2r + movq RS2r, (TEMP1, RS1) NEXT_INST .p2align 3 .CKB_VM_ASM_LABEL_OP_ADDUW: diff --git a/src/machine/asm/mod.rs b/src/machine/asm/mod.rs index a4d4b8b4..6e5a7237 100644 --- a/src/machine/asm/mod.rs +++ b/src/machine/asm/mod.rs @@ -9,8 +9,7 @@ use ckb_vm_definitions::{ RET_EBREAK, RET_ECALL, RET_INVALID_PERMISSION, RET_MAX_CYCLES_EXCEEDED, RET_OUT_OF_BOUND, RET_PAUSE, RET_SLOWPATH, }, - ISA_MOP, MEMORY_FRAMES, MEMORY_FRAME_PAGE_SHIFTS, RISCV_GENERAL_REGISTER_NUMBER, - RISCV_PAGE_SHIFTS, + ISA_MOP, MEMORY_FRAME_PAGE_SHIFTS, RISCV_GENERAL_REGISTER_NUMBER, RISCV_PAGE_SHIFTS, }; use rand::{prelude::RngCore, SeedableRng}; use std::os::raw::c_uchar; @@ -24,11 +23,11 @@ use crate::{ VERSION0, }, memory::{ - fill_page_data, get_page_indices, memset, round_page_down, round_page_up, FLAG_DIRTY, - FLAG_EXECUTABLE, FLAG_FREEZED, FLAG_WRITABLE, FLAG_WXORX_BIT, + check_no_overflow, fill_page_data, get_page_indices, memset, round_page_down, + round_page_up, FLAG_DIRTY, FLAG_EXECUTABLE, FLAG_FREEZED, FLAG_WRITABLE, FLAG_WXORX_BIT, }, CoreMachine, DefaultMachine, Error, Machine, Memory, SupportMachine, MEMORY_FRAME_SHIFTS, - RISCV_PAGES, RISCV_PAGESIZE, + RISCV_PAGESIZE, }; impl CoreMachine for Box { @@ -81,21 +80,27 @@ impl CoreMachine for Box { #[no_mangle] pub extern "C" fn inited_memory(frame_index: u64, machine: &mut AsmCoreMachine) { let addr_from = (frame_index << MEMORY_FRAME_SHIFTS) as usize; - let addr_to = ((frame_index + 1) << MEMORY_FRAME_SHIFTS) as usize; + let slice = + machine.cast_ptr_to_slice_mut(machine.memory_ptr, addr_from, 1 << MEMORY_FRAME_SHIFTS); if machine.chaos_mode != 0 { let mut gen = rand::rngs::StdRng::seed_from_u64(machine.chaos_seed.into()); - gen.fill_bytes(&mut machine.memory[addr_from..addr_to]); + gen.fill_bytes(slice); machine.chaos_seed = gen.next_u32(); } else { - memset(&mut machine.memory[addr_from..addr_to], 0); + memset(slice, 0); } } fn check_memory(machine: &mut AsmCoreMachine, page: u64) { - let frame = page >> MEMORY_FRAME_PAGE_SHIFTS; - if machine.frames[frame as usize] == 0 { - inited_memory(frame, machine); - machine.frames[frame as usize] = 1; + let frame_index = page >> MEMORY_FRAME_PAGE_SHIFTS; + unsafe { + let frames = machine.frames_ptr as *mut u8; + let frame_addr = frames.add(frame_index as usize); + let frame_flag = frame_addr.read(); + if frame_flag == 0 { + inited_memory(frame_index, machine); + frame_addr.write(0x01); + } } } @@ -198,6 +203,7 @@ struct FastMemory<'a>(&'a mut Box); impl<'a> FastMemory<'a> { fn prepare_memory(&mut self, addr: u64, size: u64) -> Result<(), Error> { + check_no_overflow(addr, size, self.0.memory_size)?; let frame_start = addr >> MEMORY_FRAME_SHIFTS << MEMORY_FRAME_SHIFTS; // There is some memory space between the start of the first memory // frame touched, and the starting address of memory to be written. We @@ -214,10 +220,13 @@ impl<'a> FastMemory<'a> { if (aligned_end + RISCV_PAGESIZE as u64) < frame_next_start { check_memory(self.0, aligned_end >> RISCV_PAGE_SHIFTS); } - let page_indices = get_page_indices(addr, size)?; + let page_indices = get_page_indices(addr, size); for page in page_indices.0..=page_indices.1 { - let frame = page >> MEMORY_FRAME_PAGE_SHIFTS; - self.0.frames[frame as usize] = 1; + let frame_index = page >> MEMORY_FRAME_PAGE_SHIFTS; + let slice = self + .0 + .cast_ptr_to_slice_mut(self.0.frames_ptr, frame_index as usize, 1); + slice[0] = 1; self.0.set_flag(page, FLAG_DIRTY)?; } Ok(()) @@ -232,7 +241,9 @@ impl<'a> Memory for FastMemory<'a> { return Ok(()); } self.prepare_memory(addr, value.len() as u64)?; - let slice = &mut self.0.memory[addr as usize..addr as usize + value.len()]; + let slice = self + .0 + .cast_ptr_to_slice_mut(self.0.memory_ptr, addr as usize, value.len()); slice.copy_from_slice(value); Ok(()) } @@ -242,10 +253,10 @@ impl<'a> Memory for FastMemory<'a> { return Ok(()); } self.prepare_memory(addr, size)?; - memset( - &mut self.0.memory[addr as usize..(addr + size) as usize], - value, - ); + let slice = self + .0 + .cast_ptr_to_slice_mut(self.0.memory_ptr, addr as usize, size as usize); + memset(slice, value); Ok(()) } @@ -337,8 +348,10 @@ impl Memory for Box { type REG = u64; fn reset_memory(&mut self) -> Result<(), Error> { - self.flags = [0; RISCV_PAGES]; - self.frames = [0; MEMORY_FRAMES]; + let slice = self.cast_ptr_to_slice_mut(self.flags_ptr, 0, self.flags_size as usize); + memset(slice, 0); + let slice = self.cast_ptr_to_slice_mut(self.frames_ptr, 0, self.frames_size as usize); + memset(slice, 0); self.load_reservation_address = u64::MAX; self.last_read_frame = u64::max_value(); self.last_write_page = u64::max_value(); @@ -390,7 +403,8 @@ impl Memory for Box { fn fetch_flag(&mut self, page: u64) -> Result { if page < self.memory_pages() as u64 { - Ok(self.flags[page as usize]) + let slice = self.cast_ptr_to_slice(self.flags_ptr, page as usize, 1); + Ok(slice[0]) } else { Err(Error::MemOutOfBound) } @@ -398,7 +412,8 @@ impl Memory for Box { fn set_flag(&mut self, page: u64, flag: u8) -> Result<(), Error> { if page < self.memory_pages() as u64 { - self.flags[page as usize] |= flag; + let slice = self.cast_ptr_to_slice_mut(self.flags_ptr, page as usize, 1); + slice[0] |= flag; // Clear last write page cache self.last_write_page = u64::max_value(); Ok(()) @@ -409,7 +424,8 @@ impl Memory for Box { fn clear_flag(&mut self, page: u64, flag: u8) -> Result<(), Error> { if page < self.memory_pages() as u64 { - self.flags[page as usize] &= !flag; + let slice = self.cast_ptr_to_slice_mut(self.flags_ptr, page as usize, 1); + slice[0] &= !flag; // Clear last write page cache self.last_write_page = u64::max_value(); Ok(()) @@ -426,13 +442,14 @@ impl Memory for Box { if value.is_empty() { return Ok(()); } - let page_indices = get_page_indices(addr, value.len() as u64)?; + check_no_overflow(addr, value.len() as u64, self.memory_size)?; + let page_indices = get_page_indices(addr, value.len() as u64); for page in page_indices.0..=page_indices.1 { check_permission(self, page, FLAG_WRITABLE)?; check_memory(self, page); self.set_flag(page, FLAG_DIRTY)?; } - let slice = &mut self.memory[addr as usize..addr as usize + value.len()]; + let slice = self.cast_ptr_to_slice_mut(self.memory_ptr, addr as usize, value.len()); slice.copy_from_slice(value); Ok(()) } @@ -441,16 +458,15 @@ impl Memory for Box { if size == 0 { return Ok(()); } - let page_indices = get_page_indices(addr, size)?; + check_no_overflow(addr, size, self.memory_size)?; + let page_indices = get_page_indices(addr, size); for page in page_indices.0..=page_indices.1 { check_permission(self, page, FLAG_WRITABLE)?; check_memory(self, page); self.set_flag(page, FLAG_DIRTY)?; } - memset( - &mut self.memory[addr as usize..(addr + size) as usize], - value, - ); + let slice = self.cast_ptr_to_slice_mut(self.memory_ptr, addr as usize, size as usize); + memset(slice, value); Ok(()) } @@ -458,90 +474,88 @@ impl Memory for Box { if size == 0 { return Ok(Bytes::new()); } - let page_indices = get_page_indices(addr, size)?; + check_no_overflow(addr, size, self.memory_size)?; + let page_indices = get_page_indices(addr, size); for page in page_indices.0..=page_indices.1 { check_memory(self, page); } - Ok(Bytes::from( - self.memory[addr as usize..(addr + size) as usize].to_vec(), - )) + let slice = unsafe { + let memory = self.memory_ptr as *mut u8; + let memory_from = memory.add(addr as usize); + std::slice::from_raw_parts(memory_from, size as usize) + }; + Ok(Bytes::from(slice)) } fn execute_load16(&mut self, addr: u64) -> Result { check_memory_executable(self, addr, 2)?; - Ok(LittleEndian::read_u16( - &self.memory[addr as usize..addr as usize + 2], - )) + let slice = self.cast_ptr_to_slice(self.memory_ptr, addr as usize, 2); + Ok(LittleEndian::read_u16(slice)) } fn execute_load32(&mut self, addr: u64) -> Result { check_memory_executable(self, addr, 4)?; - Ok(LittleEndian::read_u32( - &self.memory[addr as usize..addr as usize + 4], - )) + let slice = self.cast_ptr_to_slice(self.memory_ptr, addr as usize, 4); + Ok(LittleEndian::read_u32(slice)) } fn load8(&mut self, addr: &u64) -> Result { let addr = *addr; check_memory_inited(self, addr, 1)?; - Ok(u64::from(self.memory[addr as usize])) + let slice = self.cast_ptr_to_slice(self.memory_ptr, addr as usize, 1); + Ok(u64::from(slice[0])) } fn load16(&mut self, addr: &u64) -> Result { let addr = *addr; check_memory_inited(self, addr, 2)?; - Ok(u64::from(LittleEndian::read_u16( - &self.memory[addr as usize..addr as usize + 2], - ))) + let slice = self.cast_ptr_to_slice(self.memory_ptr, addr as usize, 2); + Ok(u64::from(LittleEndian::read_u16(slice))) } fn load32(&mut self, addr: &u64) -> Result { let addr = *addr; check_memory_inited(self, addr, 4)?; - Ok(u64::from(LittleEndian::read_u32( - &self.memory[addr as usize..addr as usize + 4], - ))) + let slice = self.cast_ptr_to_slice(self.memory_ptr, addr as usize, 4); + Ok(u64::from(LittleEndian::read_u32(slice))) } fn load64(&mut self, addr: &u64) -> Result { let addr = *addr; check_memory_inited(self, addr, 8)?; - Ok(LittleEndian::read_u64( - &self.memory[addr as usize..addr as usize + 8], - )) + let slice = self.cast_ptr_to_slice(self.memory_ptr, addr as usize, 8); + Ok(LittleEndian::read_u64(slice)) } fn store8(&mut self, addr: &u64, value: &u64) -> Result<(), Error> { let addr = *addr; check_memory_writable(self, addr, 1)?; - self.memory[addr as usize] = (*value) as u8; + let slice = self.cast_ptr_to_slice_mut(self.memory_ptr, addr as usize, 1); + slice[0] = *value as u8; Ok(()) } fn store16(&mut self, addr: &u64, value: &u64) -> Result<(), Error> { let addr = *addr; check_memory_writable(self, addr, 2)?; - LittleEndian::write_u16( - &mut self.memory[addr as usize..(addr + 2) as usize], - *value as u16, - ); + let slice = self.cast_ptr_to_slice_mut(self.memory_ptr, addr as usize, 2); + LittleEndian::write_u16(slice, *value as u16); Ok(()) } fn store32(&mut self, addr: &u64, value: &u64) -> Result<(), Error> { let addr = *addr; check_memory_writable(self, addr, 4)?; - LittleEndian::write_u32( - &mut self.memory[addr as usize..(addr + 4) as usize], - *value as u32, - ); + let slice = self.cast_ptr_to_slice_mut(self.memory_ptr, addr as usize, 4); + LittleEndian::write_u32(slice, *value as u32); Ok(()) } fn store64(&mut self, addr: &u64, value: &u64) -> Result<(), Error> { let addr = *addr; check_memory_writable(self, addr, 8)?; - LittleEndian::write_u64(&mut self.memory[addr as usize..(addr + 8) as usize], *value); + let slice = self.cast_ptr_to_slice_mut(self.memory_ptr, addr as usize, 8); + LittleEndian::write_u64(slice, *value as u64); Ok(()) } diff --git a/src/memory/flat.rs b/src/memory/flat.rs index 6613ac16..a68a48c5 100644 --- a/src/memory/flat.rs +++ b/src/memory/flat.rs @@ -1,5 +1,5 @@ -use super::super::{Error, Register, RISCV_MAX_MEMORY, RISCV_PAGESIZE}; -use super::{fill_page_data, get_page_indices, memset, set_dirty, Memory}; +use super::super::{Error, Register, DEFAULT_MEMORY_SIZE, RISCV_PAGESIZE}; +use super::{check_no_overflow, fill_page_data, get_page_indices, memset, set_dirty, Memory}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use bytes::Bytes; @@ -32,7 +32,6 @@ impl DerefMut for FlatMemory { impl FlatMemory { pub fn new_with_memory(memory_size: usize) -> Self { - assert!(memory_size <= RISCV_MAX_MEMORY); assert!(memory_size % RISCV_PAGESIZE == 0); Self { data: vec![0; memory_size], @@ -47,7 +46,7 @@ impl FlatMemory { impl Default for FlatMemory { fn default() -> Self { - Self::new_with_memory(RISCV_MAX_MEMORY) + Self::new_with_memory(DEFAULT_MEMORY_SIZE) } } @@ -161,7 +160,8 @@ impl Memory for FlatMemory { fn store8(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { let addr = addr.to_u64(); - let page_indices = get_page_indices(addr.to_u64(), 1)?; + check_no_overflow(addr, 1, self.memory_size as u64)?; + let page_indices = get_page_indices(addr, 1); set_dirty(self, &page_indices)?; let mut writer = Cursor::new(&mut self.data); writer.seek(SeekFrom::Start(addr as u64))?; @@ -171,7 +171,8 @@ impl Memory for FlatMemory { fn store16(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { let addr = addr.to_u64(); - let page_indices = get_page_indices(addr.to_u64(), 2)?; + check_no_overflow(addr, 2, self.memory_size as u64)?; + let page_indices = get_page_indices(addr, 2); set_dirty(self, &page_indices)?; let mut writer = Cursor::new(&mut self.data); writer.seek(SeekFrom::Start(addr as u64))?; @@ -181,7 +182,8 @@ impl Memory for FlatMemory { fn store32(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { let addr = addr.to_u64(); - let page_indices = get_page_indices(addr.to_u64(), 4)?; + check_no_overflow(addr, 4, self.memory_size as u64)?; + let page_indices = get_page_indices(addr, 4); set_dirty(self, &page_indices)?; let mut writer = Cursor::new(&mut self.data); writer.seek(SeekFrom::Start(addr as u64))?; @@ -191,7 +193,8 @@ impl Memory for FlatMemory { fn store64(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { let addr = addr.to_u64(); - let page_indices = get_page_indices(addr.to_u64(), 8)?; + check_no_overflow(addr, 8, self.memory_size as u64)?; + let page_indices = get_page_indices(addr, 8); set_dirty(self, &page_indices)?; let mut writer = Cursor::new(&mut self.data); writer.seek(SeekFrom::Start(addr as u64))?; @@ -204,7 +207,8 @@ impl Memory for FlatMemory { if size == 0 { return Ok(()); } - let page_indices = get_page_indices(addr.to_u64(), size)?; + check_no_overflow(addr, size, self.memory_size as u64)?; + let page_indices = get_page_indices(addr, size); set_dirty(self, &page_indices)?; let slice = &mut self[addr as usize..(addr + size) as usize]; slice.copy_from_slice(value); @@ -215,7 +219,8 @@ impl Memory for FlatMemory { if size == 0 { return Ok(()); } - let page_indices = get_page_indices(addr.to_u64(), size)?; + check_no_overflow(addr, size, self.memory_size as u64)?; + let page_indices = get_page_indices(addr, size); set_dirty(self, &page_indices)?; memset(&mut self[addr as usize..(addr + size) as usize], value); Ok(()) diff --git a/src/memory/mod.rs b/src/memory/mod.rs index b4b8437e..e53886fb 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -12,7 +12,7 @@ pub mod wxorx; pub use ckb_vm_definitions::{ memory::{FLAG_DIRTY, FLAG_EXECUTABLE, FLAG_FREEZED, FLAG_WRITABLE, FLAG_WXORX_BIT}, - MEMORY_FRAME_PAGE_SHIFTS, RISCV_MAX_MEMORY, RISCV_PAGE_SHIFTS, + DEFAULT_MEMORY_SIZE, MEMORY_FRAME_PAGE_SHIFTS, RISCV_PAGE_SHIFTS, }; #[inline(always)] @@ -100,18 +100,21 @@ pub fn fill_page_data( Ok(()) } -// `size` should be none zero u64 -pub fn get_page_indices(addr: u64, size: u64) -> Result<(u64, u64), Error> { +pub fn check_no_overflow(addr: u64, size: u64, memory_size: u64) -> Result<(), Error> { let (addr_end, overflowed) = addr.overflowing_add(size); - if overflowed { - return Err(Error::MemOutOfBound); - } - if addr_end > RISCV_MAX_MEMORY as u64 { - return Err(Error::MemOutOfBound); + if overflowed || addr_end > memory_size { + Err(Error::MemOutOfBound) + } else { + Ok(()) } +} + +// `size` should be none zero u64 +pub fn get_page_indices(addr: u64, size: u64) -> (u64, u64) { + let addr_end = addr.wrapping_add(size); let page = addr >> RISCV_PAGE_SHIFTS; let page_end = (addr_end - 1) >> RISCV_PAGE_SHIFTS; - Ok((page, page_end)) + (page, page_end) } pub fn check_permission( diff --git a/src/memory/sparse.rs b/src/memory/sparse.rs index c4320318..89500216 100644 --- a/src/memory/sparse.rs +++ b/src/memory/sparse.rs @@ -1,4 +1,4 @@ -use super::super::{Error, Register, RISCV_MAX_MEMORY, RISCV_PAGESIZE, RISCV_PAGE_SHIFTS}; +use super::super::{Error, Register, DEFAULT_MEMORY_SIZE, RISCV_PAGESIZE, RISCV_PAGE_SHIFTS}; use super::{fill_page_data, memset, round_page_down, Memory, Page, FLAG_DIRTY}; use bytes::Bytes; @@ -67,7 +67,6 @@ impl SparseMemory { } pub fn new_with_memory(memory_size: usize) -> Self { - assert!(memory_size <= RISCV_MAX_MEMORY); assert!(memory_size % RISCV_PAGESIZE == 0); Self { indices: vec![INVALID_PAGE_INDEX; memory_size / RISCV_PAGESIZE], @@ -83,7 +82,7 @@ impl SparseMemory { impl Default for SparseMemory { fn default() -> Self { - Self::new_with_memory(RISCV_MAX_MEMORY) + Self::new_with_memory(DEFAULT_MEMORY_SIZE) } } diff --git a/src/memory/wxorx.rs b/src/memory/wxorx.rs index 5a50eea5..3578d61f 100644 --- a/src/memory/wxorx.rs +++ b/src/memory/wxorx.rs @@ -1,7 +1,7 @@ use super::super::{Error, Register, RISCV_PAGESIZE}; use super::{ - check_permission, get_page_indices, round_page_down, round_page_up, Memory, FLAG_EXECUTABLE, - FLAG_FREEZED, FLAG_WRITABLE, + check_no_overflow, check_permission, get_page_indices, round_page_down, round_page_up, Memory, + FLAG_EXECUTABLE, FLAG_FREEZED, FLAG_WRITABLE, }; use bytes::Bytes; @@ -80,13 +80,15 @@ impl Memory for WXorXMemory { } fn execute_load16(&mut self, addr: u64) -> Result { - let page_indices = get_page_indices(addr, 2)?; + check_no_overflow(addr, 2, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr, 2); check_permission(self, &page_indices, FLAG_EXECUTABLE)?; self.inner.execute_load16(addr) } fn execute_load32(&mut self, addr: u64) -> Result { - let page_indices = get_page_indices(addr, 4)?; + check_no_overflow(addr, 4, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr, 4); check_permission(self, &page_indices, FLAG_EXECUTABLE)?; self.inner.execute_load32(addr) } @@ -108,25 +110,29 @@ impl Memory for WXorXMemory { } fn store8(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { - let page_indices = get_page_indices(addr.to_u64(), 1)?; + check_no_overflow(addr.to_u64(), 1, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr.to_u64(), 1); check_permission(self, &page_indices, FLAG_WRITABLE)?; self.inner.store8(addr, value) } fn store16(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { - let page_indices = get_page_indices(addr.to_u64(), 2)?; + check_no_overflow(addr.to_u64(), 2, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr.to_u64(), 2); check_permission(self, &page_indices, FLAG_WRITABLE)?; self.inner.store16(addr, value) } fn store32(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { - let page_indices = get_page_indices(addr.to_u64(), 4)?; + check_no_overflow(addr.to_u64(), 4, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr.to_u64(), 4); check_permission(self, &page_indices, FLAG_WRITABLE)?; self.inner.store32(addr, value) } fn store64(&mut self, addr: &Self::REG, value: &Self::REG) -> Result<(), Error> { - let page_indices = get_page_indices(addr.to_u64(), 8)?; + check_no_overflow(addr.to_u64(), 8, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr.to_u64(), 8); check_permission(self, &page_indices, FLAG_WRITABLE)?; self.inner.store64(addr, value) } @@ -135,7 +141,8 @@ impl Memory for WXorXMemory { if value.is_empty() { return Ok(()); } - let page_indices = get_page_indices(addr, value.len() as u64)?; + check_no_overflow(addr.to_u64(), value.len() as u64, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr, value.len() as u64); check_permission(self, &page_indices, FLAG_WRITABLE)?; self.inner.store_bytes(addr, value) } @@ -144,7 +151,8 @@ impl Memory for WXorXMemory { if size == 0 { return Ok(()); } - let page_indices = get_page_indices(addr, size)?; + check_no_overflow(addr.to_u64(), size, self.memory_size() as u64)?; + let page_indices = get_page_indices(addr, size); check_permission(self, &page_indices, FLAG_WRITABLE)?; self.inner.store_byte(addr, size, value) } diff --git a/tests/test_a_extension.rs b/tests/test_a_extension.rs index d6fa9a42..61bdbcc7 100644 --- a/tests/test_a_extension.rs +++ b/tests/test_a_extension.rs @@ -1,4 +1,6 @@ -use ckb_vm::{CoreMachine, Error, Memory}; +use ckb_vm::Error; +#[cfg(has_asm)] +use ckb_vm::{CoreMachine, Memory}; pub mod machine_build; #[test] diff --git a/tests/test_asm.rs b/tests/test_asm.rs index 34ae065b..c82544d3 100644 --- a/tests/test_asm.rs +++ b/tests/test_asm.rs @@ -493,10 +493,13 @@ fn test_fast_memory_initialization_bug() { let isa = ISA_IMC; let version = VERSION1; let buffer = fs::read("benches/data/secp256k1_bench").unwrap().into(); - let mut asm_core = AsmCoreMachine::new(isa, version, u64::max_value()); - asm_core.memory[0] = 0x1; + let asm_core = AsmCoreMachine::new(isa, version, u64::max_value()); let core = DefaultMachineBuilder::new(asm_core).build(); let mut machine = AsmMachine::new(core); + unsafe { + let memory = machine.machine.inner_mut().memory_ptr as *mut u8; + memory.write(0x01); + } machine.load_program(&buffer, &[]).unwrap(); assert_eq!(machine.machine.memory_mut().load8(&0).unwrap(), 0); } diff --git a/tests/test_misc.rs b/tests/test_misc.rs index f82ea903..25a40e51 100644 --- a/tests/test_misc.rs +++ b/tests/test_misc.rs @@ -3,8 +3,8 @@ use ckb_vm::machine::VERSION0; use ckb_vm::registers::{A0, A1, A2, A3, A4, A5, A7}; use ckb_vm::{ run, CoreMachine, Debugger, DefaultCoreMachine, DefaultMachineBuilder, Error, FlatMemory, - Memory, Register, SparseMemory, SupportMachine, Syscalls, WXorXMemory, ISA_IMC, - RISCV_MAX_MEMORY, RISCV_PAGESIZE, + Memory, Register, SparseMemory, SupportMachine, Syscalls, WXorXMemory, DEFAULT_MEMORY_SIZE, + ISA_IMC, RISCV_PAGESIZE, }; #[cfg(has_asm)] use ckb_vm_definitions::asm::AsmCoreMachine; @@ -215,10 +215,10 @@ fn assert_memory_store_empty_bytes(memory: &mut M) { pub fn test_memory_load_bytes() { let mut rng = thread_rng(); - assert_memory_load_bytes_all(&mut rng, RISCV_MAX_MEMORY, 1024 * 5, 0); - assert_memory_load_bytes_all(&mut rng, RISCV_MAX_MEMORY, 1024 * 5, 2); - assert_memory_load_bytes_all(&mut rng, RISCV_MAX_MEMORY, 1024 * 5, 1024 * 6); - assert_memory_load_bytes_all(&mut rng, RISCV_MAX_MEMORY, 0, 0); + assert_memory_load_bytes_all(&mut rng, DEFAULT_MEMORY_SIZE, 1024 * 5, 0); + assert_memory_load_bytes_all(&mut rng, DEFAULT_MEMORY_SIZE, 1024 * 5, 2); + assert_memory_load_bytes_all(&mut rng, DEFAULT_MEMORY_SIZE, 1024 * 5, 1024 * 6); + assert_memory_load_bytes_all(&mut rng, DEFAULT_MEMORY_SIZE, 0, 0); } fn assert_memory_load_bytes_all( diff --git a/tests/test_reset.rs b/tests/test_reset.rs index 5e830cd3..0f034de2 100644 --- a/tests/test_reset.rs +++ b/tests/test_reset.rs @@ -5,7 +5,7 @@ use ckb_vm::machine::asm::{AsmCoreMachine, AsmMachine}; use ckb_vm::machine::{DefaultCoreMachine, DefaultMachineBuilder, VERSION1}; use ckb_vm::{ registers::A7, Error, Register, SparseMemory, SupportMachine, Syscalls, TraceMachine, - WXorXMemory, DEFAULT_STACK_SIZE, ISA_IMC, ISA_MOP, RISCV_MAX_MEMORY, + WXorXMemory, DEFAULT_MEMORY_SIZE, ISA_IMC, ISA_MOP, }; #[allow(dead_code)] @@ -31,8 +31,8 @@ impl Syscalls for CustomSyscall { machine.load_elf(&code, true).unwrap(); machine.initialize_stack( &[], - (RISCV_MAX_MEMORY - DEFAULT_STACK_SIZE) as u64, - DEFAULT_STACK_SIZE as u64, + (DEFAULT_MEMORY_SIZE - DEFAULT_MEMORY_SIZE / 4) as u64, + (DEFAULT_MEMORY_SIZE / 4) as u64, )?; Ok(true) } diff --git a/tests/test_resume2.rs b/tests/test_resume2.rs index 93e5a854..400b4013 100644 --- a/tests/test_resume2.rs +++ b/tests/test_resume2.rs @@ -521,18 +521,18 @@ impl Machine { #[cfg(not(feature = "enable-chaos-mode-by-default"))] fn full_memory(&mut self) -> Result { - use ckb_vm::{Memory, RISCV_MAX_MEMORY}; + use ckb_vm::{Memory, DEFAULT_MEMORY_SIZE}; use Machine::*; match self { Asm(inner, _) => inner .machine .memory_mut() - .load_bytes(0, RISCV_MAX_MEMORY as u64), - Interpreter(inner, _) => inner.memory_mut().load_bytes(0, RISCV_MAX_MEMORY as u64), + .load_bytes(0, DEFAULT_MEMORY_SIZE as u64), + Interpreter(inner, _) => inner.memory_mut().load_bytes(0, DEFAULT_MEMORY_SIZE as u64), InterpreterWithTrace(inner, _) => inner .machine .memory_mut() - .load_bytes(0, RISCV_MAX_MEMORY as u64), + .load_bytes(0, DEFAULT_MEMORY_SIZE as u64), } }