diff --git a/src/c-writer.cc b/src/c-writer.cc index 7a0c548bc..e7bcff4f8 100644 --- a/src/c-writer.cc +++ b/src/c-writer.cc @@ -1369,7 +1369,24 @@ static std::string GetMemoryTypeString(const Memory& memory) { } static std::string GetMemoryAPIString(const Memory& memory, std::string api) { - return memory.page_limits.is_shared ? (api + "_shared") : api; + std::string suffix; + if (memory.page_limits.is_shared) { + suffix += "_shared"; + } + + // Memory load and store routines can be optimized for default-page-size, + // 32-bit memories (by using hardware to bounds-check memory access). + // Append "_default32" to these function names to choose the (possibly) fast + // path. + // + // We don't need to do this for runtime routines; those can check the + // wasm_rt_memory_t structure. + if (api.substr(0, 8) != "wasm_rt_" && + memory.page_size == WABT_DEFAULT_PAGE_SIZE && + memory.page_limits.is_64 == false) { + suffix += "_default32"; + } + return api + suffix; } void CWriter::WriteInitExpr(const ExprList& expr_list) { diff --git a/src/prebuilt/wasm2c_atomicops_source_declarations.cc b/src/prebuilt/wasm2c_atomicops_source_declarations.cc index 8312d5d6b..dc02591cb 100644 --- a/src/prebuilt/wasm2c_atomicops_source_declarations.cc +++ b/src/prebuilt/wasm2c_atomicops_source_declarations.cc @@ -17,25 +17,25 @@ R"w2c_template( TRAP(UNALIGNED); \ R"w2c_template( } )w2c_template" R"w2c_template( -#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ +#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ )w2c_template" -R"w2c_template( static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t1 result; \ )w2c_template" -R"w2c_template( t1 result; \ +R"w2c_template( result = atomic_load_explicit( \ )w2c_template" -R"w2c_template( result = atomic_load_explicit( \ +R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ )w2c_template" -R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ +R"w2c_template( memory_order_relaxed); \ )w2c_template" -R"w2c_template( memory_order_relaxed); \ +R"w2c_template( force_read(result); \ )w2c_template" -R"w2c_template( force_read(result); \ +R"w2c_template( return (t3)(t2)result; \ )w2c_template" -R"w2c_template( return (t3)(t2)result; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS0(name, _shared_, t1, return, t3) )w2c_template" R"w2c_template( DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT) @@ -69,9 +69,9 @@ R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ R"w2c_template( #define DEFINE_SHARED_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" @@ -81,7 +81,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr )w2c_template" R"w2c_template( memory_order_relaxed); \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name, _shared_, t1, , void, t2) )w2c_template" R"w2c_template( DEFINE_SHARED_STORE(i32_store_shared, u32, u32) @@ -105,9 +107,7 @@ R"w2c_template(DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) R"w2c_template( #define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ )w2c_template" -R"w2c_template( static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ -)w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -121,9 +121,11 @@ R"w2c_template( return (t3)(t2)result; )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) { \ +R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -137,7 +139,9 @@ R"w2c_template( force_read(result); )w2c_template" R"w2c_template( return (t3)(t2)result; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3) )w2c_template" R"w2c_template( DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT) @@ -157,9 +161,9 @@ R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ R"w2c_template( #define DEFINE_ATOMIC_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -169,11 +173,11 @@ R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, siz )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -183,7 +187,9 @@ R"w2c_template( atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof )w2c_template" R"w2c_template( wrapped); \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2) )w2c_template" R"w2c_template( DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32) @@ -203,9 +209,9 @@ R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) R"w2c_template( #define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ )w2c_template" -R"w2c_template( static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -223,11 +229,11 @@ R"w2c_template( return (t2)ret; )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -239,7 +245,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr )w2c_template" R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" R"w2c_template( DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32) @@ -319,9 +327,9 @@ R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) R"w2c_template( #define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ )w2c_template" -R"w2c_template( static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -337,11 +345,11 @@ R"w2c_template( return (t2)ret; )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -353,7 +361,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr )w2c_template" R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" R"w2c_template( DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32) @@ -371,57 +381,57 @@ R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64) R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ +#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected, \ +R"w2c_template( static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( t1 replacement) { \ +R"w2c_template( t1 expected, t1 replacement) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t2); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ +R"w2c_template( t2 expected_wrapped = (t2)expected; \ )w2c_template" -R"w2c_template( t2 expected_wrapped = (t2)expected; \ +R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ )w2c_template" -R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ +R"w2c_template( t2 ret; \ )w2c_template" -R"w2c_template( t2 ret; \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ +R"w2c_template( if (ret == expected_wrapped) { \ )w2c_template" -R"w2c_template( if (ret == expected_wrapped) { \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ +R"w2c_template( sizeof(t2)); \ )w2c_template" -R"w2c_template( sizeof(t2)); \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t1)expected_wrapped; \ )w2c_template" -R"w2c_template( return (t1)expected_wrapped; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ )w2c_template" -R"w2c_template( static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( static inline t1 name##_shared_unchecked( \ )w2c_template" -R"w2c_template( t1 expected, t1 replacement) { \ +R"w2c_template( wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t2); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ +R"w2c_template( t2 expected_wrapped = (t2)expected; \ )w2c_template" -R"w2c_template( t2 expected_wrapped = (t2)expected; \ +R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ )w2c_template" -R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ +R"w2c_template( atomic_compare_exchange_strong( \ )w2c_template" -R"w2c_template( atomic_compare_exchange_strong( \ +R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ )w2c_template" -R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ +R"w2c_template( &expected_wrapped, replacement_wrapped); \ )w2c_template" -R"w2c_template( &expected_wrapped, replacement_wrapped); \ +R"w2c_template( return (t1)expected_wrapped; \ )w2c_template" -R"w2c_template( return (t1)expected_wrapped; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1) )w2c_template" R"w2c_template( DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8); diff --git a/src/prebuilt/wasm2c_simd_source_declarations.cc b/src/prebuilt/wasm2c_simd_source_declarations.cc index 5b903b261..a43a00069 100644 --- a/src/prebuilt/wasm2c_simd_source_declarations.cc +++ b/src/prebuilt/wasm2c_simd_source_declarations.cc @@ -15,26 +15,26 @@ R"w2c_template(#endif R"w2c_template(// TODO: equivalent constraint for ARM and other architectures )w2c_template" R"w2c_template( -#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ +#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ )w2c_template" -R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ )w2c_template" -R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ +R"w2c_template( SIMD_FORCE_READ(result); \ )w2c_template" -R"w2c_template( SIMD_FORCE_READ(result); \ +R"w2c_template( return result; \ )w2c_template" -R"w2c_template( return result; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS0(name, _, t, return, v128); )w2c_template" R"w2c_template( #define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ )w2c_template" -R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr, v128 vec) { \ +R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( v128 vec) { \ )w2c_template" R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \ )w2c_template" @@ -42,29 +42,35 @@ R"w2c_template( SIMD_FORCE_READ(result); )w2c_template" R"w2c_template( return result; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name, _, t, return, v128, v128); )w2c_template" R"w2c_template( -#define DEFINE_SIMD_STORE(name, t) \ +#define DEFINE_SIMD_STORE(name, t) \ +)w2c_template" +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ +R"w2c_template( v128 value) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ )w2c_template" -R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128); )w2c_template" R"w2c_template( -#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ +#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ +)w2c_template" +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ +R"w2c_template( v128 value) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ )w2c_template" -R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128); )w2c_template" R"w2c_template( // clang-format off diff --git a/src/prebuilt/wasm2c_source_declarations.cc b/src/prebuilt/wasm2c_source_declarations.cc index 87529c404..557bf67bf 100644 --- a/src/prebuilt/wasm2c_source_declarations.cc +++ b/src/prebuilt/wasm2c_source_declarations.cc @@ -165,15 +165,32 @@ R"w2c_template( (CHECK_CALL_INDIRECT(table, ft, x), \ R"w2c_template( DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) )w2c_template" R"w2c_template( -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { )w2c_template" -R"w2c_template(#define RANGE_CHECK(mem, offset, len) \ +R"w2c_template(#if __has_builtin(__builtin_add_overflow) +)w2c_template" +R"w2c_template( return __builtin_add_overflow(a, b, resptr); +)w2c_template" +R"w2c_template(#elif defined(_MSC_VER) +)w2c_template" +R"w2c_template( return _addcarry_u64(0, a, b, resptr); +)w2c_template" +R"w2c_template(#else +)w2c_template" +R"w2c_template(#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +)w2c_template" +R"w2c_template(#endif +)w2c_template" +R"w2c_template(} +)w2c_template" +R"w2c_template( +#define RANGE_CHECK(mem, offset, len) \ )w2c_template" R"w2c_template( do { \ )w2c_template" R"w2c_template( uint64_t res; \ )w2c_template" -R"w2c_template( if (__builtin_add_overflow(offset, len, &res)) \ +R"w2c_template( if (UNLIKELY(add_overflow(offset, len, &res))) \ )w2c_template" R"w2c_template( TRAP(OOB); \ )w2c_template" @@ -183,16 +200,6 @@ R"w2c_template( TRAP(OOB); \ )w2c_template" R"w2c_template( } while (0); )w2c_template" -R"w2c_template(#else -)w2c_template" -R"w2c_template(#define RANGE_CHECK(mem, offset, len) \ -)w2c_template" -R"w2c_template( if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ -)w2c_template" -R"w2c_template( TRAP(OOB); -)w2c_template" -R"w2c_template(#endif -)w2c_template" R"w2c_template( #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS )w2c_template" @@ -215,21 +222,40 @@ R"w2c_template(#define WASM_RT_CHECK_BASE(mem) R"w2c_template(#endif )w2c_template" R"w2c_template( -#if WASM_RT_MEMCHECK_GUARD_PAGES +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +)w2c_template" +R"w2c_template(// default-page-size, 32-bit memories. It may do nothing at all +)w2c_template" +R"w2c_template(// (if hardware bounds-checking is enabled via guard pages) )w2c_template" -R"w2c_template(#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +R"w2c_template(// or it may do a slightly faster RANGE_CHECK. +)w2c_template" +R"w2c_template(#if WASM_RT_MEMCHECK_GUARD_PAGES +)w2c_template" +R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); )w2c_template" R"w2c_template(#else )w2c_template" -R"w2c_template(#define MEMCHECK(mem, a, t) \ +R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) \ +)w2c_template" +R"w2c_template( WASM_RT_CHECK_BASE(mem); \ )w2c_template" -R"w2c_template( WASM_RT_CHECK_BASE(mem); \ +R"w2c_template( if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ )w2c_template" -R"w2c_template( RANGE_CHECK(mem, a, sizeof(t)) +R"w2c_template( TRAP(OOB); )w2c_template" R"w2c_template(#endif )w2c_template" R"w2c_template( +// MEMCHECK_GENERAL can be used for any memory +)w2c_template" +R"w2c_template(#define MEMCHECK_GENERAL(mem, a, t) \ +)w2c_template" +R"w2c_template( WASM_RT_CHECK_BASE(mem); \ +)w2c_template" +R"w2c_template( RANGE_CHECK(mem, a, sizeof(t)); +)w2c_template" +R"w2c_template( #ifdef __GNUC__ )w2c_template" R"w2c_template(#define FORCE_READ_INT(var) __asm__("" ::"r"(var)); @@ -299,30 +325,103 @@ R"w2c_template( load_data(MEM_ADDR(&m, o, s), i, s); \ R"w2c_template( } while (0) )w2c_template" R"w2c_template( -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ +)w2c_template" +R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr) { \ +)w2c_template" +R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr); \ +)w2c_template" +R"w2c_template( } \ +)w2c_template" +R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ +)w2c_template" +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr); \ +)w2c_template" +R"w2c_template( } +)w2c_template" +R"w2c_template( +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ +)w2c_template" +R"w2c_template( val_type1) \ +)w2c_template" +R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr, val_type1 val1) { \ +)w2c_template" +R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \ +)w2c_template" +R"w2c_template( } \ +)w2c_template" +R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ +)w2c_template" +R"w2c_template( val_type1 val1) { \ +)w2c_template" +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \ +)w2c_template" +R"w2c_template( } +)w2c_template" +R"w2c_template( +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ +)w2c_template" +R"w2c_template( val_type1, val_type2) \ +)w2c_template" +R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr, val_type1 val1, \ +)w2c_template" +R"w2c_template( val_type2 val2) { \ )w2c_template" -R"w2c_template( static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \ )w2c_template" -R"w2c_template( t1 result; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ +R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( sizeof(t1)); \ +R"w2c_template( val_type1 val1, val_type2 val2) { \ )w2c_template" -R"w2c_template( force_read(result); \ +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ )w2c_template" -R"w2c_template( return (t3)(t2)result; \ +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \ )w2c_template" R"w2c_template( } )w2c_template" R"w2c_template( +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ +)w2c_template" +R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ +)w2c_template" +R"w2c_template( t1 result; \ +)w2c_template" +R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ +)w2c_template" +R"w2c_template( sizeof(t1)); \ +)w2c_template" +R"w2c_template( force_read(result); \ +)w2c_template" +R"w2c_template( return (t3)(t2)result; \ +)w2c_template" +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) +)w2c_template" +R"w2c_template( #define DEFINE_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" @@ -330,7 +429,9 @@ R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrappe )w2c_template" R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2) )w2c_template" R"w2c_template( DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) @@ -976,7 +1077,7 @@ R"w2c_template( return sqrtf(x); R"w2c_template(} )w2c_template" R"w2c_template( -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { )w2c_template" R"w2c_template( RANGE_CHECK(mem, d, n); )w2c_template" @@ -989,11 +1090,11 @@ static inline void memory_copy(wasm_rt_memory_t* dest, )w2c_template" R"w2c_template( const wasm_rt_memory_t* src, )w2c_template" -R"w2c_template( u32 dest_addr, +R"w2c_template( u64 dest_addr, )w2c_template" -R"w2c_template( u32 src_addr, +R"w2c_template( u64 src_addr, )w2c_template" -R"w2c_template( u32 n) { +R"w2c_template( u64 n) { )w2c_template" R"w2c_template( RANGE_CHECK(dest, dest_addr, n); )w2c_template" @@ -1010,7 +1111,7 @@ R"w2c_template( const u8* src, )w2c_template" R"w2c_template( u32 src_size, )w2c_template" -R"w2c_template( u32 dest_addr, +R"w2c_template( u64 dest_addr, )w2c_template" R"w2c_template( u32 src_addr, )w2c_template" @@ -1046,7 +1147,7 @@ R"w2c_template( const wasm_elem_segment_exp )w2c_template" R"w2c_template( u32 src_size, )w2c_template" -R"w2c_template( u32 dest_addr, +R"w2c_template( u64 dest_addr, )w2c_template" R"w2c_template( u32 src_addr, )w2c_template" @@ -1058,9 +1159,7 @@ R"w2c_template( if (UNLIKELY(src_addr + (uint64_t)n > src_size)) )w2c_template" R"w2c_template( TRAP(OOB); )w2c_template" -R"w2c_template( if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) -)w2c_template" -R"w2c_template( TRAP(OOB); +R"w2c_template( RANGE_CHECK(dest, dest_addr, n); )w2c_template" R"w2c_template( for (u32 i = 0; i < n; i++) { )w2c_template" @@ -1107,7 +1206,7 @@ R"w2c_template(static inline void externref_table_init(wasm_rt_externref_table_t )w2c_template" R"w2c_template( u32 src_size, )w2c_template" -R"w2c_template( u32 dest_addr, +R"w2c_template( u64 dest_addr, )w2c_template" R"w2c_template( u32 src_addr, )w2c_template" @@ -1117,9 +1216,7 @@ R"w2c_template( if (UNLIKELY(src_addr + (uint64_t)n > src_size)) )w2c_template" R"w2c_template( TRAP(OOB); )w2c_template" -R"w2c_template( if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) -)w2c_template" -R"w2c_template( TRAP(OOB); +R"w2c_template( RANGE_CHECK(dest, dest_addr, n); )w2c_template" R"w2c_template( for (u32 i = 0; i < n; i++) { )w2c_template" @@ -1136,17 +1233,11 @@ R"w2c_template( static inline void type##_table_copy(wasm_rt_##type##_table_t* )w2c_template" R"w2c_template( const wasm_rt_##type##_table_t* src, \ )w2c_template" -R"w2c_template( u32 dest_addr, u32 src_addr, u32 n) { \ +R"w2c_template( u64 dest_addr, u64 src_addr, u64 n) { \ )w2c_template" -R"w2c_template( if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ +R"w2c_template( RANGE_CHECK(dest, dest_addr, n); \ )w2c_template" -R"w2c_template( TRAP(OOB); \ -)w2c_template" -R"w2c_template( if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ -)w2c_template" -R"w2c_template( TRAP(OOB); \ -)w2c_template" -R"w2c_template( \ +R"w2c_template( RANGE_CHECK(src, src_addr, n); \ )w2c_template" R"w2c_template( memmove(dest->data + dest_addr, src->data + src_addr, \ )w2c_template" @@ -1164,7 +1255,7 @@ R"w2c_template( )w2c_template" R"w2c_template( static inline wasm_rt_##type##_t type##_table_get( \ )w2c_template" -R"w2c_template( const wasm_rt_##type##_table_t* table, u32 i) { \ +R"w2c_template( const wasm_rt_##type##_table_t* table, u64 i) { \ )w2c_template" R"w2c_template( if (UNLIKELY(i >= table->size)) \ )w2c_template" @@ -1184,7 +1275,7 @@ R"w2c_template( )w2c_template" R"w2c_template( static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ )w2c_template" -R"w2c_template( u32 i, const wasm_rt_##type##_t val) { \ +R"w2c_template( u64 i, const wasm_rt_##type##_t val) { \ )w2c_template" R"w2c_template( if (UNLIKELY(i >= table->size)) \ )w2c_template" @@ -1204,13 +1295,11 @@ R"w2c_template( )w2c_template" R"w2c_template( static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ )w2c_template" -R"w2c_template( u32 d, const wasm_rt_##type##_t val, \ -)w2c_template" -R"w2c_template( u32 n) { \ +R"w2c_template( u64 d, const wasm_rt_##type##_t val, \ )w2c_template" -R"w2c_template( if (UNLIKELY((uint64_t)d + n > table->size)) \ +R"w2c_template( u64 n) { \ )w2c_template" -R"w2c_template( TRAP(OOB); \ +R"w2c_template( RANGE_CHECK(table, d, n); \ )w2c_template" R"w2c_template( for (uint32_t i = d; i < d + n; i++) { \ )w2c_template" diff --git a/src/template/wasm2c.declarations.c b/src/template/wasm2c.declarations.c index 296391ed8..ecc7b6803 100644 --- a/src/template/wasm2c.declarations.c +++ b/src/template/wasm2c.declarations.c @@ -89,20 +89,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -115,14 +119,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -159,23 +173,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -523,16 +576,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -541,7 +594,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -560,14 +613,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -591,13 +643,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -606,12 +657,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -621,7 +669,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -632,7 +680,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -643,10 +691,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ diff --git a/src/template/wasm2c_atomicops.declarations.c b/src/template/wasm2c_atomicops.declarations.c index 5d9cdf904..cb25da513 100644 --- a/src/template/wasm2c_atomicops.declarations.c +++ b/src/template/wasm2c_atomicops.declarations.c @@ -9,16 +9,16 @@ TRAP(UNALIGNED); \ } -#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - result = atomic_load_explicit( \ - (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ - memory_order_relaxed); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \ + t1 result; \ + result = atomic_load_explicit( \ + (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ + memory_order_relaxed); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _shared_, t1, return, t3) DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load_shared, u64, u64, u64, FORCE_READ_INT) @@ -36,13 +36,14 @@ DEFINE_SHARED_LOAD(i64_load32_s_shared, s32, s64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ_INT) #define DEFINE_SHARED_STORE(name, t1, t2) \ - static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ atomic_store_explicit( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped, \ memory_order_relaxed); \ - } + } \ + DEF_MEM_CHECKS1(name, _shared_, t1, , void, t2) DEFINE_SHARED_STORE(i32_store_shared, u32, u32) DEFINE_SHARED_STORE(i64_store_shared, u64, u64) @@ -55,23 +56,24 @@ DEFINE_SHARED_STORE(i64_store16_shared, u16, u64) DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) #define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 result; \ wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ force_read(result); \ return (t3)(t2)result; \ } \ - static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) \ + static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 result; \ result = \ atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \ force_read(result); \ return (t3)(t2)result; \ - } + } \ + DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3) DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load, u64, u64, u64, FORCE_READ_INT) @@ -82,20 +84,21 @@ DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT) #define DEFINE_ATOMIC_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ } \ - static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t2 value) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ + static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ wrapped); \ - } + } \ + DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2) DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32) DEFINE_ATOMIC_STORE(i64_atomic_store, u64, u64) @@ -106,8 +109,8 @@ DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64) DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) #define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ - static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret; \ @@ -116,15 +119,16 @@ DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &ret, sizeof(t1)); \ return (t2)ret; \ } \ - static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t2 value) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret = atomic_##opname( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ return (t2)ret; \ - } + } \ + DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, fetch_add, +, u16, u32) @@ -167,8 +171,8 @@ DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) #define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ - static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret; \ @@ -176,15 +180,16 @@ DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ return (t2)ret; \ } \ - static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t2 value) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret = atomic_##opname( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ return (t2)ret; \ - } + } \ + DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32) DEFINE_ATOMIC_XCHG(i32_atomic_rmw16_xchg_u, exchange, u16, u32) @@ -194,32 +199,32 @@ DEFINE_ATOMIC_XCHG(i64_atomic_rmw16_xchg_u, exchange, u16, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) -#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ - static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected, \ - t1 replacement) { \ - MEMCHECK(mem, addr, t2); \ - ATOMIC_ALIGNMENT_CHECK(addr, t2); \ - t2 expected_wrapped = (t2)expected; \ - t2 replacement_wrapped = (t2)replacement; \ - t2 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ - if (ret == expected_wrapped) { \ - wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ - sizeof(t2)); \ - } \ - return (t1)expected_wrapped; \ - } \ - static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t1 expected, t1 replacement) { \ - MEMCHECK(mem, addr, t2); \ - ATOMIC_ALIGNMENT_CHECK(addr, t2); \ - t2 expected_wrapped = (t2)expected; \ - t2 replacement_wrapped = (t2)replacement; \ - atomic_compare_exchange_strong( \ - (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ - &expected_wrapped, replacement_wrapped); \ - return (t1)expected_wrapped; \ - } +#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ + static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t1 expected, t1 replacement) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t2); \ + t2 expected_wrapped = (t2)expected; \ + t2 replacement_wrapped = (t2)replacement; \ + t2 ret; \ + wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ + if (ret == expected_wrapped) { \ + wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ + sizeof(t2)); \ + } \ + return (t1)expected_wrapped; \ + } \ + DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ + static inline t1 name##_shared_unchecked( \ + wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t2); \ + t2 expected_wrapped = (t2)expected; \ + t2 replacement_wrapped = (t2)replacement; \ + atomic_compare_exchange_strong( \ + (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ + &expected_wrapped, replacement_wrapped); \ + return (t1)expected_wrapped; \ + } \ + DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1) DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8); DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw16_cmpxchg_u, u32, u16); diff --git a/src/template/wasm2c_simd.declarations.c b/src/template/wasm2c_simd.declarations.c index 0e2c95114..39eb4578e 100644 --- a/src/template/wasm2c_simd.declarations.c +++ b/src/template/wasm2c_simd.declarations.c @@ -7,33 +7,36 @@ #endif // TODO: equivalent constraint for ARM and other architectures -#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ - static inline v128 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t); \ - v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ - SIMD_FORCE_READ(result); \ - return result; \ - } +#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ + static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ + SIMD_FORCE_READ(result); \ + return result; \ + } \ + DEF_MEM_CHECKS0(name, _, t, return, v128); #define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ - static inline v128 name(wasm_rt_memory_t* mem, u64 addr, v128 vec) { \ - MEMCHECK(mem, addr, t); \ + static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 vec) { \ v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \ SIMD_FORCE_READ(result); \ return result; \ - } + } \ + DEF_MEM_CHECKS1(name, _, t, return, v128, v128); -#define DEFINE_SIMD_STORE(name, t) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ - MEMCHECK(mem, addr, t); \ - simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ - } +#define DEFINE_SIMD_STORE(name, t) \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 value) { \ + simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ + } \ + DEF_MEM_CHECKS1(name, _, t, , void, v128); -#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ - MEMCHECK(mem, addr, t); \ - func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ - } +#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 value) { \ + func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ + } \ + DEF_MEM_CHECKS1(name, _, t, , void, v128); // clang-format off #if WABT_BIG_ENDIAN diff --git a/test/run-spec-wasm2c.py b/test/run-spec-wasm2c.py index 1ff63d8fb..8dfa416ed 100755 --- a/test/run-spec-wasm2c.py +++ b/test/run-spec-wasm2c.py @@ -609,7 +609,6 @@ def main(args): if IS_WINDOWS: sys.stderr.write('skipping: wasm2c+memory64 is not yet supported under msvc\n') return SKIPPED - cflags.append('-DSUPPORT_MEMORY64=1') use_c11 = options.enable_threads diff --git a/test/wasm2c/add.txt b/test/wasm2c/add.txt index 083ca80c2..e4b5129f7 100644 --- a/test/wasm2c/add.txt +++ b/test/wasm2c/add.txt @@ -156,20 +156,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -182,14 +186,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -226,23 +240,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -590,16 +643,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -608,7 +661,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -627,14 +680,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -658,13 +710,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -673,12 +724,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -688,7 +736,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -699,7 +747,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -710,10 +758,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ diff --git a/test/wasm2c/check-imports.txt b/test/wasm2c/check-imports.txt index f162e0663..468bcfdde 100644 --- a/test/wasm2c/check-imports.txt +++ b/test/wasm2c/check-imports.txt @@ -180,20 +180,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -206,14 +210,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -250,23 +264,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -614,16 +667,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -632,7 +685,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -651,14 +704,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -682,13 +734,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -697,12 +748,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -712,7 +760,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -723,7 +771,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -734,10 +782,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ @@ -880,7 +927,7 @@ u32 w2c_test_f1(w2c_test* instance) { FUNC_PROLOGUE; u32 var_i0; var_i0 = 16u; - var_i0 = i32_load(instance->w2c_env_0x5F_linear_memory, (u64)(var_i0)); + var_i0 = i32_load_default32(instance->w2c_env_0x5F_linear_memory, (u64)(var_i0)); FUNC_EPILOGUE; return var_i0; } diff --git a/test/wasm2c/export-names.txt b/test/wasm2c/export-names.txt index cb282c46b..6e7a25b1b 100644 --- a/test/wasm2c/export-names.txt +++ b/test/wasm2c/export-names.txt @@ -180,20 +180,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -206,14 +210,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -250,23 +264,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -614,16 +667,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -632,7 +685,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -651,14 +704,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -682,13 +734,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -697,12 +748,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -712,7 +760,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -723,7 +771,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -734,10 +782,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ diff --git a/test/wasm2c/hello.txt b/test/wasm2c/hello.txt index 60d180999..e89c6150b 100644 --- a/test/wasm2c/hello.txt +++ b/test/wasm2c/hello.txt @@ -188,20 +188,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -214,14 +218,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -258,23 +272,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -622,16 +675,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -640,7 +693,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -659,14 +712,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -690,13 +742,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -705,12 +756,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -720,7 +768,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -731,7 +779,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -742,10 +790,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ @@ -901,10 +948,10 @@ void w2c_test_0x5Fstart_0(w2c_test* instance) { u32 var_i0, var_i1, var_i2, var_i3, var_i4; var_i0 = 0u; var_i1 = 8u; - i32_store(&instance->w2c_memory, (u64)(var_i0), var_i1); + i32_store_default32(&instance->w2c_memory, (u64)(var_i0), var_i1); var_i0 = 4u; var_i1 = 14u; - i32_store(&instance->w2c_memory, (u64)(var_i0), var_i1); + i32_store_default32(&instance->w2c_memory, (u64)(var_i0), var_i1); var_i0 = 1u; var_i1 = 0u; var_i2 = 1u; diff --git a/test/wasm2c/minimal.txt b/test/wasm2c/minimal.txt index 532bc141a..bbef5496f 100644 --- a/test/wasm2c/minimal.txt +++ b/test/wasm2c/minimal.txt @@ -150,20 +150,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -176,14 +180,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -220,23 +234,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -584,16 +637,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -602,7 +655,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -621,14 +674,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -652,13 +704,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -667,12 +718,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -682,7 +730,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -693,7 +741,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -704,10 +752,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ diff --git a/test/wasm2c/tail-calls.txt b/test/wasm2c/tail-calls.txt index 208bbdd5a..c34204b00 100644 --- a/test/wasm2c/tail-calls.txt +++ b/test/wasm2c/tail-calls.txt @@ -180,20 +180,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -206,14 +210,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -250,23 +264,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ } +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) + #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -614,16 +667,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -632,7 +685,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -651,14 +704,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -682,13 +734,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -697,12 +748,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -712,7 +760,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -723,7 +771,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -734,10 +782,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ diff --git a/wasm2c/examples/fac/fac.c b/wasm2c/examples/fac/fac.c index eb6e4f005..a82e7f757 100644 --- a/wasm2c/examples/fac/fac.c +++ b/wasm2c/examples/fac/fac.c @@ -108,20 +108,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, (CHECK_CALL_INDIRECT(table, ft, x), \ DO_CALL_INDIRECT(table, t, x, __VA_ARGS__)) -#ifdef SUPPORT_MEMORY64 +static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { +#if __has_builtin(__builtin_add_overflow) + return __builtin_add_overflow(a, b, resptr); +#elif defined(_MSC_VER) + return _addcarry_u64(0, a, b, resptr); +#else +#error "Missing implementation of __builtin_add_overflow or _addcarry_u64" +#endif +} + #define RANGE_CHECK(mem, offset, len) \ do { \ uint64_t res; \ - if (__builtin_add_overflow(offset, len, &res)) \ + if (UNLIKELY(add_overflow(offset, len, &res))) \ TRAP(OOB); \ if (UNLIKELY(res > mem->size)) \ TRAP(OOB); \ } while (0); -#else -#define RANGE_CHECK(mem, offset, len) \ - if (UNLIKELY(offset + (uint64_t)len > mem->size)) \ - TRAP(OOB); -#endif #if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS #include @@ -134,14 +138,24 @@ static inline bool func_types_eq(const wasm_rt_func_type_t a, #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -178,23 +192,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -542,16 +595,16 @@ static float wasm_sqrtf(float x) { return sqrtf(x); } -static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) { +static inline void memory_fill(wasm_rt_memory_t* mem, u64 d, u32 val, u64 n) { RANGE_CHECK(mem, d, n); memset(MEM_ADDR(mem, d, n), val, n); } static inline void memory_copy(wasm_rt_memory_t* dest, const wasm_rt_memory_t* src, - u32 dest_addr, - u32 src_addr, - u32 n) { + u64 dest_addr, + u64 src_addr, + u64 n) { RANGE_CHECK(dest, dest_addr, n); RANGE_CHECK(src, src_addr, n); memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n); @@ -560,7 +613,7 @@ static inline void memory_copy(wasm_rt_memory_t* dest, static inline void memory_init(wasm_rt_memory_t* dest, const u8* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) @@ -579,14 +632,13 @@ typedef struct { static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, const wasm_elem_segment_expr_t* src, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n, void* module_instance) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i]; wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]); @@ -610,13 +662,12 @@ static inline void funcref_table_init(wasm_rt_funcref_table_t* dest, // Currently wasm2c only supports initializing externref tables with ref.null. static inline void externref_table_init(wasm_rt_externref_table_t* dest, u32 src_size, - u32 dest_addr, + u64 dest_addr, u32 src_addr, u32 n) { if (UNLIKELY(src_addr + (uint64_t)n > src_size)) TRAP(OOB); - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) - TRAP(OOB); + RANGE_CHECK(dest, dest_addr, n); for (u32 i = 0; i < n; i++) { dest->data[dest_addr + i] = wasm_rt_externref_null_value; } @@ -625,12 +676,9 @@ static inline void externref_table_init(wasm_rt_externref_table_t* dest, #define DEFINE_TABLE_COPY(type) \ static inline void type##_table_copy(wasm_rt_##type##_table_t* dest, \ const wasm_rt_##type##_table_t* src, \ - u32 dest_addr, u32 src_addr, u32 n) { \ - if (UNLIKELY(dest_addr + (uint64_t)n > dest->size)) \ - TRAP(OOB); \ - if (UNLIKELY(src_addr + (uint64_t)n > src->size)) \ - TRAP(OOB); \ - \ + u64 dest_addr, u64 src_addr, u64 n) { \ + RANGE_CHECK(dest, dest_addr, n); \ + RANGE_CHECK(src, src_addr, n); \ memmove(dest->data + dest_addr, src->data + src_addr, \ n * sizeof(wasm_rt_##type##_t)); \ } @@ -640,7 +688,7 @@ DEFINE_TABLE_COPY(externref) #define DEFINE_TABLE_GET(type) \ static inline wasm_rt_##type##_t type##_table_get( \ - const wasm_rt_##type##_table_t* table, u32 i) { \ + const wasm_rt_##type##_table_t* table, u64 i) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ return table->data[i]; \ @@ -651,7 +699,7 @@ DEFINE_TABLE_GET(externref) #define DEFINE_TABLE_SET(type) \ static inline void type##_table_set(const wasm_rt_##type##_table_t* table, \ - u32 i, const wasm_rt_##type##_t val) { \ + u64 i, const wasm_rt_##type##_t val) { \ if (UNLIKELY(i >= table->size)) \ TRAP(OOB); \ table->data[i] = val; \ @@ -662,10 +710,9 @@ DEFINE_TABLE_SET(externref) #define DEFINE_TABLE_FILL(type) \ static inline void type##_table_fill(const wasm_rt_##type##_table_t* table, \ - u32 d, const wasm_rt_##type##_t val, \ - u32 n) { \ - if (UNLIKELY((uint64_t)d + n > table->size)) \ - TRAP(OOB); \ + u64 d, const wasm_rt_##type##_t val, \ + u64 n) { \ + RANGE_CHECK(table, d, n); \ for (uint32_t i = d; i < d + n; i++) { \ table->data[i] = val; \ } \ diff --git a/wasm2c/wasm-rt-mem-impl-helper.inc b/wasm2c/wasm-rt-mem-impl-helper.inc index 2c2f9b754..6d3e596c3 100644 --- a/wasm2c/wasm-rt-mem-impl-helper.inc +++ b/wasm2c/wasm-rt-mem-impl-helper.inc @@ -74,23 +74,25 @@ void MEMORY_API_NAME(wasm_rt_allocate_memory)(MEMORY_TYPE* memory, memory->is64 = is64; MEMORY_LOCK_VAR_INIT(memory->mem_lock); -#if WASM_RT_USE_MMAP - const uint64_t mmap_size = - get_alloc_size_for_mmap(memory->max_pages, memory->is64); - void* addr = os_mmap(mmap_size); - if (!addr) { - os_print_last_error("os_mmap failed."); - abort(); - } - int ret = os_mprotect(addr, byte_length); - if (ret != 0) { - os_print_last_error("os_mprotect failed."); - abort(); - } - memory->data = addr; -#else - memory->data = calloc(byte_length, 1); + if (WASM_RT_USE_MMAP && !is64) { +#if WASM_RT_USE_MMAP // mmap-related functions don't exist unless this is set + const uint64_t mmap_size = + get_alloc_size_for_mmap(memory->max_pages, memory->is64); + void* addr = os_mmap(mmap_size); + if (!addr) { + os_print_last_error("os_mmap failed."); + abort(); + } + int ret = os_mprotect(addr, byte_length); + if (ret != 0) { + os_print_last_error("os_mprotect failed."); + abort(); + } + memory->data = addr; #endif + } else { + memory->data = calloc(byte_length, 1); + } } static uint64_t MEMORY_API_NAME(grow_memory_impl)(MEMORY_TYPE* memory, @@ -106,21 +108,24 @@ static uint64_t MEMORY_API_NAME(grow_memory_impl)(MEMORY_TYPE* memory, uint64_t old_size = old_pages * WASM_PAGE_SIZE; uint64_t new_size = new_pages * WASM_PAGE_SIZE; uint64_t delta_size = delta * WASM_PAGE_SIZE; + MEMORY_CELL_TYPE new_data; + if (WASM_RT_USE_MMAP && !memory->is64) { #if WASM_RT_USE_MMAP - MEMORY_CELL_TYPE new_data = memory->data; - int ret = os_mprotect((void*)(new_data + old_size), delta_size); - if (ret != 0) { - return (uint64_t)-1; - } -#else - MEMORY_CELL_TYPE new_data = realloc((void*)memory->data, new_size); - if (new_data == NULL) { - return (uint64_t)-1; - } -#if !WABT_BIG_ENDIAN - memset((void*)(new_data + old_size), 0, delta_size); + new_data = memory->data; + int ret = os_mprotect((void*)(new_data + old_size), delta_size); + if (ret != 0) { + return (uint64_t)-1; + } #endif + } else { + new_data = realloc((void*)memory->data, new_size); + if (new_data == NULL) { + return (uint64_t)-1; + } +#if !WABT_BIG_ENDIAN + memset((void*)(new_data + old_size), 0, delta_size); #endif + } #if WABT_BIG_ENDIAN memmove((void*)(new_data + new_size - old_size), (void*)new_data, old_size); memset((void*)new_data, 0, delta_size); @@ -145,13 +150,15 @@ uint64_t MEMORY_API_NAME(wasm_rt_grow_memory)(MEMORY_TYPE* memory, } void MEMORY_API_NAME(wasm_rt_free_memory)(MEMORY_TYPE* memory) { + if (WASM_RT_USE_MMAP && !memory->is64) { #if WASM_RT_USE_MMAP - const uint64_t mmap_size = - get_alloc_size_for_mmap(memory->max_pages, memory->is64); - os_munmap((void*)memory->data, mmap_size); // ignore error -#else - free((void*)memory->data); + const uint64_t mmap_size = + get_alloc_size_for_mmap(memory->max_pages, memory->is64); + os_munmap((void*)memory->data, mmap_size); // ignore error #endif + } else { + free((void*)memory->data); + } } #undef MEMORY_LOCK_RELEASE diff --git a/wasm2c/wasm-rt.h b/wasm2c/wasm-rt.h index 057570877..c135fdcd6 100644 --- a/wasm2c/wasm-rt.h +++ b/wasm2c/wasm-rt.h @@ -129,7 +129,7 @@ extern "C" { * needed (so we can use the guard based range checks below). */ #ifndef WASM_RT_USE_MMAP -#if UINTPTR_MAX > 0xffffffff && !SUPPORT_MEMORY64 +#if UINTPTR_MAX > 0xffffffff #define WASM_RT_USE_MMAP 1 #else #define WASM_RT_USE_MMAP 0 @@ -151,8 +151,7 @@ extern "C" { */ /** Check if Guard checks are supported */ -#if UINTPTR_MAX > 0xffffffff && WASM_RT_USE_MMAP && !SUPPORT_MEMORY64 && \ - !WABT_BIG_ENDIAN +#if UINTPTR_MAX > 0xffffffff && WASM_RT_USE_MMAP && !WABT_BIG_ENDIAN #define WASM_RT_GUARD_PAGES_SUPPORTED 1 #else #define WASM_RT_GUARD_PAGES_SUPPORTED 0