From 074bce5603a33f20f7753e12bfa913f09f23a6fd Mon Sep 17 00:00:00 2001 From: scc Date: Sun, 27 Aug 2023 21:32:43 +0800 Subject: [PATCH 1/4] refactor seqlock with C11 _Atomic --- seqlock/seqlock.c | 69 +++++++++++++++++++++++------------------------ seqlock/seqlock.h | 3 ++- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/seqlock/seqlock.c b/seqlock/seqlock.c index 0b5917b..7d4e292 100644 --- a/seqlock/seqlock.c +++ b/seqlock/seqlock.c @@ -1,50 +1,46 @@ #include +#include #include #include #include +#include #include #include "seqlock.h" #define SEQLOCK_WRITER 1U -#if defined(__i386__) || defined(__x86_64__) -#define spin_wait() __builtin_ia32_pause() -#elif defined(__aarch64__) -#define spin_wait() __asm__ __volatile__("isb\n") +#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) +#define spin_wait() atomic_thread_fence(memory_order_seq_cst) #else #define spin_wait() ((void) 0) #endif -#if defined(__aarch64__) #define SEVL() sevl() static inline void sevl(void) { - __asm__ volatile("sevl" : : :); + atomic_thread_fence(memory_order_seq_cst); } #define WFE() wfe() static inline int wfe(void) { - __asm__ volatile("wfe" : : : "memory"); + atomic_thread_fence(memory_order_seq_cst); return 1; } #define LDX(a, b) ldx((a), (b)) -static inline uint32_t ldx(const uint8_t *var, int mm) +static inline uint32_t ldx(const _Atomic uint32_t *var, int mm) { uint32_t old; - if (mm == __ATOMIC_ACQUIRE) - __asm volatile("ldaxrb %w0, [%1]" : "=&r"(old) : "r"(var) : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm volatile("ldxrb %w0, [%1]" : "=&r"(old) : "r"(var) : "memory"); + + if (mm == memory_order_acquire) + old = atomic_load_explicit(var, memory_order_acquire); + else if (mm == memory_order_relaxed) + old = atomic_load_explicit(var, memory_order_relaxed); else abort(); + return old; } -#else /* generic */ -#define SEVL() (void) 0 -#define WFE() 1 -#define LDX(a, b) __atomic_load_n((a), (b)) -#endif #define UNLIKELY(x) __builtin_expect(!!(x), 0) @@ -57,7 +53,8 @@ static inline seqlock_t wait_for_no_writer(const seqlock_t *sync, int mo) { seqlock_t l; SEVL(); /* Do SEVL early to avoid excessive loop alignment (NOPs) */ - if (UNLIKELY(((l = __atomic_load_n(sync, mo)) & SEQLOCK_WRITER) != 0)) { + if (UNLIKELY(((l = atomic_load_explicit(sync, mo)) & SEQLOCK_WRITER) != + 0)) { while (WFE() && ((l = LDX(sync, mo)) & SEQLOCK_WRITER) != 0) spin_wait(); } @@ -69,7 +66,7 @@ seqlock_t seqlock_acquire_rd(const seqlock_t *sync) { /* Wait for any present writer to go away */ /* B: Synchronize with A */ - return wait_for_no_writer(sync, __ATOMIC_ACQUIRE); + return wait_for_no_writer(sync, memory_order_acquire); } bool seqlock_release_rd(const seqlock_t *sync, seqlock_t prv) @@ -77,9 +74,9 @@ bool seqlock_release_rd(const seqlock_t *sync, seqlock_t prv) /* Enforce Load/Load order as if synchronizing with a store-release or * fence-release in another thread. */ - __atomic_thread_fence(__ATOMIC_ACQUIRE); + atomic_thread_fence(memory_order_acquire); /* Test if sync remains unchanged => success */ - return __atomic_load_n(sync, __ATOMIC_RELAXED) == prv; + return atomic_load_explicit(sync, memory_order_relaxed) == prv; } void seqlock_acquire_wr(seqlock_t *sync) @@ -87,17 +84,17 @@ void seqlock_acquire_wr(seqlock_t *sync) seqlock_t l; do { /* Wait for any present writer to go away */ - l = wait_for_no_writer(sync, __ATOMIC_RELAXED); + l = wait_for_no_writer(sync, memory_order_relaxed); /* Attempt to increment, setting writer flag */ } while ( /* C: Synchronize with A */ - !__atomic_compare_exchange_n(sync, &l, l + SEQLOCK_WRITER, - /*weak=*/true, __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED)); + !atomic_compare_exchange_strong_explicit( + sync, (uint32_t *) &l, l + SEQLOCK_WRITER, memory_order_acquire, + memory_order_relaxed)); /* Enforce Store/Store order as if synchronizing with a load-acquire or * fence-acquire in another thread. */ - __atomic_thread_fence(__ATOMIC_RELEASE); + atomic_thread_fence(memory_order_release); } void seqlock_release_wr(seqlock_t *sync) @@ -110,17 +107,19 @@ void seqlock_release_wr(seqlock_t *sync) /* Increment, clearing writer flag */ /* A: Synchronize with B and C */ - __atomic_store_n(sync, cur + 1, __ATOMIC_RELEASE); + atomic_store_explicit(sync, cur + SEQLOCK_WRITER, memory_order_release); } -#define ATOMIC_COPY(_d, _s, _sz, _type) \ - ({ \ - _type val = __atomic_load_n((const _type *) (_s), __ATOMIC_RELAXED); \ - _s += sizeof(_type); \ - __atomic_store_n((_type *) (_d), val, __ATOMIC_RELAXED); \ - _d += sizeof(_type); \ - _sz -= sizeof(_type); \ - }) +#define ATOMIC_COPY(_d, _s, _sz, _type) \ + do { \ + const _Atomic _type *src_atomic = (_Atomic const _type *) (_s); \ + _type val = atomic_load_explicit(src_atomic, memory_order_relaxed); \ + _s += sizeof(_type); \ + _Atomic _type *dst_atomic = (_Atomic _type *) (_d); \ + atomic_store_explicit(dst_atomic, val, memory_order_relaxed); \ + _d += sizeof(_type); \ + _sz -= sizeof(_type); \ + } while (0) static inline void atomic_memcpy(char *dst, const char *src, size_t sz) { diff --git a/seqlock/seqlock.h b/seqlock/seqlock.h index 82c3c4b..6b30d28 100644 --- a/seqlock/seqlock.h +++ b/seqlock/seqlock.h @@ -1,10 +1,11 @@ #pragma once +#include #include #include #include -typedef uint32_t seqlock_t; +typedef _Atomic(uint32_t) seqlock_t; /* Initialise a seqlock aka reader/writer synchronization */ void seqlock_init(seqlock_t *sync); From 732ea467cff549ac20c27d5e182482b92e61a576 Mon Sep 17 00:00:00 2001 From: scc Date: Sun, 27 Aug 2023 21:33:25 +0800 Subject: [PATCH 2/4] enable all warnings also non-standard syntax --- seqlock/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqlock/Makefile b/seqlock/Makefile index f5c4125..b15df7f 100644 --- a/seqlock/Makefile +++ b/seqlock/Makefile @@ -1,5 +1,5 @@ all: - gcc -o tests -std=gnu11 -Wall -O2 seqlock.c tests.c + gcc -o tests -std=gnu11 -Wall -Wextra -pedantic -O2 seqlock.c tests.c clean: rm -f tests From 9b37fb23982af637fe5c043a93c5b8ca92f74a71 Mon Sep 17 00:00:00 2001 From: scc Date: Sun, 27 Aug 2023 22:20:26 +0800 Subject: [PATCH 3/4] Remove flag -pedantic for __builtin_expect --- seqlock/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqlock/Makefile b/seqlock/Makefile index b15df7f..07ddffe 100644 --- a/seqlock/Makefile +++ b/seqlock/Makefile @@ -1,5 +1,5 @@ all: - gcc -o tests -std=gnu11 -Wall -Wextra -pedantic -O2 seqlock.c tests.c + gcc -o tests -std=gnu11 -Wall -Wextra -O2 seqlock.c tests.c clean: rm -f tests From 5f7d6a6d221c88eeeac3379c45f0eb1c507389ec Mon Sep 17 00:00:00 2001 From: 25077667 Date: Sat, 2 Sep 2023 14:04:38 +0800 Subject: [PATCH 4/4] Add memory barrier for aarch64 --- seqlock/seqlock.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/seqlock/seqlock.c b/seqlock/seqlock.c index 7d4e292..805ade3 100644 --- a/seqlock/seqlock.c +++ b/seqlock/seqlock.c @@ -10,37 +10,43 @@ #define SEQLOCK_WRITER 1U -#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) +#if defined(__i386__) || defined(__x86_64__) #define spin_wait() atomic_thread_fence(memory_order_seq_cst) +#elif defined(__aarch64__) +#define spin_wait() __asm__ __volatile__("isb\n") #else #define spin_wait() ((void) 0) #endif +#if defined(__aarch64__) #define SEVL() sevl() static inline void sevl(void) { - atomic_thread_fence(memory_order_seq_cst); + __asm__ volatile("sevl" : : :); } #define WFE() wfe() static inline int wfe(void) { - atomic_thread_fence(memory_order_seq_cst); + __asm__ volatile("wfe" : : : "memory"); return 1; } #define LDX(a, b) ldx((a), (b)) -static inline uint32_t ldx(const _Atomic uint32_t *var, int mm) +static inline uint32_t ldx(const uint8_t *var, int mm) { uint32_t old; - if (mm == memory_order_acquire) - old = atomic_load_explicit(var, memory_order_acquire); + __asm volatile("ldaxrb %w0, [%1]" : "=&r"(old) : "r"(var) : "memory"); else if (mm == memory_order_relaxed) - old = atomic_load_explicit(var, memory_order_relaxed); + __asm volatile("ldxrb %w0, [%1]" : "=&r"(old) : "r"(var) : "memory"); else abort(); - return old; } +#else /* generic */ +#define SEVL() (void) 0 +#define WFE() 1 +#define LDX(a, b) atomic_load_explicit((a), (b)) +#endif #define UNLIKELY(x) __builtin_expect(!!(x), 0)