[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH v4 12/30] xen/riscv: introduce cmpxchg.h
The header was taken from Linux kernl 6.4.0-rc1. Addionally, were updated: * add emulation of {cmp}xchg for 1/2 byte types * replace tabs with spaces * replace __* varialbed with *__ * introduce generic version of xchg_* and cmpxchg_*. Signed-off-by: Oleksii Kurochko <oleksii.kurochko@xxxxxxxxx> --- Changes in V4: - Code style fixes. - enforce in __xchg_*() has the same type for new and *ptr, also "\n" was removed at the end of asm instruction. - dependency from https://lore.kernel.org/xen-devel/cover.1706259490.git.federico.serafini@xxxxxxxxxxx/ - switch from ASSERT_UNREACHABLE to STATIC_ASSERT_UNREACHABLE(). - drop xchg32(ptr, x) and xchg64(ptr, x) as they aren't used. - drop cmpxcg{32,64}_{local} as they aren't used. - introduce generic version of xchg_* and cmpxchg_*. - update the commit message. --- Changes in V3: - update the commit message - add emulation of {cmp}xchg_... for 1 and 2 bytes types --- Changes in V2: - update the comment at the top of the header. - change xen/lib.h to xen/bug.h. - sort inclusion of headers properly. --- xen/arch/riscv/include/asm/cmpxchg.h | 237 +++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 xen/arch/riscv/include/asm/cmpxchg.h diff --git a/xen/arch/riscv/include/asm/cmpxchg.h b/xen/arch/riscv/include/asm/cmpxchg.h new file mode 100644 index 0000000000..b751a50cbf --- /dev/null +++ b/xen/arch/riscv/include/asm/cmpxchg.h @@ -0,0 +1,237 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2014 Regents of the University of California */ + +#ifndef _ASM_RISCV_CMPXCHG_H +#define _ASM_RISCV_CMPXCHG_H + +#include <xen/compiler.h> +#include <xen/lib.h> + +#include <asm/fence.h> +#include <asm/io.h> +#include <asm/system.h> + +#define ALIGN_DOWN(addr, size) ((addr) & (~((size) - 1))) + +#define __amoswap_generic(ptr, new, ret, sfx, release_barrier, acquire_barrier) \ +({ \ + asm volatile( \ + release_barrier \ + " amoswap" sfx " %0, %2, %1\n" \ + acquire_barrier \ + : "=r" (ret), "+A" (*ptr) \ + : "r" (new) \ + : "memory" ); \ +}) + +#define emulate_xchg_1_2(ptr, new, ret, release_barrier, acquire_barrier) \ +({ \ + uint32_t *ptr_32b_aligned = (uint32_t *)ALIGN_DOWN((unsigned long)ptr, 4); \ + uint8_t mask_l = ((unsigned long)(ptr) & (0x8 - sizeof(*ptr))) * BITS_PER_BYTE; \ + uint8_t mask_size = sizeof(*ptr) * BITS_PER_BYTE; \ + uint8_t mask_h = mask_l + mask_size - 1; \ + unsigned long mask = GENMASK(mask_h, mask_l); \ + unsigned long new_ = (unsigned long)(new) << mask_l; \ + unsigned long ret_; \ + unsigned long rc; \ + \ + asm volatile( \ + release_barrier \ + "0: lr.d %0, %2\n" \ + " and %1, %0, %z4\n" \ + " or %1, %1, %z3\n" \ + " sc.d %1, %1, %2\n" \ + " bnez %1, 0b\n" \ + acquire_barrier \ + : "=&r" (ret_), "=&r" (rc), "+A" (*ptr_32b_aligned) \ + : "rJ" (new_), "rJ" (~mask) \ + : "memory"); \ + \ + ret = (__typeof__(*(ptr)))((ret_ & mask) >> mask_l); \ +}) + +#define __xchg_generic(ptr, new, size, sfx, release_barrier, acquire_barrier) \ +({ \ + __typeof__(ptr) ptr__ = (ptr); \ + __typeof__(*(ptr)) new__ = (new); \ + __typeof__(*(ptr)) ret__; \ + switch (size) \ + { \ + case 1: \ + case 2: \ + emulate_xchg_1_2(ptr__, new__, ret__, release_barrier, acquire_barrier); \ + break; \ + case 4: \ + __amoswap_generic(ptr__, new__, ret__,\ + ".w" sfx, release_barrier, acquire_barrier); \ + break; \ + case 8: \ + __amoswap_generic(ptr__, new__, ret__,\ + ".d" sfx, release_barrier, acquire_barrier); \ + break; \ + default: \ + STATIC_ASSERT_UNREACHABLE(); \ + } \ + ret__; \ +}) + +#define xchg_relaxed(ptr, x) \ +({ \ + __typeof__(*(ptr)) x_ = (x); \ + (__typeof__(*(ptr)))__xchg_generic(ptr, x_, sizeof(*(ptr)), "", "", ""); \ +}) + +#define xchg_acquire(ptr, x) \ +({ \ + __typeof__(*(ptr)) x_ = (x); \ + (__typeof__(*(ptr)))__xchg_generic(ptr, x_, sizeof(*(ptr)), \ + "", "", RISCV_ACQUIRE_BARRIER); \ +}) + +#define xchg_release(ptr, x) \ +({ \ + __typeof__(*(ptr)) x_ = (x); \ + (__typeof__(*(ptr)))__xchg_generic(ptr, x_, sizeof(*(ptr)),\ + "", RISCV_RELEASE_BARRIER, ""); \ +}) + +#define xchg(ptr,x) \ +({ \ + __typeof__(*(ptr)) ret__; \ + ret__ = (__typeof__(*(ptr))) \ + __xchg_generic(ptr, (unsigned long)(x), sizeof(*(ptr)), \ + ".aqrl", "", ""); \ + ret__; \ +}) + +#define __generic_cmpxchg(ptr, old, new, ret, lr_sfx, sc_sfx, release_barrier, acquire_barrier) \ + ({ \ + register unsigned int rc; \ + asm volatile( \ + release_barrier \ + "0: lr" lr_sfx " %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc" sc_sfx " %1, %z4, %2\n" \ + " bnez %1, 0b\n" \ + acquire_barrier \ + "1:\n" \ + : "=&r" (ret), "=&r" (rc), "+A" (*ptr) \ + : "rJ" (old), "rJ" (new) \ + : "memory"); \ + }) + +#define emulate_cmpxchg_1_2(ptr, old, new, ret, sc_sfx, release_barrier, acquire_barrier) \ +({ \ + uint32_t *ptr_32b_aligned = (uint32_t *)ALIGN_DOWN((unsigned long)ptr, 4); \ + uint8_t mask_l = ((unsigned long)(ptr) & (0x8 - sizeof(*ptr))) * BITS_PER_BYTE; \ + uint8_t mask_size = sizeof(*ptr) * BITS_PER_BYTE; \ + uint8_t mask_h = mask_l + mask_size - 1; \ + unsigned long mask = GENMASK(mask_h, mask_l); \ + unsigned long old_ = (unsigned long)(old) << mask_l; \ + unsigned long new_ = (unsigned long)(new) << mask_l; \ + unsigned long ret_; \ + unsigned long rc; \ + \ + __asm__ __volatile__ ( \ + release_barrier \ + "0: lr.d %0, %2\n" \ + " and %1, %0, %z5\n" \ + " bne %1, %z3, 1f\n" \ + " and %1, %0, %z6\n" \ + " or %1, %1, %z4\n" \ + " sc.d" sc_sfx " %1, %1, %2\n" \ + " bnez %1, 0b\n" \ + acquire_barrier \ + "1:\n" \ + : "=&r" (ret_), "=&r" (rc), "+A" (*ptr_32b_aligned) \ + : "rJ" (old_), "rJ" (new_), \ + "rJ" (mask), "rJ" (~mask) \ + : "memory"); \ + \ + ret = (__typeof__(*(ptr)))((ret_ & mask) >> mask_l); \ +}) + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + */ +#define __cmpxchg_generic(ptr, old, new, size, sc_sfx, release_barrier, acquire_barrier) \ +({ \ + __typeof__(ptr) ptr__ = (ptr); \ + __typeof__(*(ptr)) old__ = (__typeof__(*(ptr)))(old); \ + __typeof__(*(ptr)) new__ = (__typeof__(*(ptr)))(new); \ + __typeof__(*(ptr)) ret__; \ + switch (size) \ + { \ + case 1: \ + case 2: \ + emulate_cmpxchg_1_2(ptr, old, new, ret__,\ + sc_sfx, release_barrier, acquire_barrier); \ + break; \ + case 4: \ + __generic_cmpxchg(ptr__, old__, new__, ret__, \ + ".w", ".w"sc_sfx, release_barrier, acquire_barrier); \ + break; \ + case 8: \ + __generic_cmpxchg(ptr__, old__, new__, ret__, \ + ".d", ".d"sc_sfx, release_barrier, acquire_barrier); \ + break; \ + default: \ + STATIC_ASSERT_UNREACHABLE(); \ + } \ + ret__; \ +}) + +#define cmpxchg_relaxed(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) o_ = (o); \ + __typeof__(*(ptr)) n_ = (n); \ + (__typeof__(*(ptr)))__cmpxchg_generic(ptr, \ + o_, n_, sizeof(*(ptr)), "", "", ""); \ +}) + +#define cmpxchg_acquire(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) o_ = (o); \ + __typeof__(*(ptr)) n_ = (n); \ + (__typeof__(*(ptr)))__cmpxchg_generic(ptr, o_, n_, sizeof(*(ptr)), \ + "", "", RISCV_ACQUIRE_BARRIER); \ +}) + +#define cmpxchg_release(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) o_ = (o); \ + __typeof__(*(ptr)) n_ = (n); \ + (__typeof__(*(ptr)))__cmpxchg_release(ptr, o_, n_, sizeof(*(ptr)), \ + "", RISCV_RELEASE_BARRIER, ""); \ +}) + +#define cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) ret__; \ + ret__ = (__typeof__(*(ptr))) \ + __cmpxchg_generic(ptr, (unsigned long)(o), (unsigned long)(n), \ + sizeof(*(ptr)), ".rl", "", " fence rw, rw\n"); \ + ret__; \ +}) + +#define __cmpxchg(ptr, o, n, s) \ +({ \ + __typeof__(*(ptr)) ret__; \ + ret__ = (__typeof__(*(ptr))) \ + __cmpxchg_generic(ptr, (unsigned long)(o), (unsigned long)(n), \ + s, ".rl", "", " fence rw, rw\n"); \ + ret__; \ +}) + +#endif /* _ASM_RISCV_CMPXCHG_H */ + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ -- 2.43.0
|
![]() |
Lists.xenproject.org is hosted with RackSpace, monitoring our |