1 #ifndef __X86_64_SYSTEM_H__
2 #define __X86_64_SYSTEM_H__
3
4 #define cmpxchg(ptr,o,n) \
5 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o), \
6 (unsigned long)(n),sizeof(*(ptr))))
7
8 /*
9 * Atomic 16 bytes compare and exchange. Compare OLD with MEM, if
10 * identical, store NEW in MEM. Return the initial value in MEM.
11 * Success is indicated by comparing RETURN with OLD.
12 *
13 * This function can only be called when cpu_has_cx16 is true.
14 */
15
__cmpxchg16b(volatile void * ptr,const __uint128_t * oldp,const __uint128_t * newp)16 static always_inline __uint128_t __cmpxchg16b(
17 volatile void *ptr, const __uint128_t *oldp, const __uint128_t *newp)
18 {
19 union {
20 struct { uint64_t lo, hi; };
21 __uint128_t raw;
22 } new = { .raw = *newp }, old = { .raw = *oldp }, prev;
23
24 ASSERT(cpu_has_cx16);
25
26 /* Don't use "=A" here - clang can't deal with that. */
27 asm volatile ( "lock; cmpxchg16b %2"
28 : "=d" (prev.hi), "=a" (prev.lo), "+m" (*__xg(ptr))
29 : "c" (new.hi), "b" (new.lo), "0" (old.hi), "1" (old.lo) );
30
31 return prev.raw;
32 }
33
34 #define cmpxchg16b(ptr, o, n) ({ \
35 volatile void *_p = (ptr); \
36 ASSERT(!((unsigned long)_p & 0xf)); \
37 BUILD_BUG_ON(sizeof(*(o)) != sizeof(__uint128_t)); \
38 BUILD_BUG_ON(sizeof(*(n)) != sizeof(__uint128_t)); \
39 __cmpxchg16b(_p, (void *)(o), (void *)(n)); \
40 })
41
42 /*
43 * This function causes value _o to be changed to _n at location _p.
44 * If this access causes a fault then we return 1, otherwise we return 0.
45 * If no fault occurs then _o is updated to the value we saw at _p. If this
46 * is the same as the initial value of _o then _n is written to location _p.
47 */
48 #define __cmpxchg_user(_p,_o,_n,_isuff,_oppre,_regtype) \
49 stac(); \
50 asm volatile ( \
51 "1: lock; cmpxchg"_isuff" %"_oppre"2,%3\n" \
52 "2:\n" \
53 ".section .fixup,\"ax\"\n" \
54 "3: movl $1,%1\n" \
55 " jmp 2b\n" \
56 ".previous\n" \
57 _ASM_EXTABLE(1b, 3b) \
58 : "=a" (_o), "=r" (_rc) \
59 : _regtype (_n), "m" (*__xg((volatile void *)_p)), "0" (_o), "1" (0) \
60 : "memory"); \
61 clac()
62
63 #define cmpxchg_user(_p,_o,_n) \
64 ({ \
65 int _rc; \
66 switch ( sizeof(*(_p)) ) { \
67 case 1: \
68 __cmpxchg_user(_p,_o,_n,"b","b","q"); \
69 break; \
70 case 2: \
71 __cmpxchg_user(_p,_o,_n,"w","w","r"); \
72 break; \
73 case 4: \
74 __cmpxchg_user(_p,_o,_n,"l","k","r"); \
75 break; \
76 case 8: \
77 __cmpxchg_user(_p,_o,_n,"q","","r"); \
78 break; \
79 } \
80 _rc; \
81 })
82
83 #define mb() \
84 asm volatile ( "mfence" : : : "memory" )
85
86 #endif /* __X86_64_SYSTEM_H__ */
87