ObjFW
atomic_x86.h
1 /*
2  * Copyright (c) 2008-2021 Jonathan Schleifer <js@nil.im>
3  *
4  * All rights reserved.
5  *
6  * This file is part of ObjFW. It may be distributed under the terms of the
7  * Q Public License 1.0, which can be found in the file LICENSE.QPL included in
8  * the packaging of this file.
9  *
10  * Alternatively, it may be distributed under the terms of the GNU General
11  * Public License, either version 2 or 3, which can be found in the file
12  * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this
13  * file.
14  */
15 
16 OF_ASSUME_NONNULL_BEGIN
17 
18 static OF_INLINE int
19 of_atomic_int_add(volatile int *_Nonnull p, int i)
20 {
21  if (sizeof(int) == 4)
22  __asm__ __volatile__ (
23  "lock\n\t"
24  "xaddl %0, %2\n\t"
25  "addl %1, %0"
26  : "+&r"(i)
27  : "r"(i), "m"(*p)
28  );
29 #ifdef OF_X86_64
30  else if (sizeof(int) == 8)
31  __asm__ __volatile__ (
32  "lock\n\t"
33  "xaddq %0, %2\n\t"
34  "addq %1, %0"
35  : "+&r"(i)
36  : "r"(i), "m"(*p)
37  );
38 #endif
39  else
40  abort();
41 
42  return i;
43 }
44 
45 static OF_INLINE int32_t
46 of_atomic_int32_add(volatile int32_t *_Nonnull p, int32_t i)
47 {
48  __asm__ __volatile__ (
49  "lock\n\t"
50  "xaddl %0, %2\n\t"
51  "addl %1, %0"
52  : "+&r"(i)
53  : "r"(i), "m"(*p)
54  );
55 
56  return i;
57 }
58 
59 static OF_INLINE void *_Nullable
60 of_atomic_ptr_add(void *volatile _Nullable *_Nonnull p, intptr_t i)
61 {
62 #if defined(OF_X86_64)
63  __asm__ __volatile__ (
64  "lock\n\t"
65  "xaddq %0, %2\n\t"
66  "addq %1, %0"
67  : "+&r"(i)
68  : "r"(i), "m"(*p)
69  );
70 
71  return (void *)i;
72 #elif defined(OF_X86)
73  __asm__ __volatile__ (
74  "lock\n\t"
75  "xaddl %0, %2\n\t"
76  "addl %1, %0"
77  : "+&r"(i)
78  : "r"(i), "m"(*p)
79  );
80 
81  return (void *)i;
82 #endif
83 }
84 
85 static OF_INLINE int
86 of_atomic_int_sub(volatile int *_Nonnull p, int i)
87 {
88  if (sizeof(int) == 4)
89  __asm__ __volatile__ (
90  "negl %0\n\t"
91  "lock\n\t"
92  "xaddl %0, %2\n\t"
93  "subl %1, %0"
94  : "+&r"(i)
95  : "r"(i), "m"(*p)
96  );
97 #ifdef OF_X86_64
98  else if (sizeof(int) == 8)
99  __asm__ __volatile__ (
100  "negq %0\n\t"
101  "lock\n\t"
102  "xaddq %0, %2\n\t"
103  "subq %1, %0"
104  : "+&r"(i)
105  : "r"(i), "m"(*p)
106  );
107 #endif
108  else
109  abort();
110 
111  return i;
112 }
113 
114 static OF_INLINE int32_t
115 of_atomic_int32_sub(volatile int32_t *_Nonnull p, int32_t i)
116 {
117  __asm__ __volatile__ (
118  "negl %0\n\t"
119  "lock\n\t"
120  "xaddl %0, %2\n\t"
121  "subl %1, %0"
122  : "+&r"(i)
123  : "r"(i), "m"(*p)
124  );
125 
126  return i;
127 }
128 
129 static OF_INLINE void *_Nullable
130 of_atomic_ptr_sub(void *volatile _Nullable *_Nonnull p, intptr_t i)
131 {
132 #if defined(OF_X86_64)
133  __asm__ __volatile__ (
134  "negq %0\n\t"
135  "lock\n\t"
136  "xaddq %0, %2\n\t"
137  "subq %1, %0"
138  : "+&r"(i)
139  : "r"(i), "m"(*p)
140  );
141 
142  return (void *)i;
143 #elif defined(OF_X86)
144  __asm__ __volatile__ (
145  "negl %0\n\t"
146  "lock\n\t"
147  "xaddl %0, %2\n\t"
148  "subl %1, %0"
149  : "+&r"(i)
150  : "r"(i), "m"(*p)
151  );
152 
153  return (void *)i;
154 #endif
155 }
156 
157 static OF_INLINE int
158 of_atomic_int_inc(volatile int *_Nonnull p)
159 {
160  int i;
161 
162  if (sizeof(int) == 4)
163  __asm__ __volatile__ (
164  "xorl %0, %0\n\t"
165  "incl %0\n\t"
166  "lock\n\t"
167  "xaddl %0, %1\n\t"
168  "incl %0"
169  : "=&r"(i)
170  : "m"(*p)
171  );
172 #ifdef OF_X86_64
173  else if (sizeof(int) == 8)
174  __asm__ __volatile__ (
175  "xorq %0, %0\n\t"
176  "incq %0\n\t"
177  "lock\n\t"
178  "xaddq %0, %1\n\t"
179  "incq %0"
180  : "=&r"(i)
181  : "m"(*p)
182  );
183 #endif
184  else
185  abort();
186 
187  return i;
188 }
189 
190 static OF_INLINE int32_t
191 of_atomic_int32_inc(volatile int32_t *_Nonnull p)
192 {
193  int32_t i;
194 
195  __asm__ __volatile__ (
196  "xorl %0, %0\n\t"
197  "incl %0\n\t"
198  "lock\n\t"
199  "xaddl %0, %1\n\t"
200  "incl %0"
201  : "=&r"(i)
202  : "m"(*p)
203  );
204 
205  return i;
206 }
207 
208 static OF_INLINE int
209 of_atomic_int_dec(volatile int *_Nonnull p)
210 {
211  int i;
212 
213  if (sizeof(int) == 4)
214  __asm__ __volatile__ (
215  "xorl %0, %0\n\t"
216  "decl %0\n\t"
217  "lock\n\t"
218  "xaddl %0, %1\n\t"
219  "decl %0"
220  : "=&r"(i)
221  : "m"(*p)
222  );
223 #ifdef OF_X86_64
224  else if (sizeof(int) == 8)
225  __asm__ __volatile__ (
226  "xorq %0, %0\n\t"
227  "decq %0\n\t"
228  "lock\n\t"
229  "xaddq %0, %1\n\t"
230  "decq %0"
231  : "=&r"(i)
232  : "m"(*p)
233  );
234 #endif
235  else
236  abort();
237 
238  return i;
239 }
240 
241 static OF_INLINE int32_t
242 of_atomic_int32_dec(volatile int32_t *_Nonnull p)
243 {
244  int32_t i;
245 
246  __asm__ __volatile__ (
247  "xorl %0, %0\n\t"
248  "decl %0\n\t"
249  "lock\n\t"
250  "xaddl %0, %1\n\t"
251  "decl %0"
252  : "=&r"(i)
253  : "m"(*p)
254  );
255 
256  return i;
257 }
258 
259 static OF_INLINE unsigned int
260 of_atomic_int_or(volatile unsigned int *_Nonnull p, unsigned int i)
261 {
262  if (sizeof(int) == 4)
263  __asm__ __volatile__ (
264  "0:\n\t"
265  "movl %2, %0\n\t"
266  "movl %0, %%eax\n\t"
267  "orl %1, %0\n\t"
268  "lock\n\t"
269  "cmpxchg %0, %2\n\t"
270  "jne 0b"
271  : "=&r"(i)
272  : "r"(i), "m"(*p)
273  : "eax", "cc"
274  );
275 #ifdef OF_X86_64
276  else if (sizeof(int) == 8)
277  __asm__ __volatile__ (
278  "0:\n\t"
279  "movq %2, %0\n\t"
280  "movq %0, %%rax\n\t"
281  "orq %1, %0\n\t"
282  "lock\n\t"
283  "cmpxchg %0, %2\n\t"
284  "jne 0b"
285  : "=&r"(i)
286  : "r"(i), "m"(*p)
287  : "rax", "cc"
288  );
289 #endif
290  else
291  abort();
292 
293  return i;
294 }
295 
296 static OF_INLINE uint32_t
297 of_atomic_int32_or(volatile uint32_t *_Nonnull p, uint32_t i)
298 {
299  __asm__ __volatile__ (
300  "0:\n\t"
301  "movl %2, %0\n\t"
302  "movl %0, %%eax\n\t"
303  "orl %1, %0\n\t"
304  "lock\n\t"
305  "cmpxchg %0, %2\n\t"
306  "jne 0b"
307  : "=&r"(i)
308  : "r"(i), "m"(*p)
309  : "eax", "cc"
310  );
311 
312  return i;
313 }
314 
315 static OF_INLINE unsigned int
316 of_atomic_int_and(volatile unsigned int *_Nonnull p, unsigned int i)
317 {
318  if (sizeof(int) == 4)
319  __asm__ __volatile__ (
320  "0:\n\t"
321  "movl %2, %0\n\t"
322  "movl %0, %%eax\n\t"
323  "andl %1, %0\n\t"
324  "lock\n\t"
325  "cmpxchg %0, %2\n\t"
326  "jne 0b"
327  : "=&r"(i)
328  : "r"(i), "m"(*p)
329  : "eax", "cc"
330  );
331 #ifdef OF_X86_64
332  else if (sizeof(int) == 8)
333  __asm__ __volatile__ (
334  "0:\n\t"
335  "movq %2, %0\n\t"
336  "movq %0, %%rax\n\t"
337  "andq %1, %0\n\t"
338  "lock\n\t"
339  "cmpxchg %0, %2\n\t"
340  "jne 0b"
341  : "=&r"(i)
342  : "r"(i), "m"(*p)
343  : "rax", "cc"
344  );
345 #endif
346  else
347  abort();
348 
349  return i;
350 }
351 
352 static OF_INLINE uint32_t
353 of_atomic_int32_and(volatile uint32_t *_Nonnull p, uint32_t i)
354 {
355  __asm__ __volatile__ (
356  "0:\n\t"
357  "movl %2, %0\n\t"
358  "movl %0, %%eax\n\t"
359  "andl %1, %0\n\t"
360  "lock\n\t"
361  "cmpxchg %0, %2\n\t"
362  "jne 0b"
363  : "=&r"(i)
364  : "r"(i), "m"(*p)
365  : "eax", "cc"
366  );
367 
368  return i;
369 }
370 
371 static OF_INLINE unsigned int
372 of_atomic_int_xor(volatile unsigned int *_Nonnull p, unsigned int i)
373 {
374  if (sizeof(int) == 4)
375  __asm__ __volatile__ (
376  "0:\n\t"
377  "movl %2, %0\n\t"
378  "movl %0, %%eax\n\t"
379  "xorl %1, %0\n\t"
380  "lock\n\t"
381  "cmpxchg %0, %2\n\t"
382  "jne 0b"
383  : "=&r"(i)
384  : "r"(i), "m"(*p)
385  : "eax", "cc"
386  );
387 #ifdef OF_X86_64
388  else if (sizeof(int) == 8)
389  __asm__ __volatile__ (
390  "0:\n\t"
391  "movq %2, %0\n\t"
392  "movq %0, %%rax\n\t"
393  "xorq %1, %0\n\t"
394  "lock\n\t"
395  "cmpxchg %0, %2\n\t"
396  "jne 0b"
397  : "=&r"(i)
398  : "r"(i), "m"(*p)
399  : "rax", "cc"
400  );
401 #endif
402  else
403  abort();
404 
405  return i;
406 }
407 
408 static OF_INLINE uint32_t
409 of_atomic_int32_xor(volatile uint32_t *_Nonnull p, uint32_t i)
410 {
411  __asm__ __volatile__ (
412  "0:\n\t"
413  "movl %2, %0\n\t"
414  "movl %0, %%eax\n\t"
415  "xorl %1, %0\n\t"
416  "lock\n\t"
417  "cmpxchgl %0, %2\n\t"
418  "jne 0b"
419  : "=&r"(i)
420  : "r"(i), "m"(*p)
421  : "eax", "cc"
422  );
423 
424  return i;
425 }
426 
427 static OF_INLINE bool
428 of_atomic_int_cmpswap(volatile int *_Nonnull p, int o, int n)
429 {
430  int r;
431 
432  __asm__ __volatile__ (
433  "lock\n\t"
434  "cmpxchg %2, %3\n\t"
435  "sete %b0\n\t"
436  "movzbl %b0, %0"
437  : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */
438  : "r"(n), "m"(*p)
439  : "cc"
440  );
441 
442  return r;
443 }
444 
445 static OF_INLINE bool
446 of_atomic_int32_cmpswap(volatile int32_t *_Nonnull p, int32_t o, int32_t n)
447 {
448  int r;
449 
450  __asm__ __volatile__ (
451  "lock\n\t"
452  "cmpxchg %2, %3\n\t"
453  "sete %b0\n\t"
454  "movzbl %b0, %0"
455  : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */
456  : "r"(n), "m"(*p)
457  : "cc"
458  );
459 
460  return r;
461 }
462 
463 static OF_INLINE bool
464 of_atomic_ptr_cmpswap(void *volatile _Nullable *_Nonnull p,
465  void *_Nullable o, void *_Nullable n)
466 {
467  int r;
468 
469  __asm__ __volatile__ (
470  "lock\n\t"
471  "cmpxchg %2, %3\n\t"
472  "sete %b0\n\t"
473  "movzbl %b0, %0"
474  : "=&d"(r), "+a"(o) /* use d instead of r to avoid a gcc bug */
475  : "r"(n), "m"(*p)
476  : "cc"
477  );
478 
479  return r;
480 }
481 
482 static OF_INLINE void
483 of_memory_barrier(void)
484 {
485  __asm__ __volatile__ (
486  "mfence" ::: "memory"
487  );
488 }
489 
490 static OF_INLINE void
491 of_memory_barrier_acquire(void)
492 {
493  __asm__ __volatile__ ("" ::: "memory");
494 }
495 
496 static OF_INLINE void
497 of_memory_barrier_release(void)
498 {
499  __asm__ __volatile__ ("" ::: "memory");
500 }
501 
502 OF_ASSUME_NONNULL_END