atomic.c 9.93 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
#include "Rts.h"

// Fallbacks for atomic primops on byte arrays. The builtins used
// below are supported on both GCC and LLVM.
//
// Ideally these function would take StgWord8, StgWord16, etc but
// older GCC versions incorrectly assume that the register that the
// argument is passed in has been zero extended, which is incorrect
// according to the ABI and is not what GHC does when it generates
// calls to these functions.

// FetchAddByteArrayOp_Int

14
extern StgWord hs_atomic_add8(StgWord x, StgWord val);
15
StgWord
16
hs_atomic_add8(StgWord x, StgWord val)
17
{
18
  return __sync_fetch_and_add((volatile StgWord8 *) x, (StgWord8) val);
19 20
}

21
extern StgWord hs_atomic_add16(StgWord x, StgWord val);
22
StgWord
23
hs_atomic_add16(StgWord x, StgWord val)
24
{
25
  return __sync_fetch_and_add((volatile StgWord16 *) x, (StgWord16) val);
26 27
}

28
extern StgWord hs_atomic_add32(StgWord x, StgWord val);
29
StgWord
30
hs_atomic_add32(StgWord x, StgWord val)
31
{
32
  return __sync_fetch_and_add((volatile StgWord32 *) x, (StgWord32) val);
33 34
}

35
#if WORD_SIZE_IN_BITS == 64
36
extern StgWord64 hs_atomic_add64(StgWord x, StgWord64 val);
37
StgWord64
38
hs_atomic_add64(StgWord x, StgWord64 val)
39
{
40
  return __sync_fetch_and_add((volatile StgWord64 *) x, val);
41
}
42
#endif
43 44 45

// FetchSubByteArrayOp_Int

46
extern StgWord hs_atomic_sub8(StgWord x, StgWord val);
47
StgWord
48
hs_atomic_sub8(StgWord x, StgWord val)
49
{
50
  return __sync_fetch_and_sub((volatile StgWord8 *) x, (StgWord8) val);
51 52
}

53
extern StgWord hs_atomic_sub16(StgWord x, StgWord val);
54
StgWord
55
hs_atomic_sub16(StgWord x, StgWord val)
56
{
57
  return __sync_fetch_and_sub((volatile StgWord16 *) x, (StgWord16) val);
58 59
}

60
extern StgWord hs_atomic_sub32(StgWord x, StgWord val);
61
StgWord
62
hs_atomic_sub32(StgWord x, StgWord val)
63
{
64
  return __sync_fetch_and_sub((volatile StgWord32 *) x, (StgWord32) val);
65 66
}

67
#if WORD_SIZE_IN_BITS == 64
68
extern StgWord64 hs_atomic_sub64(StgWord x, StgWord64 val);
69
StgWord64
70
hs_atomic_sub64(StgWord x, StgWord64 val)
71
{
72
  return __sync_fetch_and_sub((volatile StgWord64 *) x, val);
73
}
74
#endif
75 76 77

// FetchAndByteArrayOp_Int

78
extern StgWord hs_atomic_and8(StgWord x, StgWord val);
79
StgWord
80
hs_atomic_and8(StgWord x, StgWord val)
81
{
82
  return __sync_fetch_and_and((volatile StgWord8 *) x, (StgWord8) val);
83 84
}

85
extern StgWord hs_atomic_and16(StgWord x, StgWord val);
86
StgWord
87
hs_atomic_and16(StgWord x, StgWord val)
88
{
89
  return __sync_fetch_and_and((volatile StgWord16 *) x, (StgWord16) val);
90 91
}

92
extern StgWord hs_atomic_and32(StgWord x, StgWord val);
93
StgWord
94
hs_atomic_and32(StgWord x, StgWord val)
95
{
96
  return __sync_fetch_and_and((volatile StgWord32 *) x, (StgWord32) val);
97 98
}

99
#if WORD_SIZE_IN_BITS == 64
100
extern StgWord64 hs_atomic_and64(StgWord x, StgWord64 val);
101
StgWord64
102
hs_atomic_and64(StgWord x, StgWord64 val)
103
{
104
  return __sync_fetch_and_and((volatile StgWord64 *) x, val);
105
}
106
#endif
107 108 109 110 111 112 113 114 115 116 117 118 119

// FetchNandByteArrayOp_Int

// Workaround for http://llvm.org/bugs/show_bug.cgi?id=8842
#define CAS_NAND(x, val)                                            \
  {                                                                 \
    __typeof__ (*(x)) tmp = *(x);                                   \
    while (!__sync_bool_compare_and_swap(x, tmp, ~(tmp & (val)))) { \
      tmp = *(x);                                                   \
    }                                                               \
    return tmp;                                                     \
  }

120 121 122 123 124
// This is only provided by clang
#if !defined(__has_builtin)
#define __has_builtin(x) 0
#endif

125
// Otherwise this fails with -Werror
126
#if defined(__GNUC__) && !defined(__clang__)
127 128
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsync-nand"
129
#endif
130

131
extern StgWord hs_atomic_nand8(StgWord x, StgWord val);
132
StgWord
133
hs_atomic_nand8(StgWord x, StgWord val)
134
{
135
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
136
  CAS_NAND((volatile StgWord8 *) x, (StgWord8) val)
137
#else
138
  return __sync_fetch_and_nand((volatile StgWord8 *) x, (StgWord8) val);
139 140 141
#endif
}

142
extern StgWord hs_atomic_nand16(StgWord x, StgWord val);
143
StgWord
144
hs_atomic_nand16(StgWord x, StgWord val)
145
{
146
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
147
  CAS_NAND((volatile StgWord16 *) x, (StgWord16) val);
148
#else
149
  return __sync_fetch_and_nand((volatile StgWord16 *) x, (StgWord16) val);
150 151 152
#endif
}

153
extern StgWord hs_atomic_nand32(StgWord x, StgWord val);
154
StgWord
155
hs_atomic_nand32(StgWord x, StgWord val)
156
{
157
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
158
  CAS_NAND((volatile StgWord32 *) x, (StgWord32) val);
159
#else
160
  return __sync_fetch_and_nand((volatile StgWord32 *) x, (StgWord32) val);
161 162 163
#endif
}

164
#if WORD_SIZE_IN_BITS == 64
165
extern StgWord64 hs_atomic_nand64(StgWord x, StgWord64 val);
166
StgWord64
167
hs_atomic_nand64(StgWord x, StgWord64 val)
168
{
169
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
170
  CAS_NAND((volatile StgWord64 *) x, val);
171
#else
172
  return __sync_fetch_and_nand((volatile StgWord64 *) x, val);
173 174
#endif
}
175
#endif
176

177 178
#pragma GCC diagnostic pop

179 180
// FetchOrByteArrayOp_Int

181
extern StgWord hs_atomic_or8(StgWord x, StgWord val);
182
StgWord
183
hs_atomic_or8(StgWord x, StgWord val)
184
{
185
  return __sync_fetch_and_or((volatile StgWord8 *) x, (StgWord8) val);
186 187
}

188
extern StgWord hs_atomic_or16(StgWord x, StgWord val);
189
StgWord
190
hs_atomic_or16(StgWord x, StgWord val)
191
{
192
  return __sync_fetch_and_or((volatile StgWord16 *) x, (StgWord16) val);
193 194
}

195
extern StgWord hs_atomic_or32(StgWord x, StgWord val);
196
StgWord
197
hs_atomic_or32(StgWord x, StgWord val)
198
{
199
  return __sync_fetch_and_or((volatile StgWord32 *) x, (StgWord32) val);
200 201
}

202
#if WORD_SIZE_IN_BITS == 64
203
extern StgWord64 hs_atomic_or64(StgWord x, StgWord64 val);
204
StgWord64
205
hs_atomic_or64(StgWord x, StgWord64 val)
206
{
207
  return __sync_fetch_and_or((volatile StgWord64 *) x, val);
208
}
209
#endif
210 211 212

// FetchXorByteArrayOp_Int

213
extern StgWord hs_atomic_xor8(StgWord x, StgWord val);
214
StgWord
215
hs_atomic_xor8(StgWord x, StgWord val)
216
{
217
  return __sync_fetch_and_xor((volatile StgWord8 *) x, (StgWord8) val);
218 219
}

220
extern StgWord hs_atomic_xor16(StgWord x, StgWord val);
221
StgWord
222
hs_atomic_xor16(StgWord x, StgWord val)
223
{
224
  return __sync_fetch_and_xor((volatile StgWord16 *) x, (StgWord16) val);
225 226
}

227
extern StgWord hs_atomic_xor32(StgWord x, StgWord val);
228
StgWord
229
hs_atomic_xor32(StgWord x, StgWord val)
230
{
231
  return __sync_fetch_and_xor((volatile StgWord32 *) x, (StgWord32) val);
232 233
}

234
#if WORD_SIZE_IN_BITS == 64
235
extern StgWord64 hs_atomic_xor64(StgWord x, StgWord64 val);
236
StgWord64
237
hs_atomic_xor64(StgWord x, StgWord64 val)
238
{
239
  return __sync_fetch_and_xor((volatile StgWord64 *) x, val);
240
}
241
#endif
242 243 244

// CasByteArrayOp_Int

245
extern StgWord hs_cmpxchg8(StgWord x, StgWord old, StgWord new);
246
StgWord
247
hs_cmpxchg8(StgWord x, StgWord old, StgWord new)
248
{
249
  return __sync_val_compare_and_swap((volatile StgWord8 *) x, (StgWord8) old, (StgWord8) new);
250 251
}

252
extern StgWord hs_cmpxchg16(StgWord x, StgWord old, StgWord new);
253
StgWord
254
hs_cmpxchg16(StgWord x, StgWord old, StgWord new)
255
{
256
  return __sync_val_compare_and_swap((volatile StgWord16 *) x, (StgWord16) old, (StgWord16) new);
257 258
}

259
extern StgWord hs_cmpxchg32(StgWord x, StgWord old, StgWord new);
260
StgWord
261
hs_cmpxchg32(StgWord x, StgWord old, StgWord new)
262
{
263
  return __sync_val_compare_and_swap((volatile StgWord32 *) x, (StgWord32) old, (StgWord32) new);
264 265
}

266
#if WORD_SIZE_IN_BITS == 64
267
extern StgWord hs_cmpxchg64(StgWord x, StgWord64 old, StgWord64 new);
268
StgWord
269
hs_cmpxchg64(StgWord x, StgWord64 old, StgWord64 new)
270
{
271
  return __sync_val_compare_and_swap((volatile StgWord64 *) x, old, new);
272
}
273
#endif
274 275

// AtomicReadByteArrayOp_Int
276 277 278 279
// Implies a full memory barrier (see compiler/prelude/primops.txt.pp)
// __ATOMIC_SEQ_CST: Full barrier in both directions (hoisting and sinking
// of code) and synchronizes with acquire loads and release stores in
// all threads.
280 281 282 283
//
// When we lack C11 atomics support we emulate these using the old GCC __sync
// primitives which the GCC documentation claims "usually" implies a full
// barrier.
284

285
extern StgWord hs_atomicread8(StgWord x);
286
StgWord
287
hs_atomicread8(StgWord x)
288
{
289
#if HAVE_C11_ATOMICS
290
  return __atomic_load_n((StgWord8 *) x, __ATOMIC_SEQ_CST);
291 292 293
#else
  return __sync_add_and_fetch((StgWord8 *) x, 0);
#endif
294 295
}

296
extern StgWord hs_atomicread16(StgWord x);
297
StgWord
298
hs_atomicread16(StgWord x)
299
{
300
#if HAVE_C11_ATOMICS
301
  return __atomic_load_n((StgWord16 *) x, __ATOMIC_SEQ_CST);
302 303 304
#else
  return __sync_add_and_fetch((StgWord16 *) x, 0);
#endif
305 306
}

307
extern StgWord hs_atomicread32(StgWord x);
308
StgWord
309
hs_atomicread32(StgWord x)
310
{
311
#if HAVE_C11_ATOMICS
312
  return __atomic_load_n((StgWord32 *) x, __ATOMIC_SEQ_CST);
313 314 315
#else
  return __sync_add_and_fetch((StgWord32 *) x, 0);
#endif
316 317
}

318
extern StgWord64 hs_atomicread64(StgWord x);
319
StgWord64
320
hs_atomicread64(StgWord x)
321
{
322
#if HAVE_C11_ATOMICS
323
  return __atomic_load_n((StgWord64 *) x, __ATOMIC_SEQ_CST);
324 325 326
#else
  return __sync_add_and_fetch((StgWord64 *) x, 0);
#endif
327 328 329
}

// AtomicWriteByteArrayOp_Int
330 331
// Implies a full memory barrier (see compiler/prelude/primops.txt.pp)
// __ATOMIC_SEQ_CST: Full barrier (see hs_atomicread8 above).
332

333
extern void hs_atomicwrite8(StgWord x, StgWord val);
334
void
335
hs_atomicwrite8(StgWord x, StgWord val)
336
{
337
#if HAVE_C11_ATOMICS
338
  __atomic_store_n((StgWord8 *) x, (StgWord8) val, __ATOMIC_SEQ_CST);
339 340 341
#else
  while (!__sync_bool_compare_and_swap((StgWord8 *) x, *(StgWord8 *) x, (StgWord8) val));
#endif
342 343
}

344
extern void hs_atomicwrite16(StgWord x, StgWord val);
345
void
346
hs_atomicwrite16(StgWord x, StgWord val)
347
{
348
#if HAVE_C11_ATOMICS
349
  __atomic_store_n((StgWord16 *) x, (StgWord16) val, __ATOMIC_SEQ_CST);
350 351 352
#else
  while (!__sync_bool_compare_and_swap((StgWord16 *) x, *(StgWord16 *) x, (StgWord16) val));
#endif
353 354
}

355
extern void hs_atomicwrite32(StgWord x, StgWord val);
356
void
357
hs_atomicwrite32(StgWord x, StgWord val)
358
{
359
#if HAVE_C11_ATOMICS
360
  __atomic_store_n((StgWord32 *) x, (StgWord32) val, __ATOMIC_SEQ_CST);
361 362 363
#else
  while (!__sync_bool_compare_and_swap((StgWord32 *) x, *(StgWord32 *) x, (StgWord32) val));
#endif
364 365
}

366
extern void hs_atomicwrite64(StgWord x, StgWord64 val);
367
void
368
hs_atomicwrite64(StgWord x, StgWord64 val)
369
{
370
#if HAVE_C11_ATOMICS
371
  __atomic_store_n((StgWord64 *) x, (StgWord64) val, __ATOMIC_SEQ_CST);
372 373 374
#else
  while (!__sync_bool_compare_and_swap((StgWord64 *) x, *(StgWord64 *) x, (StgWord64) val));
#endif
375
}