atomic.c 9.88 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
#include "Rts.h"

// Fallbacks for atomic primops on byte arrays. The builtins used
// below are supported on both GCC and LLVM.
//
// Ideally these function would take StgWord8, StgWord16, etc but
// older GCC versions incorrectly assume that the register that the
// argument is passed in has been zero extended, which is incorrect
// according to the ABI and is not what GHC does when it generates
// calls to these functions.

// FetchAddByteArrayOp_Int

14
extern StgWord hs_atomic_add8(StgWord x, StgWord val);
15
StgWord
16
hs_atomic_add8(StgWord x, StgWord val)
17
{
18
  return __sync_fetch_and_add((volatile StgWord8 *) x, (StgWord8) val);
19 20
}

21
extern StgWord hs_atomic_add16(StgWord x, StgWord val);
22
StgWord
23
hs_atomic_add16(StgWord x, StgWord val)
24
{
25
  return __sync_fetch_and_add((volatile StgWord16 *) x, (StgWord16) val);
26 27
}

28
extern StgWord hs_atomic_add32(StgWord x, StgWord val);
29
StgWord
30
hs_atomic_add32(StgWord x, StgWord val)
31
{
32
  return __sync_fetch_and_add((volatile StgWord32 *) x, (StgWord32) val);
33 34
}

35
#if WORD_SIZE_IN_BITS == 64
36
extern StgWord64 hs_atomic_add64(StgWord x, StgWord64 val);
37
StgWord64
38
hs_atomic_add64(StgWord x, StgWord64 val)
39
{
40
  return __sync_fetch_and_add((volatile StgWord64 *) x, val);
41
}
42
#endif
43 44 45

// FetchSubByteArrayOp_Int

46
extern StgWord hs_atomic_sub8(StgWord x, StgWord val);
47
StgWord
48
hs_atomic_sub8(StgWord x, StgWord val)
49
{
50
  return __sync_fetch_and_sub((volatile StgWord8 *) x, (StgWord8) val);
51 52
}

53
extern StgWord hs_atomic_sub16(StgWord x, StgWord val);
54
StgWord
55
hs_atomic_sub16(StgWord x, StgWord val)
56
{
57
  return __sync_fetch_and_sub((volatile StgWord16 *) x, (StgWord16) val);
58 59
}

60
extern StgWord hs_atomic_sub32(StgWord x, StgWord val);
61
StgWord
62
hs_atomic_sub32(StgWord x, StgWord val)
63
{
64
  return __sync_fetch_and_sub((volatile StgWord32 *) x, (StgWord32) val);
65 66
}

67
#if WORD_SIZE_IN_BITS == 64
68
extern StgWord64 hs_atomic_sub64(StgWord x, StgWord64 val);
69
StgWord64
70
hs_atomic_sub64(StgWord x, StgWord64 val)
71
{
72
  return __sync_fetch_and_sub((volatile StgWord64 *) x, val);
73
}
74
#endif
75 76 77

// FetchAndByteArrayOp_Int

78
extern StgWord hs_atomic_and8(StgWord x, StgWord val);
79
StgWord
80
hs_atomic_and8(StgWord x, StgWord val)
81
{
82
  return __sync_fetch_and_and((volatile StgWord8 *) x, (StgWord8) val);
83 84
}

85
extern StgWord hs_atomic_and16(StgWord x, StgWord val);
86
StgWord
87
hs_atomic_and16(StgWord x, StgWord val)
88
{
89
  return __sync_fetch_and_and((volatile StgWord16 *) x, (StgWord16) val);
90 91
}

92
extern StgWord hs_atomic_and32(StgWord x, StgWord val);
93
StgWord
94
hs_atomic_and32(StgWord x, StgWord val)
95
{
96
  return __sync_fetch_and_and((volatile StgWord32 *) x, (StgWord32) val);
97 98
}

99
#if WORD_SIZE_IN_BITS == 64
100
extern StgWord64 hs_atomic_and64(StgWord x, StgWord64 val);
101
StgWord64
102
hs_atomic_and64(StgWord x, StgWord64 val)
103
{
104
  return __sync_fetch_and_and((volatile StgWord64 *) x, val);
105
}
106
#endif
107 108 109 110 111 112 113 114 115 116 117 118 119

// FetchNandByteArrayOp_Int

// Workaround for http://llvm.org/bugs/show_bug.cgi?id=8842
#define CAS_NAND(x, val)                                            \
  {                                                                 \
    __typeof__ (*(x)) tmp = *(x);                                   \
    while (!__sync_bool_compare_and_swap(x, tmp, ~(tmp & (val)))) { \
      tmp = *(x);                                                   \
    }                                                               \
    return tmp;                                                     \
  }

120 121 122 123 124
// This is only provided by clang
#if !defined(__has_builtin)
#define __has_builtin(x) 0
#endif

125 126 127 128
// Otherwise this fails with -Werror
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsync-nand"

129
extern StgWord hs_atomic_nand8(StgWord x, StgWord val);
130
StgWord
131
hs_atomic_nand8(StgWord x, StgWord val)
132
{
133
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
134
  CAS_NAND((volatile StgWord8 *) x, (StgWord8) val)
135
#else
136
  return __sync_fetch_and_nand((volatile StgWord8 *) x, (StgWord8) val);
137 138 139
#endif
}

140
extern StgWord hs_atomic_nand16(StgWord x, StgWord val);
141
StgWord
142
hs_atomic_nand16(StgWord x, StgWord val)
143
{
144
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
145
  CAS_NAND((volatile StgWord16 *) x, (StgWord16) val);
146
#else
147
  return __sync_fetch_and_nand((volatile StgWord16 *) x, (StgWord16) val);
148 149 150
#endif
}

151
extern StgWord hs_atomic_nand32(StgWord x, StgWord val);
152
StgWord
153
hs_atomic_nand32(StgWord x, StgWord val)
154
{
155
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
156
  CAS_NAND((volatile StgWord32 *) x, (StgWord32) val);
157
#else
158
  return __sync_fetch_and_nand((volatile StgWord32 *) x, (StgWord32) val);
159 160 161
#endif
}

162
#if WORD_SIZE_IN_BITS == 64
163
extern StgWord64 hs_atomic_nand64(StgWord x, StgWord64 val);
164
StgWord64
165
hs_atomic_nand64(StgWord x, StgWord64 val)
166
{
167
#if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
168
  CAS_NAND((volatile StgWord64 *) x, val);
169
#else
170
  return __sync_fetch_and_nand((volatile StgWord64 *) x, val);
171 172
#endif
}
173
#endif
174

175 176
#pragma GCC diagnostic pop

177 178
// FetchOrByteArrayOp_Int

179
extern StgWord hs_atomic_or8(StgWord x, StgWord val);
180
StgWord
181
hs_atomic_or8(StgWord x, StgWord val)
182
{
183
  return __sync_fetch_and_or((volatile StgWord8 *) x, (StgWord8) val);
184 185
}

186
extern StgWord hs_atomic_or16(StgWord x, StgWord val);
187
StgWord
188
hs_atomic_or16(StgWord x, StgWord val)
189
{
190
  return __sync_fetch_and_or((volatile StgWord16 *) x, (StgWord16) val);
191 192
}

193
extern StgWord hs_atomic_or32(StgWord x, StgWord val);
194
StgWord
195
hs_atomic_or32(StgWord x, StgWord val)
196
{
197
  return __sync_fetch_and_or((volatile StgWord32 *) x, (StgWord32) val);
198 199
}

200
#if WORD_SIZE_IN_BITS == 64
201
extern StgWord64 hs_atomic_or64(StgWord x, StgWord64 val);
202
StgWord64
203
hs_atomic_or64(StgWord x, StgWord64 val)
204
{
205
  return __sync_fetch_and_or((volatile StgWord64 *) x, val);
206
}
207
#endif
208 209 210

// FetchXorByteArrayOp_Int

211
extern StgWord hs_atomic_xor8(StgWord x, StgWord val);
212
StgWord
213
hs_atomic_xor8(StgWord x, StgWord val)
214
{
215
  return __sync_fetch_and_xor((volatile StgWord8 *) x, (StgWord8) val);
216 217
}

218
extern StgWord hs_atomic_xor16(StgWord x, StgWord val);
219
StgWord
220
hs_atomic_xor16(StgWord x, StgWord val)
221
{
222
  return __sync_fetch_and_xor((volatile StgWord16 *) x, (StgWord16) val);
223 224
}

225
extern StgWord hs_atomic_xor32(StgWord x, StgWord val);
226
StgWord
227
hs_atomic_xor32(StgWord x, StgWord val)
228
{
229
  return __sync_fetch_and_xor((volatile StgWord32 *) x, (StgWord32) val);
230 231
}

232
#if WORD_SIZE_IN_BITS == 64
233
extern StgWord64 hs_atomic_xor64(StgWord x, StgWord64 val);
234
StgWord64
235
hs_atomic_xor64(StgWord x, StgWord64 val)
236
{
237
  return __sync_fetch_and_xor((volatile StgWord64 *) x, val);
238
}
239
#endif
240 241 242

// CasByteArrayOp_Int

243
extern StgWord hs_cmpxchg8(StgWord x, StgWord old, StgWord new);
244
StgWord
245
hs_cmpxchg8(StgWord x, StgWord old, StgWord new)
246
{
247
  return __sync_val_compare_and_swap((volatile StgWord8 *) x, (StgWord8) old, (StgWord8) new);
248 249
}

250
extern StgWord hs_cmpxchg16(StgWord x, StgWord old, StgWord new);
251
StgWord
252
hs_cmpxchg16(StgWord x, StgWord old, StgWord new)
253
{
254
  return __sync_val_compare_and_swap((volatile StgWord16 *) x, (StgWord16) old, (StgWord16) new);
255 256
}

257
extern StgWord hs_cmpxchg32(StgWord x, StgWord old, StgWord new);
258
StgWord
259
hs_cmpxchg32(StgWord x, StgWord old, StgWord new)
260
{
261
  return __sync_val_compare_and_swap((volatile StgWord32 *) x, (StgWord32) old, (StgWord32) new);
262 263
}

264
#if WORD_SIZE_IN_BITS == 64
265
extern StgWord hs_cmpxchg64(StgWord x, StgWord64 old, StgWord64 new);
266
StgWord
267
hs_cmpxchg64(StgWord x, StgWord64 old, StgWord64 new)
268
{
269
  return __sync_val_compare_and_swap((volatile StgWord64 *) x, old, new);
270
}
271
#endif
272 273

// AtomicReadByteArrayOp_Int
274 275 276 277
// Implies a full memory barrier (see compiler/prelude/primops.txt.pp)
// __ATOMIC_SEQ_CST: Full barrier in both directions (hoisting and sinking
// of code) and synchronizes with acquire loads and release stores in
// all threads.
278 279 280 281
//
// When we lack C11 atomics support we emulate these using the old GCC __sync
// primitives which the GCC documentation claims "usually" implies a full
// barrier.
282

283
extern StgWord hs_atomicread8(StgWord x);
284
StgWord
285
hs_atomicread8(StgWord x)
286
{
287
#if HAVE_C11_ATOMICS
288
  return __atomic_load_n((StgWord8 *) x, __ATOMIC_SEQ_CST);
289 290 291
#else
  return __sync_add_and_fetch((StgWord8 *) x, 0);
#endif
292 293
}

294
extern StgWord hs_atomicread16(StgWord x);
295
StgWord
296
hs_atomicread16(StgWord x)
297
{
298
#if HAVE_C11_ATOMICS
299
  return __atomic_load_n((StgWord16 *) x, __ATOMIC_SEQ_CST);
300 301 302
#else
  return __sync_add_and_fetch((StgWord16 *) x, 0);
#endif
303 304
}

305
extern StgWord hs_atomicread32(StgWord x);
306
StgWord
307
hs_atomicread32(StgWord x)
308
{
309
#if HAVE_C11_ATOMICS
310
  return __atomic_load_n((StgWord32 *) x, __ATOMIC_SEQ_CST);
311 312 313
#else
  return __sync_add_and_fetch((StgWord32 *) x, 0);
#endif
314 315
}

316
extern StgWord64 hs_atomicread64(StgWord x);
317
StgWord64
318
hs_atomicread64(StgWord x)
319
{
320
#if HAVE_C11_ATOMICS
321
  return __atomic_load_n((StgWord64 *) x, __ATOMIC_SEQ_CST);
322 323 324
#else
  return __sync_add_and_fetch((StgWord64 *) x, 0);
#endif
325 326 327
}

// AtomicWriteByteArrayOp_Int
328 329
// Implies a full memory barrier (see compiler/prelude/primops.txt.pp)
// __ATOMIC_SEQ_CST: Full barrier (see hs_atomicread8 above).
330

331
extern void hs_atomicwrite8(StgWord x, StgWord val);
332
void
333
hs_atomicwrite8(StgWord x, StgWord val)
334
{
335
#if HAVE_C11_ATOMICS
336
  __atomic_store_n((StgWord8 *) x, (StgWord8) val, __ATOMIC_SEQ_CST);
337 338 339
#else
  while (!__sync_bool_compare_and_swap((StgWord8 *) x, *(StgWord8 *) x, (StgWord8) val));
#endif
340 341
}

342
extern void hs_atomicwrite16(StgWord x, StgWord val);
343
void
344
hs_atomicwrite16(StgWord x, StgWord val)
345
{
346
#if HAVE_C11_ATOMICS
347
  __atomic_store_n((StgWord16 *) x, (StgWord16) val, __ATOMIC_SEQ_CST);
348 349 350
#else
  while (!__sync_bool_compare_and_swap((StgWord16 *) x, *(StgWord16 *) x, (StgWord16) val));
#endif
351 352
}

353
extern void hs_atomicwrite32(StgWord x, StgWord val);
354
void
355
hs_atomicwrite32(StgWord x, StgWord val)
356
{
357
#if HAVE_C11_ATOMICS
358
  __atomic_store_n((StgWord32 *) x, (StgWord32) val, __ATOMIC_SEQ_CST);
359 360 361
#else
  while (!__sync_bool_compare_and_swap((StgWord32 *) x, *(StgWord32 *) x, (StgWord32) val));
#endif
362 363
}

364
extern void hs_atomicwrite64(StgWord x, StgWord64 val);
365
void
366
hs_atomicwrite64(StgWord x, StgWord64 val)
367
{
368
#if HAVE_C11_ATOMICS
369
  __atomic_store_n((StgWord64 *) x, (StgWord64) val, __ATOMIC_SEQ_CST);
370 371 372
#else
  while (!__sync_bool_compare_and_swap((StgWord64 *) x, *(StgWord64 *) x, (StgWord64) val));
#endif
373
}