Commit 3ca03c63 authored by Ben Gamari's avatar Ben Gamari 🐢

Nonmoving: Ensure write barrier vanishes in non-threaded RTS

parent 1d7b6fac
......@@ -950,19 +950,23 @@
return (dst);
//
// Nonmoving write barrier helpers
//
// See Note [Update remembered set] in NonMovingMark.c.
#if defined(THREADED_RTS)
#define IF_WRITE_BARRIER_ENABLED \
#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
if (W_[nonmoving_write_barrier_enabled] != 0) (likely: False)
#else
// A similar measure is also taken in rts/NonMoving.h, but that isn't visible from C--
#define IF_WRITE_BARRIER_ENABLED \
#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
if (0)
#define nonmoving_write_barrier_enabled 0
#endif
// A useful helper for pushing a pointer to the update remembered set.
// See Note [Update remembered set] in NonMovingMark.c.
#define updateRemembSetPushPtr(p) \
IF_WRITE_BARRIER_ENABLED { \
IF_NONMOVING_WRITE_BARRIER_ENABLED { \
ccall updateRemembSetPushClosure_(BaseReg "ptr", p "ptr"); \
}
......@@ -21,4 +21,7 @@ void updateRemembSetPushClosure(Capability *cap, StgClosure *p);
void updateRemembSetPushThunk_(StgRegTable *reg, StgThunk *p);
// Note that RTS code should not condition on this directly by rather
// use the IF_NONMOVING_WRITE_BARRIER_ENABLED macro to ensure that
// the barrier is eliminated in the non-threaded RTS.
extern StgWord DLL_IMPORT_DATA_VAR(nonmoving_write_barrier_enabled);
......@@ -262,7 +262,7 @@ loop:
// point to the BLOCKING_QUEUE from the BLACKHOLE
write_barrier(); // make the BQ visible, see Note [Heap memory barriers].
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure*)p);
}
((StgInd*)bh)->indirectee = (StgClosure *)bq;
......@@ -293,7 +293,7 @@ loop:
}
#endif
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
// We are about to overwrite bq->queue; make sure its current value
// makes it into the update remembered set
updateRemembSetPushClosure(cap, (StgClosure*)bq->queue);
......
......@@ -478,7 +478,7 @@ stg_copyArray_barrier ( W_ hdr_size, gcptr dst, W_ dst_off, W_ n)
end = p + WDS(n);
again:
IF_WRITE_BARRIER_ENABLED {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
ccall updateRemembSetPushClosure_(BaseReg "ptr", W_[p] "ptr");
}
p = p + WDS(1);
......@@ -494,7 +494,7 @@ stg_copySmallArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n)
W_ dst_p, src_p, bytes;
if (n > 0) {
IF_WRITE_BARRIER_ENABLED {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs,
dst, dst_off, n);
}
......@@ -515,7 +515,7 @@ stg_copySmallMutableArrayzh ( gcptr src, W_ src_off, gcptr dst, W_ dst_off, W_ n
W_ dst_p, src_p, bytes;
if (n > 0) {
IF_WRITE_BARRIER_ENABLED {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
call stg_copyArray_barrier(SIZEOF_StgSmallMutArrPtrs,
dst, dst_off, n);
}
......
......@@ -297,8 +297,10 @@ static StgClosure *lock_tvar(Capability *cap,
} while (cas((void *)&(s -> current_value),
(StgWord)result, (StgWord)trec) != (StgWord)result);
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) {
updateRemembSetPushClosure(cap, result);
IF_NONMOVING_WRITE_BARRIER_ENABLED {
if (result)
updateRemembSetPushClosure(cap, result);
}
return result;
}
......@@ -323,8 +325,9 @@ static StgBool cond_lock_tvar(Capability *cap,
TRACE("%p : cond_lock_tvar(%p, %p)", trec, s, expected);
w = cas((void *)&(s -> current_value), (StgWord)expected, (StgWord)trec);
result = (StgClosure *)w;
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled && result)) {
updateRemembSetPushClosure(cap, expected);
IF_NONMOVING_WRITE_BARRIER_ENABLED {
if (result)
updateRemembSetPushClosure(cap, expected);
}
TRACE("%p : %s", trec, result ? "success" : "failure");
return (result == expected);
......
......@@ -2509,7 +2509,7 @@ resumeThread (void *task_)
incall->suspended_tso = NULL;
incall->suspended_cap = NULL;
// we will modify tso->_link
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure *)tso->_link);
}
tso->_link = END_TSO_QUEUE;
......
......@@ -334,15 +334,16 @@ threadPaused(Capability *cap, StgTSO *tso)
}
#endif
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled
&& ip_THUNK(INFO_PTR_TO_STRUCT(bh_info)))) {
// We are about to replace a thunk with a blackhole.
// Add the free variables of the closure we are about to
// overwrite to the update remembered set.
// N.B. We caught the WHITEHOLE case above.
updateRemembSetPushThunkEager(cap,
THUNK_INFO_PTR_TO_STRUCT(bh_info),
(StgThunk *) bh);
IF_NONMOVING_WRITE_BARRIER_ENABLED {
if (ip_THUNK(INFO_PTR_TO_STRUCT(bh_info))) {
// We are about to replace a thunk with a blackhole.
// Add the free variables of the closure we are about to
// overwrite to the update remembered set.
// N.B. We caught the WHITEHOLE case above.
updateRemembSetPushThunkEager(cap,
THUNK_INFO_PTR_TO_STRUCT(bh_info),
(StgThunk *) bh);
}
}
// The payload of the BLACKHOLE points to the TSO
......
......@@ -723,7 +723,7 @@ threadStackUnderflow (Capability *cap, StgTSO *tso)
barf("threadStackUnderflow: not enough space for return values");
}
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
// ensure that values that we copy into the new stack are marked
// for the nonmoving collector. Note that these values won't
// necessarily form a full closure so we need to handle them
......
......@@ -50,7 +50,7 @@
\
prim_write_barrier; \
OVERWRITING_CLOSURE(p1); \
IF_WRITE_BARRIER_ENABLED { \
IF_NONMOVING_WRITE_BARRIER_ENABLED { \
ccall updateRemembSetPushThunk_(BaseReg, p1 "ptr"); \
} \
StgInd_indirectee(p1) = p2; \
......@@ -81,7 +81,7 @@ INLINE_HEADER void updateWithIndirection (Capability *cap,
/* See Note [Heap memory barriers] in SMP.h */
write_barrier();
OVERWRITING_CLOSURE(p1);
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushThunk(cap, (StgThunk*)p1);
}
((StgInd *)p1)->indirectee = p2;
......
......@@ -143,6 +143,15 @@ extern StgIndStatic *debug_caf_list_snapshot;
extern MarkQueue *current_mark_queue;
extern bdescr *upd_rem_set_block_list;
// A similar macro is defined in includes/Cmm.h for C-- code.
#if defined(THREADED_RTS)
#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
#else
#define IF_NONMOVING_WRITE_BARRIER_ENABLED \
if (0)
#endif
void nonmovingMarkInitUpdRemSet(void);
void init_upd_rem_set(UpdRemSet *rset);
......
......@@ -478,7 +478,7 @@ lockCAF (StgRegTable *reg, StgIndStatic *caf)
// reference should be in SRTs
ASSERT(orig_info_tbl->layout.payload.ptrs == 0);
// Becuase the payload is empty we just push the SRT
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
StgThunkInfoTable *thunk_info = itbl_to_thunk_itbl(orig_info_tbl);
if (thunk_info->i.srt) {
updateRemembSetPushClosure(cap, GET_SRT(thunk_info));
......@@ -1205,7 +1205,7 @@ dirty_MUT_VAR(StgRegTable *reg, StgMutVar *mvar, StgClosure *old)
if (mvar->header.info == &stg_MUT_VAR_CLEAN_info) {
mvar->header.info = &stg_MUT_VAR_DIRTY_info;
recordClosureMutated(cap, (StgClosure *) mvar);
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure_(reg, old);
}
}
......@@ -1227,7 +1227,7 @@ dirty_TVAR(Capability *cap, StgTVar *p,
if (p->header.info == &stg_TVAR_CLEAN_info) {
p->header.info = &stg_TVAR_DIRTY_info;
recordClosureMutated(cap,(StgClosure*)p);
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled != 0)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, old);
}
}
......@@ -1244,8 +1244,9 @@ setTSOLink (Capability *cap, StgTSO *tso, StgTSO *target)
if (tso->dirty == 0) {
tso->dirty = 1;
recordClosureMutated(cap,(StgClosure*)tso);
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure *) tso->_link);
}
}
tso->_link = target;
}
......@@ -1256,8 +1257,9 @@ setTSOPrev (Capability *cap, StgTSO *tso, StgTSO *target)
if (tso->dirty == 0) {
tso->dirty = 1;
recordClosureMutated(cap,(StgClosure*)tso);
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushClosure(cap, (StgClosure *) tso->block_info.prev);
}
}
tso->block_info.prev = target;
}
......@@ -1270,8 +1272,9 @@ dirty_TSO (Capability *cap, StgTSO *tso)
recordClosureMutated(cap,(StgClosure*)tso);
}
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushTSO(cap, tso);
}
}
void
......@@ -1279,8 +1282,9 @@ dirty_STACK (Capability *cap, StgStack *stack)
{
// First push to upd_rem_set before we set stack->dirty since we
// the nonmoving collector may already be marking the stack.
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled))
IF_NONMOVING_WRITE_BARRIER_ENABLED {
updateRemembSetPushStack(cap, stack);
}
if (! (stack->dirty & STACK_DIRTY)) {
stack->dirty = STACK_DIRTY;
......@@ -1304,7 +1308,7 @@ void
update_MVAR(StgRegTable *reg, StgClosure *p, StgClosure *old_val)
{
Capability *cap = regTableToCapability(reg);
if (RTS_UNLIKELY(nonmoving_write_barrier_enabled)) {
IF_NONMOVING_WRITE_BARRIER_ENABLED {
StgMVar *mvar = (StgMVar *) p;
updateRemembSetPushClosure(cap, old_val);
updateRemembSetPushClosure(cap, (StgClosure *) mvar->head);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment