Commit 7ff503fd authored by Ben Gamari's avatar Ben Gamari 🐢

Nonmoving: Allow aging and refactor static objects logic

This commit does two things:

 * Allow aging of objects during the preparatory minor GC
 * Refactor handling of static objects to avoid the use of a hashtable
parent 431e9bf4
...@@ -69,12 +69,6 @@ alloc_for_copy (uint32_t size, uint32_t gen_no) ...@@ -69,12 +69,6 @@ alloc_for_copy (uint32_t size, uint32_t gen_no)
{ {
ASSERT(gen_no < RtsFlags.GcFlags.generations); ASSERT(gen_no < RtsFlags.GcFlags.generations);
if (RtsFlags.GcFlags.useNonmoving && major_gc) {
// unconditionally promote to non-moving heap in major gc
gct->copied += size;
return nonmovingAllocate(gct->cap, size);
}
StgPtr to; StgPtr to;
gen_workspace *ws; gen_workspace *ws;
...@@ -93,7 +87,20 @@ alloc_for_copy (uint32_t size, uint32_t gen_no) ...@@ -93,7 +87,20 @@ alloc_for_copy (uint32_t size, uint32_t gen_no)
if (RtsFlags.GcFlags.useNonmoving && gen_no == oldest_gen->no) { if (RtsFlags.GcFlags.useNonmoving && gen_no == oldest_gen->no) {
gct->copied += size; gct->copied += size;
return nonmovingAllocate(gct->cap, size); to = nonmovingAllocate(gct->cap, size);
// Add segment to the todo list unless it's already there
// current->todo_link == NULL means not in todo list
struct NonmovingSegment *seg = nonmovingGetSegment(to);
if (!seg->todo_link) {
gen_workspace *ws = &gct->gens[oldest_gen->no];
seg->todo_link = ws->todo_seg;
ws->todo_seg = seg;
}
if (major_gc)
markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, (StgClosure *) to);
return to;
} }
ws = &gct->gens[gen_no]; // zero memory references here ws = &gct->gens[gen_no]; // zero memory references here
...@@ -312,9 +319,7 @@ evacuate_large(StgPtr p) ...@@ -312,9 +319,7 @@ evacuate_large(StgPtr p)
*/ */
new_gen_no = bd->dest_no; new_gen_no = bd->dest_no;
if (RtsFlags.GcFlags.useNonmoving && major_gc) { if (new_gen_no < gct->evac_gen_no) {
new_gen_no = oldest_gen->no;
} else if (new_gen_no < gct->evac_gen_no) {
if (gct->eager_promotion) { if (gct->eager_promotion) {
new_gen_no = gct->evac_gen_no; new_gen_no = gct->evac_gen_no;
} else { } else {
...@@ -363,6 +368,13 @@ evacuate_large(StgPtr p) ...@@ -363,6 +368,13 @@ evacuate_large(StgPtr p)
STATIC_INLINE void STATIC_INLINE void
evacuate_static_object (StgClosure **link_field, StgClosure *q) evacuate_static_object (StgClosure **link_field, StgClosure *q)
{ {
if (RTS_UNLIKELY(RtsFlags.GcFlags.useNonmoving)) {
// See Note [Static objects under the nonmoving collector] in Storage.c.
if (major_gc)
markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, q);
return;
}
StgWord link = (StgWord)*link_field; StgWord link = (StgWord)*link_field;
// See Note [STATIC_LINK fields] for how the link field bits work // See Note [STATIC_LINK fields] for how the link field bits work
...@@ -603,6 +615,8 @@ loop: ...@@ -603,6 +615,8 @@ loop:
// NOTE: large objects in nonmoving heap are also marked with // NOTE: large objects in nonmoving heap are also marked with
// BF_NONMOVING. Those are moved to scavenged_large_objects list in // BF_NONMOVING. Those are moved to scavenged_large_objects list in
// mark phase. // mark phase.
if (major_gc)
markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, q);
return; return;
} }
...@@ -629,6 +643,13 @@ loop: ...@@ -629,6 +643,13 @@ loop:
// they are not) // they are not)
if (bd->flags & BF_COMPACT) { if (bd->flags & BF_COMPACT) {
evacuate_compact((P_)q); evacuate_compact((P_)q);
// We may have evacuated the block to the nonmoving generation. If so
// we need to make sure it is added to the mark queue since the only
// reference to it may be from the moving heap.
if (major_gc && bd->flags & BF_NONMOVING) {
markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, q);
}
return; return;
} }
...@@ -636,6 +657,13 @@ loop: ...@@ -636,6 +657,13 @@ loop:
*/ */
if (bd->flags & BF_LARGE) { if (bd->flags & BF_LARGE) {
evacuate_large((P_)q); evacuate_large((P_)q);
// We may have evacuated the block to the nonmoving generation. If so
// we need to make sure it is added to the mark queue since the only
// reference to it may be from the moving heap.
if (major_gc && bd->flags & BF_NONMOVING) {
markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, q);
}
return; return;
} }
...@@ -937,6 +965,8 @@ evacuate_BLACKHOLE(StgClosure **p) ...@@ -937,6 +965,8 @@ evacuate_BLACKHOLE(StgClosure **p)
ASSERT((bd->flags & BF_COMPACT) == 0); ASSERT((bd->flags & BF_COMPACT) == 0);
if (bd->flags & BF_NONMOVING) { if (bd->flags & BF_NONMOVING) {
if (major_gc)
markQueuePushClosureGC(&gct->cap->upd_rem_set.queue, q);
return; return;
} }
......
...@@ -263,7 +263,10 @@ GarbageCollect (uint32_t collect_gen, ...@@ -263,7 +263,10 @@ GarbageCollect (uint32_t collect_gen,
N = collect_gen; N = collect_gen;
major_gc = (N == RtsFlags.GcFlags.generations-1); major_gc = (N == RtsFlags.GcFlags.generations-1);
if (major_gc) { /* N.B. The nonmoving collector works a bit differently. See
* Note [Static objects under the nonmoving collector].
*/
if (major_gc && !RtsFlags.GcFlags.useNonmoving) {
prev_static_flag = static_flag; prev_static_flag = static_flag;
static_flag = static_flag =
static_flag == STATIC_FLAG_A ? STATIC_FLAG_B : STATIC_FLAG_A; static_flag == STATIC_FLAG_A ? STATIC_FLAG_B : STATIC_FLAG_A;
...@@ -736,6 +739,11 @@ GarbageCollect (uint32_t collect_gen, ...@@ -736,6 +739,11 @@ GarbageCollect (uint32_t collect_gen,
// so we need to mark those too. // so we need to mark those too.
// Note that in sequential case these lists will be appended with more // Note that in sequential case these lists will be appended with more
// weaks and threads found to be dead in mark. // weaks and threads found to be dead in mark.
#if !defined(THREADED_RTS)
// In the non-threaded runtime this is the only time we push to the
// upd_rem_set
nonmovingAddUpdRemSetBlocks(&gct->cap->upd_rem_set.queue);
#endif
nonmovingCollect(&dead_weak_ptr_list, &resurrected_threads); nonmovingCollect(&dead_weak_ptr_list, &resurrected_threads);
ACQUIRE_SM_LOCK; ACQUIRE_SM_LOCK;
} }
......
...@@ -148,14 +148,14 @@ markCAFs (evac_fn evac, void *user) ...@@ -148,14 +148,14 @@ markCAFs (evac_fn evac, void *user)
StgIndStatic *c; StgIndStatic *c;
for (c = dyn_caf_list; for (c = dyn_caf_list;
c != (StgIndStatic*)END_OF_CAF_LIST; ((StgWord) c | STATIC_FLAG_LIST) != (StgWord)END_OF_CAF_LIST;
c = (StgIndStatic *)c->static_link) c = (StgIndStatic *)c->static_link)
{ {
c = (StgIndStatic *)UNTAG_STATIC_LIST_PTR(c); c = (StgIndStatic *)UNTAG_STATIC_LIST_PTR(c);
evac(user, &c->indirectee); evac(user, &c->indirectee);
} }
for (c = revertible_caf_list; for (c = revertible_caf_list;
c != (StgIndStatic*)END_OF_CAF_LIST; ((StgWord) c | STATIC_FLAG_LIST) != (StgWord)END_OF_CAF_LIST;
c = (StgIndStatic *)c->static_link) c = (StgIndStatic *)c->static_link)
{ {
c = (StgIndStatic *)UNTAG_STATIC_LIST_PTR(c); c = (StgIndStatic *)UNTAG_STATIC_LIST_PTR(c);
......
...@@ -68,6 +68,27 @@ Mutex concurrent_coll_finished_lock; ...@@ -68,6 +68,27 @@ Mutex concurrent_coll_finished_lock;
* stopAllCapabilitiesWith(SYNC_FLUSH_UPD_REM_SET). Capabilities are held * stopAllCapabilitiesWith(SYNC_FLUSH_UPD_REM_SET). Capabilities are held
* the final mark has concluded. * the final mark has concluded.
* *
* Note that possibility of concurrent minor and non-moving collections
* requires that we handle static objects a bit specially. See
* Note [Static objects under the nonmoving collector] in Storage.c
* for details.
*
*
* Note [Aging under the non-moving collector]
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*
* The initial design of the non-moving collector mandated that all live data
* be evacuated to the non-moving heap prior to a major collection. This
* simplified certain bits of implementation and eased reasoning. However, it
* was (unsurprisingly) also found to result in significant amounts of
* unnecessary copying.
*
* Consequently, we now allow aging. We do this by teaching the moving
* collector to "evacuate" objects it encounters in the non-moving heap by
* adding them to the mark queue. Specifically, since each gc_thread holds a
* capability we push to the capability's update remembered set (implemented
* by markQueuePushClosureGC)
*
* *
* Note [Live data accounting in nonmoving collector] * Note [Live data accounting in nonmoving collector]
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
...@@ -150,6 +171,8 @@ static struct NonmovingSegment *nonmovingPopFreeSegment(void) ...@@ -150,6 +171,8 @@ static struct NonmovingSegment *nonmovingPopFreeSegment(void)
* Request a fresh segment from the free segment list or allocate one of the * Request a fresh segment from the free segment list or allocate one of the
* given node. * given node.
* *
* Caller must hold SM_MUTEX (although we take the gc_alloc_block_sync spinlock
* under the assumption that we are in a GC context).
*/ */
static struct NonmovingSegment *nonmovingAllocSegment(uint32_t node) static struct NonmovingSegment *nonmovingAllocSegment(uint32_t node)
{ {
...@@ -221,7 +244,7 @@ static struct NonmovingSegment *pop_active_segment(struct NonmovingAllocator *al ...@@ -221,7 +244,7 @@ static struct NonmovingSegment *pop_active_segment(struct NonmovingAllocator *al
} }
} }
/* sz is in words */ /* Allocate a block in the nonmoving heap. Caller must hold SM_MUTEX. sz is in words */
GNUC_ATTR_HOT GNUC_ATTR_HOT
void *nonmovingAllocate(Capability *cap, StgWord sz) void *nonmovingAllocate(Capability *cap, StgWord sz)
{ {
...@@ -239,14 +262,6 @@ void *nonmovingAllocate(Capability *cap, StgWord sz) ...@@ -239,14 +262,6 @@ void *nonmovingAllocate(Capability *cap, StgWord sz)
void *ret = nonmovingSegmentGetBlock(current, current->next_free); void *ret = nonmovingSegmentGetBlock(current, current->next_free);
ASSERT(GET_CLOSURE_TAG(ret) == 0); // check alignment ASSERT(GET_CLOSURE_TAG(ret) == 0); // check alignment
// Add segment to the todo list unless it's already there
// current->todo_link == NULL means not in todo list
if (!current->todo_link) {
gen_workspace *ws = &gct->gens[oldest_gen->no];
current->todo_link = ws->todo_seg;
ws->todo_seg = current;
}
// Advance the current segment's next_free or allocate a new segment if full // Advance the current segment's next_free or allocate a new segment if full
bool full = advance_next_free(current); bool full = advance_next_free(current);
if (full) { if (full) {
...@@ -405,6 +420,11 @@ static void nonmovingClearAllBitmaps(void) ...@@ -405,6 +420,11 @@ static void nonmovingClearAllBitmaps(void)
/* Prepare the heap bitmaps and snapshot metadata for a mark */ /* Prepare the heap bitmaps and snapshot metadata for a mark */
static void nonmovingPrepareMark(void) static void nonmovingPrepareMark(void)
{ {
// See Note [Static objects under the nonmoving collector].
prev_static_flag = static_flag;
static_flag =
static_flag == STATIC_FLAG_A ? STATIC_FLAG_B : STATIC_FLAG_A;
nonmovingClearAllBitmaps(); nonmovingClearAllBitmaps();
nonmovingBumpEpoch(); nonmovingBumpEpoch();
for (int alloca_idx = 0; alloca_idx < NONMOVING_ALLOCA_CNT; ++alloca_idx) { for (int alloca_idx = 0; alloca_idx < NONMOVING_ALLOCA_CNT; ++alloca_idx) {
...@@ -688,7 +708,7 @@ static void nonmovingMark_(MarkQueue *mark_queue, StgWeak **dead_weaks, StgTSO * ...@@ -688,7 +708,7 @@ static void nonmovingMark_(MarkQueue *mark_queue, StgWeak **dead_weaks, StgTSO *
#if defined(DEBUG) #if defined(DEBUG)
// Zap CAFs that we will sweep // Zap CAFs that we will sweep
nonmovingGcCafs(mark_queue); nonmovingGcCafs();
#endif #endif
ASSERT(mark_queue->top->head == 0); ASSERT(mark_queue->top->head == 0);
......
...@@ -205,7 +205,7 @@ static void init_mark_queue_(MarkQueue *queue); ...@@ -205,7 +205,7 @@ static void init_mark_queue_(MarkQueue *queue);
* Really the argument type should be UpdRemSet* but this would be rather * Really the argument type should be UpdRemSet* but this would be rather
* inconvenient without polymorphism. * inconvenient without polymorphism.
*/ */
static void nonmovingAddUpdRemSetBlocks(MarkQueue *rset) void nonmovingAddUpdRemSetBlocks(MarkQueue *rset)
{ {
if (markQueueIsEmpty(rset)) return; if (markQueueIsEmpty(rset)) return;
...@@ -374,6 +374,38 @@ push (MarkQueue *q, const MarkQueueEnt *ent) ...@@ -374,6 +374,38 @@ push (MarkQueue *q, const MarkQueueEnt *ent)
q->top->head++; q->top->head++;
} }
/* A variant of push to be used by the minor GC when it encounters a reference
* to an object in the non-moving heap. In contrast to the other push
* operations this uses the gc_alloc_block_sync spinlock instead of the
* SM_LOCK to allocate new blocks in the event that the mark queue is full.
*/
void
markQueuePushClosureGC (MarkQueue *q, StgClosure *p)
{
// Are we at the end of the block?
if (q->top->head == MARK_QUEUE_BLOCK_ENTRIES) {
// Yes, this block is full.
// allocate a fresh block.
ACQUIRE_SPIN_LOCK(&gc_alloc_block_sync);
bdescr *bd = allocGroup(1);
bd->link = q->blocks;
q->blocks = bd;
q->top = (MarkQueueBlock *) bd->start;
q->top->head = 0;
RELEASE_SPIN_LOCK(&gc_alloc_block_sync);
}
MarkQueueEnt ent = {
.type = MARK_CLOSURE,
.mark_closure = {
.p = UNTAG_CLOSURE(p),
.origin = NULL,
}
};
q->top->entries[q->top->head] = ent;
q->top->head++;
}
static inline static inline
void push_closure (MarkQueue *q, void push_closure (MarkQueue *q,
StgClosure *p, StgClosure *p,
...@@ -715,7 +747,6 @@ static void init_mark_queue_ (MarkQueue *queue) ...@@ -715,7 +747,6 @@ static void init_mark_queue_ (MarkQueue *queue)
void initMarkQueue (MarkQueue *queue) void initMarkQueue (MarkQueue *queue)
{ {
init_mark_queue_(queue); init_mark_queue_(queue);
queue->marked_objects = allocHashTable();
queue->is_upd_rem_set = false; queue->is_upd_rem_set = false;
} }
...@@ -723,8 +754,6 @@ void initMarkQueue (MarkQueue *queue) ...@@ -723,8 +754,6 @@ void initMarkQueue (MarkQueue *queue)
void init_upd_rem_set (UpdRemSet *rset) void init_upd_rem_set (UpdRemSet *rset)
{ {
init_mark_queue_(&rset->queue); init_mark_queue_(&rset->queue);
// Update remembered sets don't have to worry about static objects
rset->queue.marked_objects = NULL;
rset->queue.is_upd_rem_set = true; rset->queue.is_upd_rem_set = true;
} }
...@@ -739,7 +768,6 @@ void reset_upd_rem_set (UpdRemSet *rset) ...@@ -739,7 +768,6 @@ void reset_upd_rem_set (UpdRemSet *rset)
void freeMarkQueue (MarkQueue *queue) void freeMarkQueue (MarkQueue *queue)
{ {
freeChain_lock(queue->blocks); freeChain_lock(queue->blocks);
freeHashTable(queue->marked_objects, NULL);
} }
#if defined(THREADED_RTS) && defined(DEBUG) #if defined(THREADED_RTS) && defined(DEBUG)
...@@ -986,12 +1014,32 @@ mark_stack (MarkQueue *queue, StgStack *stack) ...@@ -986,12 +1014,32 @@ mark_stack (MarkQueue *queue, StgStack *stack)
mark_stack_(queue, stack->sp, stack->stack + stack->stack_size); mark_stack_(queue, stack->sp, stack->stack + stack->stack_size);
} }
/* See Note [Static objects under the nonmoving collector].
*
* Returns true if the object needs to be marked.
*/
static bool
bump_static_flag(StgClosure **link_field, StgClosure *q STG_UNUSED)
{
while (1) {
StgWord link = (StgWord) *link_field;
StgWord new = (link & ~STATIC_BITS) | static_flag;
if ((link & STATIC_BITS) == static_flag)
return false;
else if (cas((StgVolatilePtr) link_field, link, new) == link) {
return true;
}
}
}
static GNUC_ATTR_HOT void static GNUC_ATTR_HOT void
mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
{ {
(void)origin; // TODO: should be used for selector/thunk optimisations (void)origin; // TODO: should be used for selector/thunk optimisations
try_again: try_again:
;
bdescr *bd = NULL;
p = UNTAG_CLOSURE(p); p = UNTAG_CLOSURE(p);
# define PUSH_FIELD(obj, field) \ # define PUSH_FIELD(obj, field) \
...@@ -1009,45 +1057,46 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1009,45 +1057,46 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
return; return;
} }
if (lookupHashTable(queue->marked_objects, (W_)p)) {
// already marked
return;
}
insertHashTable(queue->marked_objects, (W_)p, (P_)1);
switch (type) { switch (type) {
case THUNK_STATIC: case THUNK_STATIC:
if (info->srt != 0) { if (info->srt != 0) {
markQueuePushThunkSrt(queue, info); // TODO this function repeats the check above if (bump_static_flag(THUNK_STATIC_LINK((StgClosure *)p), p)) {
markQueuePushThunkSrt(queue, info); // TODO this function repeats the check above
}
} }
return; return;
case FUN_STATIC: case FUN_STATIC:
if (info->srt != 0 || info->layout.payload.ptrs != 0) { if (info->srt != 0 || info->layout.payload.ptrs != 0) {
markQueuePushFunSrt(queue, info); // TODO this function repeats the check above if (bump_static_flag(STATIC_LINK(info, (StgClosure *)p), p)) {
markQueuePushFunSrt(queue, info); // TODO this function repeats the check above
// a FUN_STATIC can also be an SRT, so it may have pointer
// fields. See Note [SRTs] in CmmBuildInfoTables, specifically // a FUN_STATIC can also be an SRT, so it may have pointer
// the [FUN] optimisation. // fields. See Note [SRTs] in CmmBuildInfoTables, specifically
// TODO (osa) I don't understand this comment // the [FUN] optimisation.
for (StgHalfWord i = 0; i < info->layout.payload.ptrs; ++i) { // TODO (osa) I don't understand this comment
PUSH_FIELD(p, payload[i]); for (StgHalfWord i = 0; i < info->layout.payload.ptrs; ++i) {
PUSH_FIELD(p, payload[i]);
}
} }
} }
return; return;
case IND_STATIC: case IND_STATIC:
PUSH_FIELD((StgInd *) p, indirectee); if (bump_static_flag(IND_STATIC_LINK((StgClosure *)p), p)) {
PUSH_FIELD((StgInd *) p, indirectee);
}
return; return;
case CONSTR: case CONSTR:
case CONSTR_1_0: case CONSTR_1_0:
case CONSTR_2_0: case CONSTR_2_0:
case CONSTR_1_1: case CONSTR_1_1:
for (StgHalfWord i = 0; i < info->layout.payload.ptrs; ++i) { if (bump_static_flag(STATIC_LINK(info, (StgClosure *)p), p)) {
PUSH_FIELD(p, payload[i]); for (StgHalfWord i = 0; i < info->layout.payload.ptrs; ++i) {
PUSH_FIELD(p, payload[i]);
}
} }
return; return;
...@@ -1061,19 +1110,17 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1061,19 +1110,17 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
} }
} }
bdescr *bd = Bdescr((StgPtr) p); bd = Bdescr((StgPtr) p);
if (bd->gen != oldest_gen) { if (bd->gen != oldest_gen) {
// Here we have an object living outside of the non-moving heap. Since // Here we have an object living outside of the non-moving heap. While
// we moved everything to the non-moving heap before starting the major // we likely evacuated nearly everything to the nonmoving heap during
// collection, we know that we don't need to trace it: it was allocated // preparation there are nevertheless a few ways in which we might trace
// after we took our snapshot. // a reference into younger generations:
#if !defined(THREADED_RTS) //
// This should never happen in the non-concurrent case // * a mutable object might have been updated
barf("Closure outside of non-moving heap: %p", p); // * we might have aged an object
#else
return; return;
#endif
} }
ASSERTM(LOOKS_LIKE_CLOSURE_PTR(p), "invalid closure, info=%p", p->header.info); ASSERTM(LOOKS_LIKE_CLOSURE_PTR(p), "invalid closure, info=%p", p->header.info);
......
...@@ -82,10 +82,6 @@ typedef struct MarkQueue_ { ...@@ -82,10 +82,6 @@ typedef struct MarkQueue_ {
// Is this a mark queue or a capability-local update remembered set? // Is this a mark queue or a capability-local update remembered set?
bool is_upd_rem_set; bool is_upd_rem_set;
// Marked objects outside of nonmoving heap, namely large and static
// objects.
HashTable *marked_objects;
} MarkQueue; } MarkQueue;
/* While it shares its representation with MarkQueue, UpdRemSet differs in /* While it shares its representation with MarkQueue, UpdRemSet differs in
...@@ -143,8 +139,10 @@ void nonmovingResurrectThreads(struct MarkQueue_ *queue, StgTSO **resurrected_th ...@@ -143,8 +139,10 @@ void nonmovingResurrectThreads(struct MarkQueue_ *queue, StgTSO **resurrected_th
bool nonmovingIsAlive(StgClosure *p); bool nonmovingIsAlive(StgClosure *p);
void nonmovingMarkDeadWeak(struct MarkQueue_ *queue, StgWeak *w); void nonmovingMarkDeadWeak(struct MarkQueue_ *queue, StgWeak *w);
void nonmovingMarkLiveWeak(struct MarkQueue_ *queue, StgWeak *w); void nonmovingMarkLiveWeak(struct MarkQueue_ *queue, StgWeak *w);
void nonmovingAddUpdRemSetBlocks(struct MarkQueue_ *rset);
void markQueuePush(MarkQueue *q, const MarkQueueEnt *ent); void markQueuePush(MarkQueue *q, const MarkQueueEnt *ent);
void markQueuePushClosureGC(MarkQueue *q, StgClosure *p);
void markQueuePushClosure(MarkQueue *q, void markQueuePushClosure(MarkQueue *q,
StgClosure *p, StgClosure *p,
StgClosure **origin); StgClosure **origin);
......
...@@ -16,6 +16,7 @@ nonmovingScavengeOne (StgClosure *q) ...@@ -16,6 +16,7 @@ nonmovingScavengeOne (StgClosure *q)
ASSERT(LOOKS_LIKE_CLOSURE_PTR(q)); ASSERT(LOOKS_LIKE_CLOSURE_PTR(q));
StgPtr p = (StgPtr)q; StgPtr p = (StgPtr)q;
const StgInfoTable *info = get_itbl(q); const StgInfoTable *info = get_itbl(q);
const bool saved_eager_promotion = gct->eager_promotion;
switch (info->type) { switch (info->type) {
...@@ -23,9 +24,11 @@ nonmovingScavengeOne (StgClosure *q) ...@@ -23,9 +24,11 @@ nonmovingScavengeOne (StgClosure *q)
case MVAR_DIRTY: case MVAR_DIRTY:
{ {
StgMVar *mvar = ((StgMVar *)p); StgMVar *mvar = ((StgMVar *)p);
gct->eager_promotion = false;
evacuate((StgClosure **)&mvar->head); evacuate((StgClosure **)&mvar->head);
evacuate((StgClosure **)&mvar->tail); evacuate((StgClosure **)&mvar->tail);
evacuate((StgClosure **)&mvar->value); evacuate((StgClosure **)&mvar->value);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) { if (gct->failed_to_evac) {
mvar->header.info = &stg_MVAR_DIRTY_info; mvar->header.info = &stg_MVAR_DIRTY_info;
} else { } else {
...@@ -37,8 +40,10 @@ nonmovingScavengeOne (StgClosure *q) ...@@ -37,8 +40,10 @@ nonmovingScavengeOne (StgClosure *q)
case TVAR: case TVAR:
{ {
StgTVar *tvar = ((StgTVar *)p); StgTVar *tvar = ((StgTVar *)p);
gct->eager_promotion = false;
evacuate((StgClosure **)&tvar->current_value); evacuate((StgClosure **)&tvar->current_value);
evacuate((StgClosure **)&tvar->first_watch_queue_entry); evacuate((StgClosure **)&tvar->first_watch_queue_entry);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) { if (gct->failed_to_evac) {
tvar->header.info = &stg_TVAR_DIRTY_info; tvar->header.info = &stg_TVAR_DIRTY_info;
} else { } else {
...@@ -122,10 +127,21 @@ nonmovingScavengeOne (StgClosure *q) ...@@ -122,10 +127,21 @@ nonmovingScavengeOne (StgClosure *q)
break; break;
} }
case WEAK:
{
// We must evacuate the key since it may refer to an object in the
// moving heap which may be long gone by the time we call
// nonmovingTidyWeaks.
StgWeak *weak = (StgWeak *) p;
gct->eager_promotion = true;
evacuate(&weak->key);
gct->eager_promotion = saved_eager_promotion;
goto gen_obj;
}
gen_obj: gen_obj:
case CONSTR: case CONSTR:
case CONSTR_NOCAF: case CONSTR_NOCAF:
case WEAK:
case PRIM: case PRIM:
{ {
StgPtr end = (P_)((StgClosure *)p)->payload + info->layout.payload.ptrs; StgPtr end = (P_)((StgClosure *)p)->payload + info->layout.payload.ptrs;
...@@ -145,7 +161,9 @@ nonmovingScavengeOne (StgClosure *q) ...@@ -145,7 +161,9 @@ nonmovingScavengeOne (StgClosure *q)
case MUT_VAR_CLEAN: case MUT_VAR_CLEAN:
case MUT_VAR_DIRTY: case MUT_VAR_DIRTY:
gct->eager_promotion = false;
evacuate(&((StgMutVar *)p)->var); evacuate(&((StgMutVar *)p)->var);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) { if (gct->failed_to_evac) {
((StgClosure *)q)->header.info = &stg_MUT_VAR_DIRTY_info; ((StgClosure *)q)->header.info = &stg_MUT_VAR_DIRTY_info;
} else { } else {
...@@ -157,10 +175,12 @@ nonmovingScavengeOne (StgClosure *q) ...@@ -157,10 +175,12 @@ nonmovingScavengeOne (StgClosure *q)
{ {
StgBlockingQueue *bq = (StgBlockingQueue *)p; StgBlockingQueue *bq = (StgBlockingQueue *)p;
gct->eager_promotion = false;
evacuate(&bq->bh); evacuate(&bq->bh);
evacuate((StgClosure**)&bq->owner); evacuate((StgClosure**)&bq->owner);
evacuate((StgClosure**)&bq->queue); evacuate((StgClosure**)&bq->queue);
evacuate((StgClosure**)&bq->link); evacuate((StgClosure**)&bq->link);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) { if (gct->failed_to_evac) {
bq->header.info = &stg_BLOCKING_QUEUE_DIRTY_info; bq->header.info = &stg_BLOCKING_QUEUE_DIRTY_info;
...@@ -202,11 +222,9 @@ nonmovingScavengeOne (StgClosure *q) ...@@ -202,11 +222,9 @@ nonmovingScavengeOne (StgClosure *q)
case MUT_ARR_PTRS_CLEAN: case MUT_ARR_PTRS_CLEAN:
case MUT_ARR_PTRS_DIRTY: case MUT_ARR_PTRS_DIRTY:
{ {
// We don't eagerly promote objects pointed to by a mutable gct->eager_promotion = false;
// array, but if we find the array only points to objects in
// the same or an older generation, we mark it "clean" and
// avoid traversing it during minor GCs.
scavenge_mut_arr_ptrs((StgMutArrPtrs*)p); scavenge_mut_arr_ptrs((StgMutArrPtrs*)p);
gct->eager_promotion = saved_eager_promotion;
if (gct->failed_to_evac) { if (gct->failed_to_evac) {
((StgClosure *)q)->header.info = &stg_MUT_ARR_PTRS_DIRTY_info; ((StgClosure *)q)->header.info = &stg_MUT_ARR_PTRS_DIRTY_info;