Commit cafa9963 authored by Ömer Sinan Ağacan's avatar Ömer Sinan Ağacan Committed by Ben Gamari

NonMoving: Implement indirection shortcutting

This allows indirection chains residing in the non-moving heap to be
shorted-out.
parent 890b23de
...@@ -482,7 +482,7 @@ void push_closure (MarkQueue *q, ...@@ -482,7 +482,7 @@ void push_closure (MarkQueue *q,
MarkQueueEnt ent = { MarkQueueEnt ent = {
.mark_closure = { .mark_closure = {
.p = UNTAG_CLOSURE(p), .p = p,
.origin = origin, .origin = origin,
} }
}; };
...@@ -1133,13 +1133,15 @@ bump_static_flag(StgClosure **link_field, StgClosure *q STG_UNUSED) ...@@ -1133,13 +1133,15 @@ bump_static_flag(StgClosure **link_field, StgClosure *q STG_UNUSED)
} }
static GNUC_ATTR_HOT void static GNUC_ATTR_HOT void
mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) mark_closure (MarkQueue *queue, const StgClosure *p0, StgClosure **origin)
{ {
(void)origin; // TODO: should be used for selector/thunk optimisations StgClosure *p = (StgClosure*)p0;
try_again: try_again:
; ;
bdescr *bd = NULL; bdescr *bd = NULL;
StgClosure *p_next = NULL;
StgWord tag = GET_CLOSURE_TAG(p);
p = UNTAG_CLOSURE(p); p = UNTAG_CLOSURE(p);
# define PUSH_FIELD(obj, field) \ # define PUSH_FIELD(obj, field) \
...@@ -1165,7 +1167,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1165,7 +1167,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
markQueuePushThunkSrt(queue, info); // TODO this function repeats the check above markQueuePushThunkSrt(queue, info); // TODO this function repeats the check above
} }
} }
return; goto done;
case FUN_STATIC: case FUN_STATIC:
if (info->srt != 0 || info->layout.payload.ptrs != 0) { if (info->srt != 0 || info->layout.payload.ptrs != 0) {
...@@ -1181,13 +1183,13 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1181,13 +1183,13 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
} }
} }
} }
return; goto done;
case IND_STATIC: case IND_STATIC:
if (bump_static_flag(IND_STATIC_LINK((StgClosure *)p), p)) { if (bump_static_flag(IND_STATIC_LINK((StgClosure *)p), p)) {
PUSH_FIELD((StgInd *) p, indirectee); PUSH_FIELD((StgInd *) p, indirectee);
} }
return; goto done;
case CONSTR: case CONSTR:
case CONSTR_1_0: case CONSTR_1_0:
...@@ -1198,7 +1200,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1198,7 +1200,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
PUSH_FIELD(p, payload[i]); PUSH_FIELD(p, payload[i]);
} }
} }
return; goto done;
case WHITEHOLE: case WHITEHOLE:
while (get_volatile_itbl(p)->type == WHITEHOLE); while (get_volatile_itbl(p)->type == WHITEHOLE);
...@@ -1220,7 +1222,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1220,7 +1222,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
// //
// * a mutable object might have been updated // * a mutable object might have been updated
// * we might have aged an object // * we might have aged an object
return; goto done;
} }
ASSERTM(LOOKS_LIKE_CLOSURE_PTR(p), "invalid closure, info=%p", p->header.info); ASSERTM(LOOKS_LIKE_CLOSURE_PTR(p), "invalid closure, info=%p", p->header.info);
...@@ -1232,10 +1234,10 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1232,10 +1234,10 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
if (bd->flags & BF_LARGE) { if (bd->flags & BF_LARGE) {
if (! (bd->flags & BF_NONMOVING_SWEEPING)) { if (! (bd->flags & BF_NONMOVING_SWEEPING)) {
// Not in the snapshot // Not in the snapshot
return; goto done;
} }
if (bd->flags & BF_MARKED) { if (bd->flags & BF_MARKED) {
return; goto done;
} }
// Mark contents // Mark contents
...@@ -1251,7 +1253,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1251,7 +1253,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
uint8_t mark = nonmovingGetMark(seg, block_idx); uint8_t mark = nonmovingGetMark(seg, block_idx);
/* Don't mark things we've already marked (since we may loop) */ /* Don't mark things we've already marked (since we may loop) */
if (mark == nonmovingMarkEpoch) if (mark == nonmovingMarkEpoch)
return; goto done;
StgClosure *snapshot_loc = StgClosure *snapshot_loc =
(StgClosure *) nonmovingSegmentGetBlock(seg, seg->next_free_snap); (StgClosure *) nonmovingSegmentGetBlock(seg, seg->next_free_snap);
...@@ -1262,7 +1264,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1262,7 +1264,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
* things above the allocation pointer that aren't marked since * things above the allocation pointer that aren't marked since
* they may not be valid objects. * they may not be valid objects.
*/ */
return; goto done;
} }
} }
} }
...@@ -1280,7 +1282,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1280,7 +1282,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
} }
ASSERT(found_it); ASSERT(found_it);
#endif #endif
return; return; // we don't update origin here! TODO(osa): explain this
} }
else { else {
...@@ -1412,10 +1414,24 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1412,10 +1414,24 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
} }
case IND: case IND: {
case BLACKHOLE:
PUSH_FIELD((StgInd *) p, indirectee); PUSH_FIELD((StgInd *) p, indirectee);
if (origin != NULL) {
p_next = ((StgInd*)p)->indirectee;
}
break; break;
}
case BLACKHOLE: {
PUSH_FIELD((StgInd *) p, indirectee);
StgClosure *indirectee = ((StgInd*)p)->indirectee;
if (GET_CLOSURE_TAG(indirectee) == 0 || origin == NULL) {
// do nothing
} else {
p_next = indirectee;
}
break;
}
case MUT_VAR_CLEAN: case MUT_VAR_CLEAN:
case MUT_VAR_DIRTY: case MUT_VAR_DIRTY:
...@@ -1502,7 +1518,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1502,7 +1518,7 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
// the stack will be fully marked before we sweep due to the final // the stack will be fully marked before we sweep due to the final
// post-mark synchronization. Most importantly, we do not set its // post-mark synchronization. Most importantly, we do not set its
// mark bit, the mutator is responsible for this. // mark bit, the mutator is responsible for this.
return; goto done;
} }
break; break;
} }
...@@ -1561,13 +1577,27 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin) ...@@ -1561,13 +1577,27 @@ mark_closure (MarkQueue *queue, StgClosure *p, StgClosure **origin)
bd->flags |= BF_MARKED; bd->flags |= BF_MARKED;
} }
RELEASE_LOCK(&nonmoving_large_objects_mutex); RELEASE_LOCK(&nonmoving_large_objects_mutex);
} else { } else if (bd->flags & BF_NONMOVING) {
// TODO: Kill repetition // TODO: Kill repetition
struct NonmovingSegment *seg = nonmovingGetSegment((StgPtr) p); struct NonmovingSegment *seg = nonmovingGetSegment((StgPtr) p);
nonmoving_block_idx block_idx = nonmovingGetBlockIdx((StgPtr) p); nonmoving_block_idx block_idx = nonmovingGetBlockIdx((StgPtr) p);
nonmovingSetMark(seg, block_idx); nonmovingSetMark(seg, block_idx);
nonmoving_live_words += nonmovingSegmentBlockSize(seg) / sizeof(W_); nonmoving_live_words += nonmovingSegmentBlockSize(seg) / sizeof(W_);
} }
// If we found a indirection to shortcut keep going.
if (p_next) {
p = p_next;
goto try_again;
}
done:
if (origin != NULL && (!HEAP_ALLOCED(p) || bd->flags & BF_NONMOVING)) {
if (UNTAG_CLOSURE((StgClosure*)p0) != p && *origin == p0) {
if (cas((StgVolatilePtr)origin, (StgWord)p0, (StgWord)TAG_CLOSURE(tag, p)) == (StgWord)p0) {
}
}
}
} }
/* This is the main mark loop. /* This is the main mark loop.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment