Commit 5270423a authored by Simon Marlow's avatar Simon Marlow
Browse files

Make allocatePinned use local storage, and other refactorings

This is a batch of refactoring to remove some of the GC's global
state, as we move towards CPU-local GC.  

  - allocateLocal() now allocates large objects into the local
    nursery, rather than taking a global lock and allocating
    then in gen 0 step 0.

  - allocatePinned() was still allocating from global storage and
    taking a lock each time, now it uses local storage. 
    (mallocForeignPtrBytes should be faster with -threaded).
    
  - We had a gen 0 step 0, distinct from the nurseries, which are
    stored in a separate nurseries[] array.  This is slightly strange.
    I removed the g0s0 global that pointed to gen 0 step 0, and
    removed all uses of it.  I think now we don't use gen 0 step 0 at
    all, except possibly when there is only one generation.  Possibly
    more tidying up is needed here.

  - I removed the global allocate() function, and renamed
    allocateLocal() to allocate().

  - the alloc_blocks global is gone.  MAYBE_GC() and
    doYouWantToGC() now check the local nursery only.
parent 063b822b
......@@ -380,11 +380,12 @@
HP_CHK_GEN(alloc,liveness,reentry); \
TICK_ALLOC_HEAP_NOCTR(alloc);
// allocateLocal() allocates from the nursery, so we check to see
// allocate() allocates from the nursery, so we check to see
// whether the nursery is nearly empty in any function that uses
// allocateLocal() - this includes many of the primops.
// allocate() - this includes many of the primops.
#define MAYBE_GC(liveness,reentry) \
if (bdescr_link(CurrentNursery) == NULL || CInt[alloc_blocks] >= CInt[alloc_blocks_lim]) { \
if (bdescr_link(CurrentNursery) == NULL || \
step_n_large_blocks(StgRegTable_rNursery(BaseReg)) >= CInt[alloc_blocks_lim]) { \
R9 = liveness; \
R10 = reentry; \
HpAlloc = 0; \
......
......@@ -230,6 +230,7 @@ main(int argc, char *argv[])
field_offset(StgRegTable, rCurrentNursery);
field_offset(StgRegTable, rHpAlloc);
struct_field(StgRegTable, rRet);
struct_field(StgRegTable, rNursery);
def_offset("stgEagerBlackholeInfo", FUN_OFFSET(stgEagerBlackholeInfo));
def_offset("stgGCEnter1", FUN_OFFSET(stgGCEnter1));
......@@ -249,6 +250,8 @@ main(int argc, char *argv[])
struct_size(generation);
struct_field(generation, mut_list);
struct_field(step, n_large_blocks);
struct_size(CostCentreStack);
struct_field(CostCentreStack, ccsID);
struct_field(CostCentreStack, mem_alloc);
......
......@@ -75,10 +75,6 @@ typedef struct step_ {
// ------------------------------------
// Fields below are used during GC only
// During GC, if we are collecting this step, blocks and n_blocks
// are copied into the following two fields. After GC, these blocks
// are freed.
#if defined(THREADED_RTS)
char pad[128]; // make sure the following is
// on a separate cache line.
......@@ -89,6 +85,9 @@ typedef struct step_ {
int mark; // mark (not copy)? (old gen only)
int compact; // compact (not sweep)? (old gen only)
// During GC, if we are collecting this step, blocks and n_blocks
// are copied into the following two fields. After GC, these blocks
// are freed.
bdescr * old_blocks; // bdescr of first from-space block
unsigned int n_old_blocks; // number of blocks in from-space
unsigned int live_estimate; // for sweeping: estimate of live data
......@@ -125,7 +124,6 @@ typedef struct generation_ {
extern generation * generations;
extern generation * g0;
extern step * g0s0;
extern generation * oldest_gen;
extern step * all_steps;
extern nat total_steps;
......@@ -133,21 +131,14 @@ extern nat total_steps;
/* -----------------------------------------------------------------------------
Generic allocation
StgPtr allocateInGen(generation *g, nat n)
Allocates a chunk of contiguous store
n words long in generation g,
returning a pointer to the first word.
Always succeeds.
StgPtr allocate(nat n) Equaivalent to allocateInGen(g0)
StgPtr allocateLocal(Capability *cap, nat n)
StgPtr allocate(Capability *cap, nat n)
Allocates memory from the nursery in
the current Capability. This can be
done without taking a global lock,
unlike allocate().
StgPtr allocatePinned(nat n) Allocates a chunk of contiguous store
StgPtr allocatePinned(Capability *cap, nat n)
Allocates a chunk of contiguous store
n words long, which is at a fixed
address (won't be moved by GC).
Returns a pointer to the first word.
......@@ -163,27 +154,16 @@ extern nat total_steps;
allocatePinned, for the
benefit of the ticky-ticky profiler.
rtsBool doYouWantToGC(void) Returns True if the storage manager is
ready to perform a GC, False otherwise.
lnat allocatedBytes(void) Returns the number of bytes allocated
via allocate() since the last GC.
Used in the reporting of statistics.
-------------------------------------------------------------------------- */
StgPtr allocate ( lnat n );
StgPtr allocateInGen ( generation *g, lnat n );
StgPtr allocateLocal ( Capability *cap, lnat n );
StgPtr allocatePinned ( lnat n );
lnat allocatedBytes ( void );
StgPtr allocate ( Capability *cap, lnat n );
StgPtr allocatePinned ( Capability *cap, lnat n );
/* memory allocator for executable memory */
void * allocateExec(unsigned int len, void **exec_addr);
void freeExec (void *p);
// Used by GC checks in external .cmm code:
extern nat alloc_blocks;
extern nat alloc_blocks_lim;
/* -----------------------------------------------------------------------------
......
......@@ -253,6 +253,7 @@ initCapability( Capability *cap, nat i )
cap->free_trec_headers = NO_TREC;
cap->transaction_tokens = 0;
cap->context_switch = 0;
cap->pinned_object_block = NULL;
}
/* ---------------------------------------------------------------------------
......
......@@ -69,6 +69,9 @@ struct Capability_ {
bdescr **mut_lists;
bdescr **saved_mut_lists; // tmp use during GC
// block for allocating pinned objects into
bdescr *pinned_object_block;
// Context switch flag. We used to have one global flag, now one
// per capability. Locks required : none (conflicts are harmless)
int context_switch;
......
......@@ -89,7 +89,7 @@
STATIC_INLINE StgPtr
allocate_NONUPD (Capability *cap, int n_words)
{
return allocateLocal(cap, stg_max(sizeofW(StgHeader)+MIN_PAYLOAD_SIZE, n_words));
return allocate(cap, stg_max(sizeofW(StgHeader)+MIN_PAYLOAD_SIZE, n_words));
}
int rts_stop_next_breakpoint = 0;
......@@ -604,7 +604,7 @@ do_apply:
else /* arity > n */ {
// build a new PAP and return it.
StgPAP *new_pap;
new_pap = (StgPAP *)allocateLocal(cap, PAP_sizeW(pap->n_args + m));
new_pap = (StgPAP *)allocate(cap, PAP_sizeW(pap->n_args + m));
SET_HDR(new_pap,&stg_PAP_info,CCCS);
new_pap->arity = pap->arity - n;
new_pap->n_args = pap->n_args + m;
......@@ -649,7 +649,7 @@ do_apply:
// build a PAP and return it.
StgPAP *pap;
nat i;
pap = (StgPAP *)allocateLocal(cap, PAP_sizeW(m));
pap = (StgPAP *)allocate(cap, PAP_sizeW(m));
SET_HDR(pap, &stg_PAP_info,CCCS);
pap->arity = arity - n;
pap->fun = obj;
......@@ -718,7 +718,7 @@ do_apply:
run_BCO_return:
// Heap check
if (doYouWantToGC()) {
if (doYouWantToGC(cap)) {
Sp--; Sp[0] = (W_)&stg_enter_info;
RETURN_TO_SCHEDULER(ThreadInterpret, HeapOverflow);
}
......@@ -729,7 +729,7 @@ run_BCO_return:
run_BCO_return_unboxed:
// Heap check
if (doYouWantToGC()) {
if (doYouWantToGC(cap)) {
RETURN_TO_SCHEDULER(ThreadInterpret, HeapOverflow);
}
// Stack checks aren't necessary at return points, the stack use
......@@ -747,7 +747,7 @@ run_BCO_fun:
);
// Heap check
if (doYouWantToGC()) {
if (doYouWantToGC(cap)) {
Sp -= 2;
Sp[1] = (W_)obj;
Sp[0] = (W_)&stg_apply_interp_info; // placeholder, really
......@@ -863,7 +863,7 @@ run_BCO:
// stg_apply_interp_info pointer and a pointer to
// the BCO
size_words = BCO_BITMAP_SIZE(obj) + 2;
new_aps = (StgAP_STACK *) allocateLocal(cap, AP_STACK_sizeW(size_words));
new_aps = (StgAP_STACK *) allocate(cap, AP_STACK_sizeW(size_words));
SET_HDR(new_aps,&stg_AP_STACK_info,CCS_SYSTEM);
new_aps->size = size_words;
new_aps->fun = &stg_dummy_ret_closure;
......@@ -1082,7 +1082,7 @@ run_BCO:
case bci_ALLOC_AP: {
StgAP* ap;
int n_payload = BCO_NEXT;
ap = (StgAP*)allocateLocal(cap, AP_sizeW(n_payload));
ap = (StgAP*)allocate(cap, AP_sizeW(n_payload));
Sp[-1] = (W_)ap;
ap->n_args = n_payload;
SET_HDR(ap, &stg_AP_info, CCS_SYSTEM/*ToDo*/)
......@@ -1093,7 +1093,7 @@ run_BCO:
case bci_ALLOC_AP_NOUPD: {
StgAP* ap;
int n_payload = BCO_NEXT;
ap = (StgAP*)allocateLocal(cap, AP_sizeW(n_payload));
ap = (StgAP*)allocate(cap, AP_sizeW(n_payload));
Sp[-1] = (W_)ap;
ap->n_args = n_payload;
SET_HDR(ap, &stg_AP_NOUPD_info, CCS_SYSTEM/*ToDo*/)
......@@ -1105,7 +1105,7 @@ run_BCO:
StgPAP* pap;
int arity = BCO_NEXT;
int n_payload = BCO_NEXT;
pap = (StgPAP*)allocateLocal(cap, PAP_sizeW(n_payload));
pap = (StgPAP*)allocate(cap, PAP_sizeW(n_payload));
Sp[-1] = (W_)pap;
pap->n_args = n_payload;
pap->arity = arity;
......
......@@ -943,9 +943,8 @@ typedef struct _RtsSymbolVal {
SymI_HasProto(stg_writeTVarzh) \
SymI_HasProto(stg_yieldzh) \
SymI_NeedsProto(stg_interp_constr_entry) \
SymI_HasProto(alloc_blocks) \
SymI_HasProto(alloc_blocks_lim) \
SymI_HasProto(allocateLocal) \
SymI_HasProto(allocate) \
SymI_HasProto(allocateExec) \
SymI_HasProto(freeExec) \
SymI_HasProto(getAllocations) \
......
......@@ -58,7 +58,7 @@ stg_newByteArrayzh
n = R1;
payload_words = ROUNDUP_BYTES_TO_WDS(n);
words = BYTES_TO_WDS(SIZEOF_StgArrWords) + payload_words;
("ptr" p) = foreign "C" allocateLocal(MyCapability() "ptr",words) [];
("ptr" p) = foreign "C" allocate(MyCapability() "ptr",words) [];
TICK_ALLOC_PRIM(SIZEOF_StgArrWords,WDS(payload_words),0);
SET_HDR(p, stg_ARR_WORDS_info, W_[CCCS]);
StgArrWords_words(p) = payload_words;
......@@ -85,7 +85,7 @@ stg_newPinnedByteArrayzh
/* Now we convert to a number of words: */
words = ROUNDUP_BYTES_TO_WDS(bytes);
("ptr" p) = foreign "C" allocatePinned(words) [];
("ptr" p) = foreign "C" allocatePinned(MyCapability() "ptr", words) [];
TICK_ALLOC_PRIM(SIZEOF_StgArrWords,WDS(payload_words),0);
/* Now we need to move p forward so that the payload is aligned
......@@ -117,7 +117,7 @@ stg_newAlignedPinnedByteArrayzh
/* Now we convert to a number of words: */
words = ROUNDUP_BYTES_TO_WDS(bytes);
("ptr" p) = foreign "C" allocatePinned(words) [];
("ptr" p) = foreign "C" allocatePinned(MyCapability() "ptr", words) [];
TICK_ALLOC_PRIM(SIZEOF_StgArrWords,WDS(payload_words),0);
/* Now we need to move p forward so that the payload is aligned
......@@ -139,7 +139,7 @@ stg_newArrayzh
MAYBE_GC(R2_PTR,stg_newArrayzh);
words = BYTES_TO_WDS(SIZEOF_StgMutArrPtrs) + n;
("ptr" arr) = foreign "C" allocateLocal(MyCapability() "ptr",words) [R2];
("ptr" arr) = foreign "C" allocate(MyCapability() "ptr",words) [R2];
TICK_ALLOC_PRIM(SIZEOF_StgMutArrPtrs, WDS(n), 0);
SET_HDR(arr, stg_MUT_ARR_PTRS_DIRTY_info, W_[CCCS]);
......@@ -356,7 +356,7 @@ stg_mkWeakForeignEnvzh
payload_words = 4;
words = BYTES_TO_WDS(SIZEOF_StgArrWords) + payload_words;
("ptr" p) = foreign "C" allocateLocal(MyCapability() "ptr", words) [];
("ptr" p) = foreign "C" allocate(MyCapability() "ptr", words) [];
TICK_ALLOC_PRIM(SIZEOF_StgArrWords,WDS(payload_words),0);
SET_HDR(p, stg_ARR_WORDS_info, W_[CCCS]);
......
......@@ -1086,7 +1086,7 @@ heapCensus( void )
// Traverse the heap, collecting the census info
if (RtsFlags.GcFlags.generations == 1) {
heapCensusChain( census, g0s0->blocks );
heapCensusChain( census, g0->steps[0].blocks );
} else {
for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
for (s = 0; s < generations[g].n_steps; s++) {
......
......@@ -792,7 +792,7 @@ raiseAsync(Capability *cap, StgTSO *tso, StgClosure *exception,
// fun field.
//
words = frame - sp - 1;
ap = (StgAP_STACK *)allocateLocal(cap,AP_STACK_sizeW(words));
ap = (StgAP_STACK *)allocate(cap,AP_STACK_sizeW(words));
ap->size = words;
ap->fun = (StgClosure *)sp[0];
......@@ -856,7 +856,7 @@ raiseAsync(Capability *cap, StgTSO *tso, StgClosure *exception,
// we've got an exception to raise, so let's pass it to the
// handler in this frame.
//
raise = (StgThunk *)allocateLocal(cap,sizeofW(StgThunk)+1);
raise = (StgThunk *)allocate(cap,sizeofW(StgThunk)+1);
TICK_ALLOC_SE_THK(1,0);
SET_HDR(raise,&stg_raise_info,cf->header.prof.ccs);
raise->payload[0] = exception;
......
......@@ -28,7 +28,7 @@
HaskellObj
rts_mkChar (Capability *cap, HsChar c)
{
StgClosure *p = (StgClosure *)allocateLocal(cap, CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap, CONSTR_sizeW(0,1));
SET_HDR(p, Czh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgWord)(StgChar)c;
return p;
......@@ -37,7 +37,7 @@ rts_mkChar (Capability *cap, HsChar c)
HaskellObj
rts_mkInt (Capability *cap, HsInt i)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, Izh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgInt)i;
return p;
......@@ -46,7 +46,7 @@ rts_mkInt (Capability *cap, HsInt i)
HaskellObj
rts_mkInt8 (Capability *cap, HsInt8 i)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, I8zh_con_info, CCS_SYSTEM);
/* Make sure we mask out the bits above the lowest 8 */
p->payload[0] = (StgClosure *)(StgInt)i;
......@@ -56,7 +56,7 @@ rts_mkInt8 (Capability *cap, HsInt8 i)
HaskellObj
rts_mkInt16 (Capability *cap, HsInt16 i)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, I16zh_con_info, CCS_SYSTEM);
/* Make sure we mask out the relevant bits */
p->payload[0] = (StgClosure *)(StgInt)i;
......@@ -66,7 +66,7 @@ rts_mkInt16 (Capability *cap, HsInt16 i)
HaskellObj
rts_mkInt32 (Capability *cap, HsInt32 i)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, I32zh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgInt)i;
return p;
......@@ -75,7 +75,7 @@ rts_mkInt32 (Capability *cap, HsInt32 i)
HaskellObj
rts_mkInt64 (Capability *cap, HsInt64 i)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,2));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,2));
SET_HDR(p, I64zh_con_info, CCS_SYSTEM);
ASSIGN_Int64((P_)&(p->payload[0]), i);
return p;
......@@ -84,7 +84,7 @@ rts_mkInt64 (Capability *cap, HsInt64 i)
HaskellObj
rts_mkWord (Capability *cap, HsWord i)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, Wzh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgWord)i;
return p;
......@@ -94,7 +94,7 @@ HaskellObj
rts_mkWord8 (Capability *cap, HsWord8 w)
{
/* see rts_mkInt* comments */
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, W8zh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgWord)(w & 0xff);
return p;
......@@ -104,7 +104,7 @@ HaskellObj
rts_mkWord16 (Capability *cap, HsWord16 w)
{
/* see rts_mkInt* comments */
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, W16zh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgWord)(w & 0xffff);
return p;
......@@ -114,7 +114,7 @@ HaskellObj
rts_mkWord32 (Capability *cap, HsWord32 w)
{
/* see rts_mkInt* comments */
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, W32zh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgWord)(w & 0xffffffff);
return p;
......@@ -123,7 +123,7 @@ rts_mkWord32 (Capability *cap, HsWord32 w)
HaskellObj
rts_mkWord64 (Capability *cap, HsWord64 w)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,2));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,2));
/* see mk_Int8 comment */
SET_HDR(p, W64zh_con_info, CCS_SYSTEM);
ASSIGN_Word64((P_)&(p->payload[0]), w);
......@@ -134,7 +134,7 @@ rts_mkWord64 (Capability *cap, HsWord64 w)
HaskellObj
rts_mkFloat (Capability *cap, HsFloat f)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,1));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, Fzh_con_info, CCS_SYSTEM);
ASSIGN_FLT((P_)p->payload, (StgFloat)f);
return p;
......@@ -143,7 +143,7 @@ rts_mkFloat (Capability *cap, HsFloat f)
HaskellObj
rts_mkDouble (Capability *cap, HsDouble d)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,CONSTR_sizeW(0,sizeofW(StgDouble)));
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,sizeofW(StgDouble)));
SET_HDR(p, Dzh_con_info, CCS_SYSTEM);
ASSIGN_DBL((P_)p->payload, (StgDouble)d);
return p;
......@@ -152,7 +152,7 @@ rts_mkDouble (Capability *cap, HsDouble d)
HaskellObj
rts_mkStablePtr (Capability *cap, HsStablePtr s)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,sizeofW(StgHeader)+1);
StgClosure *p = (StgClosure *)allocate(cap,sizeofW(StgHeader)+1);
SET_HDR(p, StablePtr_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)s;
return p;
......@@ -161,7 +161,7 @@ rts_mkStablePtr (Capability *cap, HsStablePtr s)
HaskellObj
rts_mkPtr (Capability *cap, HsPtr a)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,sizeofW(StgHeader)+1);
StgClosure *p = (StgClosure *)allocate(cap,sizeofW(StgHeader)+1);
SET_HDR(p, Ptr_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)a;
return p;
......@@ -170,7 +170,7 @@ rts_mkPtr (Capability *cap, HsPtr a)
HaskellObj
rts_mkFunPtr (Capability *cap, HsFunPtr a)
{
StgClosure *p = (StgClosure *)allocateLocal(cap,sizeofW(StgHeader)+1);
StgClosure *p = (StgClosure *)allocate(cap,sizeofW(StgHeader)+1);
SET_HDR(p, FunPtr_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)a;
return p;
......@@ -197,7 +197,7 @@ rts_apply (Capability *cap, HaskellObj f, HaskellObj arg)
{
StgThunk *ap;
ap = (StgThunk *)allocateLocal(cap,sizeofW(StgThunk) + 2);
ap = (StgThunk *)allocate(cap,sizeofW(StgThunk) + 2);
SET_HDR(ap, (StgInfoTable *)&stg_ap_2_upd_info, CCS_SYSTEM);
ap->payload[0] = f;
ap->payload[1] = arg;
......
......@@ -412,7 +412,7 @@ static void unpark_waiters_on(Capability *cap, StgTVar *s) {
static StgInvariantCheckQueue *new_stg_invariant_check_queue(Capability *cap,
StgAtomicInvariant *invariant) {
StgInvariantCheckQueue *result;
result = (StgInvariantCheckQueue *)allocateLocal(cap, sizeofW(StgInvariantCheckQueue));
result = (StgInvariantCheckQueue *)allocate(cap, sizeofW(StgInvariantCheckQueue));
SET_HDR (result, &stg_INVARIANT_CHECK_QUEUE_info, CCS_SYSTEM);
result -> invariant = invariant;
result -> my_execution = NO_TREC;
......@@ -422,7 +422,7 @@ static StgInvariantCheckQueue *new_stg_invariant_check_queue(Capability *cap,
static StgTVarWatchQueue *new_stg_tvar_watch_queue(Capability *cap,
StgClosure *closure) {
StgTVarWatchQueue *result;
result = (StgTVarWatchQueue *)allocateLocal(cap, sizeofW(StgTVarWatchQueue));
result = (StgTVarWatchQueue *)allocate(cap, sizeofW(StgTVarWatchQueue));
SET_HDR (result, &stg_TVAR_WATCH_QUEUE_info, CCS_SYSTEM);
result -> closure = closure;
return result;
......@@ -430,7 +430,7 @@ static StgTVarWatchQueue *new_stg_tvar_watch_queue(Capability *cap,
static StgTRecChunk *new_stg_trec_chunk(Capability *cap) {
StgTRecChunk *result;
result = (StgTRecChunk *)allocateLocal(cap, sizeofW(StgTRecChunk));
result = (StgTRecChunk *)allocate(cap, sizeofW(StgTRecChunk));
SET_HDR (result, &stg_TREC_CHUNK_info, CCS_SYSTEM);
result -> prev_chunk = END_STM_CHUNK_LIST;
result -> next_entry_idx = 0;
......@@ -440,7 +440,7 @@ static StgTRecChunk *new_stg_trec_chunk(Capability *cap) {
static StgTRecHeader *new_stg_trec_header(Capability *cap,
StgTRecHeader *enclosing_trec) {
StgTRecHeader *result;
result = (StgTRecHeader *) allocateLocal(cap, sizeofW(StgTRecHeader));
result = (StgTRecHeader *) allocate(cap, sizeofW(StgTRecHeader));
SET_HDR (result, &stg_TREC_HEADER_info, CCS_SYSTEM);
result -> enclosing_trec = enclosing_trec;
......@@ -1175,7 +1175,7 @@ void stmAddInvariantToCheck(Capability *cap,
// 1. Allocate an StgAtomicInvariant, set last_execution to NO_TREC
// to signal that this is a new invariant in the current atomic block
invariant = (StgAtomicInvariant *) allocateLocal(cap, sizeofW(StgAtomicInvariant));
invariant = (StgAtomicInvariant *) allocate(cap, sizeofW(StgAtomicInvariant));
TRACE("%p : stmAddInvariantToCheck allocated invariant=%p", trec, invariant);
SET_HDR (invariant, &stg_ATOMIC_INVARIANT_info, CCS_SYSTEM);
invariant -> code = code;
......@@ -1657,7 +1657,7 @@ void stmWriteTVar(Capability *cap,
StgTVar *stmNewTVar(Capability *cap,
StgClosure *new_value) {
StgTVar *result;
result = (StgTVar *)allocateLocal(cap, sizeofW(StgTVar));
result = (StgTVar *)allocate(cap, sizeofW(StgTVar));
SET_HDR (result, &stg_TVAR_info, CCS_SYSTEM);
result -> current_value = new_value;
result -> first_watch_queue_entry = END_STM_WATCH_QUEUE;
......
......@@ -2262,7 +2262,7 @@ threadStackOverflow(Capability *cap, StgTSO *tso)
"increasing stack size from %ld words to %d.",
(long)tso->stack_size, new_stack_size);
dest = (StgTSO *)allocateLocal(cap,new_tso_size);
dest = (StgTSO *)allocate(cap,new_tso_size);
TICK_ALLOC_TSO(new_stack_size,0);
/* copy the TSO block and the old stack into the new area */
......@@ -2533,7 +2533,7 @@ raiseExceptionHelper (StgRegTable *reg, StgTSO *tso, StgClosure *exception)
// Only create raise_closure if we need to.
if (raise_closure == NULL) {
raise_closure =
(StgThunk *)allocateLocal(cap,sizeofW(StgThunk)+1);
(StgThunk *)allocate(cap,sizeofW(StgThunk)+1);
SET_HDR(raise_closure, &stg_raise_info, CCCS);
raise_closure->payload[0] = exception;
}
......
......@@ -63,7 +63,7 @@ createThread(Capability *cap, nat size)
}
size = round_to_mblocks(size);
tso = (StgTSO *)allocateLocal(cap, size);
tso = (StgTSO *)allocate(cap, size);
stack_size = size - TSO_STRUCT_SIZEW;
TICK_ALLOC_TSO(stack_size, 0);
......@@ -102,8 +102,8 @@ createThread(Capability *cap, nat size)
*/
ACQUIRE_LOCK(&sched_mutex);
tso->id = next_thread_id++; // while we have the mutex
tso->global_link = g0s0->threads;
g0s0->threads = tso;
tso->global_link = cap->r.rNursery->threads;
cap->r.rNursery->threads = tso;
RELEASE_LOCK(&sched_mutex);
// ToDo: report the stack size in the event?
......
......@@ -120,7 +120,7 @@ scheduleFinalizers(Capability *cap, StgWeak *list)
debugTrace(DEBUG_weak, "weak: batching %d finalizers", n);
arr = (StgMutArrPtrs *)allocateLocal(cap, sizeofW(StgMutArrPtrs) + n);
arr = (StgMutArrPtrs *)allocate(cap, sizeofW(StgMutArrPtrs) + n);
TICK_ALLOC_PRIM(sizeofW(StgMutArrPtrs), n, 0);
SET_HDR(arr, &stg_MUT_ARR_PTRS_FROZEN_info, CCS_SYSTEM);
arr->ptrs = n;
......
......@@ -425,9 +425,9 @@ SET_GCT(gc_threads[0]);
// g0s0->old_blocks is the old nursery
// g0s0->blocks is to-space from the previous GC
if (RtsFlags.GcFlags.generations == 1) {
if (g0s0->blocks != NULL) {
freeChain(g0s0->blocks);
g0s0->blocks = NULL;
if (g0->steps[0].blocks != NULL) {
freeChain(g0->steps[0].blocks);
g0->steps[0].blocks = NULL;
}
}
......@@ -646,18 +646,13 @@ SET_GCT(gc_threads[0]);
/* LARGE OBJECTS. The current live large objects are chained on
* scavenged_large, having been moved during garbage
* collection from large_objects. Any objects left on
* collection from large_objects. Any objects left on the
* large_objects list are therefore dead, so we free them here.
*/
for (bd = stp->large_objects; bd != NULL; bd = next) {
next = bd->link;
freeGroup(bd);
bd = next;
}
freeChain(stp->large_objects);
stp->large_objects = stp->scavenged_large_objects;
stp->n_large_blocks = stp->n_scavenged_large_blocks;
ASSERT(countBlocks(stp->large_objects) == stp->n_large_blocks);
}
else // for older generations...
{
......@@ -672,6 +667,7 @@ SET_GCT(gc_threads[0]);
// add the new blocks we promoted during this GC
stp->n_large_blocks += stp->n_scavenged_large_blocks;
ASSERT(countBlocks(stp->large_objects) == stp->n_large_blocks);
}
}
}
......@@ -685,18 +681,19 @@ SET_GCT(gc_threads[0]);
// Free the small objects allocated via allocate(), since this will
// all have been copied into G0S1 now.
if (RtsFlags.GcFlags.generations > 1) {
if (g0s0->blocks != NULL) {
freeChain(g0s0->blocks);
g0s0->blocks = NULL;
if (g0->steps[0].blocks != NULL) {
freeChain(g0->steps[0].blocks);
g0->steps[0].blocks = NULL;
}
g0s0->n_blocks = 0;
g0s0->n_words = 0;
g0->steps[0].n_blocks = 0;
g0->steps[0].n_words = 0;
}
alloc_blocks = 0;
alloc_blocks_lim = RtsFlags.GcFlags.minAllocAreaSize;