Commit ab0e778c authored by Simon Marlow's avatar Simon Marlow

Split GC.c, and move storage manager into sm/ directory

In preparation for parallel GC, split up the monolithic GC.c file into
smaller parts.  Also in this patch (and difficult to separate,
unfortunatley):
  
  - Don't include Stable.h in Rts.h, instead just include it where
    necessary.
  
  - consistently use STATIC_INLINE in source files, and INLINE_HEADER
    in header files.  STATIC_INLINE is now turned off when DEBUG is on,
    to make debugging easier.
  
  - The GC no longer takes the get_roots function as an argument.
    We weren't making use of this generalisation.
parent 2246c514
......@@ -18,6 +18,12 @@ extern "C" {
#endif
#include "Stg.h"
// Turn off inlining when debugging - it obfuscates things
#ifdef DEBUG
# undef STATIC_INLINE
# define STATIC_INLINE static
#endif
#include "RtsTypes.h"
#if __GNUC__ >= 3
......@@ -138,16 +144,13 @@ extern void _assertFail (const char *, unsigned int);
#include "OSThreads.h"
#include "SMP.h"
/* STG/Optimised-C related stuff */
#include "Block.h"
/* GNU mp library */
#include "gmp.h"
/* Macros for STG/C code */
#include "Block.h"
#include "ClosureMacros.h"
#include "StgTicky.h"
#include "Stable.h"
/* Runtime-system hooks */
#include "Hooks.h"
......
......@@ -83,9 +83,6 @@ extern void* allocateExec(unsigned int len);
Storage manager stuff exported
-------------------------------------------------------------------------- */
/* Prototype for an evacuate-like function */
typedef void (*evac_fn)(StgClosure **);
extern void performGC(void);
extern void performMajorGC(void);
extern HsInt64 getAllocations( void );
......
......@@ -66,7 +66,9 @@
#define BITS_IN(x) (BITS_PER_BYTE * sizeof(x))
/*
* 'Portable' inlining
* 'Portable' inlining:
* INLINE_HEADER is for inline functions in header files
* STATIC_INLINE is for inline functions in source files
*/
#if defined(__GNUC__) || defined( __INTEL_COMPILER)
# define INLINE_HEADER static inline
......
......@@ -185,7 +185,7 @@ extern void freeExec (void *p);
MarkRoot(StgClosure *p) Returns the new location of the root.
-------------------------------------------------------------------------- */
extern void GarbageCollect(void (*get_roots)(evac_fn),rtsBool force_major_gc);
extern void GarbageCollect(rtsBool force_major_gc);
/* -----------------------------------------------------------------------------
Generational garbage collection support
......@@ -362,7 +362,7 @@ INLINE_HEADER StgWord tso_sizeW ( StgTSO *tso )
INLINE_HEADER StgWord bco_sizeW ( StgBCO *bco )
{ return bco->size; }
STATIC_INLINE nat
INLINE_HEADER nat
closure_sizeW_ (StgClosure *p, StgInfoTable *info)
{
switch (info->type) {
......@@ -428,7 +428,7 @@ closure_sizeW_ (StgClosure *p, StgInfoTable *info)
}
// The definitive way to find the size, in words, of a heap-allocated closure
STATIC_INLINE nat
INLINE_HEADER nat
closure_sizeW (StgClosure *p)
{
return closure_sizeW_(p, get_itbl(p));
......@@ -483,6 +483,8 @@ extern lnat countNurseryBlocks ( void );
Functions from GC.c
-------------------------------------------------------------------------- */
typedef void (*evac_fn)(StgClosure **);
extern void threadPaused ( Capability *cap, StgTSO * );
extern StgClosure * isAlive ( StgClosure *p );
extern void markCAFs ( evac_fn evac );
......
......@@ -23,6 +23,7 @@
#include "Rts.h"
#include "RtsFlags.h"
#include "Storage.h"
#include "Stable.h"
#include "OSThreads.h"
#include "Capability.h"
......
......@@ -41,6 +41,7 @@ Haskell side.
#include "RtsExternal.h"
#include "RtsUtils.h"
#include "Storage.h"
#include "Stable.h"
#include <stdlib.h>
#if defined(_WIN32)
......
......@@ -20,7 +20,6 @@
#include "Rts.h"
#include "RtsUtils.h"
#include "BlockAlloc.h"
#include "Arena.h"
#include <stdlib.h>
......
......@@ -23,6 +23,7 @@
#include "STM.h"
#include "OSThreads.h"
#include "Capability.h"
#include "Storage.h"
#include "Schedule.h"
#include "Sparks.h"
#include "Trace.h"
......
......@@ -16,6 +16,7 @@
#include "RtsUtils.h"
#include "Closures.h"
#include "TSO.h"
#include "Storage.h"
#include "Schedule.h"
#include "Bytecodes.h"
......
......@@ -3,11 +3,11 @@
#include "RtsFlags.h"
#include "RtsUtils.h"
#include "StgRun.h"
#include "Storage.h"
#include "Schedule.h"
#include "Printer.h"
#include "Sanity.h"
#include "STM.h"
#include "Storage.h"
#include "SchedAPI.h"
#include "Timer.h"
#include "ProfHeap.h"
......@@ -20,3 +20,4 @@
#include "ThreadLabels.h"
#include "Threads.h"
#include "Prelude.h"
#include "Stable.h"
......@@ -8,6 +8,8 @@
#include "HsFFI.h"
#include "Rts.h"
#include "Storage.h"
#include "Stable.h"
// hs_init and hs_exit are defined in RtsStartup.c
......
......@@ -10,9 +10,9 @@
#include "RtsUtils.h"
#include "Closures.h"
#include "TSO.h"
#include "Storage.h"
#include "Schedule.h"
#include "RtsFlags.h"
#include "Storage.h"
#include "LdvProfile.h"
#include "Updates.h"
#include "Sanity.h"
......
......@@ -21,11 +21,12 @@
#include "RtsFlags.h"
#include "HsFFI.h"
#include "Hash.h"
#include "Storage.h"
#include "Stable.h"
#include "Linker.h"
#include "LinkerInternals.h"
#include "RtsUtils.h"
#include "Schedule.h"
#include "Storage.h"
#include "Sparks.h"
#include "RtsTypeable.h"
......
......@@ -12,7 +12,6 @@
#include "Rts.h"
#include "RtsAPI.h"
#include "SchedAPI.h"
#include "Schedule.h"
#include "RtsFlags.h"
#include "RtsUtils.h"
#include "Prelude.h"
......
......@@ -44,7 +44,7 @@ endif
NON_HS_PACKAGE = YES
# grab sources from these subdirectories
ALL_DIRS = hooks parallel
ALL_DIRS = hooks parallel sm
ifeq "$(HOSTPLATFORM)" "i386-unknown-mingw32"
ALL_DIRS += win32
......@@ -88,7 +88,7 @@ H_FILES = $(wildcard ../includes/*.h) $(wildcard *.h)
# gcc provides lots of useful warnings if you ask it.
# This is a pretty good list to start with - use a # to comment out
# any you don't like.
WARNING_OPTS += -Wall
WARNING_OPTS += -Wall
WARNING_OPTS += -W
WARNING_OPTS += -Wstrict-prototypes
WARNING_OPTS += -Wmissing-prototypes
......@@ -105,7 +105,7 @@ WARNING_OPTS += -Wbad-function-cast
#WARNING_OPTS += -Wredundant-decls
#WARNING_OPTS += -Wconversion
STANDARD_OPTS += -I../includes -I. -Iparallel
STANDARD_OPTS += -I../includes -I. -Iparallel -Ism
# COMPILING_RTS is only used when building Win32 DLL support.
STANDARD_OPTS += -DCOMPILING_RTS
......
......@@ -12,8 +12,8 @@
#include "Trace.h"
#include "RaiseAsync.h"
#include "SMP.h"
#include "Schedule.h"
#include "Storage.h"
#include "Schedule.h"
#include "LdvProfile.h"
#include "Updates.h"
#include "STM.h"
......
......@@ -45,7 +45,7 @@ void awakenBlockedExceptionQueue (Capability *cap, StgTSO *tso);
* indefinitely). Interruptible threads can be sent an exception with
* killThread# even if they have async exceptions blocked.
*/
STATIC_INLINE int
INLINE_HEADER int
interruptible(StgTSO *t)
{
switch (t->why_blocked) {
......
......@@ -20,15 +20,15 @@
#include "RtsUtils.h"
#include "RetainerProfile.h"
#include "RetainerSet.h"
#include "Storage.h"
#include "Schedule.h"
#include "Stable.h"
#include "Printer.h"
#include "Storage.h"
#include "RtsFlags.h"
#include "Weak.h"
#include "Sanity.h"
#include "Profiling.h"
#include "Stats.h"
#include "BlockAlloc.h"
#include "ProfHeap.h"
#include "Apply.h"
......
......@@ -17,6 +17,7 @@
#include "Prelude.h"
#include "Schedule.h"
#include "Capability.h"
#include "Stable.h"
#include <stdlib.h>
......
......@@ -11,7 +11,6 @@
#include "Rts.h"
#include "RtsFlags.h"
#include "RtsUtils.h"
#include "BlockAlloc.h"
#include "Profiling.h"
#ifdef HAVE_CTYPE_H
......
......@@ -29,6 +29,7 @@
#include "BlockAlloc.h"
#include "Trace.h"
#include "RtsTypeable.h"
#include "Stable.h"
#if defined(RTS_GTK_FRONTPANEL)
#include "FrontPanel.h"
......
......@@ -86,10 +86,10 @@
#include "Rts.h"
#include "RtsFlags.h"
#include "RtsUtils.h"
#include "Storage.h"
#include "Schedule.h"
#include "SMP.h"
#include "STM.h"
#include "Storage.h"
#include "Trace.h"
#include <stdlib.h>
......
......@@ -11,7 +11,6 @@
#include "SchedAPI.h"
#include "RtsUtils.h"
#include "RtsFlags.h"
#include "BlockAlloc.h"
#include "OSThreads.h"
#include "Storage.h"
#include "StgRun.h"
......@@ -218,11 +217,9 @@ static rtsBool scheduleHandleThreadFinished( Capability *cap, Task *task,
StgTSO *t );
static rtsBool scheduleDoHeapProfile(rtsBool ready_to_gc);
static Capability *scheduleDoGC(Capability *cap, Task *task,
rtsBool force_major,
void (*get_roots)(evac_fn));
rtsBool force_major);
static rtsBool checkBlackHoles(Capability *cap);
static void AllRoots(evac_fn evac);
static StgTSO *threadStackOverflow(Capability *cap, StgTSO *tso);
......@@ -421,7 +418,7 @@ schedule (Capability *initialCapability, Task *task)
discardSparksCap(cap);
#endif
/* scheduleDoGC() deletes all the threads */
cap = scheduleDoGC(cap,task,rtsFalse,GetRoots);
cap = scheduleDoGC(cap,task,rtsFalse);
break;
case SCHED_SHUTTING_DOWN:
debugTrace(DEBUG_sched, "SCHED_SHUTTING_DOWN");
......@@ -701,7 +698,7 @@ run_thread:
if (scheduleDoHeapProfile(ready_to_gc)) { ready_to_gc = rtsFalse; }
if (ready_to_gc) {
cap = scheduleDoGC(cap,task,rtsFalse,GetRoots);
cap = scheduleDoGC(cap,task,rtsFalse);
}
} /* end of while() */
......@@ -968,7 +965,7 @@ scheduleDetectDeadlock (Capability *cap, Task *task)
// they are unreachable and will therefore be sent an
// exception. Any threads thus released will be immediately
// runnable.
cap = scheduleDoGC (cap, task, rtsTrue/*force major GC*/, GetRoots);
cap = scheduleDoGC (cap, task, rtsTrue/*force major GC*/);
recent_activity = ACTIVITY_DONE_GC;
......@@ -1929,7 +1926,7 @@ scheduleDoHeapProfile( rtsBool ready_to_gc STG_UNUSED )
scheduleCheckBlackHoles(&MainCapability);
debugTrace(DEBUG_sched, "garbage collecting before heap census");
GarbageCollect(GetRoots, rtsTrue);
GarbageCollect(rtsTrue);
debugTrace(DEBUG_sched, "performing heap census");
heapCensus();
......@@ -1946,8 +1943,7 @@ scheduleDoHeapProfile( rtsBool ready_to_gc STG_UNUSED )
* -------------------------------------------------------------------------- */
static Capability *
scheduleDoGC (Capability *cap, Task *task USED_IF_THREADS,
rtsBool force_major, void (*get_roots)(evac_fn))
scheduleDoGC (Capability *cap, Task *task USED_IF_THREADS, rtsBool force_major)
{
StgTSO *t;
#ifdef THREADED_RTS
......@@ -2066,7 +2062,7 @@ scheduleDoGC (Capability *cap, Task *task USED_IF_THREADS,
#if defined(THREADED_RTS)
debugTrace(DEBUG_sched, "doing GC");
#endif
GarbageCollect(get_roots, force_major);
GarbageCollect(force_major);
#if defined(THREADED_RTS)
// release our stash of capabilities.
......@@ -2567,7 +2563,7 @@ exitScheduler( void )
// If we haven't killed all the threads yet, do it now.
if (sched_state < SCHED_SHUTTING_DOWN) {
sched_state = SCHED_INTERRUPTING;
scheduleDoGC(NULL,task,rtsFalse,GetRoots);
scheduleDoGC(NULL,task,rtsFalse);
}
sched_state = SCHED_SHUTTING_DOWN;
......@@ -2672,10 +2668,8 @@ GetRoots( evac_fn evac )
collect when called from Haskell via _ccall_GC.
-------------------------------------------------------------------------- */
static void (*extra_roots)(evac_fn);
static void
performGC_(rtsBool force_major, void (*get_roots)(evac_fn))
performGC_(rtsBool force_major)
{
Task *task;
// We must grab a new Task here, because the existing Task may be
......@@ -2684,27 +2678,20 @@ performGC_(rtsBool force_major, void (*get_roots)(evac_fn))
ACQUIRE_LOCK(&sched_mutex);
task = newBoundTask();
RELEASE_LOCK(&sched_mutex);
scheduleDoGC(NULL,task,force_major, get_roots);
scheduleDoGC(NULL,task,force_major);
boundTaskExiting(task);
}
void
performGC(void)
{
performGC_(rtsFalse, GetRoots);
performGC_(rtsFalse);
}
void
performMajorGC(void)
{
performGC_(rtsTrue, GetRoots);
}
static void
AllRoots(evac_fn evac)
{
GetRoots(evac); // the scheduler's roots
extra_roots(evac); // the user's roots
performGC_(rtsTrue);
}
/* -----------------------------------------------------------------------------
......
......@@ -191,7 +191,7 @@ void print_bqe (StgBlockingQueueElement *bqe);
* NOTE: tso->link should be END_TSO_QUEUE before calling this macro.
* ASSUMES: cap->running_task is the current task.
*/
STATIC_INLINE void
INLINE_HEADER void
appendToRunQueue (Capability *cap, StgTSO *tso)
{
ASSERT(tso->link == END_TSO_QUEUE);
......@@ -207,7 +207,7 @@ appendToRunQueue (Capability *cap, StgTSO *tso)
* newly awakened threads, so they get run as soon as possible.
* ASSUMES: cap->running_task is the current task.
*/
STATIC_INLINE void
INLINE_HEADER void
pushOnRunQueue (Capability *cap, StgTSO *tso)
{
tso->link = cap->run_queue_hd;
......@@ -219,7 +219,7 @@ pushOnRunQueue (Capability *cap, StgTSO *tso)
/* Pop the first thread off the runnable queue.
*/
STATIC_INLINE StgTSO *
INLINE_HEADER StgTSO *
popRunQueue (Capability *cap)
{
StgTSO *t = cap->run_queue_hd;
......@@ -235,7 +235,7 @@ popRunQueue (Capability *cap)
/* Add a thread to the end of the blocked queue.
*/
#if !defined(THREADED_RTS)
STATIC_INLINE void
INLINE_HEADER void
appendToBlockedQueue(StgTSO *tso)
{
ASSERT(tso->link == END_TSO_QUEUE);
......@@ -249,7 +249,7 @@ appendToBlockedQueue(StgTSO *tso)
#endif
#if defined(THREADED_RTS)
STATIC_INLINE void
INLINE_HEADER void
appendToWakeupQueue (Capability *cap, StgTSO *tso)
{
ASSERT(tso->link == END_TSO_QUEUE);
......@@ -264,20 +264,20 @@ appendToWakeupQueue (Capability *cap, StgTSO *tso)
/* Check whether various thread queues are empty
*/
STATIC_INLINE rtsBool
INLINE_HEADER rtsBool
emptyQueue (StgTSO *q)
{
return (q == END_TSO_QUEUE);
}
STATIC_INLINE rtsBool
INLINE_HEADER rtsBool
emptyRunQueue(Capability *cap)
{
return emptyQueue(cap->run_queue_hd);
}
#if defined(THREADED_RTS)
STATIC_INLINE rtsBool
INLINE_HEADER rtsBool
emptyWakeupQueue(Capability *cap)
{
return emptyQueue(cap->wakeup_queue_hd);
......@@ -289,7 +289,7 @@ emptyWakeupQueue(Capability *cap)
#define EMPTY_SLEEPING_QUEUE() (emptyQueue(sleeping_queue))
#endif
STATIC_INLINE rtsBool
INLINE_HEADER rtsBool
emptyThreadQueues(Capability *cap)
{
return emptyRunQueue(cap)
......@@ -301,7 +301,7 @@ emptyThreadQueues(Capability *cap)
#endif /* !IN_STG_CODE */
STATIC_INLINE void
INLINE_HEADER void
dirtyTSO (StgTSO *tso)
{
tso->flags |= TSO_DIRTY;
......
......@@ -8,9 +8,9 @@
#include "PosixSource.h"
#include "Rts.h"
#include "Storage.h"
#include "Schedule.h"
#include "SchedAPI.h"
#include "Storage.h"
#include "RtsFlags.h"
#include "RtsUtils.h"
#include "ParTicky.h"
......
......@@ -19,6 +19,7 @@
#include "RtsFlags.h"
#include "OSThreads.h"
#include "Trace.h"
#include "Stable.h"
/* Comment from ADR's implementation in old RTS:
......
......@@ -10,11 +10,11 @@
#include "RtsFlags.h"
#include "RtsUtils.h"
#include "MBlock.h"
#include "Storage.h"
#include "Schedule.h"
#include "Stats.h"
#include "ParTicky.h" /* ToDo: move into Rts.h */
#include "Profiling.h"
#include "Storage.h"
#include "GetTime.h"
/* huh? */
......
......@@ -15,6 +15,7 @@
#include "Capability.h"
#include "Stats.h"
#include "RtsFlags.h"
#include "Storage.h"
#include "Schedule.h"
#include "Hash.h"
#include "Trace.h"
......
/* -----------------------------------------------------------------------------
*
* (c) The GHC Team 1998-2006
*
* Tidying up a thread when it stops running
*
* ---------------------------------------------------------------------------*/
#include "Rts.h"
#include "Storage.h"
#include "Updates.h"
#include "RaiseAsync.h"
#include "Trace.h"
#include "RtsFlags.h"
#include <string.h> // for memmove()
/* -----------------------------------------------------------------------------
* Stack squeezing
*
* Code largely pinched from old RTS, then hacked to bits. We also do
* lazy black holing here.
*
* -------------------------------------------------------------------------- */
struct stack_gap { StgWord gap_size; struct stack_gap *next_gap; };
static void
stackSqueeze(StgTSO *tso, StgPtr bottom)
{
StgPtr frame;
rtsBool prev_was_update_frame;
StgClosure *updatee = NULL;
StgRetInfoTable *info;
StgWord current_gap_size;
struct stack_gap *gap;
// Stage 1:
// Traverse the stack upwards, replacing adjacent update frames
// with a single update frame and a "stack gap". A stack gap
// contains two values: the size of the gap, and the distance
// to the next gap (or the stack top).
frame = tso->sp;
ASSERT(frame < bottom);
prev_was_update_frame = rtsFalse;
current_gap_size = 0;
gap = (struct stack_gap *) (tso->sp - sizeofW(StgUpdateFrame));
while (frame < bottom) {
info = get_ret_itbl((StgClosure *)frame);
switch (info->i.type) {
case UPDATE_FRAME:
{
StgUpdateFrame *upd = (StgUpdateFrame *)frame;
if (prev_was_update_frame) {
TICK_UPD_SQUEEZED();
/* wasn't there something about update squeezing and ticky to be
* sorted out? oh yes: we aren't counting each enter properly
* in this case. See the log somewhere. KSW 1999-04-21
*
* Check two things: that the two update frames don't point to
* the same object, and that the updatee_bypass isn't already an
* indirection. Both of these cases only happen when we're in a
* block hole-style loop (and there are multiple update frames
* on the stack pointing to the same closure), but they can both
* screw us up if we don't check.
*/
if (upd->updatee != updatee && !closure_IND(upd->updatee)) {
UPD_IND_NOLOCK(upd->updatee, updatee);
}
// now mark this update frame as a stack gap. The gap
// marker resides in the bottom-most update frame of
// the series of adjacent frames, and covers all the
// frames in this series.
current_gap_size += sizeofW(StgUpdateFrame);
((struct stack_gap *)frame)->gap_size = current_gap_size;
((struct stack_gap *)frame)->next_gap = gap;
frame += sizeofW(StgUpdateFrame);
continue;
}
// single update frame, or the topmost update frame in a series
else {
prev_was_update_frame = rtsTrue;
updatee = upd->updatee;
frame += sizeofW(StgUpdateFrame);
continue;
}
}
default:
prev_was_update_frame = rtsFalse;
// we're not in a gap... check whether this is the end of a gap
// (an update frame can't be the end of a gap).
if (current_gap_size != 0) {
gap = (struct stack_gap *) (frame - sizeofW(StgUpdateFrame));
}
current_gap_size = 0;
frame += stack_frame_sizeW((StgClosure *)frame);
continue;
}
}
if (current_gap_size != 0) {
gap = (struct stack_gap *) (frame - sizeofW(StgUpdateFrame));
}
// Now we have a stack with gaps in it, and we have to walk down
// shoving the stack up to fill in the gaps. A diagram might
// help:
//
// +| ********* |
// | ********* | <- sp
// | |
// | | <- gap_start
// | ......... | |
// | stack_gap | <- gap | chunk_size
// | ......... | |
// | ......... | <- gap_end v
// | ********* |
// | ********* |
// | ********* |
// -| ********* |
//
// 'sp' points the the current top-of-stack
// 'gap' points to the stack_gap structure inside the gap
// ***** indicates real stack data
// ..... indicates gap
// <empty> indicates unused
//
{
void *sp;
void *gap_start, *next_gap_start, *gap_end;
nat chunk_size;
next_gap_start = (void *)((unsigned char*)gap + sizeof(StgUpdateFrame));
sp = next_gap_start;
while ((StgPtr)gap > tso->sp) {
// we're working in *bytes* now...
gap_start = next_gap_start;
gap_end = (void*) ((unsigned char*)gap_start - gap->gap_size * sizeof(W_));
gap = gap->next_gap;
next_gap_start = (void *)((unsigned char*)gap + sizeof(StgUpdateFrame));
chunk_size = (unsigned char*)gap_end - (unsigned char*)next_gap_start;
sp -= chunk_size;
memmove(sp, next_gap_start, chunk_size);
}
tso->sp = (StgPtr)sp;
}
}
/* -----------------------------------------------------------------------------
* Pausing a thread
*
* We have to prepare for GC - this means doing lazy black holing
* here. We also take the opportunity to do stack squeezing if it's
* turned on.
* -------------------------------------------------------------------------- */
void
threadPaused(Capability *cap, StgTSO *tso)
{
StgClosure *frame;
StgRetInfoTable *info;
StgClosure *bh;
StgPtr stack_end;
nat words_to_squeeze = 0;
nat weight = 0;
nat weight_pending = 0;
rtsBool prev_was_update_frame = rtsFalse;
// Check to see whether we have threads waiting to raise
// exceptions, and we're not blocking exceptions, or are blocked
// interruptibly. This is important; if a thread is running with
// TSO_BLOCKEX and becomes blocked interruptibly, this is the only
// place we ensure that the blocked_exceptions get a chance.
maybePerformBlockedException (cap, tso);