Commit 9cef40bd authored by tharris@microsoft.com's avatar tharris@microsoft.com

STM invariants

parent 87c36991
...@@ -1282,6 +1282,13 @@ primop CatchSTMOp "catchSTM#" GenPrimOp ...@@ -1282,6 +1282,13 @@ primop CatchSTMOp "catchSTM#" GenPrimOp
out_of_line = True out_of_line = True
has_side_effects = True has_side_effects = True
primop Check "check#" GenPrimOp
(State# RealWorld -> (# State# RealWorld, a #) )
-> (State# RealWorld -> (# State# RealWorld, () #) )
with
out_of_line = True
has_side_effects = True
primop NewTVarOp "newTVar#" GenPrimOp primop NewTVarOp "newTVar#" GenPrimOp
a a
-> State# s -> (# State# s, TVar# s a #) -> State# s -> (# State# s, TVar# s a #)
......
...@@ -85,13 +85,15 @@ ...@@ -85,13 +85,15 @@
#define RBH 61 #define RBH 61
#define EVACUATED 62 #define EVACUATED 62
#define REMOTE_REF 63 #define REMOTE_REF 63
#define TVAR_WAIT_QUEUE 64 #define TVAR_WATCH_QUEUE 64
#define TVAR 65 #define INVARIANT_CHECK_QUEUE 65
#define TREC_CHUNK 66 #define ATOMIC_INVARIANT 66
#define TREC_HEADER 67 #define TVAR 67
#define ATOMICALLY_FRAME 68 #define TREC_CHUNK 68
#define CATCH_RETRY_FRAME 69 #define TREC_HEADER 69
#define CATCH_STM_FRAME 70 #define ATOMICALLY_FRAME 70
#define N_CLOSURE_TYPES 71 #define CATCH_RETRY_FRAME 71
#define CATCH_STM_FRAME 72
#define N_CLOSURE_TYPES 73
#endif /* CLOSURETYPES_H */ #endif /* CLOSURETYPES_H */
...@@ -331,7 +331,7 @@ typedef struct { ...@@ -331,7 +331,7 @@ typedef struct {
* space for these data structures at the cost of more complexity in the * space for these data structures at the cost of more complexity in the
* implementation: * implementation:
* *
* - In StgTVar, current_value and first_wait_queue_entry could be held in * - In StgTVar, current_value and first_watch_queue_entry could be held in
* the same field: if any thread is waiting then its expected_value for * the same field: if any thread is waiting then its expected_value for
* the tvar is the current value. * the tvar is the current value.
* *
...@@ -345,24 +345,33 @@ typedef struct { ...@@ -345,24 +345,33 @@ typedef struct {
* (it immediately switches on frame->waiting anyway). * (it immediately switches on frame->waiting anyway).
*/ */
typedef struct StgTVarWaitQueue_ { typedef struct StgTRecHeader_ StgTRecHeader;
typedef struct StgTVarWatchQueue_ {
StgHeader header; StgHeader header;
struct StgTSO_ *waiting_tso; StgClosure *closure; // StgTSO or StgAtomicInvariant
struct StgTVarWaitQueue_ *next_queue_entry; struct StgTVarWatchQueue_ *next_queue_entry;
struct StgTVarWaitQueue_ *prev_queue_entry; struct StgTVarWatchQueue_ *prev_queue_entry;
} StgTVarWaitQueue; } StgTVarWatchQueue;
typedef struct { typedef struct {
StgHeader header; StgHeader header;
StgClosure *volatile current_value; StgClosure *volatile current_value;
StgTVarWaitQueue *volatile first_wait_queue_entry; StgTVarWatchQueue *volatile first_watch_queue_entry;
#if defined(THREADED_RTS) #if defined(THREADED_RTS)
StgInt volatile num_updates; StgInt volatile num_updates;
#endif #endif
} StgTVar; } StgTVar;
typedef struct {
StgHeader header;
StgClosure *code;
StgTRecHeader *last_execution;
StgWord lock;
} StgAtomicInvariant;
/* new_value == expected_value for read-only accesses */ /* new_value == expected_value for read-only accesses */
/* new_value is a StgTVarWaitQueue entry when trec in state TREC_WAITING */ /* new_value is a StgTVarWatchQueue entry when trec in state TREC_WAITING */
typedef struct { typedef struct {
StgTVar *tvar; StgTVar *tvar;
StgClosure *expected_value; StgClosure *expected_value;
...@@ -389,29 +398,38 @@ typedef enum { ...@@ -389,29 +398,38 @@ typedef enum {
TREC_WAITING, /* Transaction currently waiting */ TREC_WAITING, /* Transaction currently waiting */
} TRecState; } TRecState;
typedef struct StgTRecHeader_ { typedef struct StgInvariantCheckQueue_ {
StgHeader header;
StgAtomicInvariant *invariant;
StgTRecHeader *my_execution;
struct StgInvariantCheckQueue_ *next_queue_entry;
} StgInvariantCheckQueue;
struct StgTRecHeader_ {
StgHeader header; StgHeader header;
TRecState state; TRecState state;
struct StgTRecHeader_ *enclosing_trec; struct StgTRecHeader_ *enclosing_trec;
StgTRecChunk *current_chunk; StgTRecChunk *current_chunk;
} StgTRecHeader; StgInvariantCheckQueue *invariants_to_check;
};
typedef struct { typedef struct {
StgHeader header; StgHeader header;
StgClosure *code; StgClosure *code;
StgTVarWatchQueue *next_invariant_to_check;
} StgAtomicallyFrame; } StgAtomicallyFrame;
typedef struct { typedef struct {
StgHeader header; StgHeader header;
StgClosure *handler; StgClosure *code;
StgClosure *handler;
} StgCatchSTMFrame; } StgCatchSTMFrame;
typedef struct { typedef struct {
StgHeader header; StgHeader header;
StgBool running_alt_code; StgBool running_alt_code;
StgClosure *first_code; StgClosure *first_code;
StgClosure *alt_code; StgClosure *alt_code;
StgTRecHeader *first_code_trec;
} StgCatchRetryFrame; } StgCatchRetryFrame;
#if defined(PAR) || defined(GRAN) #if defined(PAR) || defined(GRAN)
......
...@@ -513,8 +513,9 @@ ...@@ -513,8 +513,9 @@
Misc junk Misc junk
-------------------------------------------------------------------------- */ -------------------------------------------------------------------------- */
#define NO_TREC stg_NO_TREC_closure #define NO_TREC stg_NO_TREC_closure
#define END_TSO_QUEUE stg_END_TSO_QUEUE_closure #define END_TSO_QUEUE stg_END_TSO_QUEUE_closure
#define END_INVARIANT_CHECK_QUEUE stg_END_INVARIANT_CHECK_QUEUE_closure
#define dirtyTSO(tso) \ #define dirtyTSO(tso) \
StgTSO_flags(tso) = StgTSO_flags(tso) | TSO_DIRTY::I32; StgTSO_flags(tso) = StgTSO_flags(tso) | TSO_DIRTY::I32;
......
...@@ -66,14 +66,13 @@ extern StgTRecHeader *stmStartNestedTransaction(Capability *cap, StgTRecHeader * ...@@ -66,14 +66,13 @@ extern StgTRecHeader *stmStartNestedTransaction(Capability *cap, StgTRecHeader *
); );
/* /*
* Exit the current transaction context, abandoning any read/write * Roll back the current transatcion context. NB: if this is a nested tx
* operations performed within it and removing the thread from any * then we merge its read set into its parents. This is because a change
* tvar wait queues if it was waitin. Note that if nested transactions * to that read set could change whether or not the tx should abort.
* are not fully supported then this may leave the enclosing
* transaction contexts doomed to abort.
*/ */
extern void stmAbortTransaction(Capability *cap, StgTRecHeader *trec); extern void stmAbortTransaction(Capability *cap, StgTRecHeader *trec);
extern void stmFreeAbortedTRec(Capability *cap, StgTRecHeader *trec);
/* /*
* Ensure that a subsequent commit / validation will fail. We use this * Ensure that a subsequent commit / validation will fail. We use this
...@@ -148,6 +147,18 @@ extern StgBool stmValidateNestOfTransactions(StgTRecHeader *trec); ...@@ -148,6 +147,18 @@ extern StgBool stmValidateNestOfTransactions(StgTRecHeader *trec);
is actually still valid. is actually still valid.
*/ */
/*
* Fill in the trec's list of invariants that might be violated by the current
* transaction.
*/
extern StgInvariantCheckQueue *stmGetInvariantsToCheck(Capability *cap,
StgTRecHeader *trec);
extern void stmAddInvariantToCheck(Capability *cap,
StgTRecHeader *trec,
StgClosure *code);
/* /*
* Test whether the current transaction context is valid and, if so, * Test whether the current transaction context is valid and, if so,
* commit its memory accesses to the heap. stmCommitTransaction must * commit its memory accesses to the heap. stmCommitTransaction must
...@@ -218,7 +229,8 @@ extern void stmWriteTVar(Capability *cap, ...@@ -218,7 +229,8 @@ extern void stmWriteTVar(Capability *cap,
/* NULLs */ /* NULLs */
#define END_STM_WAIT_QUEUE ((StgTVarWaitQueue *)(void *)&stg_END_STM_WAIT_QUEUE_closure) #define END_STM_WATCH_QUEUE ((StgTVarWatchQueue *)(void *)&stg_END_STM_WATCH_QUEUE_closure)
#define END_INVARIANT_CHECK_QUEUE ((StgInvariantCheckQueue *)(void *)&stg_END_INVARIANT_CHECK_QUEUE_closure)
#define END_STM_CHUNK_LIST ((StgTRecChunk *)(void *)&stg_END_STM_CHUNK_LIST_closure) #define END_STM_CHUNK_LIST ((StgTRecChunk *)(void *)&stg_END_STM_CHUNK_LIST_closure)
#define NO_TREC ((StgTRecHeader *)(void *)&stg_NO_TREC_closure) #define NO_TREC ((StgTRecHeader *)(void *)&stg_NO_TREC_closure)
......
...@@ -136,11 +136,14 @@ RTS_INFO(stg_AP_info); ...@@ -136,11 +136,14 @@ RTS_INFO(stg_AP_info);
RTS_INFO(stg_AP_STACK_info); RTS_INFO(stg_AP_STACK_info);
RTS_INFO(stg_dummy_ret_info); RTS_INFO(stg_dummy_ret_info);
RTS_INFO(stg_raise_info); RTS_INFO(stg_raise_info);
RTS_INFO(stg_TVAR_WAIT_QUEUE_info); RTS_INFO(stg_TVAR_WATCH_QUEUE_info);
RTS_INFO(stg_INVARIANT_CHECK_QUEUE_info);
RTS_INFO(stg_ATOMIC_INVARIANT_info);
RTS_INFO(stg_TVAR_info); RTS_INFO(stg_TVAR_info);
RTS_INFO(stg_TREC_CHUNK_info); RTS_INFO(stg_TREC_CHUNK_info);
RTS_INFO(stg_TREC_HEADER_info); RTS_INFO(stg_TREC_HEADER_info);
RTS_INFO(stg_END_STM_WAIT_QUEUE_info); RTS_INFO(stg_END_STM_WATCH_QUEUE_info);
RTS_INFO(stg_END_INVARIANT_CHECK_QUEUE_info);
RTS_INFO(stg_END_STM_CHUNK_LIST_info); RTS_INFO(stg_END_STM_CHUNK_LIST_info);
RTS_INFO(stg_NO_TREC_info); RTS_INFO(stg_NO_TREC_info);
...@@ -197,11 +200,14 @@ RTS_ENTRY(stg_AP_entry); ...@@ -197,11 +200,14 @@ RTS_ENTRY(stg_AP_entry);
RTS_ENTRY(stg_AP_STACK_entry); RTS_ENTRY(stg_AP_STACK_entry);
RTS_ENTRY(stg_dummy_ret_entry); RTS_ENTRY(stg_dummy_ret_entry);
RTS_ENTRY(stg_raise_entry); RTS_ENTRY(stg_raise_entry);
RTS_ENTRY(stg_END_STM_WAIT_QUEUE_entry); RTS_ENTRY(stg_END_STM_WATCH_QUEUE_entry);
RTS_ENTRY(stg_END_INVARIANT_CHECK_QUEUE_entry);
RTS_ENTRY(stg_END_STM_CHUNK_LIST_entry); RTS_ENTRY(stg_END_STM_CHUNK_LIST_entry);
RTS_ENTRY(stg_NO_TREC_entry); RTS_ENTRY(stg_NO_TREC_entry);
RTS_ENTRY(stg_TVAR_entry); RTS_ENTRY(stg_TVAR_entry);
RTS_ENTRY(stg_TVAR_WAIT_QUEUE_entry); RTS_ENTRY(stg_TVAR_WATCH_QUEUE_entry);
RTS_ENTRY(stg_INVARIANT_CHECK_QUEUE_entry);
RTS_ENTRY(stg_ATOMIC_INVARIANT_entry);
RTS_ENTRY(stg_TREC_CHUNK_entry); RTS_ENTRY(stg_TREC_CHUNK_entry);
RTS_ENTRY(stg_TREC_HEADER_entry); RTS_ENTRY(stg_TREC_HEADER_entry);
...@@ -224,7 +230,8 @@ RTS_CLOSURE(stg_NO_FINALIZER_closure); ...@@ -224,7 +230,8 @@ RTS_CLOSURE(stg_NO_FINALIZER_closure);
RTS_CLOSURE(stg_dummy_ret_closure); RTS_CLOSURE(stg_dummy_ret_closure);
RTS_CLOSURE(stg_forceIO_closure); RTS_CLOSURE(stg_forceIO_closure);
RTS_CLOSURE(stg_END_STM_WAIT_QUEUE_closure); RTS_CLOSURE(stg_END_STM_WATCH_QUEUE_closure);
RTS_CLOSURE(stg_END_INVARIANT_CHECK_QUEUE_closure);
RTS_CLOSURE(stg_END_STM_CHUNK_LIST_closure); RTS_CLOSURE(stg_END_STM_CHUNK_LIST_closure);
RTS_CLOSURE(stg_NO_TREC_closure); RTS_CLOSURE(stg_NO_TREC_closure);
...@@ -605,5 +612,6 @@ RTS_FUN(atomicallyzh_fast); ...@@ -605,5 +612,6 @@ RTS_FUN(atomicallyzh_fast);
RTS_FUN(newTVarzh_fast); RTS_FUN(newTVarzh_fast);
RTS_FUN(readTVarzh_fast); RTS_FUN(readTVarzh_fast);
RTS_FUN(writeTVarzh_fast); RTS_FUN(writeTVarzh_fast);
RTS_FUN(checkzh_fast);
#endif /* STGMISCCLOSURES_H */ #endif /* STGMISCCLOSURES_H */
...@@ -406,14 +406,18 @@ closure_sizeW_ (StgClosure *p, StgInfoTable *info) ...@@ -406,14 +406,18 @@ closure_sizeW_ (StgClosure *p, StgInfoTable *info)
return tso_sizeW((StgTSO *)p); return tso_sizeW((StgTSO *)p);
case BCO: case BCO:
return bco_sizeW((StgBCO *)p); return bco_sizeW((StgBCO *)p);
case TVAR_WAIT_QUEUE: case TVAR_WATCH_QUEUE:
return sizeofW(StgTVarWaitQueue); return sizeofW(StgTVarWatchQueue);
case TVAR: case TVAR:
return sizeofW(StgTVar); return sizeofW(StgTVar);
case TREC_CHUNK: case TREC_CHUNK:
return sizeofW(StgTRecChunk); return sizeofW(StgTRecChunk);
case TREC_HEADER: case TREC_HEADER:
return sizeofW(StgTRecHeader); return sizeofW(StgTRecHeader);
case ATOMIC_INVARIANT:
return sizeofW(StgAtomicInvariant);
case INVARIANT_CHECK_QUEUE:
return sizeofW(StgInvariantCheckQueue);
default: default:
return sizeW_fromITBL(info); return sizeW_fromITBL(info);
} }
......
...@@ -328,15 +328,26 @@ main(int argc, char *argv[]) ...@@ -328,15 +328,26 @@ main(int argc, char *argv[])
closure_size(StgAtomicallyFrame); closure_size(StgAtomicallyFrame);
closure_field(StgAtomicallyFrame, code); closure_field(StgAtomicallyFrame, code);
closure_field(StgAtomicallyFrame, next_invariant_to_check);
closure_field(StgInvariantCheckQueue, invariant);
closure_field(StgInvariantCheckQueue, my_execution);
closure_field(StgInvariantCheckQueue, next_queue_entry);
closure_field(StgAtomicInvariant, code);
closure_size(StgCatchSTMFrame); closure_size(StgCatchSTMFrame);
closure_field(StgCatchSTMFrame, handler); closure_field(StgCatchSTMFrame, handler);
closure_field(StgCatchSTMFrame, code);
closure_size(StgCatchRetryFrame); closure_size(StgCatchRetryFrame);
closure_field(StgCatchRetryFrame, running_alt_code); closure_field(StgCatchRetryFrame, running_alt_code);
closure_field(StgCatchRetryFrame, first_code); closure_field(StgCatchRetryFrame, first_code);
closure_field(StgCatchRetryFrame, alt_code); closure_field(StgCatchRetryFrame, alt_code);
closure_field(StgCatchRetryFrame, first_code_trec);
closure_field(StgTVarWatchQueue, closure);
closure_field(StgTVarWatchQueue, next_queue_entry);
closure_field(StgTVarWatchQueue, prev_queue_entry);
closure_size(StgWeak); closure_size(StgWeak);
closure_field(StgWeak,link); closure_field(StgWeak,link);
......
...@@ -153,7 +153,8 @@ initCapability( Capability *cap, nat i ) ...@@ -153,7 +153,8 @@ initCapability( Capability *cap, nat i )
cap->mut_lists[g] = NULL; cap->mut_lists[g] = NULL;
} }
cap->free_tvar_wait_queues = END_STM_WAIT_QUEUE; cap->free_tvar_watch_queues = END_STM_WATCH_QUEUE;
cap->free_invariant_check_queues = END_INVARIANT_CHECK_QUEUE;
cap->free_trec_chunks = END_STM_CHUNK_LIST; cap->free_trec_chunks = END_STM_CHUNK_LIST;
cap->free_trec_headers = NO_TREC; cap->free_trec_headers = NO_TREC;
cap->transaction_tokens = 0; cap->transaction_tokens = 0;
......
...@@ -89,7 +89,8 @@ struct Capability_ { ...@@ -89,7 +89,8 @@ struct Capability_ {
#endif #endif
// Per-capability STM-related data // Per-capability STM-related data
StgTVarWaitQueue *free_tvar_wait_queues; StgTVarWatchQueue *free_tvar_watch_queues;
StgInvariantCheckQueue *free_invariant_check_queues;
StgTRecChunk *free_trec_chunks; StgTRecChunk *free_trec_chunks;
StgTRecHeader *free_trec_headers; StgTRecHeader *free_trec_headers;
nat transaction_tokens; nat transaction_tokens;
......
...@@ -90,7 +90,9 @@ StgWord16 closure_flags[] = { ...@@ -90,7 +90,9 @@ StgWord16 closure_flags[] = {
/* RBH = */ ( _NS| _MUT|_UPT ), /* RBH = */ ( _NS| _MUT|_UPT ),
/* EVACUATED = */ ( 0 ), /* EVACUATED = */ ( 0 ),
/* REMOTE_REF = */ (_HNF| _NS| _UPT ), /* REMOTE_REF = */ (_HNF| _NS| _UPT ),
/* TVAR_WAIT_QUEUE = */ ( _NS| _MUT|_UPT ), /* TVAR_WATCH_QUEUE = */ ( _NS| _MUT|_UPT ),
/* INVARIANT_CHECK_QUEUE= */ ( _NS| _MUT|_UPT ),
/* ATOMIC_INVARIANT = */ ( _NS| _MUT|_UPT ),
/* TVAR = */ (_HNF| _NS| _MUT|_UPT ), /* TVAR = */ (_HNF| _NS| _MUT|_UPT ),
/* TREC_CHUNK = */ ( _NS| _MUT|_UPT ), /* TREC_CHUNK = */ ( _NS| _MUT|_UPT ),
/* TREC_HEADER = */ ( _NS| _MUT|_UPT ), /* TREC_HEADER = */ ( _NS| _MUT|_UPT ),
...@@ -99,6 +101,6 @@ StgWord16 closure_flags[] = { ...@@ -99,6 +101,6 @@ StgWord16 closure_flags[] = {
/* CATCH_STM_FRAME = */ ( _BTM ) /* CATCH_STM_FRAME = */ ( _BTM )
}; };
#if N_CLOSURE_TYPES != 71 #if N_CLOSURE_TYPES != 73
#error Closure types changed: update ClosureFlags.c! #error Closure types changed: update ClosureFlags.c!
#endif #endif
...@@ -344,12 +344,25 @@ retry_pop_stack: ...@@ -344,12 +344,25 @@ retry_pop_stack:
if (frame_type == ATOMICALLY_FRAME) { if (frame_type == ATOMICALLY_FRAME) {
/* The exception has reached the edge of a memory transaction. Check that /* The exception has reached the edge of a memory transaction. Check that
* the transaction is valid. If not then perhaps the exception should * the transaction is valid. If not then perhaps the exception should
* not have been thrown: re-run the transaction */ * not have been thrown: re-run the transaction. "trec" will either be
W_ trec; * a top-level transaction running the atomic block, or a nested
* transaction running an invariant check. In the latter case we
* abort and de-allocate the top-level transaction that encloses it
* as well (we could just abandon its transaction record, but this makes
* sure it's marked as aborted and available for re-use). */
W_ trec, outer;
W_ r; W_ r;
trec = StgTSO_trec(CurrentTSO); trec = StgTSO_trec(CurrentTSO);
r = foreign "C" stmValidateNestOfTransactions(trec "ptr"); r = foreign "C" stmValidateNestOfTransactions(trec "ptr");
"ptr" outer = foreign "C" stmGetEnclosingTRec(trec "ptr") [];
foreign "C" stmAbortTransaction(MyCapability() "ptr", trec "ptr"); foreign "C" stmAbortTransaction(MyCapability() "ptr", trec "ptr");
foreign "C" stmFreeAbortedTRec(MyCapability() "ptr", trec "ptr");
if (outer != NO_TREC) {
foreign "C" stmAbortTransaction(MyCapability() "ptr", outer "ptr");
foreign "C" stmFreeAbortedTRec(MyCapability() "ptr", outer "ptr");
}
StgTSO_trec(CurrentTSO) = NO_TREC; StgTSO_trec(CurrentTSO) = NO_TREC;
if (r != 0) { if (r != 0) {
// Transaction was valid: continue searching for a catch frame // Transaction was valid: continue searching for a catch frame
...@@ -400,6 +413,9 @@ retry_pop_stack: ...@@ -400,6 +413,9 @@ retry_pop_stack:
* If exceptions were unblocked, arrange that they are unblocked * If exceptions were unblocked, arrange that they are unblocked
* again after executing the handler by pushing an * again after executing the handler by pushing an
* unblockAsyncExceptions_ret stack frame. * unblockAsyncExceptions_ret stack frame.
*
* If we've reached an STM catch frame then roll back the nested
* transaction we were using.
*/ */
W_ frame; W_ frame;
frame = Sp; frame = Sp;
...@@ -410,6 +426,12 @@ retry_pop_stack: ...@@ -410,6 +426,12 @@ retry_pop_stack:
Sp(0) = stg_unblockAsyncExceptionszh_ret_info; Sp(0) = stg_unblockAsyncExceptionszh_ret_info;
} }
} else { } else {
W_ trec, outer;
trec = StgTSO_trec(CurrentTSO);
"ptr" outer = foreign "C" stmGetEnclosingTRec(trec "ptr") [];
foreign "C" stmAbortTransaction(MyCapability() "ptr", trec "ptr") [];
foreign "C" stmFreeAbortedTRec(MyCapability() "ptr", trec "ptr") [];
StgTSO_trec(CurrentTSO) = outer;
Sp = Sp + SIZEOF_StgCatchSTMFrame; Sp = Sp + SIZEOF_StgCatchSTMFrame;
} }
......
...@@ -2233,8 +2233,8 @@ loop: ...@@ -2233,8 +2233,8 @@ loop:
case TREC_HEADER: case TREC_HEADER:
return copy(q,sizeofW(StgTRecHeader),stp); return copy(q,sizeofW(StgTRecHeader),stp);
case TVAR_WAIT_QUEUE: case TVAR_WATCH_QUEUE:
return copy(q,sizeofW(StgTVarWaitQueue),stp); return copy(q,sizeofW(StgTVarWatchQueue),stp);
case TVAR: case TVAR:
return copy(q,sizeofW(StgTVar),stp); return copy(q,sizeofW(StgTVar),stp);
...@@ -2242,6 +2242,12 @@ loop: ...@@ -2242,6 +2242,12 @@ loop:
case TREC_CHUNK: case TREC_CHUNK:
return copy(q,sizeofW(StgTRecChunk),stp); return copy(q,sizeofW(StgTRecChunk),stp);
case ATOMIC_INVARIANT:
return copy(q,sizeofW(StgAtomicInvariant),stp);
case INVARIANT_CHECK_QUEUE:
return copy(q,sizeofW(StgInvariantCheckQueue),stp);
default: default:
barf("evacuate: strange closure type %d", (int)(info->type)); barf("evacuate: strange closure type %d", (int)(info->type));
} }
...@@ -3112,16 +3118,16 @@ scavenge(step *stp) ...@@ -3112,16 +3118,16 @@ scavenge(step *stp)
} }
#endif #endif
case TVAR_WAIT_QUEUE: case TVAR_WATCH_QUEUE:
{ {
StgTVarWaitQueue *wq = ((StgTVarWaitQueue *) p); StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p);
evac_gen = 0; evac_gen = 0;
wq->waiting_tso = (StgTSO *)evacuate((StgClosure*)wq->waiting_tso); wq->closure = (StgClosure*)evacuate((StgClosure*)wq->closure);
wq->next_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->next_queue_entry); wq->next_queue_entry = (StgTVarWatchQueue *)evacuate((StgClosure*)wq->next_queue_entry);
wq->prev_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->prev_queue_entry); wq->prev_queue_entry = (StgTVarWatchQueue *)evacuate((StgClosure*)wq->prev_queue_entry);
evac_gen = saved_evac_gen; evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable failed_to_evac = rtsTrue; // mutable
p += sizeofW(StgTVarWaitQueue); p += sizeofW(StgTVarWatchQueue);
break; break;
} }
...@@ -3130,7 +3136,7 @@ scavenge(step *stp) ...@@ -3130,7 +3136,7 @@ scavenge(step *stp)
StgTVar *tvar = ((StgTVar *) p); StgTVar *tvar = ((StgTVar *) p);
evac_gen = 0; evac_gen = 0;
tvar->current_value = evacuate((StgClosure*)tvar->current_value); tvar->current_value = evacuate((StgClosure*)tvar->current_value);
tvar->first_wait_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)tvar->first_wait_queue_entry); tvar->first_watch_queue_entry = (StgTVarWatchQueue *)evacuate((StgClosure*)tvar->first_watch_queue_entry);
evac_gen = saved_evac_gen; evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable failed_to_evac = rtsTrue; // mutable
p += sizeofW(StgTVar); p += sizeofW(StgTVar);
...@@ -3143,6 +3149,7 @@ scavenge(step *stp) ...@@ -3143,6 +3149,7 @@ scavenge(step *stp)
evac_gen = 0; evac_gen = 0;
trec->enclosing_trec = (StgTRecHeader *)evacuate((StgClosure*)trec->enclosing_trec); trec->enclosing_trec = (StgTRecHeader *)evacuate((StgClosure*)trec->enclosing_trec);
trec->current_chunk = (StgTRecChunk *)evacuate((StgClosure*)trec->current_chunk); trec->current_chunk = (StgTRecChunk *)evacuate((StgClosure*)trec->current_chunk);
trec->invariants_to_check = (StgInvariantCheckQueue *)evacuate((StgClosure*)trec->invariants_to_check);
evac_gen = saved_evac_gen; evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable failed_to_evac = rtsTrue; // mutable
p += sizeofW(StgTRecHeader); p += sizeofW(StgTRecHeader);
...@@ -3167,6 +3174,31 @@ scavenge(step *stp) ...@@ -3167,6 +3174,31 @@ scavenge(step *stp)
break; break;
} }
case ATOMIC_INVARIANT:
{
StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p);
evac_gen = 0;
invariant->code = (StgClosure *)evacuate(invariant->code);
invariant->last_execution = (StgTRecHeader *)evacuate((StgClosure*)invariant->last_execution);
evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable
p += sizeofW(StgAtomicInvariant);
break;
}
case INVARIANT_CHECK_QUEUE:
{
StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p);
evac_gen = 0;
queue->invariant = (StgAtomicInvariant *)evacuate((StgClosure*)queue->invariant);
queue->my_execution = (StgTRecHeader *)evacuate((StgClosure*)queue->my_execution);
queue->next_queue_entry = (StgInvariantCheckQueue *)evacuate((StgClosure*)queue->next_queue_entry);
evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable
p += sizeofW(StgInvariantCheckQueue);
break;
}
default: default:
barf("scavenge: unimplemented/strange closure type %d @ %p", barf("scavenge: unimplemented/strange closure type %d @ %p",
info->type, p); info->type, p);
...@@ -3496,13 +3528,13 @@ linear_scan: ...@@ -3496,13 +3528,13 @@ linear_scan:
} }
#endif /* PAR */ #endif /* PAR */
case TVAR_WAIT_QUEUE: case TVAR_WATCH_QUEUE:
{ {
StgTVarWaitQueue *wq = ((StgTVarWaitQueue *) p); StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p);
evac_gen = 0; evac_gen = 0;
wq->waiting_tso = (StgTSO *)evacuate((StgClosure*)wq->waiting_tso); wq->closure = (StgClosure*)evacuate((StgClosure*)wq->closure);
wq->next_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->next_queue_entry); wq->next_queue_entry = (StgTVarWatchQueue *)evacuate((StgClosure*)wq->next_queue_entry);
wq->prev_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->prev_queue_entry); wq->prev_queue_entry = (StgTVarWatchQueue *)evacuate((StgClosure*)wq->prev_queue_entry);
evac_gen = saved_evac_gen; evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable failed_to_evac = rtsTrue; // mutable
break; break;
...@@ -3513,7 +3545,7 @@ linear_scan: ...@@ -3513,7 +3545,7 @@ linear_scan:
StgTVar *tvar = ((StgTVar *) p); StgTVar *tvar = ((StgTVar *) p);
evac_gen = 0; evac_gen = 0;
tvar->current_value = evacuate((StgClosure*)tvar->current_value); tvar->current_value = evacuate((StgClosure*)tvar->current_value);
tvar->first_wait_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)tvar->first_wait_queue_entry); tvar->first_watch_queue_entry = (StgTVarWatchQueue *)evacuate((StgClosure*)tvar->first_watch_queue_entry);
evac_gen = saved_evac_gen; evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable failed_to_evac = rtsTrue; // mutable
break; break;
...@@ -3542,11 +3574,35 @@ linear_scan: ...@@ -3542,11 +3574,35 @@ linear_scan:
evac_gen = 0; evac_gen = 0;
trec->enclosing_trec = (StgTRecHeader *)evacuate((StgClosure*)trec->enclosing_trec); trec->enclosing_trec = (StgTRecHeader *)evacuate((StgClosure*)trec->enclosing_trec);
trec->current_chunk = (StgTRecChunk *)evacuate((StgClosure*)trec->current_chunk); trec->current_chunk = (StgTRecChunk *)evacuate((StgClosure*)trec->current_chunk);
trec->invariants_to_check = (StgInvariantCheckQueue *)evacuate((StgClosure*)trec->invariants_to_check);
evac_gen = saved_evac_gen; evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable failed_to_evac = rtsTrue; // mutable
break; break;
} }
case ATOMIC_INVARIANT:
{
StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p);
evac_gen = 0;
invariant->code = (StgClosure *)evacuate(invariant->code);
invariant->last_execution = (StgTRecHeader *)evacuate((StgClosure*)invariant->last_execution);
evac_gen = saved_evac_gen;
failed_to_evac = rtsTrue; // mutable
break;
}
case INVARIANT_CHECK_QUEUE:
{
StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p);
evac_gen = 0;
queue->invariant = (StgAtomicInvariant *)evacuate((StgClosure*)queue->invariant);
queue->my_execution = (StgTRecHeader *)evacuate((StgClosure*)queue->my_execution);
queue->next_queue_entry = (StgInvariantCheckQueue *)evacuate((StgClosure*)queue->next_queue_entry);