Compact.c 23.5 KB
Newer Older
1 2
/* -----------------------------------------------------------------------------
 *
3
 * (c) The GHC Team 2001-2008
4 5 6
 *
 * Compacting garbage collector
 *
7 8 9 10 11
 * Documentation on the architecture of the Garbage Collector can be
 * found in the online commentary:
 * 
 *   http://hackage.haskell.org/trac/ghc/wiki/Commentary/Rts/Storage/GC
 *
12 13
 * ---------------------------------------------------------------------------*/

14
#include "PosixSource.h"
15
#include "Rts.h"
Simon Marlow's avatar
Simon Marlow committed
16

Simon Marlow's avatar
Simon Marlow committed
17
#include "GCThread.h"
Simon Marlow's avatar
Simon Marlow committed
18
#include "Storage.h"
19 20
#include "RtsUtils.h"
#include "BlockAlloc.h"
21 22
#include "GC.h"
#include "Compact.h"
23
#include "Schedule.h"
24
#include "Apply.h"
Simon Marlow's avatar
Simon Marlow committed
25
#include "Trace.h"
Simon Marlow's avatar
Simon Marlow committed
26 27 28
#include "Weak.h"
#include "MarkWeak.h"
#include "Stable.h"
29

30 31
// Turn off inlining when debugging - it obfuscates things
#ifdef DEBUG
sof's avatar
sof committed
32 33
# undef  STATIC_INLINE
# define STATIC_INLINE static
34 35
#endif

36
/* ----------------------------------------------------------------------------
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
   Threading / unthreading pointers.

   The basic idea here is to chain together all the fields pointing at
   a particular object, with the root of the chain in the object's
   info table field.  The original contents of the info pointer goes
   at the end of the chain.

   Adding a new field to the chain is a matter of swapping the
   contents of the field with the contents of the object's info table
   field.

   To unthread the chain, we walk down it updating all the fields on
   the chain with the new location of the object.  We stop when we
   reach the info pointer at the end.

52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
   The main difficulty here is that we need to be able to identify the
   info pointer at the end of the chain.  We can't use the low bits of
   the pointer for this; they are already being used for
   pointer-tagging.  What's more, we need to retain the
   pointer-tagging tag bits on each pointer during the
   threading/unthreading process.

   Our solution is as follows: 
     - an info pointer (chain length zero) is identified by having tag 0
     - in a threaded chain of length > 0:
        - the pointer-tagging tag bits are attached to the info pointer
        - the first entry in the chain has tag 1
        - second and subsequent entries in the chain have tag 2

   This exploits the fact that the tag on each pointer to a given
   closure is normally the same (if they are not the same, then
   presumably the tag is not essential and it therefore doesn't matter
   if we throw away some of the tags).
   ------------------------------------------------------------------------- */
71

sof's avatar
sof committed
72
STATIC_INLINE void
73
thread (StgClosure **p)
74
{
75 76 77
    StgClosure *q0;
    StgPtr q;
    StgWord iptr;
78 79
    bdescr *bd;

80 81 82
    q0  = *p;
    q   = (StgPtr)UNTAG_CLOSURE(q0);

83 84 85
    // It doesn't look like a closure at the moment, because the info
    // ptr is possibly threaded:
    // ASSERT(LOOKS_LIKE_CLOSURE_PTR(q));
Simon Marlow's avatar
Simon Marlow committed
86
    
87
    if (HEAP_ALLOCED(q)) {
88
	bd = Bdescr(q); 
89

90
	if (bd->flags & BF_MARKED)
Simon Marlow's avatar
Simon Marlow committed
91
        {
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
            iptr = *q;
            switch (GET_CLOSURE_TAG((StgClosure *)iptr))
            {
            case 0: 
                // this is the info pointer; we are creating a new chain.
                // save the original tag at the end of the chain.
                *p = (StgClosure *)((StgWord)iptr + GET_CLOSURE_TAG(q0));
                *q = (StgWord)p + 1;
                break;
            case 1:
            case 2:
                // this is a chain of length 1 or more
                *p = (StgClosure *)iptr;
                *q = (StgWord)p + 2;
                break;
            }
        }
109 110 111
    }
}

112 113 114 115 116 117
static void
thread_root (void *user STG_UNUSED, StgClosure **p)
{
    thread(p);
}

118 119 120 121
// This version of thread() takes a (void *), used to circumvent
// warnings from gcc about pointer punning and strict aliasing.
STATIC_INLINE void thread_ (void *p) { thread((StgClosure **)p); }

sof's avatar
sof committed
122
STATIC_INLINE void
123
unthread( StgPtr p, StgWord free )
124
{
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
    StgWord q, r;
    StgPtr q0;

    q = *p;
loop:
    switch (GET_CLOSURE_TAG((StgClosure *)q))
    {
    case 0:
        // nothing to do; the chain is length zero
        return;
    case 1:
        q0 = (StgPtr)(q-1);
        r = *q0;  // r is the info ptr, tagged with the pointer-tag
        *q0 = free;
        *p = (StgWord)UNTAG_CLOSURE((StgClosure *)r);
        return;
    case 2:
        q0 = (StgPtr)(q-2);
        r = *q0;
        *q0 = free;
        q = r;
        goto loop;
    default:
        barf("unthread");
149 150 151
    }
}

152 153 154 155 156
// Traverse a threaded chain and pull out the info pointer at the end.
// The info pointer is also tagged with the appropriate pointer tag
// for this closure, which should be attached to the pointer
// subsequently passed to unthread().
STATIC_INLINE StgWord
157 158
get_threaded_info( StgPtr p )
{
159 160 161
    StgWord q;
    
    q = (W_)GET_INFO(UNTAG_CLOSURE((StgClosure *)p));
162

163 164 165 166 167 168 169 170 171
loop:
    switch (GET_CLOSURE_TAG((StgClosure *)q)) 
    {
    case 0:
        ASSERT(LOOKS_LIKE_INFO_PTR(q));
        return q;
    case 1:
    {
        StgWord r = *(StgPtr)(q-1);
Simon Marlow's avatar
Simon Marlow committed
172
        ASSERT(LOOKS_LIKE_INFO_PTR((StgWord)UNTAG_CLOSURE((StgClosure *)r)));
173 174 175 176 177 178 179
        return r;
    }
    case 2:
        q = *(StgPtr)(q-2);
        goto loop;
    default:
        barf("get_threaded_info");
180 181 182 183 184
    }
}

// A word-aligned memmove will be faster for small objects than libc's or gcc's.
// Remember, the two regions *might* overlap, but: to <= from.
sof's avatar
sof committed
185
STATIC_INLINE void
186
move(StgPtr to, StgPtr from, W_ size)
187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
{
    for(; size > 0; --size) {
	*to++ = *from++;
    }
}

static void
thread_static( StgClosure* p )
{
  const StgInfoTable *info;

  // keep going until we've threaded all the objects on the linked
  // list... 
  while (p != END_OF_STATIC_LIST) {

    info = get_itbl(p);
    switch (info->type) {
      
    case IND_STATIC:
206
	thread(&((StgInd *)p)->indirectee);
207
	p = *IND_STATIC_LINK(p);
208
	continue;
209 210
      
    case THUNK_STATIC:
211
	p = *THUNK_STATIC_LINK(p);
212
	continue;
213
    case FUN_STATIC:
214
	p = *FUN_STATIC_LINK(p);
215
	continue;
216
    case CONSTR_STATIC:
217
	p = *STATIC_LINK(info,p);
218
	continue;
219 220 221 222 223 224 225 226
      
    default:
	barf("thread_static: strange closure %d", (int)(info->type));
    }

  }
}

sof's avatar
sof committed
227
STATIC_INLINE void
228
thread_large_bitmap( StgPtr p, StgLargeBitmap *large_bitmap, W_ size )
229
{
230
    W_ i, b;
231 232 233 234 235 236
    StgWord bitmap;

    b = 0;
    bitmap = large_bitmap->bitmap[b];
    for (i = 0; i < size; ) {
	if ((bitmap & 1) == 0) {
237
	    thread((StgClosure **)p);
238 239 240 241 242 243 244 245 246 247 248 249
	}
	i++;
	p++;
	if (i % BITS_IN(W_) == 0) {
	    b++;
	    bitmap = large_bitmap->bitmap[b];
	} else {
	    bitmap = bitmap >> 1;
	}
    }
}

sof's avatar
sof committed
250
STATIC_INLINE StgPtr
251 252 253 254
thread_arg_block (StgFunInfoTable *fun_info, StgClosure **args)
{
    StgPtr p;
    StgWord bitmap;
255
    W_ size;
256 257

    p = (StgPtr)args;
258
    switch (fun_info->f.fun_type) {
259
    case ARG_GEN:
260 261
	bitmap = BITMAP_BITS(fun_info->f.b.bitmap);
	size = BITMAP_SIZE(fun_info->f.b.bitmap);
262 263
	goto small_bitmap;
    case ARG_GEN_BIG:
264 265
	size = GET_FUN_LARGE_BITMAP(fun_info)->size;
	thread_large_bitmap(p, GET_FUN_LARGE_BITMAP(fun_info), size);
266 267 268
	p += size;
	break;
    default:
269 270
	bitmap = BITMAP_BITS(stg_arg_bitmaps[fun_info->f.fun_type]);
	size = BITMAP_SIZE(stg_arg_bitmaps[fun_info->f.fun_type]);
271 272 273
    small_bitmap:
	while (size > 0) {
	    if ((bitmap & 1) == 0) {
274
		thread((StgClosure **)p);
275 276 277 278 279 280 281 282 283 284
	    }
	    p++;
	    bitmap = bitmap >> 1;
	    size--;
	}
	break;
    }
    return p;
}

285 286 287
static void
thread_stack(StgPtr p, StgPtr stack_end)
{
288
    const StgRetInfoTable* info;
ken's avatar
ken committed
289
    StgWord bitmap;
290
    W_ size;
291 292 293 294 295
    
    // highly similar to scavenge_stack, but we do pointer threading here.
    
    while (p < stack_end) {

296
	// *p must be the info pointer of an activation
297 298 299
	// record.  All activation records have 'bitmap' style layout
	// info.
	//
300
	info  = get_ret_itbl((StgClosure *)p);
301
	
302
	switch (info->i.type) {
303 304 305
	    
	    // Dynamic bitmap: the mask is stored on the stack 
	case RET_DYN:
306 307 308 309 310
	{
	    StgWord dyn;
	    dyn = ((StgRetDyn *)p)->liveness;

	    // traverse the bitmap first
311
	    bitmap = RET_DYN_LIVENESS(dyn);
312
	    p      = (P_)&((StgRetDyn *)p)->payload[0];
313
	    size   = RET_DYN_BITMAP_SIZE;
314 315
	    while (size > 0) {
		if ((bitmap & 1) == 0) {
316
		    thread((StgClosure **)p);
317 318 319 320 321
		}
		p++;
		bitmap = bitmap >> 1;
		size--;
	    }
322
	    
323
	    // skip over the non-ptr words
324
	    p += RET_DYN_NONPTRS(dyn) + RET_DYN_NONPTR_REGS_SIZE;
325 326
	    
	    // follow the ptr words
327
	    for (size = RET_DYN_PTRS(dyn); size > 0; size--) {
328
		thread((StgClosure **)p);
329 330
		p++;
	    }
331
	    continue;
332
	}
333
	    
ken's avatar
ken committed
334
	    // small bitmap (<= 32 entries, or 64 on a 64-bit machine) 
335 336 337
        case CATCH_RETRY_FRAME:
        case CATCH_STM_FRAME:
        case ATOMICALLY_FRAME:
338
	case UPDATE_FRAME:
339 340 341
        case UNDERFLOW_FRAME:
        case STOP_FRAME:
        case CATCH_FRAME:
342
	case RET_SMALL:
343 344
	    bitmap = BITMAP_BITS(info->i.layout.bitmap);
	    size   = BITMAP_SIZE(info->i.layout.bitmap);
345
	    p++;
346 347 348
	    // NOTE: the payload starts immediately after the info-ptr, we
	    // don't have an StgHeader in the same sense as a heap closure.
	    while (size > 0) {
349
		if ((bitmap & 1) == 0) {
350
		    thread((StgClosure **)p);
351 352 353
		}
		p++;
		bitmap = bitmap >> 1;
354
		size--;
355 356 357
	    }
	    continue;

358 359 360 361 362 363
	case RET_BCO: {
	    StgBCO *bco;
	    nat size;
	    
	    p++;
	    bco = (StgBCO *)*p;
364
	    thread((StgClosure **)p);
365 366 367 368 369 370 371
	    p++;
	    size = BCO_BITMAP_SIZE(bco);
	    thread_large_bitmap(p, BCO_BITMAP(bco), size);
	    p += size;
	    continue;
	}

ken's avatar
ken committed
372
	    // large bitmap (> 32 entries, or 64 on a 64-bit machine) 
373 374
	case RET_BIG:
	    p++;
375 376
	    size = GET_LARGE_BITMAP(&info->i)->size;
	    thread_large_bitmap(p, GET_LARGE_BITMAP(&info->i), size);
377 378
	    p += size;
	    continue;
379

380 381 382 383 384
	case RET_FUN:
	{
	    StgRetFun *ret_fun = (StgRetFun *)p;
	    StgFunInfoTable *fun_info;
	    
385
	    fun_info = FUN_INFO_PTR_TO_STRUCT((StgInfoTable *)UNTAG_CLOSURE((StgClosure *)
386
                           get_threaded_info((StgPtr)ret_fun->fun)));
387
	         // *before* threading it!
388
	    thread(&ret_fun->fun);
389
	    p = thread_arg_block(fun_info, ret_fun->payload);
390 391 392 393 394
	    continue;
	}

	default:
	    barf("thread_stack: weird activation record found on stack: %d", 
395
		 (int)(info->i.type));
396 397 398 399
	}
    }
}

sof's avatar
sof committed
400
STATIC_INLINE StgPtr
401
thread_PAP_payload (StgClosure *fun, StgClosure **payload, StgWord size)
402 403
{
    StgPtr p;
404
    StgWord bitmap;
405
    StgFunInfoTable *fun_info;
406

407
    fun_info = FUN_INFO_PTR_TO_STRUCT((StgInfoTable *)UNTAG_CLOSURE((StgClosure *)
408
                        get_threaded_info((StgPtr)fun)));
409 410
    ASSERT(fun_info->i.type != PAP);

411
    p = (StgPtr)payload;
412

413
    switch (fun_info->f.fun_type) {
414
    case ARG_GEN:
415
	bitmap = BITMAP_BITS(fun_info->f.b.bitmap);
416 417
	goto small_bitmap;
    case ARG_GEN_BIG:
418
	thread_large_bitmap(p, GET_FUN_LARGE_BITMAP(fun_info), size);
419 420 421
	p += size;
	break;
    case ARG_BCO:
422
	thread_large_bitmap((StgPtr)payload, BCO_BITMAP(fun), size);
423 424 425
	p += size;
	break;
    default:
426
	bitmap = BITMAP_BITS(stg_arg_bitmaps[fun_info->f.fun_type]);
427 428 429
    small_bitmap:
	while (size > 0) {
	    if ((bitmap & 1) == 0) {
430
		thread((StgClosure **)p);
431 432 433 434 435 436 437
	    }
	    p++;
	    bitmap = bitmap >> 1;
	    size--;
	}
	break;
    }
438

439 440 441 442 443 444 445 446
    return p;
}

STATIC_INLINE StgPtr
thread_PAP (StgPAP *pap)
{
    StgPtr p;
    p = thread_PAP_payload(pap->fun, pap->payload, pap->n_args);
447
    thread(&pap->fun);
448 449
    return p;
}
450 451 452 453 454 455
    
STATIC_INLINE StgPtr
thread_AP (StgAP *ap)
{
    StgPtr p;
    p = thread_PAP_payload(ap->fun, ap->payload, ap->n_args);
456
    thread(&ap->fun);
457 458
    return p;
}    
459

sof's avatar
sof committed
460
STATIC_INLINE StgPtr
461 462
thread_AP_STACK (StgAP_STACK *ap)
{
463
    thread(&ap->fun);
464 465 466 467 468 469 470
    thread_stack((P_)ap->payload, (P_)ap->payload + ap->size);
    return (P_)ap + sizeofW(StgAP_STACK) + ap->size;
}

static StgPtr
thread_TSO (StgTSO *tso)
{
471
    thread_(&tso->_link);
472
    thread_(&tso->global_link);
473 474 475

    if (   tso->why_blocked == BlockedOnMVar
	|| tso->why_blocked == BlockedOnBlackHole
476
	|| tso->why_blocked == BlockedOnMsgThrowTo
477 478
        || tso->why_blocked == NotBlocked
        ) {
479
	thread_(&tso->block_info.closure);
480
    }
481
    thread_(&tso->blocked_exceptions);
482
    thread_(&tso->bq);
483
    
484
    thread_(&tso->trec);
485

486 487
    thread_(&tso->stackobj);
    return (StgPtr)tso + sizeofW(StgTSO);
488 489 490
}


491 492 493 494 495 496 497 498
static void
update_fwd_large( bdescr *bd )
{
  StgPtr p;
  const StgInfoTable* info;

  for (; bd != NULL; bd = bd->link) {

Simon Marlow's avatar
Simon Marlow committed
499 500 501 502
    // nothing to do in a pinned block; it might not even have an object
    // at the beginning.
    if (bd->flags & BF_PINNED) continue;

503 504 505 506 507 508 509 510 511
    p = bd->start;
    info  = get_itbl((StgClosure *)p);

    switch (info->type) {

    case ARR_WORDS:
      // nothing to follow 
      continue;

512 513
    case MUT_ARR_PTRS_CLEAN:
    case MUT_ARR_PTRS_DIRTY:
514
    case MUT_ARR_PTRS_FROZEN:
515
    case MUT_ARR_PTRS_FROZEN0:
516 517
      // follow everything 
      {
518
          StgMutArrPtrs *a;
519

520 521 522 523 524
          a = (StgMutArrPtrs*)p;
          for (p = (P_)a->payload; p < (P_)&a->payload[a->ptrs]; p++) {
              thread((StgClosure **)p);
          }
          continue;
525 526
      }

527 528 529 530 531 532
    case STACK:
    {
        StgStack *stack = (StgStack*)p;
        thread_stack(stack->sp, stack->stack + stack->stack_size);
        continue;
    }
533 534 535

    case AP_STACK:
	thread_AP_STACK((StgAP_STACK *)p);
536 537 538
	continue;

    case PAP:
539
	thread_PAP((StgPAP *)p);
540 541
	continue;

542 543 544 545 546
    case TREC_CHUNK:
    {
        StgWord i;
        StgTRecChunk *tc = (StgTRecChunk *)p;
	TRecEntry *e = &(tc -> entries[0]);
547
	thread_(&tc->prev_chunk);
548
	for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
549 550 551
	  thread_(&e->tvar);
	  thread(&e->expected_value);
	  thread(&e->new_value);
552 553 554 555
	}
	continue;
    }

556 557 558 559 560 561
    default:
      barf("update_fwd_large: unknown/strange object  %d", (int)(info->type));
    }
  }
}

562 563
// ToDo: too big to inline
static /* STATIC_INLINE */ StgPtr
564 565 566
thread_obj (StgInfoTable *info, StgPtr p)
{
    switch (info->type) {
567 568 569
    case THUNK_0_1:
	return p + sizeofW(StgThunk) + 1;

570 571 572 573 574 575
    case FUN_0_1:
    case CONSTR_0_1:
	return p + sizeofW(StgHeader) + 1;
	
    case FUN_1_0:
    case CONSTR_1_0:
576
	thread(&((StgClosure *)p)->payload[0]);
577 578 579
	return p + sizeofW(StgHeader) + 1;
	
    case THUNK_1_0:
580
	thread(&((StgThunk *)p)->payload[0]);
581
	return p + sizeofW(StgThunk) + 1;
582 583
	
    case THUNK_0_2:
584 585
	return p + sizeofW(StgThunk) + 2;

586 587 588 589 590
    case FUN_0_2:
    case CONSTR_0_2:
	return p + sizeofW(StgHeader) + 2;
	
    case THUNK_1_1:
591
	thread(&((StgThunk *)p)->payload[0]);
592 593
	return p + sizeofW(StgThunk) + 2;

594 595
    case FUN_1_1:
    case CONSTR_1_1:
596
	thread(&((StgClosure *)p)->payload[0]);
597 598 599
	return p + sizeofW(StgHeader) + 2;
	
    case THUNK_2_0:
600 601
	thread(&((StgThunk *)p)->payload[0]);
	thread(&((StgThunk *)p)->payload[1]);
602 603
	return p + sizeofW(StgThunk) + 2;

604 605
    case FUN_2_0:
    case CONSTR_2_0:
606 607
	thread(&((StgClosure *)p)->payload[0]);
	thread(&((StgClosure *)p)->payload[1]);
608 609
	return p + sizeofW(StgHeader) + 2;
	
610 611
    case BCO: {
	StgBCO *bco = (StgBCO *)p;
612 613 614
	thread_(&bco->instrs);
	thread_(&bco->literals);
	thread_(&bco->ptrs);
615 616 617
	return p + bco_sizeW(bco);
    }

618
    case THUNK:
619 620 621 622 623 624
    {
	StgPtr end;
	
	end = (P_)((StgThunk *)p)->payload + 
	    info->layout.payload.ptrs;
	for (p = (P_)((StgThunk *)p)->payload; p < end; p++) {
625
	    thread((StgClosure **)p);
626 627 628 629 630
	}
	return p + info->layout.payload.nptrs;
    }

    case FUN:
631
    case CONSTR:
632 633
    case PRIM:
    case MUT_PRIM:
634 635
    case MUT_VAR_CLEAN:
    case MUT_VAR_DIRTY:
636
    case BLACKHOLE:
637
    case BLOCKING_QUEUE:
638 639 640 641 642 643
    {
	StgPtr end;
	
	end = (P_)((StgClosure *)p)->payload + 
	    info->layout.payload.ptrs;
	for (p = (P_)((StgClosure *)p)->payload; p < end; p++) {
644
	    thread((StgClosure **)p);
645 646 647 648 649 650 651
	}
	return p + info->layout.payload.nptrs;
    }
    
    case WEAK:
    {
	StgWeak *w = (StgWeak *)p;
652
	thread(&w->cfinalizer);
653 654 655
	thread(&w->key);
	thread(&w->value);
	thread(&w->finalizer);
656
	if (w->link != NULL) {
657
	    thread_(&w->link);
658 659 660 661
	}
	return p + sizeofW(StgWeak);
    }
    
662 663
    case MVAR_CLEAN:
    case MVAR_DIRTY:
664 665
    { 
	StgMVar *mvar = (StgMVar *)p;
666 667 668
	thread_(&mvar->head);
	thread_(&mvar->tail);
	thread(&mvar->value);
669 670 671
	return p + sizeofW(StgMVar);
    }
    
672 673
    case IND:
    case IND_PERM:
674
	thread(&((StgInd *)p)->indirectee);
675
	return p + sizeofW(StgInd);
676 677 678 679

    case THUNK_SELECTOR:
    { 
	StgSelector *s = (StgSelector *)p;
680
	thread(&s->selectee);
681 682 683 684 685 686 687 688
	return p + THUNK_SELECTOR_sizeW();
    }
    
    case AP_STACK:
	return thread_AP_STACK((StgAP_STACK *)p);
	
    case PAP:
	return thread_PAP((StgPAP *)p);
689 690 691

    case AP:
	return thread_AP((StgAP *)p);
692 693 694 695
	
    case ARR_WORDS:
	return p + arr_words_sizeW((StgArrWords *)p);
	
696 697
    case MUT_ARR_PTRS_CLEAN:
    case MUT_ARR_PTRS_DIRTY:
698
    case MUT_ARR_PTRS_FROZEN:
699
    case MUT_ARR_PTRS_FROZEN0:
700 701
	// follow everything 
    {
702 703 704 705
        StgMutArrPtrs *a;

        a = (StgMutArrPtrs *)p;
	for (p = (P_)a->payload; p < (P_)&a->payload[a->ptrs]; p++) {
706
	    thread((StgClosure **)p);
707
	}
708 709

	return (StgPtr)a + mut_arr_ptrs_sizeW(a);
710 711 712 713 714
    }
    
    case TSO:
	return thread_TSO((StgTSO *)p);
    
715 716 717 718 719 720 721
    case STACK:
    {
        StgStack *stack = (StgStack*)p;
        thread_stack(stack->sp, stack->stack + stack->stack_size);
        return p + stack_sizeW(stack);
    }

722 723
    case TREC_CHUNK:
    {
724
        StgWord i;
725 726
        StgTRecChunk *tc = (StgTRecChunk *)p;
	TRecEntry *e = &(tc -> entries[0]);
727
	thread_(&tc->prev_chunk);
728
	for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
729 730 731
	  thread_(&e->tvar);
	  thread(&e->expected_value);
	  thread(&e->new_value);
732 733 734 735
	}
	return p + sizeofW(StgTRecChunk);
    }

736 737
    default:
	barf("update_fwd: unknown/strange object  %d", (int)(info->type));
sof's avatar
sof committed
738
	return NULL;
739 740 741
    }
}

742 743 744 745 746 747 748 749 750 751 752 753 754 755 756
static void
update_fwd( bdescr *blocks )
{
    StgPtr p;
    bdescr *bd;
    StgInfoTable *info;

    bd = blocks;

    // cycle through all the blocks in the step
    for (; bd != NULL; bd = bd->link) {
	p = bd->start;

	// linearly scan the objects in this block
	while (p < bd->free) {
757
	    ASSERT(LOOKS_LIKE_CLOSURE_PTR(p));
758
	    info = get_itbl((StgClosure *)p);
759
	    p = thread_obj(info, p);
760 761 762 763 764 765 766 767
	}
    }
} 

static void
update_fwd_compact( bdescr *blocks )
{
    StgPtr p, q, free;
768
#if 0
769
    StgWord m;
770
#endif
771 772 773
    bdescr *bd, *free_bd;
    StgInfoTable *info;
    nat size;
774
    StgWord iptr;
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815

    bd = blocks;
    free_bd = blocks;
    free = free_bd->start;

    // cycle through all the blocks in the step
    for (; bd != NULL; bd = bd->link) {
	p = bd->start;

	while (p < bd->free ) {

	    while ( p < bd->free && !is_marked(p,bd) ) {
		p++;
	    }
	    if (p >= bd->free) {
		break;
	    }

#if 0
    next:
	m = * ((StgPtr)bd->u.bitmap + ((p - bd->start) / (BITS_IN(StgWord))));
	m >>= ((p - bd->start) & (BITS_IN(StgWord) - 1));

	while ( p < bd->free ) {

	    if ((m & 1) == 0) {
		m >>= 1;
		p++;
		if (((StgWord)p & (sizeof(W_) * BITS_IN(StgWord))) == 0) {
		    goto next;
		} else {
		    continue;
		}
	    }
#endif

	    // Problem: we need to know the destination for this cell
	    // in order to unthread its info pointer.  But we can't
	    // know the destination without the size, because we may
	    // spill into the next block.  So we have to run down the 
	    // threaded list and get the info ptr first.
816 817 818 819
            //
            // ToDo: one possible avenue of attack is to use the fact
            // that if (p&BLOCK_MASK) >= (free&BLOCK_MASK), then we
            // definitely have enough room.  Also see bug #1147.
820
            iptr = get_threaded_info(p);
821
	    info = INFO_PTR_TO_STRUCT((StgInfoTable *)UNTAG_CLOSURE((StgClosure *)iptr));
822 823 824

	    q = p;

825
	    p = thread_obj(info, p);
826 827 828

	    size = p - q;
	    if (free + size > free_bd->start + BLOCK_SIZE_W) {
829
		// set the next bit in the bitmap to indicate that
830 831 832
		// this object needs to be pushed into the next
		// block.  This saves us having to run down the
		// threaded info pointer list twice during the next pass.
833
		mark(q+1,bd);
834 835
		free_bd = free_bd->link;
		free = free_bd->start;
836
	    } else {
837
		ASSERT(!is_marked(q+1,bd));
838 839
	    }

840
	    unthread(q,(StgWord)free + GET_CLOSURE_TAG((StgClosure *)iptr));
841 842 843 844 845 846 847 848
	    free += size;
#if 0
	    goto next;
#endif
	}
    }
}

849
static W_
850
update_bkwd_compact( generation *gen )
851 852
{
    StgPtr p, free;
853
#if 0
854
    StgWord m;
855
#endif
856 857
    bdescr *bd, *free_bd;
    StgInfoTable *info;
858
    W_ size, free_blocks;
859
    StgWord iptr;
860

861
    bd = free_bd = gen->old_blocks;
862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
    free = free_bd->start;
    free_blocks = 1;

    // cycle through all the blocks in the step
    for (; bd != NULL; bd = bd->link) {
	p = bd->start;

	while (p < bd->free ) {

	    while ( p < bd->free && !is_marked(p,bd) ) {
		p++;
	    }
	    if (p >= bd->free) {
		break;
	    }

#if 0
    next:
	m = * ((StgPtr)bd->u.bitmap + ((p - bd->start) / (BITS_IN(StgWord))));
	m >>= ((p - bd->start) & (BITS_IN(StgWord) - 1));

	while ( p < bd->free ) {

	    if ((m & 1) == 0) {
		m >>= 1;
		p++;
		if (((StgWord)p & (sizeof(W_) * BITS_IN(StgWord))) == 0) {
		    goto next;
		} else {
		    continue;
		}
	    }
#endif

896
	    if (is_marked(p+1,bd)) {
897 898 899 900 901 902 903
		// don't forget to update the free ptr in the block desc.
		free_bd->free = free;
		free_bd = free_bd->link;
		free = free_bd->start;
		free_blocks++;
	    }

904 905
            iptr = get_threaded_info(p);
	    unthread(p, (StgWord)free + GET_CLOSURE_TAG((StgClosure *)iptr));
Simon Marlow's avatar
Simon Marlow committed
906
	    ASSERT(LOOKS_LIKE_INFO_PTR((StgWord)((StgClosure *)p)->header.info));
907
	    info = get_itbl((StgClosure *)p);
908
	    size = closure_sizeW_((StgClosure *)p,info);
909

910 911 912
	    if (free != p) {
		move(free,p,size);
	    }
913 914

	    // relocate TSOs
915 916
            if (info->type == STACK) {
                move_STACK((StgStack *)p, (StgStack *)free);
917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
	    }

	    free += size;
	    p += size;
#if 0
	    goto next;
#endif
	}
    }

    // free the remaining blocks and count what's left.
    free_bd->free = free;
    if (free_bd->link != NULL) {
	freeChain(free_bd->link);
	free_bd->link = NULL;
    }

    return free_blocks;
935 936
}

937
void
938
compact(StgClosure *static_objects)
939
{
940
    W_ n, g, blocks;
941
    generation *gen;
942 943

    // 1. thread the roots
944
    markCapabilities((evac_fn)thread_root, NULL);
945

Simon Marlow's avatar
Simon Marlow committed
946 947
    markScheduler((evac_fn)thread_root, NULL);

948 949
    // the weak pointer lists...
    if (weak_ptr_list != NULL) {
950
	thread((void *)&weak_ptr_list);
951 952
    }
    if (old_weak_ptr_list != NULL) {
953
	thread((void *)&old_weak_ptr_list); // tmp
954 955
    }

956 957
    // mutable lists
    for (g = 1; g < RtsFlags.GcFlags.generations; g++) {
958 959
	bdescr *bd;
	StgPtr p;
960 961 962 963 964 965 966 967
        for (n = 0; n < n_capabilities; n++) {
            for (bd = capabilities[n].mut_lists[g]; 
                 bd != NULL; bd = bd->link) {
                for (p = bd->start; p < bd->free; p++) {
                    thread((StgClosure **)p);
                }
            }
        }
968
    }
969 970

    // the global thread list
971 972
    for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
        thread((void *)&generations[g].threads);
973
    }
974

975
    // any threads resurrected during this GC
976
    thread((void *)&resurrected_threads);
977

978
    // the task list
979
    {
980
	Task *task;
981
        InCall *incall;
982
        for (task = all_tasks; task != NULL; task = task->all_next) {
983 984 985 986 987 988
            for (incall = task->incall; incall != NULL; 
                 incall = incall->prev_stack) {
                if (incall->tso) {
                    thread_(&incall->tso);
                }
            }
989 990 991
	}
    }

992
    // the static objects
993
    thread_static(static_objects /* ToDo: ok? */);
994 995

    // the stable pointer table
996
    threadStablePtrTable((evac_fn)thread_root, NULL);
997

998
    // the CAF list (used by GHCi)
999
    markCAFs((evac_fn)thread_root, NULL);
1000

1001 1002
    // 2. update forward ptrs
    for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
1003 1004 1005 1006
        gen = &generations[g];
        debugTrace(DEBUG_gc, "update_fwd:  %d", g);

        update_fwd(gen->blocks);
Simon Marlow's avatar
Simon Marlow committed
1007 1008 1009 1010
        for (n = 0; n < n_capabilities; n++) {
            update_fwd(gc_threads[n]->gens[g].todo_bd);
            update_fwd(gc_threads[n]->gens[g].part_list);
        }
1011 1012 1013 1014
        update_fwd_large(gen->scavenged_large_objects);
        if (g == RtsFlags.GcFlags.generations-1 && gen->old_blocks != NULL) {
            debugTrace(DEBUG_gc, "update_fwd:  %d (compact)", g);
            update_fwd_compact(gen->old_blocks);
1015 1016 1017 1018
	}
    }

    // 3. update backward ptrs
1019 1020 1021
    gen = oldest_gen;
    if (gen->old_blocks != NULL) {
	blocks = update_bkwd_compact(gen);
Simon Marlow's avatar
Simon Marlow committed
1022
	debugTrace(DEBUG_gc, 
1023 1024 1025
		   "update_bkwd: %d (compact, old: %d blocks, now %d blocks)",
		   gen->no, gen->n_old_blocks, blocks);
	gen->n_old_blocks = blocks;
1026 1027
    }
}