Compact.c 26.1 KB
Newer Older
1 2
/* -----------------------------------------------------------------------------
 *
3
 * (c) The GHC Team 2001-2008
4 5 6
 *
 * Compacting garbage collector
 *
7 8
 * Documentation on the architecture of the Garbage Collector can be
 * found in the online commentary:
9
 *
10
 *   http://ghc.haskell.org/trac/ghc/wiki/Commentary/Rts/Storage/GC
11
 *
12 13
 * ---------------------------------------------------------------------------*/

14
#include "PosixSource.h"
15
#include "Rts.h"
16

Simon Marlow's avatar
Simon Marlow committed
17
#include "GCThread.h"
18
#include "Storage.h"
19 20
#include "RtsUtils.h"
#include "BlockAlloc.h"
21 22
#include "GC.h"
#include "Compact.h"
23
#include "Schedule.h"
24
#include "Apply.h"
Simon Marlow's avatar
Simon Marlow committed
25
#include "Trace.h"
26 27 28
#include "Weak.h"
#include "MarkWeak.h"
#include "Stable.h"
29

30 31
// Turn off inlining when debugging - it obfuscates things
#ifdef DEBUG
32 33
# undef  STATIC_INLINE
# define STATIC_INLINE static
34 35
#endif

36
/* ----------------------------------------------------------------------------
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
   Threading / unthreading pointers.

   The basic idea here is to chain together all the fields pointing at
   a particular object, with the root of the chain in the object's
   info table field.  The original contents of the info pointer goes
   at the end of the chain.

   Adding a new field to the chain is a matter of swapping the
   contents of the field with the contents of the object's info table
   field.

   To unthread the chain, we walk down it updating all the fields on
   the chain with the new location of the object.  We stop when we
   reach the info pointer at the end.

52 53 54 55 56 57 58
   The main difficulty here is that we need to be able to identify the
   info pointer at the end of the chain.  We can't use the low bits of
   the pointer for this; they are already being used for
   pointer-tagging.  What's more, we need to retain the
   pointer-tagging tag bits on each pointer during the
   threading/unthreading process.

59
   Our solution is as follows:
60 61 62 63 64 65 66 67 68 69 70
     - an info pointer (chain length zero) is identified by having tag 0
     - in a threaded chain of length > 0:
        - the pointer-tagging tag bits are attached to the info pointer
        - the first entry in the chain has tag 1
        - second and subsequent entries in the chain have tag 2

   This exploits the fact that the tag on each pointer to a given
   closure is normally the same (if they are not the same, then
   presumably the tag is not essential and it therefore doesn't matter
   if we throw away some of the tags).
   ------------------------------------------------------------------------- */
71

72
STATIC_INLINE void
73
thread (StgClosure **p)
74
{
75 76 77
    StgClosure *q0;
    StgPtr q;
    StgWord iptr;
78 79
    bdescr *bd;

80 81 82
    q0  = *p;
    q   = (StgPtr)UNTAG_CLOSURE(q0);

83 84 85
    // It doesn't look like a closure at the moment, because the info
    // ptr is possibly threaded:
    // ASSERT(LOOKS_LIKE_CLOSURE_PTR(q));
86

87
    if (HEAP_ALLOCED(q)) {
88
        bd = Bdescr(q);
89

90
        if (bd->flags & BF_MARKED)
Simon Marlow's avatar
Simon Marlow committed
91
        {
92 93 94
            iptr = *q;
            switch (GET_CLOSURE_TAG((StgClosure *)iptr))
            {
95
            case 0:
96 97 98 99 100 101 102 103 104 105 106 107 108
                // this is the info pointer; we are creating a new chain.
                // save the original tag at the end of the chain.
                *p = (StgClosure *)((StgWord)iptr + GET_CLOSURE_TAG(q0));
                *q = (StgWord)p + 1;
                break;
            case 1:
            case 2:
                // this is a chain of length 1 or more
                *p = (StgClosure *)iptr;
                *q = (StgWord)p + 2;
                break;
            }
        }
109 110 111
    }
}

112 113 114 115 116 117
static void
thread_root (void *user STG_UNUSED, StgClosure **p)
{
    thread(p);
}

118 119 120 121
// This version of thread() takes a (void *), used to circumvent
// warnings from gcc about pointer punning and strict aliasing.
STATIC_INLINE void thread_ (void *p) { thread((StgClosure **)p); }

122
STATIC_INLINE void
123
unthread( StgPtr p, StgWord free )
124
{
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
    StgWord q, r;
    StgPtr q0;

    q = *p;
loop:
    switch (GET_CLOSURE_TAG((StgClosure *)q))
    {
    case 0:
        // nothing to do; the chain is length zero
        return;
    case 1:
        q0 = (StgPtr)(q-1);
        r = *q0;  // r is the info ptr, tagged with the pointer-tag
        *q0 = free;
        *p = (StgWord)UNTAG_CLOSURE((StgClosure *)r);
        return;
    case 2:
        q0 = (StgPtr)(q-2);
        r = *q0;
        *q0 = free;
        q = r;
        goto loop;
    default:
        barf("unthread");
149 150 151
    }
}

152 153 154 155 156
// Traverse a threaded chain and pull out the info pointer at the end.
// The info pointer is also tagged with the appropriate pointer tag
// for this closure, which should be attached to the pointer
// subsequently passed to unthread().
STATIC_INLINE StgWord
157 158
get_threaded_info( StgPtr p )
{
159
    StgWord q;
160

161
    q = (W_)GET_INFO(UNTAG_CLOSURE((StgClosure *)p));
162

163
loop:
164
    switch (GET_CLOSURE_TAG((StgClosure *)q))
165 166 167 168 169 170 171
    {
    case 0:
        ASSERT(LOOKS_LIKE_INFO_PTR(q));
        return q;
    case 1:
    {
        StgWord r = *(StgPtr)(q-1);
172 173
        ASSERT(LOOKS_LIKE_INFO_PTR((StgWord)
               UNTAG_CONST_CLOSURE((StgClosure *)r)));
174 175 176 177 178 179 180
        return r;
    }
    case 2:
        q = *(StgPtr)(q-2);
        goto loop;
    default:
        barf("get_threaded_info");
181 182 183 184 185
    }
}

// A word-aligned memmove will be faster for small objects than libc's or gcc's.
// Remember, the two regions *might* overlap, but: to <= from.
186
STATIC_INLINE void
187
move(StgPtr to, StgPtr from, StgWord size)
188 189
{
    for(; size > 0; --size) {
190
        *to++ = *from++;
191 192 193 194 195 196 197 198 199
    }
}

static void
thread_static( StgClosure* p )
{
  const StgInfoTable *info;

  // keep going until we've threaded all the objects on the linked
200
  // list...
201 202
  while (p != END_OF_STATIC_OBJECT_LIST) {
    p = UNTAG_STATIC_LIST_PTR(p);
203 204
    info = get_itbl(p);
    switch (info->type) {
205

206
    case IND_STATIC:
207 208 209 210
        thread(&((StgInd *)p)->indirectee);
        p = *IND_STATIC_LINK(p);
        continue;

211
    case THUNK_STATIC:
212 213
        p = *THUNK_STATIC_LINK(p);
        continue;
214
    case FUN_STATIC:
215 216
        p = *FUN_STATIC_LINK(p);
        continue;
217
    case CONSTR_STATIC:
218 219 220
        p = *STATIC_LINK(info,p);
        continue;

221
    default:
222
        barf("thread_static: strange closure %d", (int)(info->type));
223 224 225 226 227
    }

  }
}

228
STATIC_INLINE void
229
thread_large_bitmap( StgPtr p, StgLargeBitmap *large_bitmap, StgWord size )
230
{
231
    W_ i, b;
232 233 234 235 236
    StgWord bitmap;

    b = 0;
    bitmap = large_bitmap->bitmap[b];
    for (i = 0; i < size; ) {
237 238 239 240 241 242 243 244 245 246 247
        if ((bitmap & 1) == 0) {
            thread((StgClosure **)p);
        }
        i++;
        p++;
        if (i % BITS_IN(W_) == 0) {
            b++;
            bitmap = large_bitmap->bitmap[b];
        } else {
            bitmap = bitmap >> 1;
        }
248 249 250
    }
}

251 252 253 254 255 256 257 258 259 260 261 262 263 264
STATIC_INLINE StgPtr
thread_small_bitmap (StgPtr p, StgWord size, StgWord bitmap)
{
    while (size > 0) {
        if ((bitmap & 1) == 0) {
            thread((StgClosure **)p);
        }
        p++;
        bitmap = bitmap >> 1;
        size--;
    }
    return p;
}

265
STATIC_INLINE StgPtr
266 267 268 269
thread_arg_block (StgFunInfoTable *fun_info, StgClosure **args)
{
    StgPtr p;
    StgWord bitmap;
270
    StgWord size;
271 272

    p = (StgPtr)args;
273
    switch (fun_info->f.fun_type) {
274
    case ARG_GEN:
275 276 277
        bitmap = BITMAP_BITS(fun_info->f.b.bitmap);
        size = BITMAP_SIZE(fun_info->f.b.bitmap);
        goto small_bitmap;
278
    case ARG_GEN_BIG:
279 280 281 282
        size = GET_FUN_LARGE_BITMAP(fun_info)->size;
        thread_large_bitmap(p, GET_FUN_LARGE_BITMAP(fun_info), size);
        p += size;
        break;
283
    default:
284 285
        bitmap = BITMAP_BITS(stg_arg_bitmaps[fun_info->f.fun_type]);
        size = BITMAP_SIZE(stg_arg_bitmaps[fun_info->f.fun_type]);
286
    small_bitmap:
287
        p = thread_small_bitmap(p, size, bitmap);
288
        break;
289 290 291 292
    }
    return p;
}

293 294 295
static void
thread_stack(StgPtr p, StgPtr stack_end)
{
296
    const StgRetInfoTable* info;
297
    StgWord bitmap;
298
    StgWord size;
299

300
    // highly similar to scavenge_stack, but we do pointer threading here.
301

302 303
    while (p < stack_end) {

304 305 306 307 308 309 310 311
        // *p must be the info pointer of an activation
        // record.  All activation records have 'bitmap' style layout
        // info.
        //
        info  = get_ret_itbl((StgClosure *)p);

        switch (info->i.type) {

312
            // small bitmap (<= 32 entries, or 64 on a 64-bit machine)
313 314 315
        case CATCH_RETRY_FRAME:
        case CATCH_STM_FRAME:
        case ATOMICALLY_FRAME:
316
        case UPDATE_FRAME:
317 318 319
        case UNDERFLOW_FRAME:
        case STOP_FRAME:
        case CATCH_FRAME:
320 321 322 323 324 325
        case RET_SMALL:
            bitmap = BITMAP_BITS(info->i.layout.bitmap);
            size   = BITMAP_SIZE(info->i.layout.bitmap);
            p++;
            // NOTE: the payload starts immediately after the info-ptr, we
            // don't have an StgHeader in the same sense as a heap closure.
326
            p = thread_small_bitmap(p, size, bitmap);
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
            continue;

        case RET_BCO: {
            StgBCO *bco;

            p++;
            bco = (StgBCO *)*p;
            thread((StgClosure **)p);
            p++;
            size = BCO_BITMAP_SIZE(bco);
            thread_large_bitmap(p, BCO_BITMAP(bco), size);
            p += size;
            continue;
        }

            // large bitmap (> 32 entries, or 64 on a 64-bit machine)
        case RET_BIG:
            p++;
            size = GET_LARGE_BITMAP(&info->i)->size;
            thread_large_bitmap(p, GET_LARGE_BITMAP(&info->i), size);
            p += size;
            continue;

        case RET_FUN:
        {
            StgRetFun *ret_fun = (StgRetFun *)p;
            StgFunInfoTable *fun_info;

            fun_info = FUN_INFO_PTR_TO_STRUCT((StgInfoTable *)UNTAG_CLOSURE((StgClosure *)
356
                           get_threaded_info((StgPtr)ret_fun->fun)));
357 358 359 360 361 362 363 364 365 366
                 // *before* threading it!
            thread(&ret_fun->fun);
            p = thread_arg_block(fun_info, ret_fun->payload);
            continue;
        }

        default:
            barf("thread_stack: weird activation record found on stack: %d",
                 (int)(info->i.type));
        }
367 368 369
    }
}

370
STATIC_INLINE StgPtr
371
thread_PAP_payload (StgClosure *fun, StgClosure **payload, StgWord size)
372 373
{
    StgPtr p;
374
    StgWord bitmap;
375
    StgFunInfoTable *fun_info;
376

377
    fun_info = FUN_INFO_PTR_TO_STRUCT((StgInfoTable *)UNTAG_CLOSURE((StgClosure *)
378
                        get_threaded_info((StgPtr)fun)));
379 380
    ASSERT(fun_info->i.type != PAP);

381
    p = (StgPtr)payload;
382

383
    switch (fun_info->f.fun_type) {
384
    case ARG_GEN:
385 386
        bitmap = BITMAP_BITS(fun_info->f.b.bitmap);
        goto small_bitmap;
387
    case ARG_GEN_BIG:
388 389 390
        thread_large_bitmap(p, GET_FUN_LARGE_BITMAP(fun_info), size);
        p += size;
        break;
391
    case ARG_BCO:
392 393 394
        thread_large_bitmap((StgPtr)payload, BCO_BITMAP(fun), size);
        p += size;
        break;
395
    default:
396
        bitmap = BITMAP_BITS(stg_arg_bitmaps[fun_info->f.fun_type]);
397
    small_bitmap:
398
        p = thread_small_bitmap(p, size, bitmap);
399
        break;
400
    }
401

402 403 404 405 406 407 408 409
    return p;
}

STATIC_INLINE StgPtr
thread_PAP (StgPAP *pap)
{
    StgPtr p;
    p = thread_PAP_payload(pap->fun, pap->payload, pap->n_args);
410
    thread(&pap->fun);
411 412
    return p;
}
413

414 415 416 417 418
STATIC_INLINE StgPtr
thread_AP (StgAP *ap)
{
    StgPtr p;
    p = thread_PAP_payload(ap->fun, ap->payload, ap->n_args);
419
    thread(&ap->fun);
420
    return p;
421
}
422

423
STATIC_INLINE StgPtr
424 425
thread_AP_STACK (StgAP_STACK *ap)
{
426
    thread(&ap->fun);
427 428 429 430 431 432 433
    thread_stack((P_)ap->payload, (P_)ap->payload + ap->size);
    return (P_)ap + sizeofW(StgAP_STACK) + ap->size;
}

static StgPtr
thread_TSO (StgTSO *tso)
{
434
    thread_(&tso->_link);
435
    thread_(&tso->global_link);
436 437

    if (   tso->why_blocked == BlockedOnMVar
438
        || tso->why_blocked == BlockedOnMVarRead
439 440
        || tso->why_blocked == BlockedOnBlackHole
        || tso->why_blocked == BlockedOnMsgThrowTo
441 442
        || tso->why_blocked == NotBlocked
        ) {
443
        thread_(&tso->block_info.closure);
444
    }
445
    thread_(&tso->blocked_exceptions);
446
    thread_(&tso->bq);
447

448
    thread_(&tso->trec);
449

450 451
    thread_(&tso->stackobj);
    return (StgPtr)tso + sizeofW(StgTSO);
452 453 454
}


455 456 457 458 459 460 461 462
static void
update_fwd_large( bdescr *bd )
{
  StgPtr p;
  const StgInfoTable* info;

  for (; bd != NULL; bd = bd->link) {

Simon Marlow's avatar
Simon Marlow committed
463 464 465 466
    // nothing to do in a pinned block; it might not even have an object
    // at the beginning.
    if (bd->flags & BF_PINNED) continue;

467 468 469 470 471 472
    p = bd->start;
    info  = get_itbl((StgClosure *)p);

    switch (info->type) {

    case ARR_WORDS:
gcampax's avatar
gcampax committed
473
    case COMPACT_NFDATA:
474
      // nothing to follow
475 476
      continue;

477 478
    case MUT_ARR_PTRS_CLEAN:
    case MUT_ARR_PTRS_DIRTY:
479
    case MUT_ARR_PTRS_FROZEN:
480
    case MUT_ARR_PTRS_FROZEN0:
481
      // follow everything
482
      {
483
          StgMutArrPtrs *a;
484

485 486 487 488 489
          a = (StgMutArrPtrs*)p;
          for (p = (P_)a->payload; p < (P_)&a->payload[a->ptrs]; p++) {
              thread((StgClosure **)p);
          }
          continue;
490 491
      }

492 493 494 495
    case SMALL_MUT_ARR_PTRS_CLEAN:
    case SMALL_MUT_ARR_PTRS_DIRTY:
    case SMALL_MUT_ARR_PTRS_FROZEN:
    case SMALL_MUT_ARR_PTRS_FROZEN0:
496
      // follow everything
497 498 499 500 501 502 503 504 505 506
      {
          StgSmallMutArrPtrs *a;

          a = (StgSmallMutArrPtrs*)p;
          for (p = (P_)a->payload; p < (P_)&a->payload[a->ptrs]; p++) {
              thread((StgClosure **)p);
          }
          continue;
      }

507 508 509 510 511 512
    case STACK:
    {
        StgStack *stack = (StgStack*)p;
        thread_stack(stack->sp, stack->stack + stack->stack_size);
        continue;
    }
513 514

    case AP_STACK:
515 516
        thread_AP_STACK((StgAP_STACK *)p);
        continue;
517 518

    case PAP:
519 520
        thread_PAP((StgPAP *)p);
        continue;
521

522 523 524 525
    case TREC_CHUNK:
    {
        StgWord i;
        StgTRecChunk *tc = (StgTRecChunk *)p;
526 527 528 529 530 531 532 533
        TRecEntry *e = &(tc -> entries[0]);
        thread_(&tc->prev_chunk);
        for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
          thread_(&e->tvar);
          thread(&e->expected_value);
          thread(&e->new_value);
        }
        continue;
534 535
    }

536 537 538 539 540 541
    default:
      barf("update_fwd_large: unknown/strange object  %d", (int)(info->type));
    }
  }
}

542 543
// ToDo: too big to inline
static /* STATIC_INLINE */ StgPtr
544
thread_obj (const StgInfoTable *info, StgPtr p)
545 546
{
    switch (info->type) {
547
    case THUNK_0_1:
548
        return p + sizeofW(StgThunk) + 1;
549

550 551
    case FUN_0_1:
    case CONSTR_0_1:
552 553
        return p + sizeofW(StgHeader) + 1;

554 555
    case FUN_1_0:
    case CONSTR_1_0:
556 557 558
        thread(&((StgClosure *)p)->payload[0]);
        return p + sizeofW(StgHeader) + 1;

559
    case THUNK_1_0:
560 561 562
        thread(&((StgThunk *)p)->payload[0]);
        return p + sizeofW(StgThunk) + 1;

563
    case THUNK_0_2:
564
        return p + sizeofW(StgThunk) + 2;
565

566 567
    case FUN_0_2:
    case CONSTR_0_2:
568 569
        return p + sizeofW(StgHeader) + 2;

570
    case THUNK_1_1:
571 572
        thread(&((StgThunk *)p)->payload[0]);
        return p + sizeofW(StgThunk) + 2;
573

574 575
    case FUN_1_1:
    case CONSTR_1_1:
576 577 578
        thread(&((StgClosure *)p)->payload[0]);
        return p + sizeofW(StgHeader) + 2;

579
    case THUNK_2_0:
580 581 582
        thread(&((StgThunk *)p)->payload[0]);
        thread(&((StgThunk *)p)->payload[1]);
        return p + sizeofW(StgThunk) + 2;
583

584 585
    case FUN_2_0:
    case CONSTR_2_0:
586 587 588 589
        thread(&((StgClosure *)p)->payload[0]);
        thread(&((StgClosure *)p)->payload[1]);
        return p + sizeofW(StgHeader) + 2;

590
    case BCO: {
591 592 593 594 595
        StgBCO *bco = (StgBCO *)p;
        thread_(&bco->instrs);
        thread_(&bco->literals);
        thread_(&bco->ptrs);
        return p + bco_sizeW(bco);
596 597
    }

598
    case THUNK:
599
    {
600 601 602 603 604 605 606 607
        StgPtr end;

        end = (P_)((StgThunk *)p)->payload +
            info->layout.payload.ptrs;
        for (p = (P_)((StgThunk *)p)->payload; p < end; p++) {
            thread((StgClosure **)p);
        }
        return p + info->layout.payload.nptrs;
608 609 610
    }

    case FUN:
611
    case CONSTR:
612 613
    case PRIM:
    case MUT_PRIM:
614 615
    case MUT_VAR_CLEAN:
    case MUT_VAR_DIRTY:
616
    case TVAR:
617
    case BLACKHOLE:
618
    case BLOCKING_QUEUE:
619
    {
620 621 622 623 624 625 626 627
        StgPtr end;

        end = (P_)((StgClosure *)p)->payload +
            info->layout.payload.ptrs;
        for (p = (P_)((StgClosure *)p)->payload; p < end; p++) {
            thread((StgClosure **)p);
        }
        return p + info->layout.payload.nptrs;
628
    }
629

630 631
    case WEAK:
    {
632 633 634 635 636 637 638 639 640
        StgWeak *w = (StgWeak *)p;
        thread(&w->cfinalizers);
        thread(&w->key);
        thread(&w->value);
        thread(&w->finalizer);
        if (w->link != NULL) {
            thread_(&w->link);
        }
        return p + sizeofW(StgWeak);
641
    }
642

643 644
    case MVAR_CLEAN:
    case MVAR_DIRTY:
645 646 647 648 649 650
    {
        StgMVar *mvar = (StgMVar *)p;
        thread_(&mvar->head);
        thread_(&mvar->tail);
        thread(&mvar->value);
        return p + sizeofW(StgMVar);
651
    }
652

653
    case IND:
654 655
        thread(&((StgInd *)p)->indirectee);
        return p + sizeofW(StgInd);
656 657

    case THUNK_SELECTOR:
658 659 660 661
    {
        StgSelector *s = (StgSelector *)p;
        thread(&s->selectee);
        return p + THUNK_SELECTOR_sizeW();
662
    }
663

664
    case AP_STACK:
665 666
        return thread_AP_STACK((StgAP_STACK *)p);

667
    case PAP:
668
        return thread_PAP((StgPAP *)p);
669 670

    case AP:
671 672
        return thread_AP((StgAP *)p);

673
    case ARR_WORDS:
674
        return p + arr_words_sizeW((StgArrBytes *)p);
675

676 677
    case MUT_ARR_PTRS_CLEAN:
    case MUT_ARR_PTRS_DIRTY:
678
    case MUT_ARR_PTRS_FROZEN:
679
    case MUT_ARR_PTRS_FROZEN0:
680
        // follow everything
681
    {
682 683 684
        StgMutArrPtrs *a;

        a = (StgMutArrPtrs *)p;
685 686 687
        for (p = (P_)a->payload; p < (P_)&a->payload[a->ptrs]; p++) {
            thread((StgClosure **)p);
        }
688

689
        return (StgPtr)a + mut_arr_ptrs_sizeW(a);
690
    }
691 692 693 694 695

    case SMALL_MUT_ARR_PTRS_CLEAN:
    case SMALL_MUT_ARR_PTRS_DIRTY:
    case SMALL_MUT_ARR_PTRS_FROZEN:
    case SMALL_MUT_ARR_PTRS_FROZEN0:
696
        // follow everything
697 698 699 700
    {
        StgSmallMutArrPtrs *a;

        a = (StgSmallMutArrPtrs *)p;
701 702 703
        for (p = (P_)a->payload; p < (P_)&a->payload[a->ptrs]; p++) {
            thread((StgClosure **)p);
        }
704

705
        return (StgPtr)a + small_mut_arr_ptrs_sizeW(a);
706
    }
707

708
    case TSO:
709 710
        return thread_TSO((StgTSO *)p);

711 712 713 714 715 716 717
    case STACK:
    {
        StgStack *stack = (StgStack*)p;
        thread_stack(stack->sp, stack->stack + stack->stack_size);
        return p + stack_sizeW(stack);
    }

718 719
    case TREC_CHUNK:
    {
720
        StgWord i;
721
        StgTRecChunk *tc = (StgTRecChunk *)p;
722 723 724 725 726 727 728 729
        TRecEntry *e = &(tc -> entries[0]);
        thread_(&tc->prev_chunk);
        for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
          thread_(&e->tvar);
          thread(&e->expected_value);
          thread(&e->new_value);
        }
        return p + sizeofW(StgTRecChunk);
730 731
    }

732
    default:
733 734
        barf("update_fwd: unknown/strange object  %d", (int)(info->type));
        return NULL;
735 736 737
    }
}

738 739 740 741 742
static void
update_fwd( bdescr *blocks )
{
    StgPtr p;
    bdescr *bd;
743
    const StgInfoTable *info;
744 745 746 747 748

    bd = blocks;

    // cycle through all the blocks in the step
    for (; bd != NULL; bd = bd->link) {
749 750 751 752 753 754 755 756
        p = bd->start;

        // linearly scan the objects in this block
        while (p < bd->free) {
            ASSERT(LOOKS_LIKE_CLOSURE_PTR(p));
            info = get_itbl((StgClosure *)p);
            p = thread_obj(info, p);
        }
757
    }
758
}
759 760 761 762 763

static void
update_fwd_compact( bdescr *blocks )
{
    StgPtr p, q, free;
764
#if 0
765
    StgWord m;
766
#endif
767 768
    bdescr *bd, *free_bd;
    StgInfoTable *info;
769
    StgWord size;
770
    StgWord iptr;
771 772 773 774 775 776 777

    bd = blocks;
    free_bd = blocks;
    free = free_bd->start;

    // cycle through all the blocks in the step
    for (; bd != NULL; bd = bd->link) {
778
        p = bd->start;
779

780
        while (p < bd->free ) {
781

782 783 784 785 786 787
            while ( p < bd->free && !is_marked(p,bd) ) {
                p++;
            }
            if (p >= bd->free) {
                break;
            }
788 789 790

#if 0
    next:
791 792 793 794 795 796 797 798 799 800 801 802 803 804
        m = * ((StgPtr)bd->u.bitmap + ((p - bd->start) / (BITS_IN(StgWord))));
        m >>= ((p - bd->start) & (BITS_IN(StgWord) - 1));

        while ( p < bd->free ) {

            if ((m & 1) == 0) {
                m >>= 1;
                p++;
                if (((StgWord)p & (sizeof(W_) * BITS_IN(StgWord))) == 0) {
                    goto next;
                } else {
                    continue;
                }
            }
805 806
#endif

807 808 809 810 811
            // Problem: we need to know the destination for this cell
            // in order to unthread its info pointer.  But we can't
            // know the destination without the size, because we may
            // spill into the next block.  So we have to run down the
            // threaded list and get the info ptr first.
812 813 814 815
            //
            // ToDo: one possible avenue of attack is to use the fact
            // that if (p&BLOCK_MASK) >= (free&BLOCK_MASK), then we
            // definitely have enough room.  Also see bug #1147.
816
            iptr = get_threaded_info(p);
817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837
            info = INFO_PTR_TO_STRUCT((StgInfoTable *)UNTAG_CLOSURE((StgClosure *)iptr));

            q = p;

            p = thread_obj(info, p);

            size = p - q;
            if (free + size > free_bd->start + BLOCK_SIZE_W) {
                // set the next bit in the bitmap to indicate that
                // this object needs to be pushed into the next
                // block.  This saves us having to run down the
                // threaded info pointer list twice during the next pass.
                mark(q+1,bd);
                free_bd = free_bd->link;
                free = free_bd->start;
            } else {
                ASSERT(!is_marked(q+1,bd));
            }

            unthread(q,(StgWord)free + GET_CLOSURE_TAG((StgClosure *)iptr));
            free += size;
838
#if 0
839
            goto next;
840
#endif
841
        }
842 843 844
    }
}

845
static W_
846
update_bkwd_compact( generation *gen )
847 848
{
    StgPtr p, free;
849
#if 0
850
    StgWord m;
851
#endif
852
    bdescr *bd, *free_bd;
853
    const StgInfoTable *info;
854 855
    StgWord size;
    W_ free_blocks;
856
    StgWord iptr;
857

858
    bd = free_bd = gen->old_blocks;
859 860 861 862 863
    free = free_bd->start;
    free_blocks = 1;

    // cycle through all the blocks in the step
    for (; bd != NULL; bd = bd->link) {
864
        p = bd->start;
865

866
        while (p < bd->free ) {
867

868 869 870 871 872 873
            while ( p < bd->free && !is_marked(p,bd) ) {
                p++;
            }
            if (p >= bd->free) {
                break;
            }
874 875 876

#if 0
    next:
877 878 879 880 881 882 883 884 885 886 887 888 889 890
        m = * ((StgPtr)bd->u.bitmap + ((p - bd->start) / (BITS_IN(StgWord))));
        m >>= ((p - bd->start) & (BITS_IN(StgWord) - 1));

        while ( p < bd->free ) {

            if ((m & 1) == 0) {
                m >>= 1;
                p++;
                if (((StgWord)p & (sizeof(W_) * BITS_IN(StgWord))) == 0) {
                    goto next;
                } else {
                    continue;
                }
            }
891 892
#endif

893 894 895 896 897 898 899
            if (is_marked(p+1,bd)) {
                // don't forget to update the free ptr in the block desc.
                free_bd->free = free;
                free_bd = free_bd->link;
                free = free_bd->start;
                free_blocks++;
            }
900

901
            iptr = get_threaded_info(p);
902 903 904 905
            unthread(p, (StgWord)free + GET_CLOSURE_TAG((StgClosure *)iptr));
            ASSERT(LOOKS_LIKE_INFO_PTR((StgWord)((StgClosure *)p)->header.info));
            info = get_itbl((StgClosure *)p);
            size = closure_sizeW_((StgClosure *)p,info);
906

907 908 909
            if (free != p) {
                move(free,p,size);
            }
910

911
            // relocate TSOs
912 913
            if (info->type == STACK) {
                move_STACK((StgStack *)p, (StgStack *)free);
914
            }
915

916 917
            free += size;
            p += size;
918
#if 0
919
            goto next;
920
#endif
921
        }
922 923 924 925 926
    }

    // free the remaining blocks and count what's left.
    free_bd->free = free;
    if (free_bd->link != NULL) {
927 928
        freeChain(free_bd->link);
        free_bd->link = NULL;
929 930 931
    }

    return free_blocks;
932 933
}

934
void
935
compact(StgClosure *static_objects)
936
{
937
    W_ n, g, blocks;
938
    generation *gen;
939 940

    // 1. thread the roots
941
    markCapabilities((evac_fn)thread_root, NULL);
942

Simon Marlow's avatar
Simon Marlow committed
943 944
    markScheduler((evac_fn)thread_root, NULL);

945
    // the weak pointer lists...
946
    for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
947
        if (generations[g].weak_ptr_list != NULL) {
948
            thread((void *)&generations[g].weak_ptr_list);
949
        }
950
    }
951 952

    if (dead_weak_ptr_list != NULL) {
953
        thread((void *)&dead_weak_ptr_list); // tmp
954 955
    }

956 957
    // mutable lists
    for (g = 1; g < RtsFlags.GcFlags.generations; g++) {
958 959
        bdescr *bd;
        StgPtr p;
960
        for (n = 0; n < n_capabilities; n++) {
961
            for (bd = capabilities[n]->mut_lists[g];
962 963 964 965 966 967
                 bd != NULL; bd = bd->link) {
                for (p = bd->start; p < bd->free; p++) {
                    thread((StgClosure **)p);
                }
            }
        }
968
    }
969 970

    // the global thread list
971 972
    for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
        thread((void *)&generations[g].threads);
973
    }
974

975
    // any threads resurrected during this GC
976
    thread((void *)&resurrected_threads);
977

978
    // the task list
979
    {
980
        Task *task;
981
        InCall *incall;
982
        for (task = all_tasks; task != NULL; task = task->all_next) {
983
            for (incall = task->incall; incall != NULL;
984 985 986 987 988
                 incall = incall->prev_stack) {
                if (incall->tso) {
                    thread_(&incall->tso);
                }
            }
989
        }
990 991
    }

992
    // the static objects
993
    thread_static(static_objects /* ToDo: ok? */);
994 995

    // the stable pointer table
996
    threadStableTables((evac_fn)thread_root, NULL);
997

998
    // the CAF list (used by GHCi)
999
    markCAFs((evac_fn)thread_root, NULL);
1000

1001 1002
    // 2. update forward ptrs
    for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
1003 1004 1005 1006
        gen = &generations[g];
        debugTrace(DEBUG_gc, "update_fwd:  %d", g);

        update_fwd(gen->blocks);
Simon Marlow's avatar
Simon Marlow committed
1007 1008 1009 1010
        for (n = 0; n < n_capabilities; n++) {
            update_fwd(gc_threads[n]->gens[g].todo_bd);
            update_fwd(gc_threads[n]->gens[g].part_list);
        }
1011 1012 1013 1014
        update_fwd_large(gen->scavenged_large_objects);
        if (g == RtsFlags.GcFlags.generations-1 && gen->old_blocks != NULL) {
            debugTrace(DEBUG_gc, "update_fwd:  %d (compact)", g);
            update_fwd_compact(gen->old_blocks);
1015
        }
1016 1017 1018
    }

    // 3. update backward ptrs
1019 1020
    gen = oldest_gen;
    if (gen->old_blocks != NULL) {
1021 1022 1023 1024 1025
        blocks = update_bkwd_compact(gen);
        debugTrace(DEBUG_gc,
                   "update_bkwd: %d (compact, old: %d blocks, now %d blocks)",
                   gen->no, gen->n_old_blocks, blocks);
        gen->n_old_blocks = blocks;
1026 1027
    }
}