nonmoving: Clear filled segment bitmaps during sweep

Previously we would clear the bitmaps of all filled segments during the
preparatory pause. However, this means we would do O(heap size) work
during the preparatory pause resulting in poor pause times.

This refactors things to rather clear filled segments' bitmaps during the
(concurrent) sweep phase. This means that during the preparatory pause
we now only need to clear the bitmaps of segments that were recently
filled due to evacuation.

The idea is that instead of adding filled segments to the filled segment
list, sweep rather adds them directly back to the sweep list, where they
will be picked up by the next major GC.

rts/sm/NonMoving.c

@@ -375,7 +375,6 @@ memcount nonmoving_live_words = 0;
 #if defined(THREADED_RTS)
 static void* nonmovingConcurrentMark(void *mark_queue);
 #endif
-static void nonmovingClearBitmap(struct NonmovingSegment *seg);
 static void nonmovingMark_(MarkQueue *mark_queue, StgWeak **dead_weaks, StgTSO **resurrected_threads);
 
 static void nonmovingInitSegment(struct NonmovingSegment *seg, uint8_t log_block_size)
@@ -682,7 +681,7 @@ void nonmovingAddCapabilities(uint32_t new_n_caps)
     nonmovingHeap.n_caps = new_n_caps;
 }
 
-static inline void nonmovingClearBitmap(struct NonmovingSegment *seg)
+void nonmovingClearBitmap(struct NonmovingSegment *seg)
 {
     unsigned int n = nonmovingSegmentBlockCount(seg);
     memset(seg->bitmap, 0, n);
@@ -696,9 +695,6 @@ static void nonmovingPrepareMark(void)
     static_flag =
         static_flag == STATIC_FLAG_A ? STATIC_FLAG_B : STATIC_FLAG_A;
 
-    // Should have been cleared by the last sweep
-    ASSERT(nonmovingHeap.sweep_list == NULL);
-
     nonmovingBumpEpoch();
     for (int alloca_idx = 0; alloca_idx < NONMOVING_ALLOCA_CNT; ++alloca_idx) {
         struct NonmovingAllocator *alloca = nonmovingHeap.allocators[alloca_idx];
@@ -1087,7 +1083,6 @@ static void nonmovingMark_(MarkQueue *mark_queue, StgWeak **dead_weaks, StgTSO *
     nonmovingSweepStableNameTable();
 
     nonmovingSweep();
-    ASSERT(nonmovingHeap.sweep_list == NULL);
     debugTrace(DEBUG_nonmoving_gc, "Finished sweeping.");
     traceConcSweepEnd();
 #if defined(DEBUG)

rts/sm/NonMoving.h

@@ -130,6 +130,7 @@ void nonmovingCollect(StgWeak **dead_weaks,
 void *nonmovingAllocate(Capability *cap, StgWord sz);
 void nonmovingAddCapabilities(uint32_t new_n_caps);
 void nonmovingPushFreeSegment(struct NonmovingSegment *seg);
+void nonmovingClearBitmap(struct NonmovingSegment *seg);
 
 
 INLINE_HEADER struct NonmovingSegmentInfo *nonmovingSegmentInfo(struct NonmovingSegment *seg) {

rts/sm/NonMovingSweep.c

@@ -127,6 +127,8 @@ clear_segment_free_blocks(struct NonmovingSegment* seg)
 
 GNUC_ATTR_HOT void nonmovingSweep(void)
 {
+    struct NonmovingSegment *new_sweep_list = NULL;
+
     while (nonmovingHeap.sweep_list) {
         struct NonmovingSegment *seg = nonmovingHeap.sweep_list;
 
@@ -146,12 +148,19 @@ GNUC_ATTR_HOT void nonmovingSweep(void)
             nonmovingPushActiveSegment(seg);
             break;
         case SEGMENT_FILLED:
-            nonmovingPushFilledSegment(seg);
+            // Clear bitmap
+            nonmovingClearBitmap(seg);
+            // Set snapshot
+            nonmovingSegmentInfo(seg)->next_free_snap = seg->next_free;
+            seg->link = new_sweep_list;
+            new_sweep_list = seg;
             break;
         default:
             barf("nonmovingSweep: weird sweep return: %d\n", ret);
         }
     }
+
+    nonmovingHeap.sweep_list = new_sweep_list;
 }
 
 /* Must a closure remain on the mutable list?