heap: Implement unmarking phase

Unmarking phase must be called before major GC to clear all mark bits.

Design doc: https://bit.ly/2ZXGU20

Bug: 1029379
Change-Id: Ifcfa4af9710f72b9bdea24e89c2c613d0f34eaeb
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2003308
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Kentaro Hara <haraken@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#732357}

third_party/blink/renderer/platform/heap/heap.cc

@@ -453,14 +453,23 @@ BasePage* ThreadHeap::LookupPageForAddress(Address address) {
 
 void ThreadHeap::MakeConsistentForGC() {
   DCHECK(thread_state_->InAtomicMarkingPause());
-  for (int i = 0; i < BlinkGC::kNumberOfArenas; ++i)
-    arenas_[i]->MakeConsistentForGC();
+  for (BaseArena* arena : arenas_) {
+    arena->MakeConsistentForGC();
+  }
 }
 
 void ThreadHeap::MakeConsistentForMutator() {
   DCHECK(thread_state_->InAtomicMarkingPause());
-  for (int i = 0; i < BlinkGC::kNumberOfArenas; ++i)
-    arenas_[i]->MakeConsistentForMutator();
+  for (BaseArena* arena : arenas_) {
+    arena->MakeConsistentForMutator();
+  }
+}
+
+void ThreadHeap::Unmark() {
+  DCHECK(thread_state_->InAtomicMarkingPause());
+  for (BaseArena* arena : arenas_) {
+    arena->Unmark();
+  }
 }
 
 void ThreadHeap::Compact() {

third_party/blink/renderer/platform/heap/heap.h

@@ -344,6 +344,10 @@ class PLATFORM_EXPORT ThreadHeap {
   // the executions of mutators.
   void MakeConsistentForMutator();
 
+  // Unmarks all objects in the entire heap. This is supposed to be called in
+  // the beginning of major GC.
+  void Unmark();
+
   void Compact();
 
   enum class SweepingType : uint8_t { kMutator, kConcurrent };

third_party/blink/renderer/platform/heap/heap_page.cc

@@ -215,6 +215,15 @@ void BaseArena::MakeConsistentForMutator() {
   VerifyObjectStartBitmap();
 }
 
+void BaseArena::Unmark() {
+  DCHECK(GetThreadState()->InAtomicMarkingPause());
+  DCHECK(SweepingAndFinalizationCompleted());
+
+  for (BasePage* page : swept_pages_) {
+    page->Unmark();
+  }
+}
+
 size_t BaseArena::ObjectPayloadSizeForTesting() {
 #if DCHECK_IS_ON()
   DCHECK(IsConsistentForGC());
@@ -1479,7 +1488,9 @@ bool NormalPage::Sweep(FinalizeType finalize_type) {
       found_finalizer = false;
     }
     object_start_bit_map()->SetBit(header_address);
+#if !BUILDFLAG(BLINK_HEAP_YOUNG_GENERATION)
     header->Unmark<HeapObjectHeader::AccessMode::kAtomic>();
+#endif
     header_address += size;
     start_of_gap = header_address;
   }
@@ -1543,7 +1554,9 @@ void NormalPage::SweepAndCompact(CompactionContext& context) {
       header_address += size;
       continue;
     }
+#if !BUILDFLAG(BLINK_HEAP_YOUNG_GENERATION)
     header->Unmark();
+#endif
     // Allocate and copy over the live object.
     Address compact_frontier = current_page->Payload() + allocation_point;
     if (compact_frontier + size > current_page->PayloadEnd()) {
@@ -1635,6 +1648,29 @@ void NormalPage::MakeConsistentForMutator() {
   VerifyObjectStartBitmapIsConsistentWithPayload();
 }
 
+// This is assumed to be called from the atomic pause, so no concurrency should
+// be involved here.
+void NormalPage::Unmark() {
+  const Address current_allocation_point =
+      ArenaForNormalPage()->CurrentAllocationPoint();
+  const size_t allocation_area_size =
+      ArenaForNormalPage()->RemainingAllocationSize();
+  for (Address header_address = Payload(); header_address < PayloadEnd();) {
+    // Since unmarking can happen inside IncrementalMarkingStart, the current
+    // allocation point can be set and we need to skip over it.
+    if (header_address == current_allocation_point && allocation_area_size) {
+      header_address += allocation_area_size;
+      continue;
+    }
+    HeapObjectHeader* header =
+        reinterpret_cast<HeapObjectHeader*>(header_address);
+    if (header->IsMarked()) {
+      header->Unmark();
+    }
+    header_address += header->size();
+  }
+}
+
 #if defined(ADDRESS_SANITIZER)
 void NormalPage::PoisonUnmarkedObjects() {
   for (Address header_address = Payload(); header_address < PayloadEnd();) {
@@ -1791,17 +1827,23 @@ bool LargeObjectPage::Sweep(FinalizeType) {
   if (!ObjectHeader()->IsMarked()) {
     return true;
   }
+#if !BUILDFLAG(BLINK_HEAP_YOUNG_GENERATION)
   ObjectHeader()->Unmark();
+#endif
   return false;
 }
 
-void LargeObjectPage::MakeConsistentForMutator() {
+void LargeObjectPage::Unmark() {
   HeapObjectHeader* header = ObjectHeader();
   if (header->IsMarked()) {
     header->Unmark();
   }
 }
 
+void LargeObjectPage::MakeConsistentForMutator() {
+  Unmark();
+}
+
 void LargeObjectPage::FinalizeSweep(SweepResult action) {
   if (action == SweepResult::kPageNotEmpty) {
     MarkAsSwept();

third_party/blink/renderer/platform/heap/heap_page.h

@@ -466,6 +466,7 @@ class BasePage {
   // Does not create free list entries for empty pages.
   virtual bool Sweep(FinalizeType) = 0;
   virtual void MakeConsistentForMutator() = 0;
+  virtual void Unmark() = 0;
 
   // Calls finalizers after sweeping is done.
   virtual void FinalizeSweep(SweepResult) = 0;
@@ -653,6 +654,7 @@ class PLATFORM_EXPORT NormalPage final : public BasePage {
   void RemoveFromHeap() override;
   bool Sweep(FinalizeType) override;
   void MakeConsistentForMutator() override;
+  void Unmark() override;
   void FinalizeSweep(SweepResult) override;
 #if defined(ADDRESS_SANITIZER)
   void PoisonUnmarkedObjects() override;
@@ -879,6 +881,7 @@ class PLATFORM_EXPORT LargeObjectPage final : public BasePage {
   void RemoveFromHeap() override;
   bool Sweep(FinalizeType) override;
   void MakeConsistentForMutator() override;
+  void Unmark() override;
   void FinalizeSweep(SweepResult) override;
 
   void CollectStatistics(
@@ -945,6 +948,7 @@ class PLATFORM_EXPORT BaseArena {
   virtual void MakeIterable() {}
   virtual void MakeConsistentForGC();
   void MakeConsistentForMutator();
+  void Unmark();
 #if DCHECK_IS_ON()
   virtual bool IsConsistentForGC() = 0;
 #endif

third_party/blink/renderer/platform/heap/thread_state.cc

@@ -1614,6 +1614,13 @@ void ThreadState::MarkPhasePrologue(BlinkGC::CollectionType collection_type,
     Heap().Compaction()->Initialize(this);
   }
 
+#if BUILDFLAG(BLINK_HEAP_YOUNG_GENERATION)
+  if (collection_type == BlinkGC::CollectionType::kMajor) {
+    // Unmark heap before doing major collection cycle.
+    Heap().Unmark();
+  }
+#endif
+
   current_gc_data_.reason = reason;
   current_gc_data_.collection_type = collection_type;
   current_gc_data_.visitor =