[oilpan] Factor out common parts of the atomic phase

In a follow up, we will make CollectGarbage the bottleneck for the GC
atomic phase.

Bug: chromium:757440
Change-Id: Icf27cfde4a61375eda52207e967c84c4ed9c7c31
Reviewed-on: https://chromium-review.googlesource.com/1078214
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Kentaro Hara <haraken@chromium.org>
Cr-Commit-Position: refs/heads/master@{#563077}

third_party/blink/renderer/platform/heap/heap_stats_collector.h

@@ -36,6 +36,7 @@ class PLATFORM_EXPORT ThreadHeapStatsCollector {
     kInvokePreFinalizers,
     kLazySweepInIdle,
     kLazySweepOnAllocation,
+    kAtomicPhase,
     kAtomicPhaseMarking,
     kVisitDOMWrappers,
     kNumScopeIds,
@@ -43,6 +44,8 @@ class PLATFORM_EXPORT ThreadHeapStatsCollector {
 
   static const char* ToString(Id id) {
     switch (id) {
+      case Id::kAtomicPhase:
+        return "BlinkGC.AtomicPhase";
       case Id::kAtomicPhaseMarking:
         return "BlinkGC.AtomicPhaseMarking";
       case Id::kCompleteSweep:

third_party/blink/renderer/platform/heap/thread_state.cc

@@ -883,6 +883,15 @@ void ThreadState::RunScheduledGC(BlinkGC::StackState stack_state) {
   }
 }
 
+void ThreadState::FinishSnapshot() {
+  // Force setting NoGCScheduled to circumvent checkThread()
+  // in setGCState().
+  gc_state_ = kNoGCScheduled;
+  SetGCPhase(GCPhase::kSweeping);
+  SetGCPhase(GCPhase::kNone);
+  Heap().stats_collector()->Stop();
+}
+
 void ThreadState::PreSweep(BlinkGC::MarkingType marking_type,
                            BlinkGC::SweepingType sweeping_type) {
   DCHECK(InAtomicMarkingPause());
@@ -899,13 +908,6 @@ void ThreadState::PreSweep(BlinkGC::MarkingType marking_type,
     Heap().MakeConsistentForMutator();
 
     Heap().TakeSnapshot(ThreadHeap::SnapshotType::kFreelistSnapshot);
-
-    // Force setting NoGCScheduled to circumvent checkThread()
-    // in setGCState().
-    gc_state_ = kNoGCScheduled;
-    SetGCPhase(GCPhase::kSweeping);
-    SetGCPhase(GCPhase::kNone);
-    Heap().stats_collector()->Stop();
     return;
   }
 
@@ -1373,6 +1375,7 @@ void ThreadState::IncrementalMarkingFinalize() {
     PreSweep(current_gc_data_.marking_type, BlinkGC::kLazySweeping);
     DCHECK(IsSweepingInProgress());
     DCHECK_EQ(GcState(), kNoGCScheduled);
+    ScheduleIdleLazySweep();
   }
 }
 
@@ -1412,9 +1415,34 @@ void ThreadState::CollectGarbage(BlinkGC::StackState stack_state,
     CompleteSweep();
     SetGCState(kNoGCScheduled);
     Heap().stats_collector()->Start(reason);
+    RunAtomicPause(stack_state, marking_type, sweeping_type, reason);
+  }
+
+  double total_collect_garbage_time =
+      WTF::CurrentTimeTicksInMilliseconds() - start_total_collect_garbage_time;
+  DEFINE_THREAD_SAFE_STATIC_LOCAL(
+      CustomCountHistogram, time_for_total_collect_garbage_histogram,
+      ("BlinkGC.TimeForTotalCollectGarbage", 1, 10 * 1000, 50));
+  time_for_total_collect_garbage_histogram.Count(total_collect_garbage_time);
+  VLOG(1) << "[state:" << this << "]"
+          << " CollectGarbage: time: " << std::setprecision(2)
+          << total_collect_garbage_time << "ms"
+          << " stack: " << StackStateString(stack_state)
+          << " marking: " << MarkingTypeString(marking_type)
+          << " sweeping: " << SweepingTypeString(sweeping_type)
+          << " reason: " << GcReasonString(reason);
+}
+
+void ThreadState::RunAtomicPause(BlinkGC::StackState stack_state,
+                                 BlinkGC::MarkingType marking_type,
+                                 BlinkGC::SweepingType sweeping_type,
+                                 BlinkGC::GCReason reason) {
+  {
+    ThreadHeapStatsCollector::EnabledScope stats1(
+        Heap().stats_collector(), ThreadHeapStatsCollector::kAtomicPhase);
     AtomicPauseScope atomic_pause_scope(this);
     {
-      ThreadHeapStatsCollector::EnabledScope stats_scope(
+      ThreadHeapStatsCollector::EnabledScope stats2(
           Heap().stats_collector(),
           ThreadHeapStatsCollector::kAtomicPhaseMarking, "lazySweeping",
           sweeping_type == BlinkGC::kLazySweeping ? "yes" : "no", "gcReason",
@@ -1426,7 +1454,13 @@ void ThreadState::CollectGarbage(BlinkGC::StackState stack_state,
     }
     PreSweep(marking_type, sweeping_type);
   }
-
+  if (marking_type == BlinkGC::kTakeSnapshot) {
+    FinishSnapshot();
+    CHECK(!IsSweepingInProgress());
+    CHECK_EQ(GcState(), kNoGCScheduled);
+    return;
+  }
+  DCHECK(IsSweepingInProgress());
   if (sweeping_type == BlinkGC::kEagerSweeping) {
     // Eager sweeping should happen only in testing.
     CompleteSweep();
@@ -1435,49 +1469,41 @@ void ThreadState::CollectGarbage(BlinkGC::StackState stack_state,
     // The default behavior is lazy sweeping.
     ScheduleIdleLazySweep();
   }
+}
 
-  double total_collect_garbage_time =
-      WTF::CurrentTimeTicksInMilliseconds() - start_total_collect_garbage_time;
-  DEFINE_THREAD_SAFE_STATIC_LOCAL(
-      CustomCountHistogram, time_for_total_collect_garbage_histogram,
-      ("BlinkGC.TimeForTotalCollectGarbage", 1, 10 * 1000, 50));
-  time_for_total_collect_garbage_histogram.Count(total_collect_garbage_time);
-  VLOG(1) << "[state:" << this << "]"
-          << " CollectGarbage: time: " << std::setprecision(2)
-          << total_collect_garbage_time << "ms"
-          << " stack: " << StackStateString(stack_state)
-          << " marking: " << MarkingTypeString(marking_type)
-          << " sweeping: " << SweepingTypeString(sweeping_type)
-          << " reason: " << GcReasonString(reason);
+namespace {
+
+MarkingVisitor::MarkingMode GetMarkingMode(bool should_compact,
+                                           bool create_snapshot) {
+  CHECK(!should_compact || !create_snapshot);
+  return (create_snapshot)
+             ? MarkingVisitor::kSnapshotMarking
+             : (should_compact) ? MarkingVisitor::kGlobalMarkingWithCompaction
+                                : MarkingVisitor::kGlobalMarking;
 }
 
+}  // namespace
+
 void ThreadState::MarkPhasePrologue(BlinkGC::StackState stack_state,
                                     BlinkGC::MarkingType marking_type,
                                     BlinkGC::GCReason reason) {
   SetGCPhase(GCPhase::kMarking);
   Heap().CommitCallbackStacks();
 
+  const bool take_snapshot = marking_type == BlinkGC::kTakeSnapshot;
+  const bool should_compact =
+      !take_snapshot && Heap().Compaction()->ShouldCompact(
+                            &Heap(), stack_state, marking_type, reason);
+
+  current_gc_data_.visitor = MarkingVisitor::Create(
+      this, GetMarkingMode(should_compact, take_snapshot));
   current_gc_data_.stack_state = stack_state;
   current_gc_data_.marking_type = marking_type;
   current_gc_data_.reason = reason;
   current_gc_data_.marking_time_in_milliseconds = 0;
 
-  if (marking_type == BlinkGC::kTakeSnapshot) {
-    current_gc_data_.visitor =
-        MarkingVisitor::Create(this, MarkingVisitor::kSnapshotMarking);
-  } else {
-    DCHECK(marking_type == BlinkGC::kAtomicMarking ||
-           marking_type == BlinkGC::kIncrementalMarking);
-    if (Heap().Compaction()->ShouldCompact(&Heap(), stack_state, marking_type,
-                                           reason)) {
-      Heap().Compaction()->Initialize(this);
-      current_gc_data_.visitor = MarkingVisitor::Create(
-          this, MarkingVisitor::kGlobalMarkingWithCompaction);
-    } else {
-      current_gc_data_.visitor =
-          MarkingVisitor::Create(this, MarkingVisitor::kGlobalMarking);
-    }
-  }
+  if (should_compact)
+    Heap().Compaction()->Initialize(this);
 
   if (marking_type == BlinkGC::kTakeSnapshot)
     BlinkGCMemoryDumpProvider::Instance()->ClearProcessDumpForCurrentGC();

third_party/blink/renderer/platform/heap/thread_state.h

@@ -300,26 +300,9 @@ class PLATFORM_EXPORT ThreadState {
   void EnableIncrementalMarkingBarrier();
   void DisableIncrementalMarkingBarrier();
 
-  // A GC runs in the following sequence.
-  //
-  // 1) preGC() is called.
-  // 2) ThreadHeap::collectGarbage() is called. This marks live objects.
-  // 3) postGC() is called. This does thread-local weak processing.
-  // 4) preSweep() is called. This does pre-finalization, eager sweeping and
-  //    heap compaction.
-  // 4) Lazy sweeping sweeps heaps incrementally. completeSweep() may be called
-  //    to complete the sweeping.
-  // 5) postSweep() is called.
-  void MarkPhasePrologue(BlinkGC::StackState,
-                         BlinkGC::MarkingType,
-                         BlinkGC::GCReason);
-  void MarkPhaseVisitRoots();
-  bool MarkPhaseAdvanceMarking(double deadline_seconds);
-  void MarkPhaseEpilogue(BlinkGC::MarkingType);
-  void VerifyMarking(BlinkGC::MarkingType);
-
   void CompleteSweep();
   void PreSweep(BlinkGC::MarkingType, BlinkGC::SweepingType);
+  void FinishSnapshot();
   void PostSweep();
 
   // Support for disallowing allocation. Mainly used for sanity
@@ -592,6 +575,7 @@ class PLATFORM_EXPORT ThreadState {
   friend class incremental_marking_test::IncrementalMarkingTestDriver;
   template <typename T>
   friend class PrefinalizerRegistration;
+  friend class TestGCMarkingScope;
 
   // Number of ThreadState's that are currently in incremental marking. The
   // counter is incremented by one when some ThreadState enters incremental
@@ -604,6 +588,19 @@ class PLATFORM_EXPORT ThreadState {
   ThreadState();
   ~ThreadState();
 
+  void MarkPhasePrologue(BlinkGC::StackState,
+                         BlinkGC::MarkingType,
+                         BlinkGC::GCReason);
+  void MarkPhaseVisitRoots();
+  bool MarkPhaseAdvanceMarking(double deadline_seconds);
+  void MarkPhaseEpilogue(BlinkGC::MarkingType);
+  void VerifyMarking(BlinkGC::MarkingType);
+
+  void RunAtomicPause(BlinkGC::StackState,
+                      BlinkGC::MarkingType,
+                      BlinkGC::SweepingType,
+                      BlinkGC::GCReason);
+
   void ClearSafePointScopeMarker() {
     safe_point_stack_copy_.clear();
     safe_point_scope_marker_ = nullptr;