heap: Various marking tweaks

This CL does the following:
1) Move flushing of ephemeron pairs to the start of AdvanceMarking
  (also increase frequency by 2x and make step counters actually
  count steps).
2) Introduce additional (disabled) scopes for diagnostics.
3) Increase frequency of deadline checks when draining worklists (by
   ~8x)
4) Revise worklist draining so that the deadline is checked before
   draining starts.

Bug: 986235
Change-Id: Ie24827a52cbcb497426fd51bc052fb129c417702
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2283743
Commit-Queue: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#786229}

third_party/blink/renderer/platform/heap/heap.h

@@ -459,7 +459,7 @@ class PLATFORM_EXPORT ThreadHeap {
 
   static ThreadHeap* main_thread_heap_;
 
-  static constexpr size_t kStepsBeforeEphemeronPairsFlush = 8u;
+  static constexpr size_t kStepsBeforeEphemeronPairsFlush = 4u;
   size_t steps_since_last_ephemeron_pairs_flush_ = 0;
   static constexpr size_t kStepsBeforeEphemeronProcessing = 16u;
   size_t steps_since_last_ephemeron_processing_ = 0;

third_party/blink/renderer/platform/heap/heap_stats_collector.cc

@@ -158,7 +158,7 @@ base::TimeDelta ThreadHeapStatsCollector::Event::incremental_marking_time()
 
 base::TimeDelta
 ThreadHeapStatsCollector::Event::worklist_processing_time_foreground() const {
-  return scope_data[kMarkProcessWorklist];
+  return scope_data[kMarkProcessWorklists];
 }
 
 base::TimeDelta ThreadHeapStatsCollector::Event::atomic_marking_time() const {

third_party/blink/renderer/platform/heap/heap_stats_collector.h

@@ -40,31 +40,37 @@ class PLATFORM_EXPORT ThreadHeapStatsObserver {
   virtual void DecreaseAllocatedObjectSize(size_t) = 0;
 };
 
-#define FOR_ALL_SCOPES(V)             \
-  V(AtomicPauseCompaction)            \
-  V(AtomicPauseMarkEpilogue)          \
-  V(AtomicPauseMarkPrologue)          \
-  V(AtomicPauseMarkRoots)             \
-  V(AtomicPauseMarkTransitiveClosure) \
-  V(AtomicPauseSweepAndCompact)       \
-  V(CompleteSweep)                    \
-  V(IncrementalMarkingFinalize)       \
-  V(IncrementalMarkingStartMarking)   \
-  V(IncrementalMarkingStep)           \
-  V(InvokePreFinalizers)              \
-  V(LazySweepInIdle)                  \
-  V(LazySweepOnAllocation)            \
-  V(MarkBailOutObjects)               \
-  V(MarkInvokeEphemeronCallbacks)     \
-  V(MarkProcessWorklist)              \
-  V(MarkNotFullyConstructedObjects)   \
-  V(MarkWeakProcessing)               \
-  V(UnifiedMarkingStep)               \
-  V(VisitCrossThreadPersistents)      \
-  V(VisitPersistentRoots)             \
-  V(VisitPersistents)                 \
-  V(VisitRoots)                       \
-  V(VisitStackRoots)                  \
+#define FOR_ALL_SCOPES(V)                    \
+  V(AtomicPauseCompaction)                   \
+  V(AtomicPauseMarkEpilogue)                 \
+  V(AtomicPauseMarkPrologue)                 \
+  V(AtomicPauseMarkRoots)                    \
+  V(AtomicPauseMarkTransitiveClosure)        \
+  V(AtomicPauseSweepAndCompact)              \
+  V(CompleteSweep)                           \
+  V(IncrementalMarkingFinalize)              \
+  V(IncrementalMarkingStartMarking)          \
+  V(IncrementalMarkingStep)                  \
+  V(IncrementalMarkingWithDeadline)          \
+  V(InvokePreFinalizers)                     \
+  V(LazySweepInIdle)                         \
+  V(LazySweepOnAllocation)                   \
+  V(MarkBailOutObjects)                      \
+  V(MarkInvokeEphemeronCallbacks)            \
+  V(MarkFlushV8References)                   \
+  V(MarkFlushEphemeronPairs)                 \
+  V(MarkProcessWorklists)                    \
+  V(MarkProcessMarkingWorklist)              \
+  V(MarkProcessWriteBarrierWorklist)         \
+  V(MarkProcessNotFullyconstructeddWorklist) \
+  V(MarkNotFullyConstructedObjects)          \
+  V(MarkWeakProcessing)                      \
+  V(UnifiedMarkingStep)                      \
+  V(VisitCrossThreadPersistents)             \
+  V(VisitPersistentRoots)                    \
+  V(VisitPersistents)                        \
+  V(VisitRoots)                              \
+  V(VisitStackRoots)                         \
   V(VisitRememberedSets)
 
 #define FOR_ALL_CONCURRENT_SCOPES(V)        \

third_party/blink/renderer/platform/heap/thread_state.cc

@@ -1180,7 +1180,6 @@ void ThreadState::IncrementalMarkingStep(BlinkGC::StackState stack_state) {
     skip_incremental_marking_for_testing_ = false;
   } else {
     complete = MarkPhaseAdvanceMarking(
-        base::TimeTicks::Now() +
             marking_scheduling_->GetNextIncrementalStepDurationForTask(
                 Heap().stats_collector()->object_size_in_bytes()),
         EphemeronProcessing::kPartialProcessing);
@@ -1345,7 +1344,8 @@ void ThreadState::AtomicPauseMarkTransitiveClosure() {
       Heap().stats_collector(),
       ThreadHeapStatsCollector::kAtomicPauseMarkTransitiveClosure, "epoch",
       gc_age_, "forced", IsForcedGC(current_gc_data_.reason));
-  CHECK(MarkPhaseAdvanceMarking(base::TimeTicks::Max(),
+  // base::TimeTicks::Now() + base::TimeDelta::Max() == base::TimeTicks::Max()
+  CHECK(MarkPhaseAdvanceMarking(base::TimeDelta::Max(),
                                 EphemeronProcessing::kFullProcessing));
 }
 
@@ -1588,20 +1588,23 @@ bool ThreadState::MarkPhaseAdvanceMarkingBasedOnSchedule(
     base::TimeDelta max_deadline,
     EphemeronProcessing ephemeron_processing) {
   return MarkPhaseAdvanceMarking(
-      base::TimeTicks::Now() +
-          std::min(max_deadline,
-                   marking_scheduling_->GetNextIncrementalStepDurationForTask(
-                       Heap().stats_collector()->object_size_in_bytes())),
+      std::min(max_deadline,
+               marking_scheduling_->GetNextIncrementalStepDurationForTask(
+                   Heap().stats_collector()->object_size_in_bytes())),
       ephemeron_processing);
 }
 
 bool ThreadState::MarkPhaseAdvanceMarking(
-    base::TimeTicks deadline,
+    base::TimeDelta deadline,
     EphemeronProcessing ephemeron_processing) {
   MarkingVisitor* visitor = current_gc_data_.visitor.get();
-  const bool finished =
-      Heap().AdvanceMarking(reinterpret_cast<MarkingVisitor*>(visitor),
-                            deadline, ephemeron_processing);
+  ThreadHeapStatsCollector::Scope deadline_scope(
+      Heap().stats_collector(),
+      ThreadHeapStatsCollector::kIncrementalMarkingWithDeadline, "deadline_ms",
+      deadline.InMillisecondsF());
+  const bool finished = Heap().AdvanceMarking(
+      reinterpret_cast<MarkingVisitor*>(visitor),
+      base::TimeTicks::Now() + deadline, ephemeron_processing);
   // visitor->marked_bytes() can also include bytes marked during roots
   // visitation which is not counted in worklist_processing_time_foreground.
   // Since the size of the roots is usually small relative to the size of

third_party/blink/renderer/platform/heap/thread_state.h

@@ -516,7 +516,7 @@ class PLATFORM_EXPORT ThreadState final {
   void MarkPhaseVisitNotFullyConstructedObjects();
   bool MarkPhaseAdvanceMarkingBasedOnSchedule(base::TimeDelta,
                                               EphemeronProcessing);
-  bool MarkPhaseAdvanceMarking(base::TimeTicks, EphemeronProcessing);
+  bool MarkPhaseAdvanceMarking(base::TimeDelta, EphemeronProcessing);
   void VerifyMarking(BlinkGC::MarkingType);
 
   // Visit the stack after pushing registers onto the stack.