[oilpan] Faster write barrier guard

Add a general counter for threads that are currently in incremental
marking.

This guard can be checked as first reason for bailing out in any write
barrier as it only requires a single load and does not need to go
through TLS or page lookups.

Bug: chromium:757440
Change-Id: Ida63e560bb52ccd66da739139c0510229b05a550
Reviewed-on: https://chromium-review.googlesource.com/995272
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Kentaro Hara <haraken@chromium.org>
Cr-Commit-Position: refs/heads/master@{#548033}

third_party/WebKit/Source/platform/heap/Heap.cpp

@@ -753,18 +753,6 @@ bool ThreadHeap::AdvanceLazySweep(double deadline_seconds) {
   return true;
 }
 
-void ThreadHeap::EnableIncrementalMarkingBarrier() {
-  thread_state_->SetIncrementalMarking(true);
-  for (int i = 0; i < BlinkGC::kNumberOfArenas; ++i)
-    arenas_[i]->EnableIncrementalMarkingBarrier();
-}
-
-void ThreadHeap::DisableIncrementalMarkingBarrier() {
-  thread_state_->SetIncrementalMarking(false);
-  for (int i = 0; i < BlinkGC::kNumberOfArenas; ++i)
-    arenas_[i]->DisableIncrementalMarkingBarrier();
-}
-
 void ThreadHeap::WriteBarrier(const void* value) {
   if (!value || !thread_state_->IsIncrementalMarking())
     return;
@@ -774,7 +762,6 @@ void ThreadHeap::WriteBarrier(const void* value) {
 
 void ThreadHeap::WriteBarrierInternal(BasePage* page, const void* value) {
   DCHECK(thread_state_->IsIncrementalMarking());
-  DCHECK(page->IsIncrementalMarking());
   DCHECK(value);
   HeapObjectHeader* const header =
       page->IsLargeObjectPage()

third_party/WebKit/Source/platform/heap/Heap.h

@@ -472,11 +472,6 @@ class PLATFORM_EXPORT ThreadHeap {
   enum SnapshotType { kHeapSnapshot, kFreelistSnapshot };
   void TakeSnapshot(SnapshotType);
 
-  // Enables or disables the incremental marking barrier that intercepts
-  // writes to Member<T> objects.
-  void EnableIncrementalMarkingBarrier();
-  void DisableIncrementalMarkingBarrier();
-
   // Write barrier used after adding an object to the graph.
   void WriteBarrier(const void* value);
 

third_party/WebKit/Source/platform/heap/HeapAllocator.h

@@ -130,7 +130,7 @@ class PLATFORM_EXPORT HeapAllocator {
 
   static void BackingWriteBarrier(void* address) {
 #if BUILDFLAG(BLINK_HEAP_INCREMENTAL_MARKING)
-    if (!address)
+    if (!address || !ThreadState::IsAnyIncrementalMarking())
       return;
     ThreadState* state = PageFromObject(address)->Arena()->GetThreadState();
     state->Heap().WriteBarrier(address);
@@ -214,6 +214,8 @@ class PLATFORM_EXPORT HeapAllocator {
   template <typename T, typename Traits>
   static void NotifyNewObject(T* object) {
 #if BUILDFLAG(BLINK_HEAP_INCREMENTAL_MARKING)
+    if (!ThreadState::IsAnyIncrementalMarking())
+      return;
     // The object may have been in-place constructed as part of a large object.
     // It is not safe to retrieve the page from the object here.
     ThreadState* const thread_state = ThreadState::Current();
@@ -239,6 +241,8 @@ class PLATFORM_EXPORT HeapAllocator {
   template <typename T, typename Traits>
   static void NotifyNewObjects(T* array, size_t len) {
 #if BUILDFLAG(BLINK_HEAP_INCREMENTAL_MARKING)
+    if (!ThreadState::IsAnyIncrementalMarking())
+      return;
     // The object may have been in-place constructed as part of a large object.
     // It is not safe to retrieve the page from the object here.
     ThreadState* const thread_state = ThreadState::Current();

third_party/WebKit/Source/platform/heap/HeapPage.cpp

@@ -416,18 +416,6 @@ bool BaseArena::WillObjectBeLazilySwept(BasePage* page,
   return true;
 }
 
-void BaseArena::EnableIncrementalMarkingBarrier() {
-  DCHECK(SweepingCompleted());
-  for (BasePage* page = first_page_; page; page = page->Next())
-    page->SetIncrementalMarking(true);
-}
-
-void BaseArena::DisableIncrementalMarkingBarrier() {
-  DCHECK(SweepingCompleted());
-  for (BasePage* page = first_page_; page; page = page->Next())
-    page->SetIncrementalMarking(false);
-}
-
 NormalPageArena::NormalPageArena(ThreadState* state, int index)
     : BaseArena(state, index),
       current_allocation_point_(nullptr),
@@ -1276,8 +1264,7 @@ BasePage::BasePage(PageMemory* storage, BaseArena* arena)
       storage_(storage),
       arena_(arena),
       next_(nullptr),
-      swept_(true),
-      incremental_marking_(arena->GetThreadState()->IsIncrementalMarking()) {
+      swept_(true) {
 #if DCHECK_IS_ON()
   DCHECK(IsPageHeaderAddress(reinterpret_cast<Address>(this)));
 #endif

third_party/WebKit/Source/platform/heap/HeapPage.h

@@ -411,10 +411,6 @@ class BasePage {
     swept_ = false;
   }
 
-  void SetIncrementalMarking(bool value) { incremental_marking_ = value; }
-
-  bool IsIncrementalMarking() const { return incremental_marking_; }
-
   // Returns true if magic number is valid.
   bool IsValid() const;
 
@@ -432,8 +428,7 @@ class BasePage {
   // Track the sweeping state of a page. Set to false at the start of a sweep,
   // true  upon completion of lazy sweeping.
   bool swept_;
-  // Track whether incremental marking is currently running.
-  bool incremental_marking_;
+
   friend class BaseArena;
 };
 
@@ -779,9 +774,6 @@ class PLATFORM_EXPORT BaseArena {
 
   bool WillObjectBeLazilySwept(BasePage*, void*) const;
 
-  void EnableIncrementalMarkingBarrier();
-  void DisableIncrementalMarkingBarrier();
-
   virtual void Verify(){};
   virtual void VerifyMarking(){};
 

third_party/WebKit/Source/platform/heap/IncrementalMarkingTest.cpp

@@ -51,7 +51,7 @@ class IncrementalMarkingScope : public IncrementalMarkingScopeBase {
     ThreadState::AtomicPauseScope atomic_pause_scope_(thread_state_);
     EXPECT_TRUE(marking_worklist_->IsGlobalEmpty());
     EXPECT_TRUE(not_fully_constructed_worklist_->IsGlobalEmpty());
-    heap_.EnableIncrementalMarkingBarrier();
+    thread_state->EnableIncrementalMarkingBarrier();
     thread_state->current_gc_data_.visitor =
         MarkingVisitor::Create(thread_state, MarkingVisitor::kGlobalMarking);
   }
@@ -59,7 +59,7 @@ class IncrementalMarkingScope : public IncrementalMarkingScopeBase {
   ~IncrementalMarkingScope() {
     EXPECT_TRUE(marking_worklist_->IsGlobalEmpty());
     EXPECT_TRUE(not_fully_constructed_worklist_->IsGlobalEmpty());
-    heap_.DisableIncrementalMarkingBarrier();
+    thread_state_->DisableIncrementalMarkingBarrier();
     // Need to clear out unused worklists that might have been polluted during
     // test.
     heap_.GetPostMarkingWorklist()->Clear();
@@ -193,14 +193,11 @@ class Object : public GarbageCollected<Object> {
 TEST(IncrementalMarkingTest, EnableDisableBarrier) {
   Object* object = Object::Create();
   BasePage* page = PageFromObject(object);
-  ThreadHeap& heap = ThreadState::Current()->Heap();
-  EXPECT_FALSE(page->IsIncrementalMarking());
   EXPECT_FALSE(ThreadState::Current()->IsIncrementalMarking());
-  heap.EnableIncrementalMarkingBarrier();
-  EXPECT_TRUE(page->IsIncrementalMarking());
+  ThreadState::Current()->EnableIncrementalMarkingBarrier();
   EXPECT_TRUE(ThreadState::Current()->IsIncrementalMarking());
-  heap.DisableIncrementalMarkingBarrier();
-  EXPECT_FALSE(page->IsIncrementalMarking());
+  EXPECT_TRUE(ThreadState::IsAnyIncrementalMarking());
+  ThreadState::Current()->DisableIncrementalMarkingBarrier();
   EXPECT_FALSE(ThreadState::Current()->IsIncrementalMarking());
 }
 

third_party/WebKit/Source/platform/heap/Member.h

@@ -272,13 +272,14 @@ class Member : public MemberBase<T, TracenessMemberConfiguration::kTraced> {
  protected:
   ALWAYS_INLINE void WriteBarrier(const T* value) const {
 #if BUILDFLAG(BLINK_HEAP_INCREMENTAL_MARKING)
-    if (LIKELY(value && !this->IsHashTableDeletedValue())) {
+    if (LIKELY(value && !this->IsHashTableDeletedValue()) &&
+        ThreadState::IsAnyIncrementalMarking()) {
+      ThreadState* const thread_state = ThreadState::Current();
+      if (thread_state->IsIncrementalMarking()) {
         // The following method for retrieving a page works as allocation of
         // mixins on large object pages is prohibited.
         BasePage* const page = PageFromObject(value);
-      if (page->IsIncrementalMarking()) {
-        DCHECK(ThreadState::Current()->IsIncrementalMarking());
-        ThreadState::Current()->Heap().WriteBarrierInternal(page, value);
+        thread_state->Heap().WriteBarrierInternal(page, value);
       }
     }
 #endif  // BUILDFLAG(BLINK_HEAP_INCREMENTAL_MARKING)

third_party/WebKit/Source/platform/heap/ThreadState.cpp

@@ -37,6 +37,7 @@
 #include <limits>
 #include <memory>
 
+#include "base/atomicops.h"
 #include "base/location.h"
 #include "base/trace_event/process_memory_dump.h"
 #include "build/build_config.h"
@@ -1233,6 +1234,21 @@ void ThreadState::InvokePreFinalizers() {
   }
 }
 
+// static
+base::subtle::AtomicWord ThreadState::incremental_marking_counter_ = 0;
+
+void ThreadState::EnableIncrementalMarkingBarrier() {
+  CHECK(!IsIncrementalMarking());
+  base::subtle::Barrier_AtomicIncrement(&incremental_marking_counter_, 1);
+  SetIncrementalMarking(true);
+}
+
+void ThreadState::DisableIncrementalMarkingBarrier() {
+  CHECK(IsIncrementalMarking());
+  base::subtle::Barrier_AtomicIncrement(&incremental_marking_counter_, -1);
+  SetIncrementalMarking(false);
+}
+
 void ThreadState::IncrementalMarkingStart() {
   VLOG(2) << "[state:" << this << "] "
           << "IncrementalMarking: Start";
@@ -1243,7 +1259,7 @@ void ThreadState::IncrementalMarkingStart() {
   MarkPhasePrologue(BlinkGC::kNoHeapPointersOnStack,
                     BlinkGC::kIncrementalMarking, BlinkGC::kIdleGC);
   MarkPhaseVisitRoots();
-  Heap().EnableIncrementalMarkingBarrier();
+  EnableIncrementalMarkingBarrier();
   ScheduleIncrementalMarkingStep();
   DCHECK(IsMarkingInProgress());
 }
@@ -1268,6 +1284,7 @@ void ThreadState::IncrementalMarkingFinalize() {
   VLOG(2) << "[state:" << this << "] "
           << "IncrementalMarking: Finalize";
   SetGCState(kNoGCScheduled);
+  DisableIncrementalMarkingBarrier();
   AtomicPauseScope atomic_pause_scope(this);
   DCHECK(IsMarkingInProgress());
   RecursiveMutexLocker persistent_lock(
@@ -1276,7 +1293,6 @@ void ThreadState::IncrementalMarkingFinalize() {
   bool complete =
       MarkPhaseAdvanceMarking(std::numeric_limits<double>::infinity());
   CHECK(complete);
-  Heap().DisableIncrementalMarkingBarrier();
   MarkPhaseEpilogue(current_gc_data_.marking_type);
   PreSweep(current_gc_data_.marking_type, BlinkGC::kLazySweeping);
   DCHECK(IsSweepingInProgress());

third_party/WebKit/Source/platform/heap/ThreadState.h

@@ -33,6 +33,7 @@
 
 #include <memory>
 
+#include "base/atomicops.h"
 #include "base/macros.h"
 #include "platform/PlatformExport.h"
 #include "platform/bindings/ScriptForbiddenScope.h"
@@ -215,6 +216,14 @@ class PLATFORM_EXPORT ThreadState {
     ThreadState* state_;
   };
 
+  // Returns true if any thread is currently incremental marking its heap and
+  // false otherwise. For an exact check use
+  // ThreadState::IsIncrementalMarking().
+  static bool IsAnyIncrementalMarking() {
+    // Stores use full barrier to allow using the simplest relaxed load here.
+    return base::subtle::NoBarrier_Load(&incremental_marking_counter_) > 0;
+  }
+
   static void AttachMainThread();
 
   // Associate ThreadState object with the current thread. After this
@@ -277,6 +286,9 @@ class PLATFORM_EXPORT ThreadState {
   void IncrementalMarkingStep();
   void IncrementalMarkingFinalize();
 
+  void EnableIncrementalMarkingBarrier();
+  void DisableIncrementalMarkingBarrier();
+
   // A GC runs in the following sequence.
   //
   // 1) preGC() is called.
@@ -575,6 +587,11 @@ class PLATFORM_EXPORT ThreadState {
   template <typename T>
   friend class PrefinalizerRegistration;
 
+  // Number of ThreadState's that are currently in incremental marking. The
+  // counter is incremented by one when some ThreadState enters incremental
+  // marking and decremented upon finishing.
+  static base::subtle::AtomicWord incremental_marking_counter_;
+
   ThreadState();
   ~ThreadState();