concurrent-marking: Inverse bit for objects in construction

The bit is set whenever writing free list entries.

Bug: 945806
Change-Id: Ifd5b7e6a8c4d7e4d557afd02fa79b1f1d60f4ba8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1609912
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Kentaro Hara <haraken@chromium.org>
Cr-Commit-Position: refs/heads/master@{#660020}

third_party/blink/renderer/platform/heap/heap.h

@@ -523,10 +523,9 @@ T* MakeGarbageCollected(Args&&... args) {
                 "T needs to be a garbage collected object");
   void* memory = T::AllocateObject(sizeof(T), IsEagerlyFinalizedType<T>::value);
   HeapObjectHeader* header = HeapObjectHeader::FromPayload(memory);
-  header->MarkIsInConstruction();
   // Placement new as regular operator new() is deleted.
   T* object = ::new (memory) T(std::forward<Args>(args)...);
-  header->UnmarkIsInConstruction();
+  header->MarkFullyConstructed();
   return object;
 }
 
@@ -545,10 +544,9 @@ T* MakeGarbageCollected(AdditionalBytes additional_bytes, Args&&... args) {
   void* memory = T::AllocateObject(sizeof(T) + additional_bytes.value,
                                    IsEagerlyFinalizedType<T>::value);
   HeapObjectHeader* header = HeapObjectHeader::FromPayload(memory);
-  header->MarkIsInConstruction();
   // Placement new as regular operator new() is deleted.
   T* object = ::new (memory) T(std::forward<Args>(args)...);
-  header->UnmarkIsInConstruction();
+  header->MarkFullyConstructed();
   return object;
 }
 

third_party/blink/renderer/platform/heap/heap_allocator.h

@@ -75,7 +75,7 @@ class PLATFORM_EXPORT HeapAllocator {
     uint32_t gc_info_index = GCInfoTrait<HeapVectorBacking<T>>::Index();
     NormalPageArena* arena = static_cast<NormalPageArena*>(
         state->Heap().VectorBackingArena(gc_info_index));
-    return reinterpret_cast<T*>(arena->AllocateObject(
+    return MarkAsConstructed<T>(arena->AllocateObject(
         ThreadHeap::AllocationSizeFromSize(size), gc_info_index));
   }
   template <typename T>
@@ -86,7 +86,7 @@ class PLATFORM_EXPORT HeapAllocator {
     uint32_t gc_info_index = GCInfoTrait<HeapVectorBacking<T>>::Index();
     NormalPageArena* arena = static_cast<NormalPageArena*>(
         state->Heap().ExpandedVectorBackingArena(gc_info_index));
-    return reinterpret_cast<T*>(arena->AllocateObject(
+    return MarkAsConstructed<T>(arena->AllocateObject(
         ThreadHeap::AllocationSizeFromSize(size), gc_info_index));
   }
   static void FreeVectorBacking(void*);
@@ -100,7 +100,7 @@ class PLATFORM_EXPORT HeapAllocator {
     ThreadState* state =
         ThreadStateFor<ThreadingTrait<T>::kAffinity>::GetState();
     const char* type_name = WTF_HEAP_PROFILER_TYPE_NAME(HeapVectorBacking<T>);
-    return reinterpret_cast<T*>(state->Heap().AllocateOnArenaIndex(
+    return MarkAsConstructed<T>(state->Heap().AllocateOnArenaIndex(
         state, size, BlinkGC::kInlineVectorArenaIndex, gc_info_index,
         type_name));
   }
@@ -118,7 +118,7 @@ class PLATFORM_EXPORT HeapAllocator {
         ThreadStateFor<ThreadingTrait<T>::kAffinity>::GetState();
     const char* type_name =
         WTF_HEAP_PROFILER_TYPE_NAME(HeapHashTableBacking<HashTable>);
-    return reinterpret_cast<T*>(state->Heap().AllocateOnArenaIndex(
+    return MarkAsConstructed<T>(state->Heap().AllocateOnArenaIndex(
         state, size, BlinkGC::kHashTableArenaIndex, gc_info_index, type_name));
   }
   template <typename T, typename HashTable>
@@ -148,7 +148,7 @@ class PLATFORM_EXPORT HeapAllocator {
 
   template <typename Return, typename Metadata>
   static Return Malloc(size_t size, const char* type_name) {
-    return reinterpret_cast<Return>(ThreadHeap::Allocate<Metadata>(
+    return MarkAsConstructed<Return>(ThreadHeap::Allocate<Metadata>(
         size, IsEagerlyFinalizedType<Metadata>::value));
   }
 
@@ -298,6 +298,13 @@ class PLATFORM_EXPORT HeapAllocator {
   }
 
  private:
+  template <typename T>
+  static T* MarkAsConstructed(Address address) {
+    HeapObjectHeader::FromPayload(reinterpret_cast<void*>(address))
+        ->MarkFullyConstructed();
+    return reinterpret_cast<T*>(address);
+  }
+
   static void BackingFree(void*);
   static bool BackingExpand(void*, size_t);
   static bool BackingShrink(void*,

third_party/blink/renderer/platform/heap/heap_page.h

@@ -149,7 +149,7 @@ uint32_t ComputeRandomMagic();
 //
 // | random magic value (32 bits) | Only present on 64-bit platforms.
 // | gc_info_index (14 bits)      |
-// | in construction (1 bit)      |
+// | in construction (1 bit)      | true: bit not set; false bit set
 // | size (14 bits)               | Actually 17 bits because sizes are aligned.
 // | wrapper mark bit (1 bit)     |
 // | unused (1 bit)               |
@@ -227,9 +227,8 @@ class PLATFORM_EXPORT HeapObjectHeader {
   void Unmark();
   bool TryMark();
 
-  void MarkIsInConstruction();
-  void UnmarkIsInConstruction();
   bool IsInConstruction() const;
+  void MarkFullyConstructed();
 
   // The payload starts directly after the HeapObjectHeader, and the payload
   // size does not include the sizeof(HeapObjectHeader).
@@ -931,17 +930,12 @@ NO_SANITIZE_ADDRESS inline void HeapObjectHeader::SetSize(size_t size) {
 }
 
 NO_SANITIZE_ADDRESS inline bool HeapObjectHeader::IsInConstruction() const {
-  return encoded_ & kHeaderIsInConstructionMask;
+  return (encoded_ & kHeaderIsInConstructionMask) == 0;
 }
 
-NO_SANITIZE_ADDRESS inline void HeapObjectHeader::MarkIsInConstruction() {
-  DCHECK(!IsInConstruction());
-  encoded_ = encoded_ | kHeaderIsInConstructionMask;
-}
-
-NO_SANITIZE_ADDRESS inline void HeapObjectHeader::UnmarkIsInConstruction() {
+NO_SANITIZE_ADDRESS inline void HeapObjectHeader::MarkFullyConstructed() {
   DCHECK(IsInConstruction());
-  encoded_ = encoded_ & ~kHeaderIsInConstructionMask;
+  encoded_ |= kHeaderIsInConstructionMask;
 }
 
 NO_SANITIZE_ADDRESS inline bool HeapObjectHeader::IsValid() const {
@@ -1127,9 +1121,9 @@ NO_SANITIZE_ADDRESS inline HeapObjectHeader::HeapObjectHeader(
   magic_ = GetMagic();
 #endif
 
-  DCHECK(gc_info_index < GCInfoTable::kMaxIndex);
+  DCHECK_LT(gc_info_index, GCInfoTable::kMaxIndex);
   DCHECK_LT(size, kNonLargeObjectPageSizeMax);
-  DCHECK(!(size & kAllocationMask));
+  DCHECK_EQ(0u, size & kAllocationMask);
   encoded_ =
       static_cast<uint32_t>((gc_info_index << kHeaderGCInfoIndexShift) | size);
   if (header_location == kNormalPage) {
@@ -1140,6 +1134,7 @@ NO_SANITIZE_ADDRESS inline HeapObjectHeader::HeapObjectHeader(
   } else {
     DCHECK(PageFromObject(this)->IsLargeObjectPage());
   }
+  DCHECK(IsInConstruction());
 }
 
 }  // namespace blink