Reland "Basic implementation and tests for HangWatcher."

This fixes the compilation problems on ios and corrects
a couple typos. Compare patch sets 1 and 4 to see the differences between
the original commit and the relands.

Original change's description:
> Basic implementation and tests for HangWatcher.
>
> This class is a successor to ThreadWatcher which aims to be more
> versatile and provide a larger coverage for hang watching.
>
> Bug: 1034046
> Change-Id: I365fe8824eb34c37c75e6ddf6874ff26ecd494c6
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2008189
> Commit-Queue: Oliver Li <olivierli@chromium.org>
> Reviewed-by: François Doray <fdoray@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#736080}

Change-Id: I4785c8c489678fd537a0105d65419673fe383aa4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2026316
Auto-Submit: Oliver Li <olivierli@chromium.org>
Reviewed-by: François Doray <fdoray@chromium.org>
Commit-Queue: Oliver Li <olivierli@chromium.org>
Cr-Commit-Position: refs/heads/master@{#736521}

base/BUILD.gn

@@ -702,6 +702,8 @@ jumbo_component("base") {
     "third_party/nspr/prtime.h",
     "third_party/superfasthash/superfasthash.c",
     "thread_annotations.h",
+    "threading/hang_watcher.cc",
+    "threading/hang_watcher.h",
     "threading/platform_thread.cc",
     "threading/platform_thread.h",
     "threading/post_task_and_reply_impl.cc",
@@ -2705,6 +2707,7 @@ test("base_unittests") {
     "test/test_pending_task_unittest.cc",
     "test/trace_event_analyzer_unittest.cc",
     "thread_annotations_unittest.cc",
+    "threading/hang_watcher_unittest.cc",
     "threading/platform_thread_unittest.cc",
     "threading/post_task_and_reply_impl_unittest.cc",
     "threading/scoped_blocking_call_unittest.cc",

base/threading/hang_watcher.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/threading/hang_watcher.h"
+#include "base/bind.h"
+#include "base/callback_helpers.h"
+#include "base/no_destructor.h"
+#include "base/synchronization/lock.h"
+#include "base/threading/thread_checker.h"
+#include "base/threading/thread_restrictions.h"
+#include "base/time/time.h"
+
+namespace base {
+
+namespace {
+HangWatcher* g_instance = nullptr;
+}
+
+HangWatchScope::HangWatchScope(TimeDelta timeout) {
+  internal::HangWatchState* current_hang_watch_state =
+      internal::HangWatchState::GetHangWatchStateForCurrentThread()->Get();
+
+  DCHECK(current_hang_watch_state)
+      << "A scope can only be used on a thread that "
+         "registered for hang watching with HangWatcher::RegisterThread.";
+
+#if DCHECK_IS_ON()
+  previous_scope_ = current_hang_watch_state->GetCurrentHangWatchScope();
+  current_hang_watch_state->SetCurrentHangWatchScope(this);
+#endif
+
+  // TODO(crbug.com/1034046): Check whether we are over deadline already for the
+  // previous scope here by issuing only one TimeTicks::Now() and resuing the
+  // value.
+
+  previous_deadline_ = current_hang_watch_state->GetDeadline();
+  TimeTicks deadline = TimeTicks::Now() + timeout;
+  current_hang_watch_state->SetDeadline(deadline);
+}
+
+HangWatchScope::~HangWatchScope() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+  internal::HangWatchState* current_hang_watch_state =
+      internal::HangWatchState::GetHangWatchStateForCurrentThread()->Get();
+
+#if DCHECK_IS_ON()
+  // Verify that no Scope was destructed out of order.
+  DCHECK_EQ(this, current_hang_watch_state->GetCurrentHangWatchScope());
+  current_hang_watch_state->SetCurrentHangWatchScope(previous_scope_);
+#endif
+
+  // Reset the deadline to the value it had before entering this scope.
+  current_hang_watch_state->SetDeadline(previous_deadline_);
+  // TODO(crbug.com/1034046): Log when a HangWatchScope exits after its deadline
+  // and that went undetected by the HangWatcher.
+}
+
+HangWatcher::HangWatcher(RepeatingClosure on_hang_closure)
+    : on_hang_closure_(std::move(on_hang_closure)) {
+  DCHECK(!g_instance);
+  g_instance = this;
+}
+
+HangWatcher::~HangWatcher() {
+  DCHECK_EQ(g_instance, this);
+  DCHECK(watch_states_.empty());
+  g_instance = nullptr;
+}
+
+// static
+HangWatcher* HangWatcher::GetInstance() {
+  return g_instance;
+}
+
+ScopedClosureRunner HangWatcher::RegisterThread() {
+  AutoLock auto_lock(watch_state_lock_);
+
+  watch_states_.push_back(
+      internal::HangWatchState::CreateHangWatchStateForCurrentThread());
+
+  return ScopedClosureRunner(BindOnce(&HangWatcher::UnregisterThread,
+                                      Unretained(HangWatcher::GetInstance())));
+}
+
+void HangWatcher::Monitor() {
+  bool must_invoke_hang_closure = false;
+  {
+    AutoLock auto_lock(watch_state_lock_);
+    for (const auto& watch_state : watch_states_) {
+      if (watch_state->IsOverDeadline()) {
+        must_invoke_hang_closure = true;
+        break;
+      }
+    }
+  }
+
+  if (must_invoke_hang_closure) {
+    // Invoke the closure outside the scope of |watch_state_lock_|
+    // to prevent lock reentrancy.
+    on_hang_closure_.Run();
+  }
+}
+
+void HangWatcher::UnregisterThread() {
+  AutoLock auto_lock(watch_state_lock_);
+
+  internal::HangWatchState* current_hang_watch_state =
+      internal::HangWatchState::GetHangWatchStateForCurrentThread()->Get();
+
+  auto it =
+      std::find_if(watch_states_.cbegin(), watch_states_.cend(),
+                   [current_hang_watch_state](
+                       const std::unique_ptr<internal::HangWatchState>& state) {
+                     return state.get() == current_hang_watch_state;
+                   });
+
+  // Thread should be registered to get unregistered.
+  DCHECK(it != watch_states_.end());
+
+  watch_states_.erase(it);
+}
+
+namespace internal {
+
+// |deadline_| starts at Max() to avoid validation problems
+// when setting the first legitimate value.
+HangWatchState::HangWatchState() : deadline_(TimeTicks::Max()) {}
+
+HangWatchState::~HangWatchState() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+#if DCHECK_IS_ON()
+  // Destroying the HangWatchState should not be done if there are live
+  // HangWatchScopes.
+  DCHECK(!current_hang_watch_scope_);
+#endif
+}
+
+// static
+std::unique_ptr<HangWatchState>
+HangWatchState::CreateHangWatchStateForCurrentThread() {
+  // There should not exist a state object for this thread already.
+  DCHECK(!GetHangWatchStateForCurrentThread()->Get());
+
+  // Allocate a watch state object for this thread.
+  std::unique_ptr<HangWatchState> hang_state =
+      std::make_unique<HangWatchState>();
+
+  // Bind the new instance to this thread.
+  GetHangWatchStateForCurrentThread()->Set(hang_state.get());
+
+  // Setting the thread local worked.
+  DCHECK(GetHangWatchStateForCurrentThread()->Get());
+
+  // Transfer ownership to caller.
+  return hang_state;
+}
+
+TimeTicks HangWatchState::GetDeadline() const {
+  return deadline_.load();
+}
+
+TimeTicks HangWatchState::SetDeadline(TimeTicks deadline) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+  return deadline_.exchange(deadline);
+}
+
+bool HangWatchState::IsOverDeadline() const {
+  return TimeTicks::Now() > deadline_.load();
+}
+
+#if DCHECK_IS_ON()
+void HangWatchState::SetCurrentHangWatchScope(HangWatchScope* scope) {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+  current_hang_watch_scope_ = scope;
+}
+
+HangWatchScope* HangWatchState::GetCurrentHangWatchScope() {
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+  return current_hang_watch_scope_;
+}
+#endif
+
+// static
+ThreadLocalPointer<HangWatchState>*
+HangWatchState::GetHangWatchStateForCurrentThread() {
+  static NoDestructor<ThreadLocalPointer<HangWatchState>> hang_watch_state;
+  return hang_watch_state.get();
+}
+
+}  // namespace internal
+
+}  // namespace base

base/threading/hang_watcher.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_THREADING_HANG_WATCHER_H_
+#define BASE_THREADING_HANG_WATCHER_H_
+
+#include <exception>
+#include <memory>
+
+#include "base/atomicops.h"
+#include "base/callback.h"
+#include "base/callback_helpers.h"
+#include "base/threading/thread_checker.h"
+#include "base/threading/thread_local.h"
+#include "base/time/time.h"
+
+namespace base {
+class HangWatchScope;
+namespace internal {
+class HangWatchState;
+}  // namespace internal
+}  // namespace base
+
+namespace base {
+
+// Instantiate a HangWatchScope in a scope to register to be
+// watched for hangs of more than |timeout| by the HangWatcher.
+//
+// Example usage:
+//
+//  void FooBar(){
+//    HangWatchScope scope(base::TimeDelta::FromSeconds(5));
+//    DoSomeWork();
+//  }
+//
+// If DoSomeWork() takes more than 5s to run and the HangWatcher
+// inspects the thread state before Foobar returns a hang will be
+// reported. Instances of this object should live on the stack only as they are
+// intrinsically linked to the execution scopes that contain them.
+// Keeping a HangWatchScope alive after the scope in which it was created has
+// exited would lead to non-actionable hang reports.
+class BASE_EXPORT HangWatchScope {
+ public:
+  // Constructing/destructing thread must be the same thread.
+  explicit HangWatchScope(TimeDelta timeout);
+  ~HangWatchScope();
+
+  HangWatchScope(const HangWatchScope&) = delete;
+  HangWatchScope& operator=(const HangWatchScope&) = delete;
+
+ private:
+  // This object should always be constructed and destructed on the same thread.
+  THREAD_CHECKER(thread_checker_);
+
+  // The deadline set by the previous HangWatchScope created on this thread.
+  // Stored so it can be restored when this HangWatchScope is destroyed.
+  TimeTicks previous_deadline_;
+
+#if DCHECK_IS_ON()
+  // The previous HangWatchScope created on this thread.
+  HangWatchScope* previous_scope_;
+#endif
+};
+
+// Monitors registered threads for hangs by inspecting their associated
+// HangWatchStates for deadline overruns. Only one instance of HangWatcher can
+// exist at a time within a single process. This instance must outlive all
+// monitored threads.
+class BASE_EXPORT HangWatcher {
+ public:
+  // The first invocation of the constructor will set the global instance
+  // accessible through GetInstance(). This means that only one instance can
+  // exist at a time.
+  explicit HangWatcher(RepeatingClosure on_hang_closure);
+
+  // Clears the global instance for the class.
+  ~HangWatcher();
+
+  HangWatcher(const HangWatcher&) = delete;
+  HangWatcher& operator=(const HangWatcher&) = delete;
+
+  // Returns a non-owning pointer to the global HangWatcher instance.
+  static HangWatcher* GetInstance();
+
+  // Sets up the calling thread to be monitored for threads. Returns a
+  // ScopedClosureRunner that unregisters the thread. This closure has to be
+  // called from the registered thread before it's joined.
+  ScopedClosureRunner RegisterThread() WARN_UNUSED_RESULT;
+
+  // Inspects the state of all registered threads to check if they are hung.
+  void Monitor();
+
+ private:
+  // Stops hang watching on the calling thread by removing the entry from the
+  // watch list.
+  void UnregisterThread();
+
+  const RepeatingClosure on_hang_closure_;
+  Lock watch_state_lock_;
+
+  std::vector<std::unique_ptr<internal::HangWatchState>> watch_states_
+      GUARDED_BY(watch_state_lock_);
+};
+
+// Classes here are exposed in the header only for testing. They are not
+// intended to be used outside of base.
+namespace internal {
+
+// Contains the information necessary for hang watching a specific
+// thread. Instances of this class are accessed concurrently by the associated
+// thread and the HangWatcher. The HangWatcher owns instances of this
+// class and outside of it they are accessed through
+// GetHangWatchStateForCurrentThread().
+class BASE_EXPORT HangWatchState {
+ public:
+  HangWatchState();
+  ~HangWatchState();
+
+  HangWatchState(const HangWatchState&) = delete;
+  HangWatchState& operator=(const HangWatchState&) = delete;
+
+  // Allocates a new state object bound to the calling thread and returns an
+  // owning pointer to it.
+  static std::unique_ptr<HangWatchState> CreateHangWatchStateForCurrentThread();
+
+  // Retrieves the hang watch state associated with the calling thread.
+  // Returns nullptr if no HangWatchState exists for the current thread (see
+  // CreateHangWatchStateForCurrentThread()).
+  static ThreadLocalPointer<HangWatchState>*
+  GetHangWatchStateForCurrentThread();
+
+  // Returns the value of the current deadline. Use this function if you need to
+  // store the value. To test if the deadline has expired use IsOverDeadline().
+  TimeTicks GetDeadline() const;
+
+  // Atomically sets the deadline to a new value and return the previous value.
+  TimeTicks SetDeadline(TimeTicks deadline);
+
+  // Tests whether the associated thread's execution has gone over the deadline.
+  bool IsOverDeadline() const;
+
+#if DCHECK_IS_ON()
+  // Saves the supplied HangWatchScope as the currently active scope.
+  void SetCurrentHangWatchScope(HangWatchScope* scope);
+
+  // Retrieve the currently active scope.
+  HangWatchScope* GetCurrentHangWatchScope();
+#endif
+
+ private:
+  // The thread that creates the instance should be the class that updates
+  // the deadline.
+  THREAD_CHECKER(thread_checker_);
+
+  // If the deadline fails to be updated before TimeTicks::Now() ever
+  // reaches the value contained in it this constistutes a hang.
+  std::atomic<TimeTicks> deadline_;
+
+#if DCHECK_IS_ON()
+  // Used to keep track of the current HangWatchScope and detect improper usage.
+  // Scopes should always be destructed in reverse order from the one they were
+  // constructed in. Example of improper use:
+  //
+  // {
+  //   std::unique_ptr<Scope> scope = std::make_unique<Scope>(...);
+  //   Scope other_scope;
+  //   |scope| gets deallocated first, violating reverse destruction order.
+  //   scope.reset();
+  // }
+  HangWatchScope* current_hang_watch_scope_{nullptr};
+#endif
+};
+
+}  // namespace internal
+}  // namespace base
+
+#endif  // BASE_THREADING_HANG_WATCHER_H_

base/threading/hang_watcher_unittest.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/threading/hang_watcher.h"
+#include <memory>
+
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/callback_helpers.h"
+#include "base/synchronization/lock.h"
+#include "base/synchronization/waitable_event.h"
+#include "base/test/task_environment.h"
+#include "base/threading/platform_thread.h"
+#include "base/time/time.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace base {
+namespace {
+
+const base::TimeDelta kTimeout = base::TimeDelta::FromSeconds(10);
+const base::TimeDelta kHangTime = kTimeout + base::TimeDelta::FromSeconds(1);
+
+// Waits on provided WaitableEvent before executing and signals when done.
+class BlockingThread : public PlatformThread::Delegate {
+ public:
+  explicit BlockingThread(base::WaitableEvent* unblock_thread)
+      : unblock_thread_(unblock_thread) {}
+
+  void ThreadMain() override {
+    // (Un)Register the thread here instead of in ctor/dtor so that the action
+    // happens on the right thread.
+    base::ScopedClosureRunner unregister_closure =
+        base::HangWatcher::GetInstance()->RegisterThread();
+
+    HangWatchScope scope(kTimeout);
+    wait_until_entered_scope_.Signal();
+
+    unblock_thread_->Wait();
+    run_event_.Signal();
+  }
+
+  bool IsDone() { return run_event_.IsSignaled(); }
+
+  // Block until this thread registered itself for hang watching and has entered
+  // a HangWatchScope.
+  void WaitUntilScopeEntered() { wait_until_entered_scope_.Wait(); }
+
+ private:
+  // Will be signaled once the thread is properly registered for watching and
+  // scope has been entered.
+  WaitableEvent wait_until_entered_scope_;
+
+  // Will be signaled once ThreadMain has run.
+  WaitableEvent run_event_;
+
+  base::WaitableEvent* const unblock_thread_;
+};
+
+class HangWatcherTest : public testing::Test {
+ public:
+  HangWatcherTest()
+      : hang_watcher_(std::make_unique<HangWatcher>(
+            base::BindRepeating(&WaitableEvent::Signal,
+                                base::Unretained(&hang_event_)))),
+        thread_(&unblock_thread_) {}
+
+  void StartBlockedThread() {
+    // Thread has not run yet.
+    ASSERT_FALSE(thread_.IsDone());
+
+    // Start the thread. It will block since |unblock_thread_| was not
+    // signaled yet.
+    ASSERT_TRUE(PlatformThread::Create(0, &thread_, &handle));
+
+    thread_.WaitUntilScopeEntered();
+  }
+
+  void MonitorHangsAndJoinThread() {
+    // Try to detect a hang if any.
+    HangWatcher::GetInstance()->Monitor();
+
+    unblock_thread_.Signal();
+
+    // Thread is joinable since we signaled |unblock_thread_|.
+    PlatformThread::Join(handle);
+
+    // If thread is done then it signaled.
+    ASSERT_TRUE(thread_.IsDone());
+  }
+
+ protected:
+  std::unique_ptr<HangWatcher> hang_watcher_;
+
+  // Used exclusively for MOCK_TIME. No tasks will be run on the environment.
+  test::TaskEnvironment task_environment_{
+      test::TaskEnvironment::TimeSource::MOCK_TIME};
+
+  PlatformThreadHandle handle;
+
+  WaitableEvent unblock_thread_;
+
+  BlockingThread thread_;
+
+  // Signaled when a hang is detected.
+  WaitableEvent hang_event_;
+};
+}  // namespace
+
+TEST_F(HangWatcherTest, NoScopes) {
+  HangWatcher::GetInstance()->Monitor();
+  ASSERT_FALSE(hang_event_.IsSignaled());
+}
+
+TEST_F(HangWatcherTest, NestedScopes) {
+  // Create a state object for the test thread since this test is single
+  // threaded.
+  auto current_hang_watch_state =
+      base::internal::HangWatchState::CreateHangWatchStateForCurrentThread();
+
+  ASSERT_FALSE(current_hang_watch_state->IsOverDeadline());
+  base::TimeTicks original_deadline = current_hang_watch_state->GetDeadline();
+
+  constexpr base::TimeDelta kFirstTimeout(
+      base::TimeDelta::FromMilliseconds(500));
+  base::TimeTicks first_deadline = base::TimeTicks::Now() + kFirstTimeout;
+
+  constexpr base::TimeDelta kSecondTimeout(
+      base::TimeDelta::FromMilliseconds(250));
+  base::TimeTicks second_deadline = base::TimeTicks::Now() + kSecondTimeout;
+
+  // At this point we have not set any timeouts.
+  {
+    // Create a first timeout which is more restrictive than the default.
+    HangWatchScope first_scope(kFirstTimeout);
+
+    // We are on mock time. There is no time advancement and as such no hangs.
+    ASSERT_FALSE(current_hang_watch_state->IsOverDeadline());
+    ASSERT_EQ(current_hang_watch_state->GetDeadline(), first_deadline);
+    {
+      // Set a yet more restrictive deadline. Still no hang.
+      HangWatchScope second_scope(kSecondTimeout);
+      ASSERT_FALSE(current_hang_watch_state->IsOverDeadline());
+      ASSERT_EQ(current_hang_watch_state->GetDeadline(), second_deadline);
+    }
+    // First deadline we set should be restored.
+    ASSERT_FALSE(current_hang_watch_state->IsOverDeadline());
+    ASSERT_EQ(current_hang_watch_state->GetDeadline(), first_deadline);
+  }
+
+  // Original deadline should now be restored.
+  ASSERT_FALSE(current_hang_watch_state->IsOverDeadline());
+  ASSERT_EQ(current_hang_watch_state->GetDeadline(), original_deadline);
+}
+
+TEST_F(HangWatcherTest, Hang) {
+  StartBlockedThread();
+
+  // Simulate hang.
+  task_environment_.FastForwardBy(kHangTime);
+
+  MonitorHangsAndJoinThread();
+  ASSERT_TRUE(hang_event_.IsSignaled());
+}
+
+TEST_F(HangWatcherTest, NoHang) {
+  StartBlockedThread();
+
+  MonitorHangsAndJoinThread();
+  ASSERT_FALSE(hang_event_.IsSignaled());
+}
+
+}  // namespace base