Multi-Process Tracking Support

Fully support having multiple processes write information to the same breadcrumbs file.

This involves some process-specific operations plus the ability for a controlling process to clean-up after its dead children.

BUG=620813
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.win:win10_chromium_x64_rel_ng

Review-Url: https://codereview.chromium.org/2680123003
Cr-Commit-Position: refs/heads/master@{#457502}

base/debug/activity_tracker_unittest.cc

@@ -84,45 +84,73 @@ class ActivityTrackerTest : public testing::Test {
     return GlobalActivityTracker::Get()->user_data_allocator_.cache_used();
   }
 
+  void HandleProcessExit(int64_t id,
+                         int64_t stamp,
+                         int code,
+                         GlobalActivityTracker::ProcessPhase phase,
+                         std::string&& command,
+                         ActivityUserData::Snapshot&& data) {
+    exit_id = id;
+    exit_stamp = stamp;
+    exit_code = code;
+    exit_phase = phase;
+    exit_command = std::move(command);
+    exit_data = std::move(data);
+  }
+
   static void DoNothing() {}
+
+  int64_t exit_id = 0;
+  int64_t exit_stamp;
+  int exit_code;
+  GlobalActivityTracker::ProcessPhase exit_phase;
+  std::string exit_command;
+  ActivityUserData::Snapshot exit_data;
 };
 
 TEST_F(ActivityTrackerTest, UserDataTest) {
   char buffer[256];
   memset(buffer, 0, sizeof(buffer));
   ActivityUserData data(buffer, sizeof(buffer));
-  const size_t space = sizeof(buffer) - 8;
+  size_t space = sizeof(buffer) - sizeof(ActivityUserData::MemoryHeader);
   ASSERT_EQ(space, data.available_);
 
   data.SetInt("foo", 1);
-  ASSERT_EQ(space - 24, data.available_);
+  space -= 24;
+  ASSERT_EQ(space, data.available_);
 
   data.SetUint("b", 1U);  // Small names fit beside header in a word.
-  ASSERT_EQ(space - 24 - 16, data.available_);
+  space -= 16;
+  ASSERT_EQ(space, data.available_);
 
   data.Set("c", buffer, 10);
-  ASSERT_EQ(space - 24 - 16 - 24, data.available_);
+  space -= 24;
+  ASSERT_EQ(space, data.available_);
 
   data.SetString("dear john", "it's been fun");
-  ASSERT_EQ(space - 24 - 16 - 24 - 32, data.available_);
+  space -= 32;
+  ASSERT_EQ(space, data.available_);
 
   data.Set("c", buffer, 20);
-  ASSERT_EQ(space - 24 - 16 - 24 - 32, data.available_);
+  ASSERT_EQ(space, data.available_);
 
   data.SetString("dear john", "but we're done together");
-  ASSERT_EQ(space - 24 - 16 - 24 - 32, data.available_);
+  ASSERT_EQ(space, data.available_);
 
   data.SetString("dear john", "bye");
-  ASSERT_EQ(space - 24 - 16 - 24 - 32, data.available_);
+  ASSERT_EQ(space, data.available_);
 
   data.SetChar("d", 'x');
-  ASSERT_EQ(space - 24 - 16 - 24 - 32 - 8, data.available_);
+  space -= 8;
+  ASSERT_EQ(space, data.available_);
 
   data.SetBool("ee", true);
-  ASSERT_EQ(space - 24 - 16 - 24 - 32 - 8 - 16, data.available_);
+  space -= 16;
+  ASSERT_EQ(space, data.available_);
 
   data.SetString("f", "");
-  ASSERT_EQ(space - 24 - 16 - 24 - 32 - 8 - 16 - 8, data.available_);
+  space -= 8;
+  ASSERT_EQ(space, data.available_);
 }
 
 TEST_F(ActivityTrackerTest, PushPopTest) {
@@ -250,6 +278,16 @@ TEST_F(ActivityTrackerTest, CreateWithFileTest) {
 
 // GlobalActivityTracker tests below.
 
+TEST_F(ActivityTrackerTest, BasicTest) {
+  GlobalActivityTracker::CreateWithLocalMemory(kMemorySize, 0, "", 3);
+  GlobalActivityTracker* global = GlobalActivityTracker::Get();
+
+  // Ensure the data repositories have backing store, indicated by non-zero ID.
+  EXPECT_NE(0U, global->process_data().id());
+  EXPECT_NE(0U, global->global_data().id());
+  EXPECT_NE(global->process_data().id(), global->global_data().id());
+}
+
 class SimpleActivityThread : public SimpleThread {
  public:
   SimpleActivityThread(const std::string& name,
@@ -336,5 +374,107 @@ TEST_F(ActivityTrackerTest, ThreadDeathTest) {
   EXPECT_EQ(starting_inactive + 1, GetGlobalInactiveTrackerCount());
 }
 
+TEST_F(ActivityTrackerTest, ProcessDeathTest) {
+  // This doesn't actually create and destroy a process. Instead, it uses for-
+  // testing interfaces to simulate data created by other processes.
+  const ProcessId other_process_id = GetCurrentProcId() + 1;
+
+  GlobalActivityTracker::CreateWithLocalMemory(kMemorySize, 0, "", 3);
+  GlobalActivityTracker* global = GlobalActivityTracker::Get();
+  ThreadActivityTracker* thread = global->GetOrCreateTrackerForCurrentThread();
+
+  // Get callbacks for process exit.
+  global->SetProcessExitCallback(
+      Bind(&ActivityTrackerTest::HandleProcessExit, Unretained(this)));
+
+  // Pretend than another process has started.
+  global->RecordProcessLaunch(other_process_id, FILE_PATH_LITERAL("foo --bar"));
+
+  // Do some activities.
+  PendingTask task(FROM_HERE, base::Bind(&DoNothing));
+  ScopedTaskRunActivity activity(task);
+  ActivityUserData& user_data = activity.user_data();
+  ASSERT_NE(0U, user_data.id());
+
+  // Get the memory-allocator references to that data.
+  PersistentMemoryAllocator::Reference proc_data_ref =
+      global->allocator()->GetAsReference(
+          global->process_data().GetBaseAddress(),
+          GlobalActivityTracker::kTypeIdProcessDataRecord);
+  ASSERT_TRUE(proc_data_ref);
+  PersistentMemoryAllocator::Reference tracker_ref =
+      global->allocator()->GetAsReference(
+          thread->GetBaseAddress(),
+          GlobalActivityTracker::kTypeIdActivityTracker);
+  ASSERT_TRUE(tracker_ref);
+  PersistentMemoryAllocator::Reference user_data_ref =
+      global->allocator()->GetAsReference(
+          user_data.GetBaseAddress(),
+          GlobalActivityTracker::kTypeIdUserDataRecord);
+  ASSERT_TRUE(user_data_ref);
+
+  // Make a copy of the thread-tracker state so it can be restored later.
+  const size_t tracker_size = global->allocator()->GetAllocSize(tracker_ref);
+  std::unique_ptr<char[]> tracker_copy(new char[tracker_size]);
+  memcpy(tracker_copy.get(), thread->GetBaseAddress(), tracker_size);
+
+  // Change the objects to appear to be owned by another process.
+  ProcessId owning_id;
+  int64_t stamp;
+  ASSERT_TRUE(ActivityUserData::GetOwningProcessId(
+      global->process_data().GetBaseAddress(), &owning_id, &stamp));
+  EXPECT_NE(other_process_id, owning_id);
+  ASSERT_TRUE(ThreadActivityTracker::GetOwningProcessId(
+      thread->GetBaseAddress(), &owning_id, &stamp));
+  EXPECT_NE(other_process_id, owning_id);
+  ASSERT_TRUE(ActivityUserData::GetOwningProcessId(user_data.GetBaseAddress(),
+                                                   &owning_id, &stamp));
+  EXPECT_NE(other_process_id, owning_id);
+  global->process_data().SetOwningProcessIdForTesting(other_process_id, stamp);
+  thread->SetOwningProcessIdForTesting(other_process_id, stamp);
+  user_data.SetOwningProcessIdForTesting(other_process_id, stamp);
+  ASSERT_TRUE(ActivityUserData::GetOwningProcessId(
+      global->process_data().GetBaseAddress(), &owning_id, &stamp));
+  EXPECT_EQ(other_process_id, owning_id);
+  ASSERT_TRUE(ThreadActivityTracker::GetOwningProcessId(
+      thread->GetBaseAddress(), &owning_id, &stamp));
+  EXPECT_EQ(other_process_id, owning_id);
+  ASSERT_TRUE(ActivityUserData::GetOwningProcessId(user_data.GetBaseAddress(),
+                                                   &owning_id, &stamp));
+  EXPECT_EQ(other_process_id, owning_id);
+
+  // Check that process exit will perform callback and free the allocations.
+  ASSERT_EQ(0, exit_id);
+  ASSERT_EQ(GlobalActivityTracker::kTypeIdProcessDataRecord,
+            global->allocator()->GetType(proc_data_ref));
+  ASSERT_EQ(GlobalActivityTracker::kTypeIdActivityTracker,
+            global->allocator()->GetType(tracker_ref));
+  ASSERT_EQ(GlobalActivityTracker::kTypeIdUserDataRecord,
+            global->allocator()->GetType(user_data_ref));
+  global->RecordProcessExit(other_process_id, 0);
+  EXPECT_EQ(other_process_id, exit_id);
+  EXPECT_EQ("foo --bar", exit_command);
+  EXPECT_EQ(GlobalActivityTracker::kTypeIdProcessDataRecordFree,
+            global->allocator()->GetType(proc_data_ref));
+  EXPECT_EQ(GlobalActivityTracker::kTypeIdActivityTrackerFree,
+            global->allocator()->GetType(tracker_ref));
+  EXPECT_EQ(GlobalActivityTracker::kTypeIdUserDataRecordFree,
+            global->allocator()->GetType(user_data_ref));
+
+  // Restore memory contents and types so things don't crash when doing real
+  // process clean-up.
+  memcpy(const_cast<void*>(thread->GetBaseAddress()), tracker_copy.get(),
+         tracker_size);
+  global->allocator()->ChangeType(
+      proc_data_ref, GlobalActivityTracker::kTypeIdProcessDataRecord,
+      GlobalActivityTracker::kTypeIdUserDataRecordFree, false);
+  global->allocator()->ChangeType(
+      tracker_ref, GlobalActivityTracker::kTypeIdActivityTracker,
+      GlobalActivityTracker::kTypeIdActivityTrackerFree, false);
+  global->allocator()->ChangeType(
+      user_data_ref, GlobalActivityTracker::kTypeIdUserDataRecord,
+      GlobalActivityTracker::kTypeIdUserDataRecordFree, false);
+}
+
 }  // namespace debug
 }  // namespace base

base/process/launch_win.cc

@@ -17,6 +17,7 @@
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/command_line.h"
+#include "base/debug/activity_tracker.h"
 #include "base/debug/stack_trace.h"
 #include "base/logging.h"
 #include "base/message_loop/message_loop.h"
@@ -94,7 +95,12 @@ bool GetAppOutputInternal(const StringPiece16& cl,
     NOTREACHED() << "Failed to start process";
     return false;
   }
+
   base::win::ScopedProcessInformation proc_info(temp_process_info);
+  base::debug::GlobalActivityTracker* tracker =
+      base::debug::GlobalActivityTracker::Get();
+  if (tracker)
+    tracker->RecordProcessLaunch(proc_info.process_id(), cl.as_string());
 
   // Close our writing end of pipe now. Otherwise later read would not be able
   // to detect end of child's output.
@@ -119,6 +125,8 @@ bool GetAppOutputInternal(const StringPiece16& cl,
   int exit_code;
   base::TerminationStatus status = GetTerminationStatus(
       proc_info.process_handle(), &exit_code);
+  base::debug::GlobalActivityTracker::RecordProcessExitIfEnabled(
+      proc_info.process_id(), exit_code);
   return status != base::TERMINATION_STATUS_PROCESS_CRASHED &&
          status != base::TERMINATION_STATUS_ABNORMAL_TERMINATION;
 }
@@ -317,6 +325,8 @@ Process LaunchProcess(const string16& cmdline,
   if (options.wait)
     WaitForSingleObject(process_info.process_handle(), INFINITE);
 
+  base::debug::GlobalActivityTracker::RecordProcessLaunchIfEnabled(
+      process_info.process_id(), cmdline);
   return Process(process_info.TakeProcessHandle());
 }
 
@@ -344,6 +354,8 @@ Process LaunchElevatedProcess(const CommandLine& cmdline,
   if (options.wait)
     WaitForSingleObject(shex_info.hProcess, INFINITE);
 
+  base::debug::GlobalActivityTracker::RecordProcessLaunchIfEnabled(
+      GetProcessId(shex_info.hProcess), file, arguments);
   return Process(shex_info.hProcess);
 }
 

base/process/process_win.cc

@@ -139,6 +139,10 @@ bool Process::Terminate(int exit_code, bool wait) const {
   } else if (!result) {
     DPLOG(ERROR) << "Unable to terminate process";
   }
+  if (result) {
+    base::debug::GlobalActivityTracker::RecordProcessExitIfEnabled(Pid(),
+                                                                   exit_code);
+  }
   return result;
 }
 
@@ -162,6 +166,9 @@ bool Process::WaitForExitWithTimeout(TimeDelta timeout, int* exit_code) const {
 
   if (exit_code)
     *exit_code = temp_code;
+
+  base::debug::GlobalActivityTracker::RecordProcessExitIfEnabled(
+      Pid(), static_cast<int>(temp_code));
   return true;
 }
 

content/app/content_main.cc

@@ -6,6 +6,7 @@
 
 #include <memory>
 
+#include "base/debug/activity_tracker.h"
 #include "content/public/app/content_main_runner.h"
 
 namespace content {
@@ -14,13 +15,34 @@ int ContentMain(const ContentMainParams& params) {
   std::unique_ptr<ContentMainRunner> main_runner(ContentMainRunner::Create());
 
   int exit_code = main_runner->Initialize(params);
-  if (exit_code >= 0)
+  if (exit_code >= 0) {
+    base::debug::GlobalActivityTracker* tracker =
+        base::debug::GlobalActivityTracker::Get();
+    if (tracker) {
+      tracker->SetProcessPhase(
+          base::debug::GlobalActivityTracker::PROCESS_LAUNCH_FAILED);
+      tracker->process_data().SetInt("exit-code", exit_code);
+    }
     return exit_code;
+  }
 
   exit_code = main_runner->Run();
 
   main_runner->Shutdown();
 
+  base::debug::GlobalActivityTracker* tracker =
+      base::debug::GlobalActivityTracker::Get();
+  if (tracker) {
+    if (exit_code == 0) {
+      tracker->SetProcessPhaseIfEnabled(
+          base::debug::GlobalActivityTracker::PROCESS_EXITED_CLEANLY);
+    } else {
+      tracker->SetProcessPhaseIfEnabled(
+          base::debug::GlobalActivityTracker::PROCESS_EXITED_WITH_CODE);
+      tracker->process_data().SetInt("exit-code", exit_code);
+    }
+  }
+
   return exit_code;
 }
 

content/common/sandbox_win.cc

@@ -10,6 +10,7 @@
 
 #include "base/base_switches.h"
 #include "base/command_line.h"
+#include "base/debug/activity_tracker.h"
 #include "base/debug/profiler.h"
 #include "base/files/file_util.h"
 #include "base/hash.h"
@@ -836,6 +837,13 @@ sandbox::ResultCode StartSandboxedProcess(
 
   TRACE_EVENT_END0("startup", "StartProcessWithAccess::LAUNCHPROCESS");
 
+  base::debug::GlobalActivityTracker* tracker =
+      base::debug::GlobalActivityTracker::Get();
+  if (tracker) {
+    tracker->RecordProcessLaunch(target.process_id(),
+                                 cmd_line->GetCommandLineString());
+  }
+
   if (sandbox::SBOX_ALL_OK != result) {
     UMA_HISTOGRAM_SPARSE_SLOWLY("Process.Sandbox.Launch.Error", last_error);
     if (result == sandbox::SBOX_ERROR_GENERIC)