[iOS] Attach breadcrumbs to breakpad crash reports

Breadcrumbs need to be attached to crash reports before upload. Ideally,
|BreadcrumbPersistentStorageKeyedService| instances would store
breadcrumbs associated with each browser state in their respective
storage directories. However, this would then require using
|BreadcrumbController setParametersToAddAtUploadTime:| which must be
called before breakpad is started and the state of the
|kLogBreadcrumbs| feature flag is not yet available for use before
breakpad is started. Attaching breadcrumbs in this way would also
require merging and sorting multiple breadcrumb files during the primary
launch flow of the application which may impact time to cold launch
performance.

Therefore, in order to attach beadcrumb events to breakpad crash logs,
create a |CrashReporterBreadcrumbObserver| crash report helper object.
Although it is not ideal that this object is a singleton, it follows
the pattern of other similar classes defined in crash_report_helper.mm.
The class was added to its own file to allow for the creation of some
minimal unittests. In order to support this class a
BreadcrumbManagerObserverBridge was added. This process can be improved
once crashpad is adopted as it supports upload streaming of files on
disk.

Bug: 1003922
Change-Id: I9eb21d390a27a0c69e64397fbb82cabcc1ea2979
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1979449Reviewed-by: Eugene But <eugenebut@chromium.org>
Commit-Queue: Mike Dougherty <michaeldo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#727289}

ios/chrome/browser/crash_report/BUILD.gn

@@ -12,6 +12,8 @@ source_set("crash_report") {
     "crash_report_multi_parameter.mm",
     "crash_report_user_application_state.h",
     "crash_report_user_application_state.mm",
+    "crash_reporter_breadcrumb_observer.h",
+    "crash_reporter_breadcrumb_observer.mm",
     "crash_upload_list.cc",
     "crash_upload_list.h",
     "main_thread_freeze_detector.h",
@@ -26,6 +28,8 @@ source_set("crash_report") {
     "//components/crash/core/common",
     "//components/upload_list",
     "//ios/chrome/browser",
+    "//ios/chrome/browser/browser_state",
+    "//ios/chrome/browser/crash_report/breadcrumbs",
     "//ios/web",
     "//third_party/breakpad:client",
   ]
@@ -79,6 +83,7 @@ source_set("unit_tests") {
   sources = [
     "breakpad_helper_unittest.mm",
     "crash_loop_detection_util_unittest.mm",
+    "crash_reporter_breadcrumb_observer_unittest.mm",
     "crash_restore_helper_unittest.mm",
   ]
   deps = [
@@ -88,6 +93,7 @@ source_set("unit_tests") {
     "//base/test:test_support",
     "//ios/chrome/browser/browser_state",
     "//ios/chrome/browser/browser_state:test_support",
+    "//ios/chrome/browser/crash_report/breadcrumbs",
     "//ios/chrome/browser/sessions:serialisation",
     "//ios/chrome/browser/sessions:session_service",
     "//ios/chrome/test/ocmock",

ios/chrome/browser/crash_report/breadcrumbs/BUILD.gn

@@ -30,6 +30,8 @@ source_set("breadcrumbs") {
     "breadcrumb_manager_keyed_service_factory.cc",
     "breadcrumb_manager_keyed_service_factory.h",
     "breadcrumb_manager_observer.h",
+    "breadcrumb_manager_observer_bridge.h",
+    "breadcrumb_manager_observer_bridge.mm",
     "breadcrumb_manager_tab_helper.h",
     "breadcrumb_manager_tab_helper.mm",
     "breadcrumb_persistent_storage_keyed_service.cc",
@@ -54,10 +56,12 @@ source_set("unit_tests") {
     "//ios/chrome/test:test_support",
     "//ios/web/public/test",
     "//testing/gtest",
+    "//third_party/ocmock",
   ]
 
   sources = [
     "breadcrumb_manager_keyed_service_unittest.mm",
+    "breadcrumb_manager_observer_bridge_unittest.mm",
     "breadcrumb_manager_observer_unittest.mm",
     "breadcrumb_manager_tab_helper_unittest.mm",
     "breadcrumb_persistent_storage_keyed_service_unittest.mm",

ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_observer_bridge.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IOS_CHROME_BROWSER_CRASH_REPORT_BREADCRUMBS_BREADCRUMB_MANAGER_OBSERVER_BRIDGE_H_
+#define IOS_CHROME_BROWSER_CRASH_REPORT_BREADCRUMBS_BREADCRUMB_MANAGER_OBSERVER_BRIDGE_H_
+
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_observer.h"
+
+class BreadcrumbManagerKeyedService;
+
+// Protocol mirroring BreadcrumbManagerObserver
+@protocol BreadcrumbManagerObserving
+- (void)breadcrumbManager:(BreadcrumbManagerKeyedService*)manager
+              didAddEvent:(NSString*)string;
+@end
+
+// A C++ bridge class to handle receiving notifications from the C++ class
+// that observes the connection type.
+class BreadcrumbManagerObserverBridge : public BreadcrumbManagerObserver {
+ public:
+  explicit BreadcrumbManagerObserverBridge(
+      BreadcrumbManagerKeyedService* breadcrumb_manager_keyed_service,
+      id<BreadcrumbManagerObserving> observer);
+  ~BreadcrumbManagerObserverBridge() override;
+
+ private:
+  BreadcrumbManagerObserverBridge(const BreadcrumbManagerObserverBridge&) =
+      delete;
+  BreadcrumbManagerObserverBridge& operator=(
+      const BreadcrumbManagerObserverBridge&) = delete;
+
+  // BreadcrumbManagerObserver implementation:
+  void EventAdded(BreadcrumbManagerKeyedService* manager,
+                  const std::string& event) override;
+
+  BreadcrumbManagerKeyedService* breadcrumb_manager_keyed_service_ = nullptr;
+  __weak id<BreadcrumbManagerObserving> observer_ = nil;
+};
+
+#endif  // IOS_CHROME_BROWSER_CRASH_REPORT_BREADCRUMBS_BREADCRUMB_MANAGER_OBSERVER_BRIDGE_H_

ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_observer_bridge.mm

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_observer_bridge.h"
+
+#include "base/logging.h"
+#include "base/strings/sys_string_conversions.h"
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_keyed_service.h"
+
+#if !defined(__has_feature) || !__has_feature(objc_arc)
+#error "This file requires ARC support."
+#endif
+
+BreadcrumbManagerObserverBridge::BreadcrumbManagerObserverBridge(
+    BreadcrumbManagerKeyedService* breadcrumb_manager_keyed_service,
+    id<BreadcrumbManagerObserving> observer)
+    : breadcrumb_manager_keyed_service_(breadcrumb_manager_keyed_service),
+      observer_(observer) {
+  DCHECK(observer_);
+  breadcrumb_manager_keyed_service_->AddObserver(this);
+}
+
+BreadcrumbManagerObserverBridge::~BreadcrumbManagerObserverBridge() {
+  breadcrumb_manager_keyed_service_->RemoveObserver(this);
+}
+
+void BreadcrumbManagerObserverBridge::EventAdded(
+    BreadcrumbManagerKeyedService* manager,
+    const std::string& event) {
+  [observer_ breadcrumbManager:manager
+                   didAddEvent:base::SysUTF8ToNSString(event)];
+}

ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_observer_bridge_unittest.mm

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_observer_bridge.h"
+
+#import "base/strings/sys_string_conversions.h"
+#include "base/test/task_environment.h"
+#include "ios/chrome/browser/browser_state/test_chrome_browser_state.h"
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_keyed_service.h"
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_keyed_service_factory.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#import "testing/gtest_mac.h"
+#include "testing/platform_test.h"
+#import "third_party/ocmock/OCMock/OCMock.h"
+#include "third_party/ocmock/gtest_support.h"
+
+#if !defined(__has_feature) || !__has_feature(objc_arc)
+#error "This file requires ARC support."
+#endif
+
+// Test fixture to test BreadcrumbManagerObserverBridge class.
+class BreadcrumbManagerObserverBridgeTest : public PlatformTest {
+ protected:
+  BreadcrumbManagerObserverBridgeTest() {
+    TestChromeBrowserState::Builder test_cbs_builder;
+    chrome_browser_state_ = test_cbs_builder.Build();
+
+    breadcrumb_service_ =
+        BreadcrumbManagerKeyedServiceFactory::GetForBrowserState(
+            chrome_browser_state_.get());
+
+    mock_observer_ = OCMProtocolMock(@protocol(BreadcrumbManagerObserving));
+    observer_bridge_ = std::make_unique<BreadcrumbManagerObserverBridge>(
+        breadcrumb_service_, mock_observer_);
+  }
+
+  base::test::TaskEnvironment task_environment_;
+  std::unique_ptr<TestChromeBrowserState> chrome_browser_state_;
+  BreadcrumbManagerKeyedService* breadcrumb_service_;
+  id mock_observer_;
+  std::unique_ptr<BreadcrumbManagerObserverBridge> observer_bridge_;
+};
+
+// Tests |breadcrumbManager:didAddEvent:| forwarding.
+TEST_F(BreadcrumbManagerObserverBridgeTest, EventAdded) {
+  std::string event("sample event");
+
+  id event_parameter_validator = [OCMArg checkWithBlock:^BOOL(id value) {
+    // The manager will prepended a timestamp to the event so verify that the
+    // end matches |event|.
+    return [value isKindOfClass:[NSString class]] &&
+           [value hasSuffix:base::SysUTF8ToNSString(event)];
+  }];
+  OCMExpect([mock_observer_ breadcrumbManager:breadcrumb_service_
+                                  didAddEvent:event_parameter_validator]);
+
+  breadcrumb_service_->AddEvent(event);
+
+  [mock_observer_ verify];
+}

ios/chrome/browser/crash_report/breakpad_helper.h

@@ -124,6 +124,9 @@ void SetGridToVisibleTabAnimation(NSString* to_view_controller,
 // tab.
 void RemoveGridToVisibleTabAnimation();
 
+// Sets a key with the given |breadcrumbs|.
+void SetBreadcrumbEvents(NSString* breadcrumbs);
+
 // Sets a key in browser to store the playback state of media player (audio or
 // video). This function records a new start. This function is called for each
 // stream in the media (once or twice for audio, two or three times for video).

ios/chrome/browser/crash_report/breakpad_helper.mm

@@ -51,6 +51,7 @@ NSString* const kMemoryWarningCount = @"memory_warning_count";
 NSString* const kUptimeAtRestoreInMs = @"uptime_at_restore_in_ms";
 NSString* const kUploadedInRecoveryMode = @"uploaded_in_recovery_mode";
 NSString* const kGridToVisibleTabAnimation = @"grid_to_visible_tab_animation";
+NSString* const kBrowserStateBreadcrumbs = @"browser_state_breadcrumbs";
 
 // Multiple state information are combined into one CrachReportMultiParameter
 // to save limited and finite number of ReportParameters.
@@ -372,6 +373,10 @@ void RemoveGridToVisibleTabAnimation() {
   RemoveReportParameter(kGridToVisibleTabAnimation);
 }
 
+void SetBreadcrumbEvents(NSString* breadcrumbs) {
+  AddReportParameter(kBrowserStateBreadcrumbs, breadcrumbs, true);
+}
+
 void MediaStreamPlaybackDidStart() {
   [[CrashReportUserApplicationState sharedInstance]
       incrementValue:kVideoPlaying];

ios/chrome/browser/crash_report/breakpad_helper_unittest.mm

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 #import "ios/chrome/browser/crash_report/breakpad_helper.h"
-#import "base/test/ios/wait_util.h"
 #include "ios/chrome/browser/crash_report/main_thread_freeze_detector.h"
 #import "ios/chrome/test/ocmock/OCMockObject+BreakpadControllerTesting.h"
 #import "ios/testing/scoped_block_swizzler.h"
@@ -58,15 +57,32 @@ TEST_F(BreakpadHelperTest, CrashReportUserApplicationStateAllKeys) {
   // single breakpad record. This test should include all keys for
   // CrashReportUserApplicationState, since the whole dictionary is considered a
   // single breakpad record.
+  breakpad_helper::SetCurrentlyInBackground(true);
+  breakpad_helper::SetCurrentlySignedIn(true);
+  breakpad_helper::SetMemoryWarningCount(2);
+  breakpad_helper::SetMemoryWarningInProgress(true);
+  breakpad_helper::SetHangReport(true);
+  breakpad_helper::SetCurrentFreeMemoryInKB(1234);
+  breakpad_helper::SetCurrentFreeDiskInKB(12345);
+  breakpad_helper::SetCurrentTabIsPDF(true);
   breakpad_helper::SetCurrentOrientation(3, 7);
   breakpad_helper::SetCurrentHorizontalSizeClass(2);
   breakpad_helper::SetCurrentUserInterfaceStyle(2);
   breakpad_helper::SetRegularTabCount(999);
   breakpad_helper::SetIncognitoTabCount(999);
   breakpad_helper::SetDestroyingAndRebuildingIncognitoBrowserState(true);
+  breakpad_helper::SetGridToVisibleTabAnimation(
+      @"to_view_controller", @"presenting_view_controller",
+      @"presented_view_controller", @"parent_view_controller");
   breakpad_helper::MediaStreamPlaybackDidStart();
-  breakpad_helper::SetCurrentTabIsPDF(true);
-  breakpad_helper::SetCurrentlySignedIn(true);
+
+  // Build a sample breadcrumbs string greater than the maximum value size as
+  // defined in Breakpad.h at the BreakpadSetKeyValue function comment.
+  NSMutableString* breadcrumbs = [[NSMutableString alloc] init];
+  while (breadcrumbs.length < 255) {
+    [breadcrumbs appendString:@"12:01 Fake Breadcrumb Event/n"];
+  }
+  breakpad_helper::SetBreadcrumbEvents(breadcrumbs);
 }
 
 TEST_F(BreakpadHelperTest, GetCrashReportCount) {

ios/chrome/browser/crash_report/crash_report_helper.h

@@ -7,6 +7,10 @@
 
 @class NSString;
 
+namespace ios {
+class ChromeBrowserState;
+}  // namespace ios
+
 namespace web {
 class WebState;
 }  // namespace web
@@ -41,6 +45,17 @@ void StopMonitoringTabStateForWebStateList(WebStateList* web_state_list);
 // Clear any state about the urls loaded in the given WebStateList; this should
 // be called when the WebStateList is deactivated.
 void ClearStateForWebStateList(WebStateList* web_state_list);
+
+// Starts listening for breadcrumbs logged to |browser_state|'s
+// BreadcrumbManagerKeyedService. Collected breadcrumbs will be attached to
+// crash reports.
+void MonitorBreadcrumbsForBrowserState(ios::ChromeBrowserState* browser_state);
+
+// Stops listening for breadcrumbs logged to |browser_state|'s
+// BreadcrumbManagerKeyedService.
+void StopMonitoringBreadcrumbsForBrowserState(
+    ios::ChromeBrowserState* browser_state);
+
 }  // namespace breakpad
 
 #endif  // IOS_CHROME_BROWSER_CRASH_REPORT_CRASH_REPORT_HELPER_H_

ios/chrome/browser/crash_report/crash_report_helper.mm

@@ -19,6 +19,7 @@
 #include "ios/chrome/browser/chrome_paths.h"
 #include "ios/chrome/browser/crash_report/breakpad_helper.h"
 #import "ios/chrome/browser/crash_report/crash_report_user_application_state.h"
+#import "ios/chrome/browser/crash_report/crash_reporter_breadcrumb_observer.h"
 #import "ios/chrome/browser/web/tab_id_tab_helper.h"
 #import "ios/chrome/browser/web_state_list/all_web_state_observation_forwarder.h"
 #import "ios/chrome/browser/web_state_list/web_state_list.h"
@@ -438,4 +439,15 @@ void ClearStateForWebStateList(WebStateList* web_state_list) {
   }
 }
 
+void MonitorBreadcrumbsForBrowserState(ios::ChromeBrowserState* browser_state) {
+  [[CrashReporterBreadcrumbObserver uniqueInstance]
+      observeBrowserState:browser_state];
+}
+
+void StopMonitoringBreadcrumbsForBrowserState(
+    ios::ChromeBrowserState* browser_state) {
+  [[CrashReporterBreadcrumbObserver uniqueInstance]
+      stopObservingBrowserState:browser_state];
+}
+
 }  // namespace breakpad

ios/chrome/browser/crash_report/crash_reporter_breadcrumb_observer.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IOS_CHROME_BROWSER_CRASH_REPORT_CRASH_REPORTER_BREADCRUMB_OBSERVER_H_
+#define IOS_CHROME_BROWSER_CRASH_REPORT_CRASH_REPORTER_BREADCRUMB_OBSERVER_H_
+
+#include <map>
+#include <memory>
+
+#import <Foundation/Foundation.h>
+
+#import "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_observer_bridge.h"
+
+namespace ios {
+class ChromeBrowserState;
+}  // namespace ios
+
+// Combines breadcrumbs from multiple ChromeBrowserState instances and sends the
+// merged breadcrumb events to breakpad for attachment to crash reports.
+@interface CrashReporterBreadcrumbObserver
+    : NSObject <BreadcrumbManagerObserving> {
+}
+
+// Creates a singleton instance.
++ (CrashReporterBreadcrumbObserver*)uniqueInstance;
+
+// Starts collecting breadcrumb events associated with |browserState|.
+- (void)observeBrowserState:(ios::ChromeBrowserState*)browserState;
+// Stops collecting breadcrumb events associated with |browserState|.
+- (void)stopObservingBrowserState:(ios::ChromeBrowserState*)browserState;
+
+@end
+
+#endif  // IOS_CHROME_BROWSER_CRASH_REPORT_CRASH_REPORTER_BREADCRUMB_OBSERVER_H_

ios/chrome/browser/crash_report/crash_reporter_breadcrumb_observer.mm

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ios/chrome/browser/crash_report/crash_reporter_breadcrumb_observer.h"
+
+#include "ios/chrome/browser/browser_state/chrome_browser_state.h"
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_keyed_service.h"
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_keyed_service_factory.h"
+#include "ios/chrome/browser/crash_report/breakpad_helper.h"
+
+#if !defined(__has_feature) || !__has_feature(objc_arc)
+#error "This file requires ARC support."
+#endif
+
+namespace {
+// The maximum string length of combined breadcrumb events to store at any given
+// time. Breakpad truncates long values, so keeping the string stored here short
+// will save on used memory. This will not affect the amount of data attached to
+// crash reports (as long as the value matches or exceeds the breakpad maximum).
+const int kMaxCombinedBreadcrumbLength = 255;
+}
+
+@interface CrashReporterBreadcrumbObserver () {
+  // Map associating the observed ChromeBrowserStates with the corresponding
+  // observer bridge instances.
+  std::map<ios::ChromeBrowserState*,
+           std::unique_ptr<BreadcrumbManagerObserverBridge>>
+      _breadcrumbManagerObservers;
+  // A string which stores the received breadcrumbs. Since breakpad will
+  // truncate this string anyway, it is truncated when a new event is added in
+  // order to reduce overall memory usage.
+  NSMutableString* _breadcrumbsString;
+}
+@end
+
+@implementation CrashReporterBreadcrumbObserver
+
++ (CrashReporterBreadcrumbObserver*)uniqueInstance {
+  static CrashReporterBreadcrumbObserver* instance =
+      [[CrashReporterBreadcrumbObserver alloc] init];
+  return instance;
+}
+
+- (instancetype)init {
+  if ((self = [super init])) {
+    _breadcrumbsString = [[NSMutableString alloc] init];
+  }
+  return self;
+}
+
+- (void)observeBrowserState:(ios::ChromeBrowserState*)browserState {
+  DCHECK(!_breadcrumbManagerObservers[browserState]);
+
+  BreadcrumbManagerKeyedService* service =
+      BreadcrumbManagerKeyedServiceFactory::GetForBrowserState(browserState);
+  _breadcrumbManagerObservers[browserState] =
+      std::make_unique<BreadcrumbManagerObserverBridge>(service, self);
+}
+
+- (void)stopObservingBrowserState:(ios::ChromeBrowserState*)browserState {
+  _breadcrumbManagerObservers[browserState] = nullptr;
+}
+
+#pragma mark - BreadcrumbManagerObserving protocol
+
+- (void)breadcrumbManager:(BreadcrumbManagerKeyedService*)manager
+              didAddEvent:(NSString*)event {
+  NSString* eventWithSeperator = [NSString stringWithFormat:@"%@\n", event];
+  [_breadcrumbsString insertString:eventWithSeperator atIndex:0];
+
+  if (_breadcrumbsString.length > kMaxCombinedBreadcrumbLength) {
+    NSRange trimRange =
+        NSMakeRange(kMaxCombinedBreadcrumbLength,
+                    _breadcrumbsString.length - kMaxCombinedBreadcrumbLength);
+    [_breadcrumbsString deleteCharactersInRange:trimRange];
+  }
+  breakpad_helper::SetBreadcrumbEvents(_breadcrumbsString);
+}
+
+@end

ios/chrome/browser/crash_report/crash_reporter_breadcrumb_observer_unittest.mm

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "ios/chrome/browser/crash_report/crash_reporter_breadcrumb_observer.h"
+
+#import "base/strings/sys_string_conversions.h"
+#import "base/test/ios/wait_util.h"
+#include "base/test/task_environment.h"
+#include "ios/chrome/browser/browser_state/test_chrome_browser_state.h"
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_keyed_service.h"
+#include "ios/chrome/browser/crash_report/breadcrumbs/breadcrumb_manager_keyed_service_factory.h"
+#import "ios/chrome/browser/crash_report/breakpad_helper.h"
+#import "ios/chrome/test/ocmock/OCMockObject+BreakpadControllerTesting.h"
+#import "ios/testing/scoped_block_swizzler.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#import "testing/gtest_mac.h"
+#include "testing/platform_test.h"
+#import "third_party/breakpad/breakpad/src/client/ios/BreakpadController.h"
+#import "third_party/ocmock/OCMock/OCMock.h"
+#include "third_party/ocmock/gtest_support.h"
+
+#if !defined(__has_feature) || !__has_feature(objc_arc)
+#error "This file requires ARC support."
+#endif
+
+namespace {
+// Returns an OCMArg validator which checks that the parameter value is a string
+// containing |count| occurances of |substring|.
+id StringParameterValidatorWithCountOfSubstring(NSUInteger count,
+                                                NSString* substring) {
+  return [OCMArg checkWithBlock:^(id value) {
+    if (![value isKindOfClass:[NSString class]]) {
+      return NO;
+    }
+    NSError* error = nil;
+    NSRegularExpression* regex = [NSRegularExpression
+        regularExpressionWithPattern:substring
+                             options:NSRegularExpressionCaseInsensitive
+                               error:&error];
+    if (error) {
+      return NO;
+    }
+    return count == [regex
+                        numberOfMatchesInString:value
+                                        options:0
+                                          range:NSMakeRange(0, [value length])];
+  }];
+}
+}
+
+// Tests that CrashReporterBreadcrumbObserver attaches observed breadcrumb
+// events to crash reports.
+class CrashReporterBreadcrumbObserverTest : public PlatformTest {
+ public:
+  void SetUp() override {
+    PlatformTest::SetUp();
+
+    TestChromeBrowserState::Builder test_cbs_builder;
+    chrome_browser_state_ = test_cbs_builder.Build();
+
+    mock_breakpad_controller_ =
+        [OCMockObject mockForClass:[BreakpadController class]];
+
+    // Swizzle +[BreakpadController sharedInstance] to return
+    // |mock_breakpad_controller_| instead of the normal singleton instance.
+    id implementation_block = ^BreakpadController*(id self) {
+      return mock_breakpad_controller_;
+    };
+    breakpad_controller_shared_instance_swizzler_.reset(new ScopedBlockSwizzler(
+        [BreakpadController class], @selector(sharedInstance),
+        implementation_block));
+  }
+
+  void TearDown() override {
+    [[mock_breakpad_controller_ stub] stop];
+    breakpad_helper::SetEnabled(false);
+    PlatformTest::TearDown();
+  }
+
+ protected:
+  id mock_breakpad_controller_;
+  std::unique_ptr<ScopedBlockSwizzler>
+      breakpad_controller_shared_instance_swizzler_;
+
+  base::test::TaskEnvironment task_environment_;
+  std::unique_ptr<TestChromeBrowserState> chrome_browser_state_;
+};
+
+// Tests that breadcrumb events logged to a single BreadcrumbManagerKeyedService
+// are collected by the CrashReporterBreadcrumbObserver and attached to crash
+// reports.
+TEST_F(CrashReporterBreadcrumbObserverTest, EventsAttachedToCrashReport) {
+  [[mock_breakpad_controller_ expect] start:NO];
+  breakpad_helper::SetEnabled(true);
+
+  CrashReporterBreadcrumbObserver* crash_reporter_breadcrumb_observer =
+      [[CrashReporterBreadcrumbObserver alloc] init];
+  [crash_reporter_breadcrumb_observer
+      observeBrowserState:chrome_browser_state_.get()];
+
+  const std::string event = std::string("Breadcrumb Event");
+  BreadcrumbManagerKeyedService* breadcrumb_manager =
+      BreadcrumbManagerKeyedServiceFactory::GetForBrowserState(
+          chrome_browser_state_.get());
+
+  id breadcrumbs_param_vaidation_block = [OCMArg checkWithBlock:^(id value) {
+    if (![value isKindOfClass:[NSString class]]) {
+      return NO;
+    }
+    std::list<std::string> events = breadcrumb_manager->GetEvents(0);
+    std::string expected_breadcrumbs;
+    for (const auto& event : events) {
+      expected_breadcrumbs += event + "\n";
+    }
+    return
+        [value isEqualToString:base::SysUTF8ToNSString(expected_breadcrumbs)];
+  }];
+  [[mock_breakpad_controller_ expect]
+      addUploadParameter:breadcrumbs_param_vaidation_block
+                  forKey:@"browser_state_breadcrumbs"];
+
+  breadcrumb_manager->AddEvent(event);
+  EXPECT_OCMOCK_VERIFY(mock_breakpad_controller_);
+}
+
+// Tests that breadcrumb events logged to multiple BreadcrumbManagerKeyedService
+// instances are collected by the CrashReporterBreadcrumbObserver and attached
+// to crash reports.
+TEST_F(CrashReporterBreadcrumbObserverTest,
+       MultipleBrowserStatesAttachedToCrashReport) {
+  [[mock_breakpad_controller_ expect] start:NO];
+  breakpad_helper::SetEnabled(true);
+
+  const std::string event = std::string("Breadcrumb Event");
+  NSString* event_nsstring = base::SysUTF8ToNSString(event);
+
+  CrashReporterBreadcrumbObserver* crash_reporter_breadcrumb_observer =
+      [[CrashReporterBreadcrumbObserver alloc] init];
+
+  [crash_reporter_breadcrumb_observer
+      observeBrowserState:chrome_browser_state_.get()];
+  BreadcrumbManagerKeyedService* breadcrumb_manager =
+      BreadcrumbManagerKeyedServiceFactory::GetForBrowserState(
+          chrome_browser_state_.get());
+  [[mock_breakpad_controller_ expect]
+      addUploadParameter:StringParameterValidatorWithCountOfSubstring(
+                             1, event_nsstring)
+                  forKey:@"browser_state_breadcrumbs"];
+  breadcrumb_manager->AddEvent(event);
+
+  ios::ChromeBrowserState* otr_browser_state =
+      chrome_browser_state_->GetOffTheRecordChromeBrowserState();
+  [crash_reporter_breadcrumb_observer observeBrowserState:otr_browser_state];
+  BreadcrumbManagerKeyedService* otr_breadcrumb_manager =
+      BreadcrumbManagerKeyedServiceFactory::GetForBrowserState(
+          otr_browser_state);
+  [[mock_breakpad_controller_ expect]
+      addUploadParameter:StringParameterValidatorWithCountOfSubstring(
+                             2, event_nsstring)
+                  forKey:@"browser_state_breadcrumbs"];
+  otr_breadcrumb_manager->AddEvent(event);
+
+  TestChromeBrowserState::Builder test_cbs_builder;
+  std::unique_ptr<TestChromeBrowserState> chrome_browser_state_2 =
+      test_cbs_builder.Build();
+  [crash_reporter_breadcrumb_observer
+      observeBrowserState:chrome_browser_state_2.get()];
+  BreadcrumbManagerKeyedService* breadcrumb_manager_2 =
+      BreadcrumbManagerKeyedServiceFactory::GetForBrowserState(
+          chrome_browser_state_2.get());
+  [[mock_breakpad_controller_ expect]
+      addUploadParameter:StringParameterValidatorWithCountOfSubstring(
+                             3, event_nsstring)
+                  forKey:@"browser_state_breadcrumbs"];
+  breadcrumb_manager_2->AddEvent(event);
+
+  EXPECT_OCMOCK_VERIFY(mock_breakpad_controller_);
+
+  // Manually clear observer reference before the Browsers are deconstructed.
+  crash_reporter_breadcrumb_observer = nil;
+}

ios/chrome/browser/ui/main/BUILD.gn

@@ -63,6 +63,7 @@ source_set("main") {
     "//ios/chrome/browser/browser_state",
     "//ios/chrome/browser/browsing_data",
     "//ios/chrome/browser/crash_report:crash_report_internal",
+    "//ios/chrome/browser/crash_report/breadcrumbs:feature_flags",
     "//ios/chrome/browser/device_sharing",
     "//ios/chrome/browser/download",
     "//ios/chrome/browser/main",

ios/chrome/browser/ui/main/browser_view_wrangler.mm

@@ -4,10 +4,12 @@
 
 #import "ios/chrome/browser/ui/main/browser_view_wrangler.h"
 
+#include "base/feature_list.h"
 #include "base/files/file_path.h"
 #include "base/strings/sys_string_conversions.h"
 #include "ios/chrome/browser/application_context.h"
 #include "ios/chrome/browser/browser_state/chrome_browser_state.h"
+#import "ios/chrome/browser/crash_report/breadcrumbs/features.h"
 #include "ios/chrome/browser/crash_report/crash_report_helper.h"
 #import "ios/chrome/browser/device_sharing/device_sharing_manager.h"
 #import "ios/chrome/browser/main/browser.h"
@@ -160,6 +162,9 @@
                          (id<BrowserStateStorageSwitching>)storageSwitcher {
   if ((self = [super init])) {
     _browserState = browserState;
+    if (base::FeatureList::IsEnabled(kLogBreadcrumbs)) {
+      breakpad::MonitorBreadcrumbsForBrowserState(_browserState);
+    }
     _applicationCommandEndpoint = applicationCommandEndpoint;
     _browsingDataCommandEndpoint = browsingDataCommandEndpoint;
     _appURLLoadingService = appURLLoadingService;
@@ -340,6 +345,10 @@
   // Stop watching the OTR webStateList's state for crashes.
   breakpad::StopMonitoringTabStateForWebStateList(
       self.otrBrowser->GetWebStateList());
+  if (base::FeatureList::IsEnabled(kLogBreadcrumbs)) {
+    breakpad::StopMonitoringBreadcrumbsForBrowserState(
+        self.otrBrowser->GetBrowserState());
+  }
 
   // At this stage, a new incognitoBrowserCoordinator shouldn't be lazily
   // constructed by calling the property getter.
@@ -393,6 +402,13 @@
   [self setMainBrowser:nullptr];
   [self setOtrBrowser:nullptr];
 
+  if (base::FeatureList::IsEnabled(kLogBreadcrumbs)) {
+    if (_browserState->HasOffTheRecordChromeBrowserState()) {
+      breakpad::StopMonitoringBreadcrumbsForBrowserState(
+          _browserState->GetOffTheRecordChromeBrowserState());
+    }
+    breakpad::StopMonitoringBreadcrumbsForBrowserState(_browserState);
+  }
   _browserState = nullptr;
 }
 
@@ -404,6 +420,9 @@
   ios::ChromeBrowserState* otrBrowserState =
       _browserState->GetOffTheRecordChromeBrowserState();
   DCHECK(otrBrowserState);
+  if (base::FeatureList::IsEnabled(kLogBreadcrumbs)) {
+    breakpad::MonitorBreadcrumbsForBrowserState(otrBrowserState);
+  }
 
   std::unique_ptr<Browser> browser = Browser::Create(otrBrowserState);
   [self setUpTabModel:browser->GetTabModel()