Access Context Auditing: Add SQLite database backend

CL adds the SQLite database backend implementation to support recording
first-party contexts in which client-side storage APIs are accessed.

The database is not currently instanstiated or accessible from
production code.

Bug: 1083384
Change-Id: I0fa83b09c11c988364957ef58d25b51eae0a16ec
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2218072Reviewed-by: Martin Šrámek <msramek@chromium.org>
Commit-Queue: Theodore Olsauskas-Warren <sauski@google.com>
Cr-Commit-Position: refs/heads/master@{#774044}

chrome/browser/BUILD.gn

@@ -3181,6 +3181,8 @@ static_library("browser") {
       "banners/app_banner_manager_desktop.h",
       "bookmarks/bookmark_html_writer.cc",
       "bookmarks/bookmark_html_writer.h",
+      "browsing_data/access_context_audit_database.cc",
+      "browsing_data/access_context_audit_database.h",
       "certificate_viewer.h",
       "chrome_browser_field_trials_desktop.cc",
       "chrome_browser_field_trials_desktop.h",

chrome/browser/browsing_data/access_context_audit_database.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/browsing_data/access_context_audit_database.h"
+
+#include "sql/database.h"
+#include "sql/recovery.h"
+#include "sql/statement.h"
+#include "sql/transaction.h"
+
+namespace {
+
+const base::FilePath::CharType kDatabaseName[] =
+    FILE_PATH_LITERAL("AccessContextAudit");
+const char kCookieTableName[] = "cookies";
+const char kStorageAPITableName[] = "originStorageAPIs";
+
+// Callback that is fired upon an SQLite error, attempts to automatically
+// recover the database if it appears possible to do so.
+// TODO(crbug.com/1087272): Remove duplication of this function in the codebase.
+void DatabaseErrorCallback(sql::Database* db,
+                           const base::FilePath& db_path,
+                           int extended_error,
+                           sql::Statement* stmt) {
+  if (sql::Recovery::ShouldRecover(extended_error)) {
+    // Prevent reentrant calls.
+    db->reset_error_callback();
+
+    // After this call, the |db| handle is poisoned so that future calls will
+    // return errors until the handle is re-opened.
+    sql::Recovery::RecoverDatabase(db, db_path);
+
+    // The DLOG(WARNING) below is intended to draw immediate attention to errors
+    // in newly-written code.  Database corruption is generally a result of OS
+    // or hardware issues, not coding errors at the client level, so displaying
+    // the error would probably lead to confusion.  The ignored call signals the
+    // test-expectation framework that the error was handled.
+    ignore_result(sql::Database::IsExpectedSqliteError(extended_error));
+    return;
+  }
+
+  // The default handling is to assert on debug and to ignore on release.
+  if (!sql::Database::IsExpectedSqliteError(extended_error))
+    DLOG(FATAL) << db->GetErrorMessage();
+}
+
+}  // namespace
+
+AccessContextAuditDatabase::AccessRecord::AccessRecord(
+    const GURL& top_frame_origin,
+    const std::string& name,
+    const std::string& domain,
+    const std::string& path,
+    const base::Time& last_access_time)
+    : top_frame_origin(top_frame_origin),
+      type(StorageAPIType::kCookie),
+      name(name),
+      domain(domain),
+      path(path),
+      last_access_time(last_access_time) {}
+
+AccessContextAuditDatabase::AccessRecord::AccessRecord(
+    const GURL& top_frame_origin,
+    const StorageAPIType& type,
+    const GURL& origin,
+    const base::Time& last_access_time)
+    : top_frame_origin(top_frame_origin),
+      type(type),
+      origin(origin),
+      last_access_time(last_access_time) {
+  DCHECK(type != StorageAPIType::kCookie);
+}
+
+AccessContextAuditDatabase::AccessRecord::AccessRecord(
+    const AccessRecord& other) = default;
+
+AccessContextAuditDatabase::AccessRecord::~AccessRecord() = default;
+
+AccessContextAuditDatabase::AccessContextAuditDatabase(
+    const base::FilePath& path_to_database_dir)
+    : db_file_path_(path_to_database_dir.Append(kDatabaseName)) {}
+
+bool AccessContextAuditDatabase::Init() {
+  db_.set_histogram_tag("Access Context Audit");
+
+  db_.set_error_callback(
+      base::BindRepeating(&DatabaseErrorCallback, &db_, db_file_path_));
+
+  // Cache values generated assuming ~5000 individual pieces of client storage
+  // API data, each accessed in an average of 3 different contexts (complete
+  // speculation, most will be 1, some will be >50), with an average of 40bytes
+  // per audit entry.
+  // TODO(crbug.com/1083384): Revist these numbers.
+  db_.set_page_size(4096);
+  db_.set_cache_size(128);
+
+  db_.set_exclusive_locking();
+
+  return db_.Open(db_file_path_) && InitializeSchema();
+}
+
+bool AccessContextAuditDatabase::InitializeSchema() {
+  std::string create_table;
+  create_table.append("CREATE TABLE IF NOT EXISTS ");
+  create_table.append(kCookieTableName);
+  create_table.append(
+      "(top_frame_origin TEXT NOT NULL,"
+      "name TEXT NOT NULL,"
+      "domain TEXT NOT NULL,"
+      "path TEXT NOT NULL,"
+      "access_utc INTEGER NOT NULL,"
+      "PRIMARY KEY (top_frame_origin, name, domain, path))");
+
+  if (!db_.Execute(create_table.c_str()))
+    return false;
+
+  create_table.clear();
+  create_table.append("CREATE TABLE IF NOT EXISTS ");
+  create_table.append(kStorageAPITableName);
+  create_table.append(
+      "(top_frame_origin TEXT NOT NULL,"
+      "type INTEGER NOT NULL,"
+      "origin TEXT NOT NULL,"
+      "access_utc INTEGER NOT NULL,"
+      "PRIMARY KEY (top_frame_origin, origin, type))");
+
+  return db_.Execute(create_table.c_str());
+}
+
+bool AccessContextAuditDatabase::AddRecords(
+    const std::vector<AccessContextAuditDatabase::AccessRecord>& records) {
+  sql::Transaction transaction(&db_);
+  if (!transaction.Begin())
+    return false;
+
+  // Create both insert statements ahead of iterating over records. These are
+  // highly likely to both be used, and should be in the statement cache.
+  std::string insert;
+  insert.append("INSERT OR REPLACE INTO ");
+  insert.append(kCookieTableName);
+  insert.append(
+      "(top_frame_origin, name, domain, path, access_utc) "
+      "VALUES (?, ?, ?, ?, ?)");
+  sql::Statement insert_cookie(
+      db_.GetCachedStatement(SQL_FROM_HERE, insert.c_str()));
+
+  insert.clear();
+  insert.append("INSERT OR REPLACE INTO ");
+  insert.append(kStorageAPITableName);
+  insert.append(
+      "(top_frame_origin, type, origin, access_utc) "
+      "VALUES (?, ?, ?, ?)");
+  sql::Statement insert_storage_api(
+      db_.GetCachedStatement(SQL_FROM_HERE, insert.c_str()));
+
+  for (const auto& record : records) {
+    if (record.type == StorageAPIType::kCookie) {
+      insert_cookie.BindString(0, record.top_frame_origin.GetOrigin().spec());
+      insert_cookie.BindString(1, record.name);
+      insert_cookie.BindString(2, record.domain);
+      insert_cookie.BindString(3, record.path);
+      insert_cookie.BindInt64(
+          4,
+          record.last_access_time.ToDeltaSinceWindowsEpoch().InMicroseconds());
+
+      if (!insert_cookie.Run())
+        return false;
+
+      insert_cookie.Reset(true);
+    } else {
+      insert_storage_api.BindString(0,
+                                    record.top_frame_origin.GetOrigin().spec());
+      insert_storage_api.BindInt(1, static_cast<int>(record.type));
+      insert_storage_api.BindString(2, record.origin.GetOrigin().spec());
+      insert_storage_api.BindInt64(
+          3,
+          record.last_access_time.ToDeltaSinceWindowsEpoch().InMicroseconds());
+
+      if (!insert_storage_api.Run())
+        return false;
+
+      insert_storage_api.Reset(true);
+    }
+  }
+
+  return transaction.Commit();
+}
+
+bool AccessContextAuditDatabase::RemoveRecord(const AccessRecord& record) {
+  sql::Statement remove_statement;
+
+  std::string remove;
+  remove.append("DELETE FROM ");
+  if (record.type == StorageAPIType::kCookie) {
+    remove.append(kCookieTableName);
+    remove.append(
+        " WHERE top_frame_origin = ? AND name = ? AND domain = ? AND path = ?");
+    remove_statement.Assign(
+        db_.GetCachedStatement(SQL_FROM_HERE, remove.c_str()));
+    remove_statement.BindString(0, record.top_frame_origin.GetOrigin().spec());
+    remove_statement.BindString(1, record.name);
+    remove_statement.BindString(2, record.domain);
+    remove_statement.BindString(3, record.path);
+  } else {
+    remove.append(kStorageAPITableName);
+    remove.append(" WHERE top_frame_origin = ? AND type = ? AND origin = ?");
+    remove_statement.Assign(
+        db_.GetCachedStatement(SQL_FROM_HERE, remove.c_str()));
+    remove_statement.BindString(0, record.top_frame_origin.GetOrigin().spec());
+    remove_statement.BindInt(1, static_cast<int>(record.type));
+    remove_statement.BindString(2, record.origin.GetOrigin().spec());
+  }
+  return remove_statement.Run();
+}
+
+std::vector<AccessContextAuditDatabase::AccessRecord>
+AccessContextAuditDatabase::GetAllRecords() {
+  std::vector<AccessContextAuditDatabase::AccessRecord> records;
+
+  std::string select;
+  select.append(
+      "SELECT top_frame_origin, name, domain, path, access_utc FROM ");
+  select.append(kCookieTableName);
+  sql::Statement select_cookies(
+      db_.GetCachedStatement(SQL_FROM_HERE, select.c_str()));
+
+  while (select_cookies.Step()) {
+    records.emplace_back(
+        GURL(select_cookies.ColumnString(0)), select_cookies.ColumnString(1),
+        select_cookies.ColumnString(2), select_cookies.ColumnString(3),
+        base::Time::FromDeltaSinceWindowsEpoch(
+            base::TimeDelta::FromMicroseconds(select_cookies.ColumnInt64(4))));
+  }
+
+  select.clear();
+  select.append("SELECT top_frame_origin, type, origin, access_utc FROM ");
+  select.append(kStorageAPITableName);
+  sql::Statement select_storage_api(
+      db_.GetCachedStatement(SQL_FROM_HERE, select.c_str()));
+
+  while (select_storage_api.Step()) {
+    records.emplace_back(
+        GURL(select_storage_api.ColumnString(0)),
+        static_cast<StorageAPIType>(select_storage_api.ColumnInt(1)),
+        GURL(select_storage_api.ColumnString(2)),
+        base::Time::FromDeltaSinceWindowsEpoch(
+            base::TimeDelta::FromMicroseconds(
+                select_storage_api.ColumnInt64(3))));
+  }
+
+  return records;
+}

chrome/browser/browsing_data/access_context_audit_database.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_BROWSING_DATA_ACCESS_CONTEXT_AUDIT_DATABASE_H_
+#define CHROME_BROWSER_BROWSING_DATA_ACCESS_CONTEXT_AUDIT_DATABASE_H_
+
+#include "base/files/file_path.h"
+#include "base/memory/ref_counted.h"
+#include "net/cookies/canonical_cookie.h"
+#include "sql/database.h"
+#include "sql/init_status.h"
+#include "sql/test/test_helpers.h"
+#include "url/gurl.h"
+
+// Provides the backend SQLite storage to support access context auditing. This
+// requires storing information associating individual client-side storage API
+// accesses (e.g. cookies, indexedDBs, etc.) with the top level frame origins
+// at the time of their access.
+class AccessContextAuditDatabase {
+ public:
+  // All client-side storage API types supported by the database.
+  enum class StorageAPIType : int {
+    kCookie = 0,
+    kLocalStorage = 1,
+    kSessionStorage,
+    kFileSystem,
+    kWebDatabase,
+    kServiceWorker,
+    kCacheStorage,
+    kIndexedDB,
+    kAppCache,
+  };
+
+  // An individual record of a Storage API access, associating the individual
+  // API usage with a top level frame origin.
+  struct AccessRecord {
+    AccessRecord(const GURL& top_frame_origin,
+                 const std::string& name,
+                 const std::string& domain,
+                 const std::string& path,
+                 const base::Time& last_access_time);
+    AccessRecord(const GURL& top_frame_origin,
+                 const StorageAPIType& type,
+                 const GURL& origin,
+                 const base::Time& last_access_time);
+    AccessRecord(const AccessRecord& other);
+    ~AccessRecord();
+    GURL top_frame_origin;
+    StorageAPIType type;
+
+    // Identifies a canonical cookie, only used when |type| is kCookie.
+    std::string name;
+    std::string domain;
+    std::string path;
+
+    // Identifies an origin-keyed storage API, used when |type| is NOT kCookie.
+    GURL origin;
+
+    base::Time last_access_time;
+  };
+
+  explicit AccessContextAuditDatabase(
+      const base::FilePath& path_to_database_dir);
+
+  // Initialises internal database. Must be called prior to any other usage.
+  bool Init();
+
+  // Persists the provided list of |records| in the database.
+  bool AddRecords(const std::vector<AccessRecord>& records);
+
+  // Returns all entries in the database. No ordering is enforced.
+  std::vector<AccessRecord> GetAllRecords();
+
+  // Removes a record from the database and from future calls to GetAllRecords.
+  bool RemoveRecord(const AccessRecord& record);
+
+ private:
+  bool InitializeSchema();
+
+  sql::Database db_;
+  base::FilePath db_file_path_;
+};
+
+#endif

chrome/browser/browsing_data/access_context_audit_database_unittest.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/browsing_data/access_context_audit_database.h"
+
+#include "base/files/scoped_temp_dir.h"
+#include "sql/database.h"
+#include "sql/test/scoped_error_expecter.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace {
+
+// Define an ordering based on ascending timestamps to allow sorting of
+// AccessRecords for easier testing, as no ordering is guaranteed by the
+// database.
+bool TimestampOrdering(const AccessContextAuditDatabase::AccessRecord& a,
+                       const AccessContextAuditDatabase::AccessRecord& b) {
+  return a.last_access_time < b.last_access_time;
+}
+
+void ExpectAccessRecordsEqual(
+    const AccessContextAuditDatabase::AccessRecord& a,
+    const AccessContextAuditDatabase::AccessRecord& b) {
+  EXPECT_EQ(a.top_frame_origin, b.top_frame_origin);
+  EXPECT_EQ(a.type, b.type);
+  EXPECT_EQ(a.last_access_time, b.last_access_time);
+
+  if (a.type == AccessContextAuditDatabase::StorageAPIType::kCookie) {
+    EXPECT_EQ(a.name, b.name);
+    EXPECT_EQ(a.domain, b.domain);
+    EXPECT_EQ(a.path, b.path);
+  } else {
+    EXPECT_EQ(a.origin, b.origin);
+  }
+}
+
+void ValidateDatabaseRecords(
+    AccessContextAuditDatabase* database,
+    const std::vector<AccessContextAuditDatabase::AccessRecord>&
+        expected_records) {
+  auto stored_records = database->GetAllRecords();
+  std::sort(stored_records.begin(), stored_records.end(), TimestampOrdering);
+
+  EXPECT_EQ(stored_records.size(), expected_records.size());
+  for (size_t i = 0;
+       i < std::min(stored_records.size(), expected_records.size()); i++) {
+    ExpectAccessRecordsEqual(stored_records[i], expected_records[i]);
+  }
+}
+
+}  // namespace
+
+class AccessContextAuditDatabaseTest : public testing::Test {
+ public:
+  AccessContextAuditDatabaseTest() = default;
+
+  void SetUp() override { ASSERT_TRUE(temp_directory_.CreateUniqueTempDir()); }
+
+  void OpenDatabase() {
+    database_.reset();
+    database_ =
+        std::make_unique<AccessContextAuditDatabase>(temp_directory_.GetPath());
+    EXPECT_TRUE(database_->Init());
+  }
+
+  void CloseDatabase() { database_.reset(); }
+
+  base::FilePath db_path() {
+    return temp_directory_.GetPath().Append(
+        FILE_PATH_LITERAL("AccessContextAudit"));
+  }
+
+  AccessContextAuditDatabase* database() { return database_.get(); }
+
+  std::vector<AccessContextAuditDatabase::AccessRecord> GetTestRecords() {
+    return {
+        AccessContextAuditDatabase::AccessRecord(
+            GURL("https://test.com"),
+            AccessContextAuditDatabase::StorageAPIType::kLocalStorage,
+            GURL("https://test.com"),
+            base::Time::FromDeltaSinceWindowsEpoch(
+                base::TimeDelta::FromHours(1))),
+        AccessContextAuditDatabase::AccessRecord(
+            GURL("https://test2.com:8000"),
+            AccessContextAuditDatabase::StorageAPIType::kLocalStorage,
+            GURL("https://test.com"),
+            base::Time::FromDeltaSinceWindowsEpoch(
+                base::TimeDelta::FromHours(2))),
+        AccessContextAuditDatabase::AccessRecord(
+            GURL("https://test2.com"), "cookie1", "test.com", "/",
+            base::Time::FromDeltaSinceWindowsEpoch(
+                base::TimeDelta::FromHours(3))),
+        AccessContextAuditDatabase::AccessRecord(
+            GURL("https://test2.com"), "cookie2", "test.com", "/",
+            base::Time::FromDeltaSinceWindowsEpoch(
+                base::TimeDelta::FromHours(4))),
+    };
+  }
+
+ private:
+  base::ScopedTempDir temp_directory_;
+  std::unique_ptr<AccessContextAuditDatabase> database_;
+};
+
+TEST_F(AccessContextAuditDatabaseTest, DatabaseInitialization) {
+  // Check that tables are created and at least have the appropriate number of
+  // columns.
+  OpenDatabase();
+  CloseDatabase();
+  sql::Database raw_db;
+  EXPECT_TRUE(raw_db.Open(db_path()));
+
+  // [cookies] and [storageapi].
+  EXPECT_EQ(2u, sql::test::CountSQLTables(&raw_db));
+
+  // [top_frame_origin, name, domain, path, access_utc]
+  EXPECT_EQ(5u, sql::test::CountTableColumns(&raw_db, "cookies"));
+
+  // [top_frame_origin, type, origin, access_utc]
+  EXPECT_EQ(4u, sql::test::CountTableColumns(&raw_db, "originStorageAPIs"));
+}
+
+TEST_F(AccessContextAuditDatabaseTest, DataPersisted) {
+  // Check that data is retrievable both before and after a database reopening.
+  auto test_records = GetTestRecords();
+  OpenDatabase();
+  database()->AddRecords(test_records);
+  ValidateDatabaseRecords(database(), test_records);
+
+  CloseDatabase();
+  OpenDatabase();
+
+  ValidateDatabaseRecords(database(), test_records);
+  CloseDatabase();
+}
+
+TEST_F(AccessContextAuditDatabaseTest, RecoveredOnOpen) {
+  // Check that a database recovery is performed when opening a corrupted file.
+  auto test_records = GetTestRecords();
+  OpenDatabase();
+  database()->AddRecords(test_records);
+  ValidateDatabaseRecords(database(), test_records);
+  CloseDatabase();
+
+  // Corrupt the database.
+  EXPECT_TRUE(sql::test::CorruptSizeInHeader(db_path()));
+
+  sql::test::ScopedErrorExpecter expecter;
+  expecter.ExpectError(SQLITE_CORRUPT);
+
+  // Open that database and ensure that it does not fail.
+  EXPECT_NO_FATAL_FAILURE(OpenDatabase());
+
+  // Data should be recovered.
+  ValidateDatabaseRecords(database(), test_records);
+
+  EXPECT_TRUE(expecter.SawExpectedErrors());
+}
+
+TEST_F(AccessContextAuditDatabaseTest, RemoveRecord) {
+  // Check that entries are removed from the database such that they are both
+  // not returned by GetAllRecords and are removed from the database file.
+  auto test_records = GetTestRecords();
+  OpenDatabase();
+  database()->AddRecords(test_records);
+  while (test_records.size() > 0) {
+    database()->RemoveRecord(test_records[0]);
+    test_records.erase(test_records.begin());
+    ValidateDatabaseRecords(database(), test_records);
+  }
+  CloseDatabase();
+
+  // Verify that everything is deleted.
+  sql::Database raw_db;
+  EXPECT_TRUE(raw_db.Open(db_path()));
+
+  size_t cookie_rows;
+  size_t storage_api_rows;
+  sql::test::CountTableRows(&raw_db, "cookies", &cookie_rows);
+  sql::test::CountTableRows(&raw_db, "originStorageAPIs", &storage_api_rows);
+
+  EXPECT_EQ(0u, cookie_rows);
+  EXPECT_EQ(0u, storage_api_rows);
+}
+
+TEST_F(AccessContextAuditDatabaseTest, RepeatedAccesses) {
+  // Check that additional access records, only differing by timestamp to
+  // previous entries, update those entries rather than creating new ones.
+  auto test_records = GetTestRecords();
+  OpenDatabase();
+  database()->AddRecords(test_records);
+
+  for (auto& record : test_records) {
+    record.last_access_time += base::TimeDelta::FromHours(1);
+  }
+
+  database()->AddRecords(test_records);
+  ValidateDatabaseRecords(database(), test_records);
+  CloseDatabase();
+
+  // Verify that extra entries are not present in the database.
+  size_t num_test_cookie_entries = std::count_if(
+      test_records.begin(), test_records.end(),
+      [](const AccessContextAuditDatabase::AccessRecord& record) {
+        return record.type ==
+               AccessContextAuditDatabase::StorageAPIType::kCookie;
+      });
+  size_t num_test_storage_entries =
+      test_records.size() - num_test_cookie_entries;
+
+  sql::Database raw_db;
+  EXPECT_TRUE(raw_db.Open(db_path()));
+
+  size_t cookie_rows;
+  size_t storage_api_rows;
+  sql::test::CountTableRows(&raw_db, "cookies", &cookie_rows);
+  sql::test::CountTableRows(&raw_db, "originStorageAPIs", &storage_api_rows);
+
+  EXPECT_EQ(num_test_cookie_entries, cookie_rows);
+  EXPECT_EQ(num_test_storage_entries, storage_api_rows);
+}

chrome/test/BUILD.gn

@@ -3565,6 +3565,7 @@ test("unit_tests") {
 
   if (!is_android) {
     sources += [
+      "../browser/browsing_data/access_context_audit_database_unittest.cc",
       "../browser/component_updater/soda_component_installer_unittest.cc",
       "../browser/content_settings/generated_cookie_prefs_unittest.cc",
       "../browser/device_identity/device_oauth2_token_service_unittest.cc",