Refactor content verification hashes into ContentHash class.

There are two hash files that are involved for content
verification: verified_contents.json and computed_hashes.json.
The new class aims to pull all operations related to these
files into ContentHash to improve content verification code
and also facilitates further improvements to the code.

This CL moves both hash files' reading code into ContentHash
class avoiding the need to read verified_contents.json from
multiple places. This CL also moves computed_hashes.json file
generation/writing to ContentHash. The Cancel() behavior of
ContentHashFetcherJob is retained in this CL, and is moved
primarily to ContentHash. ContentHash uses CancellationStatus
interface to consult ContentHashFetcherJob about cross thread
cancellation of ContentHashFetcherJob.

Update ContentHashReader and ContentHashFetcher accordingly
to use ContentHash to deal with the hashes. Fetching
verified_contents.json from network is still kept in
ContentHashFetcher to keep the change simpler.

Behavior changes:
We used to write computed_hashes.json after computing hashes but
didn't read that file at that step, this CL reads them back to
ContentHash::Reader right after writing it. This avoids special
cases and makes ContentHash always have both hashes ready. Other
than that, expect no behavior change in this CL.

Bug: 796395
Change-Id: I00e03cf208982e1cbd8be7951e06da97173a3ffc
Reviewed-on: https://chromium-review.googlesource.com/831341
Commit-Queue: Istiaque Ahmed <lazyboy@chromium.org>
Reviewed-by: Devlin <rdevlin.cronin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#530422}

extensions/browser/BUILD.gn

@@ -78,6 +78,8 @@ source_set("browser_sources") {
     "content_hash_tree.h",
     "content_verifier.cc",
     "content_verifier.h",
+    "content_verifier/content_hash.cc",
+    "content_verifier/content_hash.h",
     "content_verifier_delegate.h",
     "content_verifier_io_data.cc",
     "content_verifier_io_data.h",

extensions/browser/content_hash_reader.cc

@@ -4,9 +4,7 @@
 
 #include "extensions/browser/content_hash_reader.h"
 
-#include "base/base64.h"
 #include "base/files/file_util.h"
-#include "base/json/json_reader.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/strings/string_util.h"
 #include "base/timer/elapsed_timer.h"
@@ -14,13 +12,8 @@
 #include "crypto/sha2.h"
 #include "extensions/browser/computed_hashes.h"
 #include "extensions/browser/content_hash_tree.h"
+#include "extensions/browser/content_verifier/content_hash.h"
 #include "extensions/browser/verified_contents.h"
-#include "extensions/common/extension.h"
-#include "extensions/common/file_util.h"
-
-using base::DictionaryValue;
-using base::ListValue;
-using base::Value;
 
 namespace extensions {
 
@@ -42,30 +35,18 @@ bool ContentHashReader::Init() {
   base::ElapsedTimer timer;
   DCHECK_EQ(status_, NOT_INITIALIZED);
   status_ = FAILURE;
-  base::FilePath verified_contents_path =
-      file_util::GetVerifiedContentsPath(extension_root_);
-  if (!base::PathExists(verified_contents_path))
-    return false;
-
-  VerifiedContents verified_contents(key_.data, key_.size);
-  if (!verified_contents.InitFrom(verified_contents_path) ||
-      !verified_contents.valid_signature() ||
-      verified_contents.version() != extension_version_ ||
-      verified_contents.extension_id() != extension_id_) {
-    return false;
-  }
 
-  base::FilePath computed_hashes_path =
-      file_util::GetComputedHashesPath(extension_root_);
-  if (!base::PathExists(computed_hashes_path))
-    return false;
+  std::unique_ptr<ContentHash> content_hash =
+      ContentHash::Create(ContentHash::ExtensionKey(
+          extension_id_, extension_root_, extension_version_, key_));
 
-  ComputedHashes::Reader reader;
-  if (!reader.InitFromFile(computed_hashes_path))
+  if (!content_hash->succeeded())
     return false;
 
   has_content_hashes_ = true;
 
+  const VerifiedContents& verified_contents = content_hash->verified_contents();
+
   // Extensions sometimes request resources that do not have an entry in
   // verified_contents.json. This can happen when an extension sends an XHR to a
   // resource.
@@ -82,9 +63,11 @@ bool ContentHashReader::Init() {
     return false;
   }
 
+  const ComputedHashes::Reader& reader = content_hash->computed_hashes();
   if (!reader.GetHashes(relative_path_, &block_size_, &hashes_) ||
-      block_size_ % crypto::kSHA256Length != 0)
+      block_size_ % crypto::kSHA256Length != 0) {
     return false;
+  }
 
   std::string root =
       ComputeTreeHashRoot(hashes_, block_size_ / crypto::kSHA256Length);

extensions/browser/content_verifier/content_hash.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "extensions/browser/content_verifier/content_hash.h"
+
+#include "base/files/file_enumerator.h"
+#include "base/files/file_util.h"
+#include "base/memory/ptr_util.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/threading/thread_restrictions.h"
+#include "base/timer/elapsed_timer.h"
+#include "crypto/sha2.h"
+#include "extensions/browser/content_hash_tree.h"
+#include "extensions/common/file_util.h"
+
+namespace {
+
+using SortedFilePathSet = std::set<base::FilePath>;
+
+}  // namespace
+
+namespace extensions {
+
+ContentHash::ExtensionKey::ExtensionKey(const ExtensionId& extension_id,
+                                        const base::FilePath& extension_root,
+                                        const base::Version& extension_version,
+                                        const ContentVerifierKey& verifier_key)
+    : extension_id(extension_id),
+      extension_root(extension_root),
+      extension_version(extension_version),
+      verifier_key(verifier_key) {}
+
+ContentHash::ExtensionKey::ExtensionKey(
+    const ContentHash::ExtensionKey& other) = default;
+
+ContentHash::ExtensionKey& ContentHash::ExtensionKey::operator=(
+    const ContentHash::ExtensionKey& other) = default;
+
+// static
+std::unique_ptr<ContentHash> ContentHash::Create(const ExtensionKey& key) {
+  return ContentHash::CreateImpl(key, 0 /* mode */,
+                                 false /* create_computed_hashes_file */,
+                                 IsCancelledCallback());
+}
+
+// static
+std::unique_ptr<ContentHash> ContentHash::Create(
+    const ExtensionKey& key,
+    int mode,
+    const IsCancelledCallback& is_cancelled) {
+  return ContentHash::CreateImpl(
+      key, mode, true /* create_computed_hashes_file */, is_cancelled);
+}
+
+ContentHash::~ContentHash() = default;
+
+const VerifiedContents& ContentHash::verified_contents() const {
+  DCHECK(status_ >= Status::kHasVerifiedContents && verified_contents_);
+  return *verified_contents_;
+}
+
+const ComputedHashes::Reader& ContentHash::computed_hashes() const {
+  DCHECK(status_ == Status::kSucceeded && computed_hashes_);
+  return *computed_hashes_;
+}
+
+ContentHash::ContentHash(
+    const ExtensionKey& key,
+    std::unique_ptr<VerifiedContents> verified_contents,
+    std::unique_ptr<ComputedHashes::Reader> computed_hashes)
+    : key_(key),
+      verified_contents_(std::move(verified_contents)),
+      computed_hashes_(std::move(computed_hashes)) {
+  if (!verified_contents_)
+    status_ = Status::kInvalid;
+  else if (!computed_hashes_)
+    status_ = Status::kHasVerifiedContents;
+  else
+    status_ = Status::kSucceeded;
+}
+
+// static
+std::unique_ptr<ContentHash> ContentHash::CreateImpl(
+    const ExtensionKey& key,
+    int mode,
+    bool create_computed_hashes_file,
+    const IsCancelledCallback& is_cancelled) {
+  base::AssertBlockingAllowed();
+
+  // verified_contents.json:
+  base::FilePath verified_contents_path =
+      file_util::GetVerifiedContentsPath(key.extension_root);
+  if (!base::PathExists(verified_contents_path)) {
+    // kInvalid.
+    return base::WrapUnique(new ContentHash(key, nullptr, nullptr));
+  }
+
+  auto verified_contents = std::make_unique<VerifiedContents>(
+      key.verifier_key.data, key.verifier_key.size);
+  if (!verified_contents->InitFrom(verified_contents_path)) {
+    if (mode & Mode::kDeleteInvalidVerifiedContents) {
+      if (!base::DeleteFile(verified_contents_path, false))
+        LOG(WARNING) << "Failed to delete " << verified_contents_path.value();
+    }
+    // kInvalid.
+    return base::WrapUnique(new ContentHash(key, nullptr, nullptr));
+  }
+
+  // computed_hashes.json: create if necessary.
+  base::FilePath computed_hashes_path =
+      file_util::GetComputedHashesPath(key.extension_root);
+
+  std::unique_ptr<ContentHash> hash = base::WrapUnique(
+      new ContentHash(key, std::move(verified_contents), nullptr));
+  if (create_computed_hashes_file) {
+    // If force recreate wasn't requested, existence of computed_hashes.json is
+    // enough, otherwise create computed_hashes.json.
+    bool need_to_create =
+        (mode & Mode::kForceRecreateExistingComputedHashesFile) != 0 ||
+        !base::PathExists(computed_hashes_path);
+    if (need_to_create)
+      hash->CreateHashes(computed_hashes_path, is_cancelled);
+  }
+
+  // computed_hashes.json: read.
+  if (!base::PathExists(computed_hashes_path))
+    return hash;
+
+  auto computed_hashes = std::make_unique<ComputedHashes::Reader>();
+  if (!computed_hashes->InitFromFile(computed_hashes_path))
+    return hash;
+
+  hash->status_ = Status::kSucceeded;
+  hash->computed_hashes_ = std::move(computed_hashes);
+
+  return hash;
+}
+
+bool ContentHash::CreateHashes(const base::FilePath& hashes_file,
+                               const IsCancelledCallback& is_cancelled) {
+  base::ElapsedTimer timer;
+  // Make sure the directory exists.
+  if (!base::CreateDirectoryAndGetError(hashes_file.DirName(), nullptr))
+    return false;
+
+  base::FileEnumerator enumerator(key_.extension_root, true, /* recursive */
+                                  base::FileEnumerator::FILES);
+  // First discover all the file paths and put them in a sorted set.
+  SortedFilePathSet paths;
+  for (;;) {
+    if (is_cancelled && is_cancelled.Run())
+      return false;
+
+    base::FilePath full_path = enumerator.Next();
+    if (full_path.empty())
+      break;
+    paths.insert(full_path);
+  }
+
+  // Now iterate over all the paths in sorted order and compute the block hashes
+  // for each one.
+  ComputedHashes::Writer writer;
+  for (SortedFilePathSet::iterator i = paths.begin(); i != paths.end(); ++i) {
+    if (is_cancelled && is_cancelled.Run())
+      return false;
+
+    const base::FilePath& full_path = *i;
+    base::FilePath relative_unix_path;
+    key_.extension_root.AppendRelativePath(full_path, &relative_unix_path);
+    relative_unix_path = relative_unix_path.NormalizePathSeparatorsTo('/');
+
+    if (!verified_contents_->HasTreeHashRoot(relative_unix_path))
+      continue;
+
+    std::string contents;
+    if (!base::ReadFileToString(full_path, &contents)) {
+      LOG(ERROR) << "Could not read " << full_path.MaybeAsASCII();
+      continue;
+    }
+
+    // Iterate through taking the hash of each block of size (block_size_) of
+    // the file.
+    std::vector<std::string> hashes;
+    ComputedHashes::ComputeHashesForContent(contents, block_size_, &hashes);
+    std::string root =
+        ComputeTreeHashRoot(hashes, block_size_ / crypto::kSHA256Length);
+    if (!verified_contents_->TreeHashRootEquals(relative_unix_path, root)) {
+      VLOG(1) << "content mismatch for " << relative_unix_path.AsUTF8Unsafe();
+      hash_mismatch_unix_paths_.insert(relative_unix_path);
+      continue;
+    }
+
+    writer.AddHashes(relative_unix_path, block_size_, hashes);
+  }
+  bool result = writer.WriteToFile(hashes_file);
+  UMA_HISTOGRAM_TIMES("ExtensionContentHashFetcher.CreateHashesTime",
+                      timer.Elapsed());
+
+  if (result)
+    status_ = Status::kSucceeded;
+
+  return result;
+}
+
+}  // namespace extensions

extensions/browser/content_verifier/content_hash.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef EXTENSIONS_BROWSER_CONTENT_VERIFIER_CONTENT_HASH_H_
+#define EXTENSIONS_BROWSER_CONTENT_VERIFIER_CONTENT_HASH_H_
+
+#include <set>
+
+#include "base/files/file_path.h"
+#include "base/version.h"
+#include "extensions/browser/computed_hashes.h"
+#include "extensions/browser/content_verifier_delegate.h"
+#include "extensions/browser/verified_contents.h"
+#include "extensions/common/extension_id.h"
+
+namespace extensions {
+
+// Represents content verification hashes for an extension.
+//
+// Instances can be created using Create() factory methods on sequences with
+// blocking IO access. If hash retrieval succeeds then ContentHash::succeeded()
+// will return true and
+// a. ContentHash::verified_contents() will return structured representation of
+//    verified_contents.json
+// b. ContentHash::computed_hashes() will return structured representation of
+//    computed_hashes.json.
+//
+// Additionally if computed_hashes.json was required to be written to disk and
+// it was successful, ContentHash::hash_mismatch_unix_paths() will return all
+// FilePaths from the extension directory that had content verification
+// mismatch.
+class ContentHash {
+ public:
+  // Key to identify an extension.
+  struct ExtensionKey {
+    ExtensionId extension_id;
+    base::FilePath extension_root;
+    base::Version extension_version;
+    // The key used to validate verified_contents.json.
+    ContentVerifierKey verifier_key;
+
+    ExtensionKey(const ExtensionId& extension_id,
+                 const base::FilePath& extension_root,
+                 const base::Version& extension_version,
+                 const ContentVerifierKey& verifier_key);
+
+    ExtensionKey(const ExtensionKey& other);
+    ExtensionKey& operator=(const ExtensionKey& other);
+  };
+
+  // Specifiable modes for ContentHash instantiation.
+  enum Mode {
+    // Deletes verified_contents.json if the file on disk is invalid.
+    kDeleteInvalidVerifiedContents = 1 << 0,
+
+    // Always creates computed_hashes.json (as opposed to only when the file is
+    // non-existent).
+    kForceRecreateExistingComputedHashesFile = 1 << 1,
+  };
+
+  using IsCancelledCallback = base::RepeatingCallback<bool(void)>;
+
+  // Factories:
+  // These will never return nullptr, but verified_contents or computed_hashes
+  // may be empty if something fails.
+  // Reads existing hashes from disk.
+  static std::unique_ptr<ContentHash> Create(const ExtensionKey& key);
+  // Reads existing hashes from disk, with specifying flags from |Mode|.
+  static std::unique_ptr<ContentHash> Create(
+      const ExtensionKey& key,
+      int mode,
+      const IsCancelledCallback& is_cancelled);
+
+  ~ContentHash();
+
+  const VerifiedContents& verified_contents() const;
+  const ComputedHashes::Reader& computed_hashes() const;
+
+  bool has_verified_contents() const {
+    return status_ >= Status::kHasVerifiedContents;
+  }
+  bool succeeded() const { return status_ >= Status::kSucceeded; }
+
+  // If ContentHash creation writes computed_hashes.json, then this returns the
+  // FilePaths whose content hash didn't match expected hashes.
+  const std::set<base::FilePath>& hash_mismatch_unix_paths() {
+    return hash_mismatch_unix_paths_;
+  }
+
+ private:
+  enum class Status {
+    // Retrieving hashes failed.
+    kInvalid,
+    // Retrieved valid verified_contents.json, but there was no
+    // computed_hashes.json.
+    kHasVerifiedContents,
+    // Both verified_contents.json and computed_hashes.json were read
+    // correctly.
+    kSucceeded,
+  };
+
+  ContentHash(const ExtensionKey& key,
+              std::unique_ptr<VerifiedContents> verified_contents,
+              std::unique_ptr<ComputedHashes::Reader> computed_hashes);
+
+  static std::unique_ptr<ContentHash> CreateImpl(
+      const ExtensionKey& key,
+      int mode,
+      bool create_computed_hashes_file,
+      const IsCancelledCallback& is_cancelled);
+
+  // Computes hashes for all files in |key_.extension_root|, and uses
+  // a ComputedHashes::Writer to write that information into |hashes_file|.
+  // Returns true on success.
+  // The verified contents file from the webstore only contains the treehash
+  // root hash, but for performance we want to cache the individual block level
+  // hashes. This function will create that cache with block-level hashes for
+  // each file in the extension if needed (the treehash root hash for each of
+  // these should equal what is in the verified contents file from the
+  // webstore).
+  bool CreateHashes(const base::FilePath& hashes_file,
+                    const IsCancelledCallback& is_cancelled);
+
+  ExtensionKey key_;
+
+  Status status_;
+
+  // TODO(lazyboy): Avoid dynamic allocations here, |this| already supports
+  // move.
+  std::unique_ptr<VerifiedContents> verified_contents_;
+  std::unique_ptr<ComputedHashes::Reader> computed_hashes_;
+
+  // Paths that were found to have a mismatching hash.
+  std::set<base::FilePath> hash_mismatch_unix_paths_;
+
+  // The block size to use for hashing.
+  // TODO(asargent) - use the value from verified_contents.json for each
+  // file, instead of using a constant.
+  int block_size_ = 4096;
+
+  DISALLOW_COPY_AND_ASSIGN(ContentHash);
+};
+
+}  // namespace extensions
+
+#endif  // EXTENSIONS_BROWSER_CONTENT_VERIFIER_CONTENT_HASH_H_