Gave the GetOriginBoundCertificate an asynchronous interface because certificate

generation is a blocking operation.

BUG=88782
TEST=None

Review URL: http://codereview.chromium.org/7565023

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@96229 0039d316-1c4b-4281-b951-d872f2087c98

net/base/net_error_list.h

@@ -574,6 +574,18 @@ NET_ERROR(PKCS12_IMPORT_INVALID_FILE, -708)
 // PKCS #12 import failed due to unsupported features.
 NET_ERROR(PKCS12_IMPORT_UNSUPPORTED, -709)
 
+// Key generation failed.
+NET_ERROR(KEY_GENERATION_FAILED, -710)
+
+// Origin-bound certificate generation failed.
+NET_ERROR(ORIGIN_BOUND_CERT_GENERATION_FAILED, -711)
+
+// Failure to export private key.
+NET_ERROR(PRIVATE_KEY_EXPORT_FAILED, -712)
+
+// Failure to get certificate bytes.
+NET_ERROR(GET_CERT_BYTES_FAILED, -713)
+
 // DNS error codes.
 
 // DNS resolver received a malformed response.

net/base/origin_bound_cert_service.cc

@@ -6,14 +6,23 @@
 
 #include <limits>
 
+#include "base/compiler_specific.h"
 #include "base/logging.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_ptr.h"
+#include "base/message_loop.h"
 #include "base/rand_util.h"
+#include "base/stl_util.h"
+#include "base/threading/worker_pool.h"
 #include "crypto/rsa_private_key.h"
+#include "net/base/net_errors.h"
 #include "net/base/origin_bound_cert_store.h"
 #include "net/base/x509_certificate.h"
 
+#if defined(USE_NSS)
+#include <private/pprthred.h>  // PR_DetachThread
+#endif
+
 namespace net {
 
 namespace {
@@ -23,43 +32,312 @@ const int kValidityPeriodInDays = 365;
 
 }  // namespace
 
+// Represents the output and result callback of a request.
+class OriginBoundCertServiceRequest {
+ public:
+  OriginBoundCertServiceRequest(CompletionCallback* callback,
+                                std::string* private_key,
+                                std::string* cert)
+      : callback_(callback),
+        private_key_(private_key),
+        cert_(cert) {
+  }
+
+  // Ensures that the result callback will never be made.
+  void Cancel() {
+    callback_ = NULL;
+    private_key_ = NULL;
+    cert_ = NULL;
+  }
+
+  // Copies the contents of |private_key| and |cert| to the caller's output
+  // arguments and calls the callback.
+  void Post(int error,
+            const std::string& private_key,
+            const std::string& cert) {
+    if (callback_) {
+      *private_key_ = private_key;
+      *cert_ = cert;
+      callback_->Run(error);
+    }
+    delete this;
+  }
+
+  bool canceled() const { return !callback_; }
+
+ private:
+  CompletionCallback* callback_;
+  std::string* private_key_;
+  std::string* cert_;
+};
+
+// OriginBoundCertServiceWorker runs on a worker thread and takes care of the
+// blocking process of performing key generation. Deletes itself eventually
+// if Start() succeeds.
+class OriginBoundCertServiceWorker {
+ public:
+  OriginBoundCertServiceWorker(
+      const std::string& origin,
+      OriginBoundCertService* origin_bound_cert_service)
+      : origin_(origin),
+        serial_number_(base::RandInt(0, std::numeric_limits<int>::max())),
+        origin_loop_(MessageLoop::current()),
+        origin_bound_cert_service_(origin_bound_cert_service),
+        canceled_(false),
+        error_(ERR_FAILED) {
+  }
+
+  bool Start() {
+    DCHECK_EQ(MessageLoop::current(), origin_loop_);
+
+    return base::WorkerPool::PostTask(
+        FROM_HERE,
+        NewRunnableMethod(this, &OriginBoundCertServiceWorker::Run),
+        true /* task is slow */);
+  }
+
+  // Cancel is called from the origin loop when the OriginBoundCertService is
+  // getting deleted.
+  void Cancel() {
+    DCHECK_EQ(MessageLoop::current(), origin_loop_);
+    base::AutoLock locked(lock_);
+    canceled_ = true;
+  }
+
+ private:
+  void Run() {
+    // Runs on a worker thread.
+    error_ = OriginBoundCertService::GenerateCert(origin_,
+                                                  serial_number_,
+                                                  &private_key_,
+                                                  &cert_);
+#if defined(USE_NSS)
+    // Detach the thread from NSPR.
+    // Calling NSS functions attaches the thread to NSPR, which stores
+    // the NSPR thread ID in thread-specific data.
+    // The threads in our thread pool terminate after we have called
+    // PR_Cleanup. Unless we detach them from NSPR, net_unittests gets
+    // segfaults on shutdown when the threads' thread-specific data
+    // destructors run.
+    PR_DetachThread();
+#endif
+    Finish();
+  }
+
+  // DoReply runs on the origin thread.
+  void DoReply() {
+    DCHECK_EQ(MessageLoop::current(), origin_loop_);
+    {
+      // We lock here because the worker thread could still be in Finished,
+      // after the PostTask, but before unlocking |lock_|. If we do not lock in
+      // this case, we will end up deleting a locked Lock, which can lead to
+      // memory leaks or worse errors.
+      base::AutoLock locked(lock_);
+      if (!canceled_) {
+        origin_bound_cert_service_->HandleResult(origin_, error_,
+                                                 private_key_, cert_);
+      }
+    }
+    delete this;
+  }
+
+  void Finish() {
+    // Runs on the worker thread.
+    // We assume that the origin loop outlives the OriginBoundCertService. If
+    // the OriginBoundCertService is deleted, it will call Cancel on us. If it
+    // does so before the Acquire, we'll delete ourselves and return. If it's
+    // trying to do so concurrently, then it'll block on the lock and we'll
+    // call PostTask while the OriginBoundCertService (and therefore the
+    // MessageLoop) is still alive. If it does so after this function, we
+    // assume that the MessageLoop will process pending tasks. In which case
+    // we'll notice the |canceled_| flag in DoReply.
+
+    bool canceled;
+    {
+      base::AutoLock locked(lock_);
+      canceled = canceled_;
+      if (!canceled) {
+        origin_loop_->PostTask(
+            FROM_HERE,
+            NewRunnableMethod(this, &OriginBoundCertServiceWorker::DoReply));
+      }
+    }
+    if (canceled)
+      delete this;
+  }
+
+  const std::string origin_;
+  // Note that serial_number_ must be initialized on a non-worker thread
+  // (see documentation for OriginBoundCertService::GenerateCert).
+  uint32 serial_number_;
+  MessageLoop* const origin_loop_;
+  OriginBoundCertService* const origin_bound_cert_service_;
+
+  // lock_ protects canceled_.
+  base::Lock lock_;
+
+  // If canceled_ is true,
+  // * origin_loop_ cannot be accessed by the worker thread,
+  // * origin_bound_cert_service_ cannot be accessed by any thread.
+  bool canceled_;
+
+  int error_;
+  std::string private_key_;
+  std::string cert_;
+
+  DISALLOW_COPY_AND_ASSIGN(OriginBoundCertServiceWorker);
+};
+
+// An OriginBoundCertServiceJob is a one-to-one counterpart of an
+// OriginBoundCertServiceWorker. It lives only on the OriginBoundCertService's
+// origin message loop.
+class OriginBoundCertServiceJob {
+ public:
+  explicit OriginBoundCertServiceJob(OriginBoundCertServiceWorker* worker)
+      : worker_(worker) {
+  }
+
+  ~OriginBoundCertServiceJob() {
+    if (worker_) {
+      worker_->Cancel();
+      DeleteAllCanceled();
+    }
+  }
+
+  void AddRequest(OriginBoundCertServiceRequest* request) {
+    requests_.push_back(request);
+  }
+
+  void HandleResult(int error,
+                    const std::string& private_key,
+                    const std::string& cert) {
+    worker_ = NULL;
+    PostAll(error, private_key, cert);
+  }
+
+ private:
+  void PostAll(int error,
+               const std::string& private_key,
+               const std::string& cert) {
+    std::vector<OriginBoundCertServiceRequest*> requests;
+    requests_.swap(requests);
+
+    for (std::vector<OriginBoundCertServiceRequest*>::iterator
+         i = requests.begin(); i != requests.end(); i++) {
+      (*i)->Post(error, private_key, cert);
+      // Post() causes the OriginBoundCertServiceRequest to delete itself.
+    }
+  }
+
+  void DeleteAllCanceled() {
+    for (std::vector<OriginBoundCertServiceRequest*>::iterator
+         i = requests_.begin(); i != requests_.end(); i++) {
+      if ((*i)->canceled()) {
+        delete *i;
+      } else {
+        LOG(DFATAL) << "OriginBoundCertServiceRequest leaked!";
+      }
+    }
+  }
+
+  std::vector<OriginBoundCertServiceRequest*> requests_;
+  OriginBoundCertServiceWorker* worker_;
+};
+
 OriginBoundCertService::OriginBoundCertService(
     OriginBoundCertStore* origin_bound_cert_store)
-    : origin_bound_cert_store_(origin_bound_cert_store) {}
+    : origin_bound_cert_store_(origin_bound_cert_store),
+      requests_(0),
+      cert_store_hits_(0),
+      inflight_joins_(0) {}
+
+OriginBoundCertService::~OriginBoundCertService() {
+  STLDeleteValues(&inflight_);
+}
 
-OriginBoundCertService::~OriginBoundCertService() {}
+int OriginBoundCertService::GetOriginBoundCert(const std::string& origin,
+                                               std::string* private_key,
+                                               std::string* cert,
+                                               CompletionCallback* callback,
+                                               RequestHandle* out_req) {
+  DCHECK(CalledOnValidThread());
 
-bool OriginBoundCertService::GetOriginBoundCert(const std::string& origin,
-                                                std::string* private_key_result,
-                                                std::string* cert_result) {
-  // Check if origin bound cert already exists for this origin.
+  if (!callback || !private_key || !cert || origin.empty()) {
+    *out_req = NULL;
+    return ERR_INVALID_ARGUMENT;
+  }
+
+  requests_++;
+
+  // Check if an origin bound cert already exists for this origin.
   if (origin_bound_cert_store_->GetOriginBoundCert(origin,
-                                                   private_key_result,
-                                                   cert_result))
-    return true;
+                                                   private_key,
+                                                   cert)) {
+    cert_store_hits_++;
+    *out_req = NULL;
+    return OK;
+  }
+
+  // |origin_bound_cert_store_| has no cert for this origin. See if an
+  // identical request is currently in flight.
+  OriginBoundCertServiceJob* job;
+  std::map<std::string, OriginBoundCertServiceJob*>::const_iterator j;
+  j = inflight_.find(origin);
+  if (j != inflight_.end()) {
+    // An identical request is in flight already. We'll just attach our
+    // callback.
+    inflight_joins_++;
+    job = j->second;
+  } else {
+    // Need to make a new request.
+    OriginBoundCertServiceWorker* worker =
+        new OriginBoundCertServiceWorker(origin, this);
+    job = new OriginBoundCertServiceJob(worker);
+    if (!worker->Start()) {
+      delete job;
+      delete worker;
+      *out_req = NULL;
+      // TODO(rkn): Log to the NetLog.
+      LOG(ERROR) << "OriginBoundCertServiceWorker couldn't be started.";
+      return ERR_INSUFFICIENT_RESOURCES;  // Just a guess.
+    }
+    inflight_[origin] = job;
+  }
+
+  OriginBoundCertServiceRequest* request =
+      new OriginBoundCertServiceRequest(callback, private_key, cert);
+  job->AddRequest(request);
+  *out_req = request;
+  return ERR_IO_PENDING;
+}
 
-  // No origin bound cert exists, we have to create one.
+// static
+int OriginBoundCertService::GenerateCert(const std::string& origin,
+                                         uint32 serial_number,
+                                         std::string* private_key,
+                                         std::string* cert) {
   std::string subject = "CN=OBC";
   scoped_ptr<crypto::RSAPrivateKey> key(
       crypto::RSAPrivateKey::Create(kKeySizeInBits));
   if (!key.get()) {
     LOG(WARNING) << "Unable to create key pair for client";
-    return false;
+    return ERR_KEY_GENERATION_FAILED;
   }
+
   scoped_refptr<X509Certificate> x509_cert = X509Certificate::CreateSelfSigned(
       key.get(),
       subject,
-      base::RandInt(0, std::numeric_limits<int>::max()),
+      serial_number,
       base::TimeDelta::FromDays(kValidityPeriodInDays));
   if (!x509_cert) {
     LOG(WARNING) << "Unable to create x509 cert for client";
-    return false;
+    return ERR_ORIGIN_BOUND_CERT_GENERATION_FAILED;
   }
 
   std::vector<uint8> private_key_info;
   if (!key->ExportPrivateKey(&private_key_info)) {
     LOG(WARNING) << "Unable to export private key";
-    return false;
+    return ERR_PRIVATE_KEY_EXPORT_FAILED;
   }
   // TODO(rkn): Perhaps ExportPrivateKey should be changed to output a
   // std::string* to prevent this copying.
@@ -67,24 +345,49 @@ bool OriginBoundCertService::GetOriginBoundCert(const std::string& origin,
 
   std::string der_cert;
   if (!x509_cert->GetDEREncoded(&der_cert)) {
-    LOG(WARNING) << "Unable to get DER-enconded cert";
-    return false;
+    LOG(WARNING) << "Unable to get DER-encoded cert";
+    return ERR_GET_CERT_BYTES_FAILED;
   }
 
-  if (!origin_bound_cert_store_->SetOriginBoundCert(origin,
-                                                    key_out,
-                                                    der_cert)) {
-    LOG(WARNING) << "Unable to set origin bound certificate";
-    return false;
+  private_key->swap(key_out);
+  cert->swap(der_cert);
+  return OK;
+}
+
+void OriginBoundCertService::CancelRequest(RequestHandle req) {
+  DCHECK(CalledOnValidThread());
+  OriginBoundCertServiceRequest* request =
+      reinterpret_cast<OriginBoundCertServiceRequest*>(req);
+  request->Cancel();
+}
+
+// HandleResult is called by OriginBoundCertServiceWorker on the origin message
+// loop. It deletes OriginBoundCertServiceJob.
+void OriginBoundCertService::HandleResult(const std::string& origin,
+                                         int error,
+                                         const std::string& private_key,
+                                         const std::string& cert) {
+  DCHECK(CalledOnValidThread());
+
+  origin_bound_cert_store_->SetOriginBoundCert(origin, private_key, cert);
+
+  std::map<std::string, OriginBoundCertServiceJob*>::iterator j;
+  j = inflight_.find(origin);
+  if (j == inflight_.end()) {
+    NOTREACHED();
+    return;
   }
+  OriginBoundCertServiceJob* job = j->second;
+  inflight_.erase(j);
 
-  private_key_result->swap(key_out);
-  cert_result->swap(der_cert);
-  return true;
+  job->HandleResult(error, private_key, cert);
+  delete job;
 }
 
-int OriginBoundCertService::GetCertCount() {
+int OriginBoundCertService::cert_count() {
   return origin_bound_cert_store_->GetCertCount();
 }
 
 }  // namespace net
+
+DISABLE_RUNNABLE_METHOD_REFCOUNT(net::OriginBoundCertServiceWorker);

net/base/origin_bound_cert_service.h

@@ -6,20 +6,30 @@
 #define NET_BASE_ORIGIN_BOUND_CERT_SERVICE_H_
 #pragma once
 
+#include <map>
 #include <string>
 
-#include "base/memory/ref_counted.h"
+#include "base/basictypes.h"
 #include "base/memory/scoped_ptr.h"
+#include "base/threading/non_thread_safe.h"
+#include "net/base/completion_callback.h"
 #include "net/base/net_api.h"
 
 namespace net {
 
+class OriginBoundCertServiceJob;
+class OriginBoundCertServiceWorker;
 class OriginBoundCertStore;
 
 // A class for creating and fetching origin bound certs.
+// Inherits from NonThreadSafe in order to use the function
+// |CalledOnValidThread|.
 class NET_API OriginBoundCertService
-    : public base::RefCountedThreadSafe<OriginBoundCertService> {
+    : NON_EXPORTED_BASE(public base::NonThreadSafe) {
  public:
+  // Opaque type used to cancel a request.
+  typedef void* RequestHandle;
+
   // This object owns origin_bound_cert_store.
   explicit OriginBoundCertService(
       OriginBoundCertStore* origin_bound_cert_store);
@@ -27,21 +37,68 @@ class NET_API OriginBoundCertService
   ~OriginBoundCertService();
 
   // TODO(rkn): Specify certificate type (RSA or DSA).
-  // TODO(rkn): Key generation can be time consuming, so this should have an
-  // asynchronous interface.
+  //
   // Fetches the origin bound cert for the specified origin if one exists
-  // and creates one otherwise. On success, |private_key_result| stores a
-  // DER-encoded PrivateKeyInfo struct, and |cert_result| stores a DER-encoded
-  // certificate.
-  bool GetOriginBoundCert(const std::string& origin,
-                          std::string* private_key_result,
-                          std::string* cert_result);
+  // and creates one otherwise. Returns OK if successful or an error code upon
+  // failure.
+  //
+  // On successful completion, |private_key| stores a DER-encoded
+  // PrivateKeyInfo struct, and |cert| stores a DER-encoded certificate.
+  //
+  // |callback| must not be null. ERR_IO_PENDING is returned if the operation
+  // could not be completed immediately, in which case the result code will
+  // be passed to the callback when available.
+  //
+  // If |out_req| is non-NULL, then |*out_req| will be filled with a handle to
+  // the async request. This handle is not valid after the request has
+  // completed.
+  int GetOriginBoundCert(const std::string& origin,
+                         std::string* private_key,
+                         std::string* cert,
+                         CompletionCallback* callback,
+                         RequestHandle* out_req);
+
+  // Cancels the specified request. |req| is the handle returned by
+  // GetOriginBoundCert(). After a request is canceled, its completion
+  // callback will not be called.
+  void CancelRequest(RequestHandle req);
 
   // Public only for unit testing.
-  int GetCertCount();
+  int cert_count();
+  uint64 requests() const { return requests_; }
+  uint64 cert_store_hits() const { return cert_store_hits_; }
+  uint64 inflight_joins() const { return inflight_joins_; }
 
  private:
+  friend class OriginBoundCertServiceWorker;  // Calls HandleResult.
+
+  // On success, |private_key| stores a DER-encoded PrivateKeyInfo
+  // struct, and |cert| stores a DER-encoded certificate. Returns
+  // OK if successful and an error code otherwise.
+  // |serial_number| is passed in because it is created with the function
+  // base::RandInt, which opens the file /dev/urandom. /dev/urandom is opened
+  // with a LazyInstance, which is not allowed on a worker thread.
+  static int GenerateCert(const std::string& origin,
+                          uint32 serial_number,
+                          std::string* private_key,
+                          std::string* cert);
+
+  void HandleResult(const std::string& origin,
+                    int error,
+                    const std::string& private_key,
+                    const std::string& cert);
+
   scoped_ptr<OriginBoundCertStore> origin_bound_cert_store_;
+
+  // inflight_ maps from an origin to an active generation which is taking
+  // place.
+  std::map<std::string, OriginBoundCertServiceJob*> inflight_;
+
+  uint64 requests_;
+  uint64 cert_store_hits_;
+  uint64 inflight_joins_;
+
+  DISALLOW_COPY_AND_ASSIGN(OriginBoundCertService);
 };
 
 }  // namespace net

net/base/origin_bound_cert_service_unittest.cc

@@ -7,57 +7,165 @@
 #include <string>
 #include <vector>
 
-#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
 #include "crypto/rsa_private_key.h"
 #include "net/base/default_origin_bound_cert_store.h"
+#include "net/base/net_errors.h"
+#include "net/base/test_completion_callback.h"
 #include "net/base/x509_certificate.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace net {
 
+class OriginBoundCertServiceTest : public testing::Test {
+};
+
+class ExplodingCallback : public CallbackRunner<Tuple1<int> > {
+ public:
+  virtual void RunWithParams(const Tuple1<int>& params) {
+    FAIL();
+  }
+};
+
 // See http://crbug.com/91512 - implement OpenSSL version of CreateSelfSigned.
 #if !defined(USE_OPENSSL)
 
-TEST(OriginBoundCertServiceTest, DuplicateCertTest) {
-  scoped_refptr<OriginBoundCertService> service(
+TEST_F(OriginBoundCertServiceTest, CacheHit) {
+  scoped_ptr<OriginBoundCertService> service(
+      new OriginBoundCertService(new DefaultOriginBoundCertStore(NULL)));
+  std::string origin("https://encrypted.google.com:443");
+
+  int error;
+  TestCompletionCallback callback;
+  OriginBoundCertService::RequestHandle request_handle;
+
+  // Asynchronous completion.
+  std::string private_key_info1, der_cert1;
+  EXPECT_EQ(0, service->cert_count());
+  error = service->GetOriginBoundCert(
+      origin, &private_key_info1, &der_cert1, &callback, &request_handle);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle != NULL);
+  error = callback.WaitForResult();
+  EXPECT_EQ(OK, error);
+  EXPECT_EQ(1, service->cert_count());
+  EXPECT_FALSE(private_key_info1.empty());
+  EXPECT_FALSE(der_cert1.empty());
+
+  // Synchronous completion.
+  std::string private_key_info2, der_cert2;
+  error = service->GetOriginBoundCert(
+      origin, &private_key_info2, &der_cert2, &callback, &request_handle);
+  EXPECT_TRUE(request_handle == NULL);
+  EXPECT_EQ(OK, error);
+  EXPECT_EQ(1, service->cert_count());
+
+  EXPECT_EQ(private_key_info1, private_key_info2);
+  EXPECT_EQ(der_cert1, der_cert2);
+
+  EXPECT_EQ(2u, service->requests());
+  EXPECT_EQ(1u, service->cert_store_hits());
+  EXPECT_EQ(0u, service->inflight_joins());
+}
+
+TEST_F(OriginBoundCertServiceTest, StoreCerts) {
+  scoped_ptr<OriginBoundCertService> service(
       new OriginBoundCertService(new DefaultOriginBoundCertStore(NULL)));
-  std::string origin1("https://encrypted.google.com/");
-  std::string origin2("https://www.verisign.com/");
-
-  // The store should start out empty and should increment appropriately.
-  std::string private_key_info_1a, der_cert_1a;
-  EXPECT_EQ(0, service->GetCertCount());
-  EXPECT_TRUE(service->GetOriginBoundCert(
-      origin1, &private_key_info_1a, &der_cert_1a));
-  EXPECT_EQ(1, service->GetCertCount());
-
-  // We should get the same cert and key for the same origin.
-  std::string private_key_info_1b, der_cert_1b;
-  EXPECT_TRUE(service->GetOriginBoundCert(
-      origin1, &private_key_info_1b, &der_cert_1b));
-  EXPECT_EQ(1, service->GetCertCount());
-  EXPECT_EQ(private_key_info_1a, private_key_info_1b);
-  EXPECT_EQ(der_cert_1a, der_cert_1b);
-
-  // We should get a different cert and key for different origins.
-  std::string private_key_info_2, der_cert_2;
-  EXPECT_TRUE(service->GetOriginBoundCert(
-      origin2, &private_key_info_2, &der_cert_2));
-  EXPECT_EQ(2, service->GetCertCount());
-  EXPECT_NE(private_key_info_1a, private_key_info_2);
-  EXPECT_NE(der_cert_1a, der_cert_2);
+  int error;
+  TestCompletionCallback callback;
+  OriginBoundCertService::RequestHandle request_handle;
+
+  std::string origin1("https://encrypted.google.com:443");
+  std::string private_key_info1, der_cert1;
+  EXPECT_EQ(0, service->cert_count());
+  error = service->GetOriginBoundCert(
+      origin1, &private_key_info1, &der_cert1, &callback, &request_handle);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle != NULL);
+  error = callback.WaitForResult();
+  EXPECT_EQ(OK, error);
+  EXPECT_EQ(1, service->cert_count());
+
+  std::string origin2("https://www.verisign.com:443");
+  std::string private_key_info2, der_cert2;
+  error = service->GetOriginBoundCert(
+      origin2, &private_key_info2, &der_cert2, &callback, &request_handle);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle != NULL);
+  error = callback.WaitForResult();
+  EXPECT_EQ(OK, error);
+  EXPECT_EQ(2, service->cert_count());
+
+  std::string origin3("https://www.twitter.com:443");
+  std::string private_key_info3, der_cert3;
+  error = service->GetOriginBoundCert(
+      origin3, &private_key_info3, &der_cert3, &callback, &request_handle);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle != NULL);
+  error = callback.WaitForResult();
+  EXPECT_EQ(OK, error);
+  EXPECT_EQ(3, service->cert_count());
+
+  EXPECT_NE(private_key_info1, private_key_info2);
+  EXPECT_NE(der_cert1, der_cert2);
+  EXPECT_NE(private_key_info1, private_key_info3);
+  EXPECT_NE(der_cert1, der_cert3);
+  EXPECT_NE(private_key_info2, private_key_info3);
+  EXPECT_NE(der_cert2, der_cert3);
 }
 
-TEST(OriginBoundCertServiceTest, ExtractValuesFromBytes) {
-  scoped_refptr<OriginBoundCertService> service(
+// Tests an inflight join.
+TEST_F(OriginBoundCertServiceTest, InflightJoin) {
+  scoped_ptr<OriginBoundCertService> service(
       new OriginBoundCertService(new DefaultOriginBoundCertStore(NULL)));
-  std::string origin("https://encrypted.google.com/");
+  std::string origin("https://encrypted.google.com:443");
+  int error;
+
+  std::string private_key_info1, der_cert1;
+  TestCompletionCallback callback1;
+  OriginBoundCertService::RequestHandle request_handle1;
+
+  std::string private_key_info2, der_cert2;
+  TestCompletionCallback callback2;
+  OriginBoundCertService::RequestHandle request_handle2;
+
+  error = service->GetOriginBoundCert(
+      origin, &private_key_info1, &der_cert1, &callback1, &request_handle1);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle1 != NULL);
+  error = service->GetOriginBoundCert(
+      origin, &private_key_info2, &der_cert2, &callback2, &request_handle2);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle2 != NULL);
+
+  error = callback1.WaitForResult();
+  EXPECT_EQ(OK, error);
+  error = callback2.WaitForResult();
+  EXPECT_EQ(OK, error);
+
+  EXPECT_EQ(2u, service->requests());
+  EXPECT_EQ(0u, service->cert_store_hits());
+  EXPECT_EQ(1u, service->inflight_joins());
+}
+
+TEST_F(OriginBoundCertServiceTest, ExtractValuesFromBytes) {
+  scoped_ptr<OriginBoundCertService> service(
+      new OriginBoundCertService(new DefaultOriginBoundCertStore(NULL)));
+  std::string origin("https://encrypted.google.com:443");
   std::string private_key_info, der_cert;
-  EXPECT_TRUE(service->GetOriginBoundCert(
-      origin, &private_key_info, &der_cert));
-  std::vector<uint8> key_vec(private_key_info.begin(), private_key_info.end());
+  int error;
+  TestCompletionCallback callback;
+  OriginBoundCertService::RequestHandle request_handle;
+
+  error = service->GetOriginBoundCert(
+      origin, &private_key_info, &der_cert, &callback, &request_handle);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle != NULL);
+  error = callback.WaitForResult();
+  EXPECT_EQ(OK, error);
 
-  // Check that we can retrieve the key pair from the bytes.
+  // Check that we can retrieve the key from the bytes.
+  std::vector<uint8> key_vec(private_key_info.begin(), private_key_info.end());
   scoped_ptr<crypto::RSAPrivateKey> private_key(
       crypto::RSAPrivateKey::CreateFromPrivateKeyInfo(key_vec));
   EXPECT_TRUE(private_key != NULL);
@@ -68,6 +176,42 @@ TEST(OriginBoundCertServiceTest, ExtractValuesFromBytes) {
   EXPECT_TRUE(x509cert != NULL);
 }
 
+// Tests that the callback of a canceled request is never made.
+TEST_F(OriginBoundCertServiceTest, CancelRequest) {
+  scoped_ptr<OriginBoundCertService> service(
+      new OriginBoundCertService(new DefaultOriginBoundCertStore(NULL)));
+  std::string origin("https://encrypted.google.com:443");
+  std::string private_key_info, der_cert;
+  int error;
+  ExplodingCallback exploding_callback;
+  OriginBoundCertService::RequestHandle request_handle;
+
+  error = service->GetOriginBoundCert(origin,
+                                      &private_key_info,
+                                      &der_cert,
+                                      &exploding_callback,
+                                      &request_handle);
+  EXPECT_EQ(ERR_IO_PENDING, error);
+  EXPECT_TRUE(request_handle != NULL);
+  service->CancelRequest(request_handle);
+
+  // Issue a few more requests to the worker pool and wait for their
+  // completion, so that the task of the canceled request (which runs on a
+  // worker thread) is likely to complete by the end of this test.
+  TestCompletionCallback callback;
+  for (int i = 0; i < 5; ++i) {
+    error = service->GetOriginBoundCert(
+        "https://encrypted.google.com:" + std::string(1, (char) ('1' + i)),
+        &private_key_info,
+        &der_cert,
+        &callback,
+        &request_handle);
+    EXPECT_EQ(ERR_IO_PENDING, error);
+    EXPECT_TRUE(request_handle != NULL);
+    error = callback.WaitForResult();
+  }
+}
+
 #endif  // !defined(USE_OPENSSL)
 
 }  // namespace net