Initial implementation for CertNetFetcher.

CertNetFetcher is a class for issuing and cancelling network fetches, that will be used by CertVerifier. It fetches http:// AIA and CRL URLs.

This initial implementation has some remaining TODOs around:
  - Add POST parameters
  - Add cache bypass controls
  - Add maximum requests thresholds
  - Add more tests for cancellation/de-duplication

BUG=455366
NOPRESUBMIT=true

Review URL: https://codereview.chromium.org/908863004

Cr-Commit-Position: refs/heads/master@{#324261}

net/BUILD.gn

@@ -1564,6 +1564,7 @@ if (!is_android && !is_win && !is_mac) {
         "spdy/fuzzing/hpack_fuzz_util_test.cc",
 
         # Need TestServer.
+        "cert_net/cert_net_fetcher_impl_unittest.cc",
         "proxy/proxy_script_fetcher_impl_unittest.cc",
         "socket/ssl_client_socket_unittest.cc",
         "url_request/url_fetcher_impl_unittest.cc",

net/cert/cert_net_fetcher.h

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_CERT_CERT_NET_FETCHER_H_
+#define NET_CERT_CERT_NET_FETCHER_H_
+
+#include <vector>
+
+#include "base/callback.h"
+#include "net/base/net_errors.h"
+#include "net/base/net_export.h"
+
+class GURL;
+
+namespace net {
+
+class URLRequestContext;
+
+// CertNetFetcher is an asynchronous interface for fetching AIA URLs and CRL
+// URLs.
+//
+// -------------------------
+// Cancellation of requests
+// -------------------------
+//
+//  * Network requests started by the CertNetFetcher can be cancelled by
+//    deleting the Request object. Cancellation means the request's callback
+//    will no longer be invoked.
+//
+//  * If the CertNetFetcher is deleted then any outstanding
+//    requests are automatically cancelled.
+//
+//  * Cancelling a request within the execution of a callback is allowed.
+//
+//  * Deleting the CertNetFetcher from within the execution of a callback is
+//    allowed.
+//
+// -------------------------
+// Threading
+// -------------------------
+//
+// The CertNetFetcher is expected to be operated from a single thread, which has
+// an IO message loop. The URLRequestContext will be accessed from this same
+// thread, and callbacks will be posted to this message loop.
+//
+// For more details see the design document:
+//   https://docs.google.com/a/chromium.org/document/d/1CdS9YOnPdAyVZBJqHY7ZJ6tUlU71OCvX8kHnaVhf144/edit
+class NET_EXPORT CertNetFetcher {
+ public:
+  class Request {
+   public:
+    virtual ~Request() {}
+  };
+
+  // Callback invoked on request completion. If the Error is OK, then the
+  // vector contains the response bytes.
+  using FetchCallback =
+      base::Callback<void(Error, const std::vector<uint8_t>&)>;
+
+  // This value can be used in place of timeout or max size limits.
+  enum { DEFAULT = -1 };
+
+  CertNetFetcher() {}
+
+  // Deletion implicitly cancels any outstanding requests.
+  virtual ~CertNetFetcher() {}
+
+  // The Fetch*() methods start an asynchronous request which can be cancelled
+  // by deleting the returned Request. Here is the meaning of the common
+  // parameters:
+  //
+  //   * url -- The http:// URL to fetch.
+  //   * timeout_seconds -- The maximum allowed duration for the fetch job. If
+  //         this delay is exceeded then the request will fail. To use a default
+  //         timeout pass DEFAULT.
+  //   * max_response_bytes -- The maximum size of the response body. If this
+  //     size is exceeded then the request will fail. To use a default timeout
+  //     pass DEFAULT.
+  //   * callback -- The callback that will be invoked on completion of the job.
+
+  virtual WARN_UNUSED_RESULT scoped_ptr<Request> FetchCaIssuers(
+      const GURL& url,
+      int timeout_milliseconds,
+      int max_response_bytes,
+      const FetchCallback& callback) = 0;
+
+  virtual WARN_UNUSED_RESULT scoped_ptr<Request> FetchCrl(
+      const GURL& url,
+      int timeout_milliseconds,
+      int max_response_bytes,
+      const FetchCallback& callback) = 0;
+
+  virtual WARN_UNUSED_RESULT scoped_ptr<Request> FetchOcsp(
+      const GURL& url,
+      int timeout_milliseconds,
+      int max_response_bytes,
+      const FetchCallback& callback) = 0;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(CertNetFetcher);
+};
+
+}  // namespace net
+
+#endif  // NET_CERT_NET_CERT_NET_FETCHER_H_

net/cert_net/README

+cert_net/ contains certificate functionality that depends on network loading (OCSP, CRL, AIA fetching).
+
+Conceptually certificates (net/cert/) is a separable concept from net/ and may
+end up becoming its own build target. This file organization encourages not
+adding dependencies in cert/ for doing network loading. Instead that code
+should be placed here.

net/cert_net/cert_net_fetcher_impl.h

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_CERT_NET_CERT_NET_FETCHER_H_
+#define NET_CERT_NET_CERT_NET_FETCHER_H_
+
+#include <set>
+
+#include "base/callback.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/threading/thread_checker.h"
+#include "net/base/net_errors.h"
+#include "net/base/net_export.h"
+#include "net/cert/cert_net_fetcher.h"
+
+namespace net {
+
+class URLRequestContext;
+
+// CertNetFetcherImpl is an implementation of CertNetFetcher that uses the
+// network stack.
+//
+// For more details refer to the documentation for the interface.
+class NET_EXPORT CertNetFetcherImpl : public CertNetFetcher {
+ public:
+  // Initializes CertNetFetcherImpl using the specified URLRequestContext for
+  // issuing requests. |context| must remain valid for the entire lifetime of
+  // the CertNetFetcherImpl.
+  explicit CertNetFetcherImpl(URLRequestContext* context);
+
+  // Deletion implicitly cancels any outstanding requests.
+  ~CertNetFetcherImpl() override;
+
+  WARN_UNUSED_RESULT scoped_ptr<Request> FetchCaIssuers(
+      const GURL& url,
+      int timeout_milliseconds,
+      int max_response_bytes,
+      const FetchCallback& callback) override;
+
+  WARN_UNUSED_RESULT scoped_ptr<Request> FetchCrl(
+      const GURL& url,
+      int timeout_milliseconds,
+      int max_response_bytes,
+      const FetchCallback& callback) override;
+
+  WARN_UNUSED_RESULT scoped_ptr<Request> FetchOcsp(
+      const GURL& url,
+      int timeout_milliseconds,
+      int max_response_bytes,
+      const FetchCallback& callback) override;
+
+ private:
+  class RequestImpl;
+  class Job;
+  struct JobToRequestParamsComparator;
+  struct RequestParams;
+
+  struct JobComparator {
+    bool operator()(const Job* job1, const Job* job2) const;
+  };
+
+  // Owns the jobs.
+  using JobSet = std::set<Job*, JobComparator>;
+
+  // Starts an asynchronous request to fetch the given URL. On completion
+  // |callback| will be invoked.
+  //
+  // Completion of the request will never occur synchronously. In other words it
+  // is guaranteed that |callback| will only be invoked once the Fetch*() method
+  // has returned.
+  WARN_UNUSED_RESULT scoped_ptr<Request> Fetch(
+      scoped_ptr<RequestParams> request_params,
+      const FetchCallback& callback);
+
+  // Finds a job with a matching RequestPararms or returns nullptr if there was
+  // no match.
+  Job* FindJob(const RequestParams& params);
+
+  // Removes |job| from the in progress jobs and transfers ownership to the
+  // caller.
+  scoped_ptr<Job> RemoveJob(Job* job);
+
+  // Indicates which Job is currently executing inside of OnJobCompleted().
+  void SetCurrentlyCompletingJob(Job* job);
+  void ClearCurrentlyCompletingJob(Job* job);
+
+  // The in-progress jobs. This set does not contain the job which is actively
+  // invoking callbacks (OnJobCompleted). Instead that is tracked by
+  // |currently_completing_job_|.
+  JobSet jobs_;
+
+  // The Job that is currently executing OnJobCompleted(). There can be at most
+  // one such job. This pointer is not owned.
+  Job* currently_completing_job_;
+
+  // Not owned. CertNetFetcherImpl must outlive the URLRequestContext.
+  URLRequestContext* context_;
+
+  base::ThreadChecker thread_checker_;
+
+  DISALLOW_COPY_AND_ASSIGN(CertNetFetcherImpl);
+};
+
+}  // namespace net
+
+#endif  // NET_CERT_NET_CERT_NET_FETCHER_H_

net/data/cert_net_fetcher_impl_unittest/404.html

+-404.html-

net/data/cert_net_fetcher_impl_unittest/404.html.mock-http-headers

+HTTP/1.1 404 OK
+Content-Type: text/html

net/data/cert_net_fetcher_impl_unittest/500.html

+-500.html-

net/data/cert_net_fetcher_impl_unittest/500.html.mock-http-headers

+HTTP/1.1 500 OK
+Content-Type: text/html

net/data/cert_net_fetcher_impl_unittest/cacheable_1hr.crt

+-cacheable_1hr.crt-

net/data/cert_net_fetcher_impl_unittest/cacheable_1hr.crt.mock-http-headers

+HTTP/1.1 200 OK
+Content-Type: application/pkix-cert
+Cache-Control: public, max-age=3600

net/data/cert_net_fetcher_impl_unittest/cert.crt

+-cert.crt-

net/data/cert_net_fetcher_impl_unittest/cert.crt.mock-http-headers

+HTTP/1.1 200 OK
+Content-Type: application/pkix-cert

net/data/cert_net_fetcher_impl_unittest/certs.p7c

+-certs.p7c-

net/data/cert_net_fetcher_impl_unittest/certs.p7c.mock-http-headers

+HTTP/1.1 200 OK
+Content-Type: application/pkcs7-mime

net/data/cert_net_fetcher_impl_unittest/downloadable.js

+-downloadable.js-

net/data/cert_net_fetcher_impl_unittest/downloadable.js.mock-http-headers

+HTTP/1.1 200 OK
+Content-Type: text/javascript
+Content-Disposition: attachment; filename="download-me.exe"

net/data/cert_net_fetcher_impl_unittest/foo.txt

+-foo.txt-

net/data/cert_net_fetcher_impl_unittest/foo.txt.mock-http-headers

+HTTP/1.1 200 OK
+Content-Type: text/plain

net/data/cert_net_fetcher_impl_unittest/gzipped_crl.mock-http-headers

+HTTP/1.1 200 OK
+Content-Type: application/pkix-crl
+Content-Encoding: gzip

net/data/cert_net_fetcher_impl_unittest/redirect_https

+-redirected_https-

net/data/cert_net_fetcher_impl_unittest/redirect_https.mock-http-headers

+HTTP/1.1 302 Found
+Location: https://www.foo/

net/data/cert_net_fetcher_impl_unittest/root.crl

+-root.crl-

net/data/cert_net_fetcher_impl_unittest/root.crl.mock-http-headers

+HTTP/1.1 200 OK
+Content-Type: application/pkix-crl

net/net.gyp

@@ -380,6 +380,7 @@
               # Need to read input data files.
               'filter/gzip_filter_unittest.cc',
               # Need TestServer.
+              "cert_net/cert_net_fetcher_impl_unittest.cc",
               'proxy/proxy_script_fetcher_impl_unittest.cc',
               'socket/ssl_client_socket_unittest.cc',
               'socket/ssl_server_socket_unittest.cc',

net/net.gypi

@@ -303,6 +303,7 @@
       'cert/cert_database_mac.cc',
       'cert/cert_database_nss.cc',
       'cert/cert_database_win.cc',
+      'cert/cert_net_fetcher.h',
       'cert/cert_trust_anchor_provider.h',
       'cert/cert_verify_proc.cc',
       'cert/cert_verify_proc.h',
@@ -375,6 +376,8 @@
       'cert/x509_util_mac.h',
       'cert/x509_util_nss.cc',
       'cert/x509_util_nss.h',
+      'cert_net/cert_net_fetcher_impl.cc',
+      'cert_net/cert_net_fetcher_impl.h',
       'cookies/canonical_cookie.cc',
       'cookies/canonical_cookie.h',
       'cookies/cookie_constants.cc',
@@ -1314,6 +1317,7 @@
       'cert/x509_util_nss_unittest.cc',
       'cert/x509_util_openssl_unittest.cc',
       'cert/x509_util_unittest.cc',
+      'cert_net/cert_net_fetcher_impl_unittest.cc',
       'cookies/canonical_cookie_unittest.cc',
       'cookies/cookie_constants_unittest.cc',
       'cookies/cookie_monster_unittest.cc',