introduce new proto messages for in-progress downloads

Splits downloadEntry into UkmInfo and InProgressInfo,
added new information needed to resume the download item after chrome crashes.


Bug: 803135
Change-Id: Ided64d9801a3e8e1d8390a4190d17aa3b58d652c
Reviewed-on: https://chromium-review.googlesource.com/1036539
Commit-Queue: Min Qin <qinmin@chromium.org>
Reviewed-by: David Trainor <dtrainor@chromium.org>
Reviewed-by: Xing Liu <xingliu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#556179}

components/download/downloader/in_progress/BUILD.gn

@@ -16,6 +16,10 @@ source_set("in_progress") {
     "in_progress_cache_impl.h",
     "in_progress_conversions.cc",
     "in_progress_conversions.h",
+    "in_progress_info.cc",
+    "in_progress_info.h",
+    "ukm_info.cc",
+    "ukm_info.h",
   ]
 
   deps = [

components/download/downloader/in_progress/in_progress_conversions.cc

@@ -6,6 +6,7 @@
 
 #include <utility>
 #include "base/logging.h"
+#include "base/pickle.h"
 
 namespace download {
 
@@ -37,7 +38,6 @@ metadata_pb::DownloadEntry InProgressConversions::DownloadEntryToProto(
     auto* proto_header = proto.add_request_headers();
     *proto_header = HttpRequestHeaderToProto(header);
   }
-
   return proto;
 }
 
@@ -139,4 +139,102 @@ InProgressConversions::HttpRequestHeaderFromProto(
   return std::make_pair(proto.key(), proto.value());
 }
 
+// static
+metadata_pb::InProgressInfo InProgressConversions::InProgressInfoToProto(
+    const InProgressInfo& in_progress_info) {
+  metadata_pb::InProgressInfo proto;
+  for (size_t i = 0; i < in_progress_info.url_chain.size(); ++i)
+    proto.add_url_chain(in_progress_info.url_chain[i].spec());
+  proto.set_fetch_error_body(in_progress_info.fetch_error_body);
+  for (const auto& header : in_progress_info.request_headers) {
+    auto* proto_header = proto.add_request_headers();
+    *proto_header = HttpRequestHeaderToProto(header);
+  }
+  proto.set_etag(in_progress_info.etag);
+  proto.set_last_modified(in_progress_info.last_modified);
+  proto.set_total_bytes(in_progress_info.total_bytes);
+  base::Pickle current_path;
+  in_progress_info.current_path.WriteToPickle(&current_path);
+  proto.set_current_path(current_path.data(), current_path.size());
+  base::Pickle target_path;
+  in_progress_info.target_path.WriteToPickle(&target_path);
+  proto.set_target_path(target_path.data(), target_path.size());
+  proto.set_received_bytes(in_progress_info.received_bytes);
+  proto.set_end_time(
+      in_progress_info.end_time.ToDeltaSinceWindowsEpoch().InMilliseconds());
+  for (size_t i = 0; i < in_progress_info.received_slices.size(); ++i) {
+    metadata_pb::ReceivedSlice* slice = proto.add_received_slices();
+    slice->set_received_bytes(
+        in_progress_info.received_slices[i].received_bytes);
+    slice->set_offset(in_progress_info.received_slices[i].offset);
+    slice->set_finished(in_progress_info.received_slices[i].finished);
+  }
+  proto.set_hash(in_progress_info.hash);
+  proto.set_transient(in_progress_info.transient);
+  proto.set_state(in_progress_info.state);
+  proto.set_danger_type(in_progress_info.danger_type);
+  proto.set_interrupt_reason(in_progress_info.interrupt_reason);
+  proto.set_paused(in_progress_info.paused);
+  proto.set_metered(in_progress_info.metered);
+  proto.set_request_origin(in_progress_info.request_origin);
+  proto.set_bytes_wasted(in_progress_info.bytes_wasted);
+  return proto;
+}
+
+// static
+InProgressInfo InProgressConversions::InProgressInfoFromProto(
+    const metadata_pb::InProgressInfo& proto) {
+  InProgressInfo info;
+  for (const auto& url : proto.url_chain())
+    info.url_chain.emplace_back(url);
+  info.fetch_error_body = proto.fetch_error_body();
+  for (const auto& header : proto.request_headers())
+    info.request_headers.emplace_back(HttpRequestHeaderFromProto(header));
+  info.etag = proto.etag();
+  info.last_modified = proto.last_modified();
+  info.total_bytes = proto.total_bytes();
+  base::PickleIterator current_path(
+      base::Pickle(proto.current_path().data(), proto.current_path().size()));
+  info.current_path.ReadFromPickle(&current_path);
+  base::PickleIterator target_path(
+      base::Pickle(proto.target_path().data(), proto.target_path().size()));
+  info.target_path.ReadFromPickle(&target_path);
+  info.received_bytes = proto.received_bytes();
+  info.end_time = base::Time::FromDeltaSinceWindowsEpoch(
+      base::TimeDelta::FromMilliseconds(proto.end_time()));
+
+  for (int i = 0; i < proto.received_slices_size(); ++i) {
+    info.received_slices.emplace_back(proto.received_slices(i).offset(),
+                                      proto.received_slices(i).received_bytes(),
+                                      proto.received_slices(i).finished());
+  }
+  info.hash = proto.hash();
+  info.transient = proto.transient();
+  info.state = static_cast<DownloadItem::DownloadState>(proto.state());
+  info.danger_type = static_cast<DownloadDangerType>(proto.danger_type());
+  info.interrupt_reason =
+      static_cast<DownloadInterruptReason>(proto.interrupt_reason());
+  info.paused = proto.paused();
+  info.metered = proto.metered();
+  info.request_origin = proto.request_origin();
+  info.bytes_wasted = proto.bytes_wasted();
+  return info;
+}
+
+UkmInfo InProgressConversions::UkmInfoFromProto(
+    const metadata_pb::UkmInfo& proto) {
+  UkmInfo info;
+  info.download_source = DownloadSourceFromProto(proto.download_source());
+  info.ukm_download_id = proto.ukm_download_id();
+  return info;
+}
+
+metadata_pb::UkmInfo InProgressConversions::UkmInfoToProto(
+    const UkmInfo& info) {
+  metadata_pb::UkmInfo proto;
+  proto.set_download_source(DownloadSourceToProto(info.download_source));
+  proto.set_ukm_download_id(info.ukm_download_id);
+  return proto;
+}
+
 }  // namespace download

components/download/downloader/in_progress/in_progress_conversions.h

@@ -7,8 +7,10 @@
 
 #include "base/macros.h"
 #include "components/download/downloader/in_progress/download_entry.h"
+#include "components/download/downloader/in_progress/in_progress_info.h"
 #include "components/download/downloader/in_progress/proto/download_entry.pb.h"
 #include "components/download/downloader/in_progress/proto/download_source.pb.h"
+#include "components/download/downloader/in_progress/ukm_info.h"
 
 namespace download {
 
@@ -37,6 +39,16 @@ class InProgressConversions {
 
   static std::pair<std::string, std::string> HttpRequestHeaderFromProto(
       const metadata_pb::HttpRequestHeader& proto);
+
+  static metadata_pb::InProgressInfo InProgressInfoToProto(
+      const InProgressInfo& in_progress_info);
+
+  static InProgressInfo InProgressInfoFromProto(
+      const metadata_pb::InProgressInfo& proto);
+
+  static metadata_pb::UkmInfo UkmInfoToProto(const UkmInfo& ukm_info);
+
+  static UkmInfo UkmInfoFromProto(const metadata_pb::UkmInfo& proto);
 };
 
 }  // namespace download

components/download/downloader/in_progress/in_progress_conversions_unittest.cc

@@ -79,4 +79,44 @@ TEST_F(InProgressConversionsTest, HttpRequestHeaders) {
             HttpRequestHeaderFromProto(HttpRequestHeaderToProto(header)));
 }
 
+TEST_F(InProgressConversionsTest, InProgressInfo) {
+  // InProgressInfo with no fields.
+  InProgressInfo info;
+  EXPECT_EQ(false, info.fetch_error_body);
+  EXPECT_TRUE(info.request_headers.empty());
+  EXPECT_EQ(info, InProgressInfoFromProto(InProgressInfoToProto(info)));
+
+  // InProgressInfo with valid fields.
+  info.current_path = base::FilePath(FILE_PATH_LITERAL("/tmp.crdownload"));
+  info.target_path = base::FilePath(FILE_PATH_LITERAL("/tmp"));
+  info.url_chain.emplace_back("http://foo");
+  info.url_chain.emplace_back("http://foo2");
+  info.end_time = base::Time::NowFromSystemTime().LocalMidnight();
+  info.etag = "A";
+  info.last_modified = "Wed, 1 Oct 2018 07:00:00 GMT";
+  info.received_bytes = 1000;
+  info.total_bytes = 10000;
+  info.state = DownloadItem::IN_PROGRESS;
+  info.danger_type = DOWNLOAD_DANGER_TYPE_NOT_DANGEROUS;
+  info.interrupt_reason = DOWNLOAD_INTERRUPT_REASON_NONE;
+  info.transient = false;
+  info.paused = false;
+  info.hash = "abcdefg";
+  info.metered = true;
+  info.received_slices.emplace_back(0, 500, false);
+  info.received_slices.emplace_back(5000, 500, false);
+  info.request_origin = "request origin";
+  info.bytes_wasted = 1234;
+  info.fetch_error_body = true;
+  info.request_headers.emplace_back(
+      std::make_pair<std::string, std::string>("123", "456"));
+  info.request_headers.emplace_back(
+      std::make_pair<std::string, std::string>("ABC", "def"));
+  EXPECT_EQ(info, InProgressInfoFromProto(InProgressInfoToProto(info)));
+}
+
+TEST_F(InProgressConversionsTest, UkmInfo) {
+  UkmInfo info(DownloadSource::FROM_RENDERER, 100);
+  EXPECT_EQ(info, UkmInfoFromProto(UkmInfoToProto(info)));
+}
 }  // namespace download

components/download/downloader/in_progress/in_progress_info.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/download/downloader/in_progress/in_progress_info.h"
+
+namespace download {
+
+InProgressInfo::InProgressInfo() = default;
+
+InProgressInfo::InProgressInfo(const InProgressInfo& other) = default;
+
+InProgressInfo::~InProgressInfo() = default;
+
+bool InProgressInfo::operator==(const InProgressInfo& other) const {
+  return url_chain == other.url_chain &&
+         fetch_error_body == other.fetch_error_body &&
+         request_headers == other.request_headers && etag == other.etag &&
+         last_modified == other.last_modified &&
+         total_bytes == other.total_bytes &&
+         current_path == other.current_path &&
+         target_path == other.target_path &&
+         received_bytes == other.received_bytes && end_time == other.end_time &&
+         received_slices == other.received_slices && hash == other.hash &&
+         transient == other.transient && state == other.state &&
+         danger_type == other.danger_type &&
+         interrupt_reason == other.interrupt_reason && paused == other.paused &&
+         metered == other.metered && request_origin == other.request_origin &&
+         bytes_wasted == other.bytes_wasted;
+}
+
+}  // namespace download

components/download/downloader/in_progress/in_progress_info.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_DOWNLOAD_DOWNLOADER_IN_PROGRESS_IN_PROGRESS_INFO_H_
+#define COMPONENTS_DOWNLOAD_DOWNLOADER_IN_PROGRESS_IN_PROGRESS_INFO_H_
+
+#include <string>
+#include <vector>
+
+#include "components/download/public/common/download_danger_type.h"
+#include "components/download/public/common/download_item.h"
+#include "components/download/public/common/download_url_parameters.h"
+#include "url/gurl.h"
+
+namespace download {
+
+// Contains information to reconstruct an interrupted download item for
+// resumption.
+struct InProgressInfo {
+ public:
+  InProgressInfo();
+  InProgressInfo(const InProgressInfo& other);
+  ~InProgressInfo();
+
+  bool operator==(const InProgressInfo& other) const;
+
+  //  request info  ------------------------------------------------------------
+
+  // The url chain.
+  std::vector<GURL> url_chain;
+
+  // If the entity body of unsuccessful HTTP response, like HTTP 404, will be
+  // downloaded.
+  bool fetch_error_body = false;
+
+  // Request header key/value pairs that will be added to the download HTTP
+  // request.
+  DownloadUrlParameters::RequestHeadersType request_headers;
+
+  //  response info  -----------------------------------------------------------
+
+  // Contents of most recently seen ETag header.
+  std::string etag;
+
+  // Contents of most recently seen Last-Modified header.
+  std::string last_modified;
+
+  // The total number of bytes in the download.
+  int64_t total_bytes = 0;
+
+  //  destination info  --------------------------------------------------------
+
+  // The current path to the download (potentially different from final if
+  // download is in progress or interrupted).
+  base::FilePath current_path;
+
+  // The target path where the download will go when it's complete.
+  base::FilePath target_path;
+
+  // The number of bytes received (so far).
+  int64_t received_bytes = 0;
+
+  // The time when the download completed.
+  base::Time end_time;
+
+  // Data slices that have been downloaded so far. The slices must be ordered
+  // by their offset.
+  std::vector<DownloadItem::ReceivedSlice> received_slices;
+
+  // Hash of the downloaded content.
+  std::string hash;
+
+  //  state info  --------------------------------------------------------------
+
+  // Whether this download is transient. Transient items are cleaned up after
+  // completion and not shown in the UI.
+  bool transient = false;
+
+  // The current state of the download.
+  DownloadItem::DownloadState state = DownloadItem::DownloadState::IN_PROGRESS;
+
+  // Whether and how the download is dangerous.
+  DownloadDangerType danger_type =
+      DownloadDangerType::DOWNLOAD_DANGER_TYPE_NOT_DANGEROUS;
+
+  // The reason the download was interrupted, if state == kStateInterrupted.
+  DownloadInterruptReason interrupt_reason = DOWNLOAD_INTERRUPT_REASON_NONE;
+
+  // Whether this download is paused.
+  bool paused = false;
+
+  // Whether the download is initiated on a metered network
+  bool metered = false;
+
+  // Represents the origin information for this download. Used by offline pages.
+  std::string request_origin;
+
+  // Count for how many (extra) bytes were used (including resumption).
+  int64_t bytes_wasted = 0;
+};
+
+}  // namespace download
+
+#endif  // COMPONENTS_DOWNLOAD_DOWNLOADER_IN_PROGRESS_IN_PROGRESS_INFO_H_

components/download/downloader/in_progress/proto/download_entry.proto

@@ -15,6 +15,13 @@ message HttpRequestHeader {
   optional string value = 2;
 }
 
+// Slice information for parallel downloading.
+message ReceivedSlice {
+  optional int64 offset = 1;
+  optional int64 received_bytes = 2;
+  optional bool finished = 3;
+}
+
 // Stores various in-progress metadata related to a download.
 message DownloadEntry {
   optional string guid = 1;
@@ -30,3 +37,48 @@ message DownloadEntry {
 message DownloadEntries {
   repeated DownloadEntry entries = 1;
 }
+
+// Information for ukm reporting
+message UkmInfo {
+  optional DownloadSource download_source = 1;
+  optional int64 ukm_download_id = 2;
+}
+
+// Information about an in progress download.
+message InProgressInfo {
+  repeated string url_chain = 1;
+  optional bool fetch_error_body = 2;
+  repeated HttpRequestHeader request_headers = 3;
+  optional string etag = 4;
+  optional string last_modified = 5;
+  optional int64 total_bytes = 6;
+  optional bytes current_path = 7;  // Serialized pickles to support string16
+  optional bytes target_path = 8;   // Serialized pickles to support string16
+  optional int64 received_bytes = 9;
+  optional int64 end_time = 10;
+  repeated ReceivedSlice received_slices = 11;
+  optional string hash = 12;
+  optional bool transient = 13;
+  optional int32 state = 14;
+  optional int32 danger_type = 15;
+  optional int32 interrupt_reason = 16;
+  optional bool paused = 17;
+  optional bool metered = 18;
+  optional string request_origin = 19;
+  optional int64 bytes_wasted = 20;
+}
+
+// Stores various in-progress metadata related to a download.
+// WIP and will replace DownloadEntry.
+message DownloadInfo {
+  optional string guid = 1;
+  optional UkmInfo ukm_info = 2;
+  optional InProgressInfo in_progress_info = 3;
+}
+
+// database entry for download information.
+message DownloadDBEntry {
+  optional string id = 1;
+  // Add field for offline page download.
+  oneof entry { DownloadInfo download_info = 2; }
+}
\ No newline at end of file

components/download/downloader/in_progress/ukm_info.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/download/downloader/in_progress/ukm_info.h"
+
+namespace download {
+
+UkmInfo::UkmInfo() = default;
+
+UkmInfo::UkmInfo(const UkmInfo& other) = default;
+
+UkmInfo::UkmInfo(DownloadSource download_source, int64_t ukm_download_id)
+    : download_source(download_source), ukm_download_id(ukm_download_id) {}
+
+UkmInfo::~UkmInfo() = default;
+
+bool UkmInfo::operator==(const UkmInfo& other) const {
+  return download_source == other.download_source &&
+         ukm_download_id == other.ukm_download_id;
+}
+
+}  // namespace download

components/download/downloader/in_progress/ukm_info.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_DOWNLOAD_DOWNLOADER_IN_PROGRESS_UKM_INFO_H_
+#define COMPONENTS_DOWNLOAD_DOWNLOADER_IN_PROGRESS_UKM_INFO_H_
+
+#include <stdint.h>
+
+#include "components/download/public/common/download_source.h"
+
+namespace download {
+
+// Contains information for UKM reporting.
+struct UkmInfo {
+ public:
+  UkmInfo();
+  UkmInfo(DownloadSource download_source, int64_t ukm_download_id);
+  UkmInfo(const UkmInfo& other);
+  ~UkmInfo();
+
+  bool operator==(const UkmInfo& other) const;
+
+  // The source that triggered the download.
+  DownloadSource download_source = DownloadSource::UNKNOWN;
+
+  // Unique ID that tracks the download UKM entry, where 0 means the
+  // download_id is not yet initialized.
+  uint64_t ukm_download_id = 0;
+};
+
+}  // namespace download
+
+#endif  // COMPONENTS_DOWNLOAD_DOWNLOADER_IN_PROGRESS_UKM_INFO_H_

components/download/public/common/download_item.h

@@ -112,7 +112,7 @@ class COMPONENTS_DOWNLOAD_EXPORT DownloadItem : public base::SupportsUserData {
   // parallel downloading is enabled. Slices should have different offsets
   // so that they don't overlap. |finished| will be marked as true when the
   // download stream is successfully completed.
-  struct COMPONENTS_DOWNLOAD_EXPORT ReceivedSlice {
+  struct ReceivedSlice {
     ReceivedSlice(int64_t offset, int64_t received_bytes)
         : offset(offset), received_bytes(received_bytes), finished(false) {}