Commit 11583d4b authored by Xing Liu's avatar Xing Liu Committed by Commit Bot

Download service: In memory download backend.

This CL introduces a backend that uses URLFetcher and save in memory
download data to a blob.

Currently there is an issue that only content/browser uses
storage::BlobStorageContext and content/public does not expose it.
Only BrowserContext::CreateMemoryBackedBlob does the browser side save
blob task for media API, which only expose the UUID in
content::BlobHandle.

Bug: 95809
Change-Id: Iac93952dd9a73a8c3eba6e6a1d1bf642a9a0d50f
Reviewed-on: https://chromium-review.googlesource.com/833026
Commit-Queue: Xing Liu <xingliu@chromium.org>
Reviewed-by: default avatarKinuko Yasuda <kinuko@chromium.org>
Reviewed-by: default avatarDavid Trainor <dtrainor@chromium.org>
Cr-Commit-Position: refs/heads/master@{#526733}
parent 7cc1ca38
......@@ -5,15 +5,12 @@
group("unit_tests") {
testonly = true
deps = [
"//components/download/internal:unit_tests",
]
if (!is_ios) {
deps += [
deps = [
"//components/download/content/internal:unit_tests",
"//components/download/content/public:unit_tests",
"//components/download/downloader/in_progress:unit_tests",
"//components/download/internal:unit_tests",
]
}
......
......@@ -10,6 +10,7 @@ DownloadServiceModelImplTest.*
DownloadStoreTest.*
FileMonitorTest.*
InProgressConversionsTest.*
InMemoryDownloadTest.*
NavigationMonitorImplTest.*
NetworkListenerTest.*
ProtoConversionsTest.*
......
......@@ -16,6 +16,8 @@ static_library("internal") {
]
sources = [
"blob_task_proxy.cc",
"blob_task_proxy.h",
"client_set.cc",
"client_set.h",
"config.cc",
......@@ -39,6 +41,8 @@ static_library("internal") {
"file_monitor.h",
"file_monitor_impl.cc",
"file_monitor_impl.h",
"in_memory_download.cc",
"in_memory_download.h",
"log_sink.h",
"log_source.h",
"logger_impl.cc",
......@@ -75,6 +79,7 @@ static_library("internal") {
"//components/download/public",
"//components/leveldb_proto",
"//net",
"//storage/browser",
]
if (is_android) {
......@@ -122,6 +127,7 @@ source_set("unit_tests") {
"download_store_unittest.cc",
"entry_utils_unittest.cc",
"file_monitor_unittest.cc",
"in_memory_download_unittest.cc",
"model_impl_unittest.cc",
"navigation_monitor_impl_unittests.cc",
"proto_conversions_unittest.cc",
......@@ -138,6 +144,7 @@ source_set("unit_tests") {
"//components/download/public/test:test_support",
"//components/leveldb_proto:test_support",
"//net:test_support",
"//storage/browser",
"//testing/gmock",
"//testing/gtest",
]
......
......@@ -5,4 +5,6 @@ include_rules = [
"+base",
"+jni",
"+net",
"+storage/browser",
"+storage/common",
]
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/download/internal/blob_task_proxy.h"
#include "base/guid.h"
#include "base/task_runner_util.h"
#include "storage/browser/blob/blob_data_builder.h"
#include "storage/browser/blob/blob_data_handle.h"
#include "storage/browser/blob/blob_storage_context.h"
namespace download {
// static
std::unique_ptr<BlobTaskProxy> BlobTaskProxy::Create(
BlobContextGetter blob_context_getter,
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner) {
return std::make_unique<BlobTaskProxy>(std::move(blob_context_getter),
io_task_runner);
}
BlobTaskProxy::BlobTaskProxy(
BlobContextGetter blob_context_getter,
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner)
: main_task_runner_(base::ThreadTaskRunnerHandle::Get()),
io_task_runner_(io_task_runner),
weak_ptr_factory_(this) {
// Unretained the raw pointer because owner on UI thread should destroy this
// object on IO thread.
io_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&BlobTaskProxy::InitializeOnIO, base::Unretained(this),
std::move(blob_context_getter)));
}
BlobTaskProxy::~BlobTaskProxy() {
io_task_runner_->BelongsToCurrentThread();
}
void BlobTaskProxy::InitializeOnIO(BlobContextGetter blob_context_getter) {
io_task_runner_->BelongsToCurrentThread();
blob_storage_context_ = std::move(blob_context_getter).Run();
}
void BlobTaskProxy::SaveAsBlob(std::unique_ptr<std::string> data,
BlobDataHandleCallback callback) {
// Unretained the raw pointer because owner on UI thread should destroy this
// object on IO thread.
io_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&BlobTaskProxy::SaveAsBlobOnIO, base::Unretained(this),
std::move(data), std::move(callback)));
}
void BlobTaskProxy::SaveAsBlobOnIO(std::unique_ptr<std::string> data,
BlobDataHandleCallback callback) {
io_task_runner_->BelongsToCurrentThread();
// Build blob data.
std::string blob_uuid = base::GenerateGUID();
storage::BlobDataBuilder builder(blob_uuid);
builder.AppendData(*data.get());
blob_data_handle_ = blob_storage_context_->AddFinishedBlob(builder);
// Wait for blob data construction complete.
auto cb = base::BindRepeating(&BlobTaskProxy::BlobSavedOnIO,
weak_ptr_factory_.GetWeakPtr(),
base::Passed(std::move(callback)));
blob_data_handle_->RunOnConstructionComplete(cb);
}
void BlobTaskProxy::BlobSavedOnIO(BlobDataHandleCallback callback,
storage::BlobStatus status) {
io_task_runner_->BelongsToCurrentThread();
// Relay BlobDataHandle and |status| back to main thread.
auto cb = base::BindOnce(std::move(callback),
base::Passed(std::move(blob_data_handle_)), status);
main_task_runner_->PostTask(FROM_HERE, std::move(cb));
}
} // namespace download
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef COMPONENTS_DOWNLOAD_INTERNAL_BLOB_TASK_PROXY_H_
#define COMPONENTS_DOWNLOAD_INTERNAL_BLOB_TASK_PROXY_H_
#include <memory>
#include <string>
#include "base/callback.h"
#include "base/macros.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "storage/common/blob_storage/blob_storage_constants.h"
namespace storage {
class BlobDataHandle;
class BlobStorageContext;
} // namespace storage
namespace download {
// Proxy for blob related task on IO thread.
// Created on main thread and do work on IO thread, destroyed on IO thread.
class BlobTaskProxy {
public:
using BlobContextGetter =
base::OnceCallback<base::WeakPtr<storage::BlobStorageContext>()>;
using BlobDataHandleCallback =
base::OnceCallback<void(std::unique_ptr<storage::BlobDataHandle>,
storage::BlobStatus status)>;
static std::unique_ptr<BlobTaskProxy> Create(
BlobContextGetter blob_context_getter,
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner);
BlobTaskProxy(BlobContextGetter blob_context_getter,
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner);
~BlobTaskProxy();
// Save blob data on UI thread. |callback| will be called on main thread after
// blob construction completes.
void SaveAsBlob(std::unique_ptr<std::string> data,
BlobDataHandleCallback callback);
private:
void InitializeOnIO(BlobContextGetter blob_context_getter);
void SaveAsBlobOnIO(std::unique_ptr<std::string> data,
BlobDataHandleCallback callback);
void BlobSavedOnIO(BlobDataHandleCallback callback,
storage::BlobStatus status);
scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
// Used to build blob data, must accessed on |io_task_runner_|.
base::WeakPtr<storage::BlobStorageContext> blob_storage_context_;
// Used to access blob storage context.
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
// BlobDataHandle that will be eventually passed to main thread.
std::unique_ptr<storage::BlobDataHandle> blob_data_handle_;
// Bounded to IO thread task runner.
base::WeakPtrFactory<BlobTaskProxy> weak_ptr_factory_;
DISALLOW_COPY_AND_ASSIGN(BlobTaskProxy);
};
} // namespace download
#endif // COMPONENTS_DOWNLOAD_INTERNAL_BLOB_TASK_PROXY_H_
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/download/internal/in_memory_download.h"
#include <memory>
#include "base/bind.h"
#include "base/strings/string_util.h"
#include "components/download/internal/blob_task_proxy.h"
#include "net/base/io_buffer.h"
#include "net/base/net_errors.h"
#include "net/http/http_status_code.h"
#include "net/traffic_annotation/network_traffic_annotation.h"
#include "net/url_request/url_fetcher.h"
#include "storage/browser/blob/blob_data_handle.h"
#include "storage/browser/blob/blob_storage_context.h"
namespace download {
namespace {
// Converts a string to HTTP method used by URLFetcher.
net::URLFetcher::RequestType ToRequestType(const std::string& method) {
// Only supports GET and POST.
if (base::EqualsCaseInsensitiveASCII(method, "GET"))
return net::URLFetcher::RequestType::GET;
if (base::EqualsCaseInsensitiveASCII(method, "POST"))
return net::URLFetcher::RequestType::POST;
NOTREACHED();
return net::URLFetcher::RequestType::GET;
}
} // namespace
InMemoryDownload::InMemoryDownload(
const std::string& guid,
const RequestParams& request_params,
const net::NetworkTrafficAnnotationTag& traffic_annotation,
Delegate* delegate,
scoped_refptr<net::URLRequestContextGetter> request_context_getter,
BlobTaskProxy::BlobContextGetter blob_context_getter,
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner)
: guid_(guid),
request_params_(request_params),
traffic_annotation_(traffic_annotation),
request_context_getter_(request_context_getter),
blob_task_proxy_(BlobTaskProxy::Create(std::move(blob_context_getter),
io_task_runner)),
io_task_runner_(io_task_runner),
state_(State::INITIAL),
delegate_(delegate),
bytes_downloaded_(0u),
weak_ptr_factory_(this) {
DCHECK(!guid_.empty());
}
InMemoryDownload::~InMemoryDownload() {
io_task_runner_->DeleteSoon(FROM_HERE, blob_task_proxy_.release());
}
void InMemoryDownload::Start() {
DCHECK(state_ == State::INITIAL);
url_fetcher_ = net::URLFetcher::Create(request_params_.url,
ToRequestType(request_params_.method),
this, traffic_annotation_);
url_fetcher_->SetRequestContext(request_context_getter_.get());
url_fetcher_->SetExtraRequestHeaders(
request_params_.request_headers.ToString());
url_fetcher_->Start();
state_ = State::IN_PROGRESS;
}
std::unique_ptr<storage::BlobDataHandle> InMemoryDownload::ResultAsBlob() {
DCHECK(state_ == State::COMPLETE || state_ == State::FAILED);
// Return a copy.
return std::make_unique<storage::BlobDataHandle>(*blob_data_handle_);
}
void InMemoryDownload::OnURLFetchDownloadProgress(
const net::URLFetcher* source,
int64_t current,
int64_t total,
int64_t current_network_bytes) {
bytes_downloaded_ = current;
if (delegate_)
delegate_->OnDownloadProgress(this);
}
void InMemoryDownload::OnURLFetchComplete(const net::URLFetcher* source) {
switch (source->GetStatus().status()) {
case net::URLRequestStatus::Status::SUCCESS:
if (HandleResponseCode(source->GetResponseCode())) {
SaveAsBlob();
return;
}
state_ = State::FAILED;
NotifyDelegateDownloadComplete();
return;
case net::URLRequestStatus::Status::IO_PENDING:
return;
case net::URLRequestStatus::Status::CANCELED:
case net::URLRequestStatus::Status::FAILED:
state_ = State::FAILED;
NotifyDelegateDownloadComplete();
break;
}
}
bool InMemoryDownload::HandleResponseCode(int response_code) {
switch (response_code) {
case -1: // Non-HTTP request.
case net::HTTP_OK:
case net::HTTP_NON_AUTHORITATIVE_INFORMATION:
case net::HTTP_PARTIAL_CONTENT:
case net::HTTP_CREATED:
case net::HTTP_ACCEPTED:
case net::HTTP_NO_CONTENT:
case net::HTTP_RESET_CONTENT:
return true;
// All other codes are considered as failed.
default:
return false;
}
}
void InMemoryDownload::SaveAsBlob() {
DCHECK(url_fetcher_);
// This will copy the internal memory in |url_fetcher| into |data|.
// TODO(xingliu): Use response writer to avoid one extra copies. And destroy
// |url_fetcher_| at the correct time.
std::unique_ptr<std::string> data = std::make_unique<std::string>();
DCHECK(url_fetcher_->GetResponseAsString(data.get()));
auto callback = base::BindOnce(&InMemoryDownload::OnSaveBlobDone,
weak_ptr_factory_.GetWeakPtr());
blob_task_proxy_->SaveAsBlob(std::move(data), std::move(callback));
}
void InMemoryDownload::OnSaveBlobDone(
std::unique_ptr<storage::BlobDataHandle> blob_handle,
storage::BlobStatus status) {
// |status| is valid on IO thread, consumer of |blob_handle| should validate
// the data when using the blob data.
state_ =
(status == storage::BlobStatus::DONE) ? State::COMPLETE : State::FAILED;
blob_data_handle_ = std::move(blob_handle);
NotifyDelegateDownloadComplete();
}
void InMemoryDownload::NotifyDelegateDownloadComplete() {
if (delegate_)
delegate_->OnDownloadComplete(this);
}
} // namespace download
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef COMPONENTS_DOWNLOAD_INTERNAL_IN_MEMORY_DOWNLOAD_H_
#define COMPONENTS_DOWNLOAD_INTERNAL_IN_MEMORY_DOWNLOAD_H_
#include <memory>
#include <string>
#include "base/callback.h"
#include "base/macros.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "components/download/internal/blob_task_proxy.h"
#include "components/download/public/download_params.h"
#include "net/base/completion_callback.h"
#include "net/url_request/url_fetcher_delegate.h"
#include "net/url_request/url_request_context_getter.h"
namespace net {
class URLFetcher;
struct NetworkTrafficAnnotationTag;
} // namespace net
namespace storage {
class BlobDataHandle;
} // namespace storage
namespace download {
struct RequestParams;
// Object to start a single download and hold in-memory download data.
// Used by download service in Incognito mode, where download files shouldn't
// be persisted to disk.
//
// Life cycle: The object is created before creating the network request.
// Call Start() to send the network request.
//
// Threading contract:
// 1. This object lives on the main thread.
// 2. Reading/writing IO buffer from network is done on another thread,
// based on |request_context_getter_|. When complete, main thread is notified.
// 3. After network IO is done, Blob related work is done on IO thread with
// |blob_task_proxy_|, then notify the result to main thread.
class InMemoryDownload : public net::URLFetcherDelegate {
public:
// Report download progress with in-memory download backend.
class Delegate {
public:
virtual void OnDownloadProgress(InMemoryDownload* download) = 0;
virtual void OnDownloadComplete(InMemoryDownload* download) = 0;
protected:
virtual ~Delegate() = default;
};
// States of the download.
enum class State {
// The object is created but network request has not been sent.
INITIAL,
// Download is in progress, including the following procedures.
// 1. Transfer network data.
// 2. Save to blob storage.
IN_PROGRESS,
// The download can fail due to:
// 1. network layer failure or unsuccessful HTTP server response code.
// 2. Blob system failures after blob construction is done.
FAILED,
// Download is completed, and data is successfully saved as a blob.
// 1. We guarantee the states of network responses.
// 2. Do not guarantee the state of blob data. The consumer of blob
// should validate its state when using it on IO thread.
COMPLETE,
};
InMemoryDownload(
const std::string& guid,
const RequestParams& request_params,
const net::NetworkTrafficAnnotationTag& traffic_annotation,
Delegate* delegate,
scoped_refptr<net::URLRequestContextGetter> request_context_getter,
BlobTaskProxy::BlobContextGetter blob_context_getter,
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner);
~InMemoryDownload() override;
// Send the download request.
void Start();
// Get a copy of blob data handle.
std::unique_ptr<storage::BlobDataHandle> ResultAsBlob();
const std::string& guid() const { return guid_; }
uint64_t bytes_downloaded() const { return bytes_downloaded_; }
State state() const { return state_; }
private:
// net::URLFetcherDelegate implementation.
void OnURLFetchDownloadProgress(const net::URLFetcher* source,
int64_t current,
int64_t total,
int64_t current_network_bytes) override;
void OnURLFetchComplete(const net::URLFetcher* source) override;
// Handles response code and change the state accordingly.
// Returns if the response code is considered as successful code.
bool HandleResponseCode(int response_code);
// Saves the download data into blob storage.
void SaveAsBlob();
void OnSaveBlobDone(std::unique_ptr<storage::BlobDataHandle> blob_handle,
storage::BlobStatus status);
// Notifies the delegate about completion.
void NotifyDelegateDownloadComplete();
// GUID of the download.
const std::string guid_;
// Request parameters of the download.
const RequestParams request_params_;
// Traffic annotation of the request.
const net::NetworkTrafficAnnotationTag traffic_annotation_;
// Used to send requests to servers. Also contains the download data in its
// string buffer. We should avoid extra copy on the data and release the
// memory when needed.
std::unique_ptr<net::URLFetcher> url_fetcher_;
// Request context getter used by |url_fetcher_|.
scoped_refptr<net::URLRequestContextGetter> request_context_getter_;
// Worker that does blob related task on IO thread.
std::unique_ptr<BlobTaskProxy> blob_task_proxy_;
// Owned blob data handle, so that blob system keeps at least one reference
// count of the underlying data.
std::unique_ptr<storage::BlobDataHandle> blob_data_handle_;
// Used to access blob storage context.
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
State state_;
Delegate* delegate_;
uint64_t bytes_downloaded_;
// Bounded to main thread task runner.
base::WeakPtrFactory<InMemoryDownload> weak_ptr_factory_;
DISALLOW_COPY_AND_ASSIGN(InMemoryDownload);
};
} // namespace download
#endif // COMPONENTS_DOWNLOAD_INTERNAL_IN_MEMORY_DOWNLOAD_H_
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/download/internal/in_memory_download.h"
#include "base/files/file_util.h"
#include "base/guid.h"
#include "base/path_service.h"
#include "base/run_loop.h"
#include "base/threading/thread.h"
#include "net/test/embedded_test_server/embedded_test_server.h"
#include "net/traffic_annotation/network_traffic_annotation_test_helper.h"
#include "net/url_request/url_request_test_util.h"
#include "storage/browser/blob/blob_storage_context.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace download {
namespace {
class MockDelegate : public InMemoryDownload::Delegate {
public:
MockDelegate() = default;
void WaitForCompletion() {
DCHECK(!run_loop_.running());
run_loop_.Run();
}
// InMemoryDownload::Delegate implementation.
MOCK_METHOD1(OnDownloadProgress, void(InMemoryDownload*));
void OnDownloadComplete(InMemoryDownload* download) {
if (run_loop_.running())
run_loop_.Quit();
}
private:
base::RunLoop run_loop_;
DISALLOW_COPY_AND_ASSIGN(MockDelegate);
};
// Must run on IO thread task runner.
base::WeakPtr<storage::BlobStorageContext> BlobStorageContextGetter(
storage::BlobStorageContext* blob_context) {
DCHECK(blob_context);
return blob_context->AsWeakPtr();
}
class InMemoryDownloadTest : public testing::Test {
public:
InMemoryDownloadTest() = default;
~InMemoryDownloadTest() override = default;
void SetUp() override {
test_server_.ServeFilesFromDirectory(GetTestDataDirectory());
ASSERT_TRUE(test_server_.Start());
io_thread_.reset(new base::Thread("Network and Blob IO thread"));
base::Thread::Options options(base::MessageLoop::TYPE_IO, 0);
io_thread_->StartWithOptions(options);
request_context_getter_ =
new net::TestURLRequestContextGetter(io_thread_->task_runner());
blob_storage_context_ = std::make_unique<storage::BlobStorageContext>();
}
void TearDown() override {
// Say goodbye to |blob_storage_context_| on IO thread.
io_thread_->task_runner()->DeleteSoon(FROM_HERE,
blob_storage_context_.release());
}
protected:
base::FilePath GetTestDataDirectory() {
base::FilePath test_data_dir;
EXPECT_TRUE(base::PathService::Get(base::DIR_SOURCE_ROOT, &test_data_dir));
return test_data_dir.AppendASCII("components/test/data/download");
}
// Helper method to create a download with request_params.
void CreateDownload(const RequestParams& request_params) {
download_ = std::make_unique<InMemoryDownload>(
base::GenerateGUID(), request_params, TRAFFIC_ANNOTATION_FOR_TESTS,
delegate(), request_context_getter_,
base::BindOnce(&BlobStorageContextGetter, blob_storage_context_.get()),
io_thread_->task_runner());
}
InMemoryDownload* download() { return download_.get(); }
MockDelegate* delegate() { return &mock_delegate_; }
net::EmbeddedTestServer* test_server() { return &test_server_; }
private:
// IO thread used by network and blob IO tasks.
std::unique_ptr<base::Thread> io_thread_;
// Message loop for the main thread.
base::MessageLoop main_loop;
std::unique_ptr<InMemoryDownload> download_;
MockDelegate mock_delegate_;
scoped_refptr<net::TestURLRequestContextGetter> request_context_getter_;
std::unique_ptr<storage::BlobStorageContext> blob_storage_context_;
net::EmbeddedTestServer test_server_;
DISALLOW_COPY_AND_ASSIGN(InMemoryDownloadTest);
};
TEST_F(InMemoryDownloadTest, DownloadTest) {
RequestParams request_params;
request_params.url = test_server()->GetURL("/text_data.json");
CreateDownload(request_params);
download()->Start();
delegate()->WaitForCompletion();
base::FilePath path = GetTestDataDirectory().AppendASCII("text_data.json");
std::string data;
EXPECT_TRUE(ReadFileToString(path, &data));
EXPECT_EQ(InMemoryDownload::State::COMPLETE, download()->state());
// TODO(xingliu): Read the blob and verify data.
}
} // namespace
} // namespace download
{
"data":"In earlier tellings, the dog had a better reputation than the cat, however the president veto it."
}
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment