[fsp] Introduce BufferingFileStreamReader to read files in bigger chunks.

Default chunk size is 32KB and it is too small for provided file systems, which
require IPC calls for each chunk. Such small chunks cause the IPC overhead to
slow down the reading process drastically.

Moreover, since there is no streaming support in XMLHttpRequest, returning
data in chunks in providing extensions, requires making separate HTTP requests
with appriopriate Range header. That introduces another overhead.

This patch introduces a wrapper on a FileStreamReader, which converts 32KB
chunk requests to 512KB chunk requests to the underlying file stream  reader.

As a result, the overhead of IPC and HTTP is decreased significantly.

TEST=unit_tests: *FileSystemProvider*BufferingFileStreamReader*
BUG=248427

Review URL: https://codereview.chromium.org/318563002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@282290 0039d316-1c4b-4281-b951-d872f2087c98

chrome/browser/chromeos/file_system_provider/fileapi/backend_delegate.cc

@@ -5,6 +5,7 @@
 #include "chrome/browser/chromeos/file_system_provider/fileapi/backend_delegate.h"
 
 #include "base/memory/scoped_ptr.h"
+#include "chrome/browser/chromeos/file_system_provider/fileapi/buffering_file_stream_reader.h"
 #include "chrome/browser/chromeos/file_system_provider/fileapi/file_stream_reader.h"
 #include "chrome/browser/chromeos/file_system_provider/fileapi/provider_async_file_util.h"
 #include "content/public/browser/browser_thread.h"
@@ -16,6 +17,13 @@ using content::BrowserThread;
 
 namespace chromeos {
 namespace file_system_provider {
+namespace {
+
+// Size of the stream reader internal buffer. At most this number of bytes will
+// be read ahead of the requested data.
+const int kReaderBufferSize = 512 * 1024;  // 512KB.
+
+}  // namespace
 
 BackendDelegate::BackendDelegate()
     : async_file_util_(new internal::ProviderAsyncFileUtil) {}
@@ -39,7 +47,10 @@ BackendDelegate::CreateFileStreamReader(
   DCHECK_EQ(fileapi::kFileSystemTypeProvided, url.type());
 
   return scoped_ptr<webkit_blob::FileStreamReader>(
-      new FileStreamReader(context, url, offset, expected_modification_time));
+      new BufferingFileStreamReader(
+          scoped_ptr<webkit_blob::FileStreamReader>(new FileStreamReader(
+              context, url, offset, expected_modification_time)),
+          kReaderBufferSize));
 }
 
 scoped_ptr<fileapi::FileStreamWriter> BackendDelegate::CreateFileStreamWriter(

chrome/browser/chromeos/file_system_provider/fileapi/buffering_file_stream_reader.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/chromeos/file_system_provider/fileapi/buffering_file_stream_reader.h"
+
+#include <algorithm>
+
+#include "net/base/io_buffer.h"
+#include "net/base/net_errors.h"
+
+namespace chromeos {
+namespace file_system_provider {
+
+BufferingFileStreamReader::BufferingFileStreamReader(
+    scoped_ptr<webkit_blob::FileStreamReader> file_stream_reader,
+    int buffer_size)
+    : file_stream_reader_(file_stream_reader.Pass()),
+      buffer_size_(buffer_size),
+      preloading_buffer_(new net::IOBuffer(buffer_size_)),
+      preloading_buffer_offset_(0),
+      buffered_bytes_(0),
+      weak_ptr_factory_(this) {
+}
+
+BufferingFileStreamReader::~BufferingFileStreamReader() {
+}
+
+int BufferingFileStreamReader::Read(net::IOBuffer* buffer,
+                                    int buffer_length,
+                                    const net::CompletionCallback& callback) {
+  // Return as much as available in the internal buffer. It may be less than
+  // |buffer_length|, what is valid.
+  const int bytes_read =
+      CopyFromPreloadingBuffer(make_scoped_refptr(buffer), buffer_length);
+  if (bytes_read)
+    return bytes_read;
+
+  // If the internal buffer is empty, and more bytes than the internal buffer
+  // size is requested, then call the internal file stream reader directly.
+  if (buffer_length >= buffer_size_) {
+    const int result =
+        file_stream_reader_->Read(buffer, buffer_length, callback);
+    DCHECK_EQ(result, net::ERR_IO_PENDING);
+    return result;
+  }
+
+  // Nothing copied, so contents have to be preloaded.
+  Preload(base::Bind(&BufferingFileStreamReader::OnPreloadCompleted,
+                     weak_ptr_factory_.GetWeakPtr(),
+                     make_scoped_refptr(buffer),
+                     buffer_length,
+                     callback));
+
+  return net::ERR_IO_PENDING;
+}
+
+int64 BufferingFileStreamReader::GetLength(
+    const net::Int64CompletionCallback& callback) {
+  const int64 result = file_stream_reader_->GetLength(callback);
+  DCHECK_EQ(net::ERR_IO_PENDING, result);
+
+  return result;
+}
+
+int BufferingFileStreamReader::CopyFromPreloadingBuffer(
+    scoped_refptr<net::IOBuffer> buffer,
+    int buffer_length) {
+  const int read_bytes = std::min(buffer_length, buffered_bytes_);
+
+  memcpy(buffer->data(),
+         preloading_buffer_->data() + preloading_buffer_offset_,
+         read_bytes);
+  preloading_buffer_offset_ += read_bytes;
+  buffered_bytes_ -= read_bytes;
+
+  return read_bytes;
+}
+
+void BufferingFileStreamReader::Preload(
+    const net::CompletionCallback& callback) {
+  // TODO(mtomasz): Dynamically calculate the chunk size. Start from a small
+  // one, then increase for consecutive requests. That would improve performance
+  // when reading just small chunks, instead of the entire file.
+  const int preload_bytes = buffer_size_;
+
+  const int result =
+      file_stream_reader_->Read(preloading_buffer_, preload_bytes, callback);
+  DCHECK_EQ(result, net::ERR_IO_PENDING);
+}
+
+void BufferingFileStreamReader::OnPreloadCompleted(
+    scoped_refptr<net::IOBuffer> buffer,
+    int buffer_length,
+    const net::CompletionCallback& callback,
+    int result) {
+  if (result < 0) {
+    callback.Run(result);
+    return;
+  }
+
+  preloading_buffer_offset_ = 0;
+  buffered_bytes_ = result;
+
+  callback.Run(CopyFromPreloadingBuffer(buffer, buffer_length));
+}
+
+}  // namespace file_system_provider
+}  // namespace chromeos

chrome/browser/chromeos/file_system_provider/fileapi/buffering_file_stream_reader.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_CHROMEOS_FILE_SYSTEM_PROVIDER_FILEAPI_BUFFERING_FILE_STREAM_READER_H_
+#define CHROME_BROWSER_CHROMEOS_FILE_SYSTEM_PROVIDER_FILEAPI_BUFFERING_FILE_STREAM_READER_H_
+
+#include "base/basictypes.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/weak_ptr.h"
+#include "webkit/browser/blob/file_stream_reader.h"
+
+namespace net {
+class IOBuffer;
+}  // namespace net
+
+namespace chromeos {
+namespace file_system_provider {
+
+// Wraps the file stream reader implementation with a prefetching buffer.
+// Reads data from the internal file stream reader in chunks of size at least
+// |buffer_size| bytes (or less for the last chunk, because of EOF).
+//
+// The underlying inner file stream reader *must not* return any values
+// synchronously. Instead, results must be returned by a callback, including
+// errors.
+class BufferingFileStreamReader : public webkit_blob::FileStreamReader {
+ public:
+  BufferingFileStreamReader(
+      scoped_ptr<webkit_blob::FileStreamReader> file_stream_reader,
+      int buffer_size);
+
+  virtual ~BufferingFileStreamReader();
+
+  // webkit_blob::FileStreamReader overrides.
+  virtual int Read(net::IOBuffer* buf,
+                   int buf_len,
+                   const net::CompletionCallback& callback) OVERRIDE;
+  virtual int64 GetLength(
+      const net::Int64CompletionCallback& callback) OVERRIDE;
+
+ private:
+  // Copies data from the preloading buffer and updates the internal iterator.
+  // Returns number of bytes successfully copied.
+  int CopyFromPreloadingBuffer(scoped_refptr<net::IOBuffer> buffer,
+                               int buffer_length);
+
+  // Preloads data from the internal stream reader and calls the |callback|.
+  void Preload(const net::CompletionCallback& callback);
+
+  // Called when preloading of a buffer chunk is finished. Updates state of the
+  // preloading buffer and copied requested data to the |buffer|.
+  void OnPreloadCompleted(scoped_refptr<net::IOBuffer> buffer,
+                          int buffer_length,
+                          const net::CompletionCallback& callback,
+                          int result);
+
+  scoped_ptr<webkit_blob::FileStreamReader> file_stream_reader_;
+  int buffer_size_;
+  scoped_refptr<net::IOBuffer> preloading_buffer_;
+  int preloading_buffer_offset_;
+  int buffered_bytes_;
+
+  base::WeakPtrFactory<BufferingFileStreamReader> weak_ptr_factory_;
+  DISALLOW_COPY_AND_ASSIGN(BufferingFileStreamReader);
+};
+
+}  // namespace file_system_provider
+}  // namespace chromeos
+
+#endif  // CHROME_BROWSER_CHROMEOS_FILE_SYSTEM_PROVIDER_FILEAPI_BUFFERING_FILE_STREAM_READER_H_

chrome/browser/chromeos/file_system_provider/fileapi/file_stream_reader.h

@@ -7,14 +7,11 @@
 
 #include "base/basictypes.h"
 #include "base/files/file_path.h"
+#include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "webkit/browser/blob/file_stream_reader.h"
 #include "webkit/browser/fileapi/file_system_url.h"
 
-namespace fileapi {
-class AsyncFileUtil;
-}  // namespace fileapi
-
 namespace chromeos {
 namespace file_system_provider {
 

chrome/chrome_browser_chromeos.gypi

@@ -380,6 +380,8 @@
         'browser/chromeos/file_manager/zip_file_creator.h',
         'browser/chromeos/file_system_provider/fileapi/backend_delegate.cc',
         'browser/chromeos/file_system_provider/fileapi/backend_delegate.h',
+        'browser/chromeos/file_system_provider/fileapi/buffering_file_stream_reader.cc',
+        'browser/chromeos/file_system_provider/fileapi/buffering_file_stream_reader.h',
         'browser/chromeos/file_system_provider/fileapi/file_stream_reader.cc',
         'browser/chromeos/file_system_provider/fileapi/file_stream_reader.h',
         'browser/chromeos/file_system_provider/fileapi/provider_async_file_util.cc',

chrome/chrome_tests_unit.gypi

@@ -738,6 +738,7 @@
         'browser/chromeos/file_manager/volume_manager_unittest.cc',
         'browser/chromeos/file_system_provider/fake_provided_file_system.cc',
         'browser/chromeos/file_system_provider/fake_provided_file_system.h',
+        'browser/chromeos/file_system_provider/fileapi/buffering_file_stream_reader_unittest.cc',
         'browser/chromeos/file_system_provider/fileapi/file_stream_reader_unittest.cc',
         'browser/chromeos/file_system_provider/fileapi/provider_async_file_util_unittest.cc',
         'browser/chromeos/file_system_provider/mount_path_util_unittest.cc',

chrome/test/data/extensions/api_test/file_system_provider/big_file/test.js

@@ -104,10 +104,9 @@ function onReadFileRequested(options, onSuccess, onError) {
 
   if (filePath == '/' + TESTING_6GB_FILE.name) {
     if (options.offset < TESTING_TEXT_OFFSET ||
-        options.offset + options.length >
-            TESTING_TEXT_OFFSET + TESTING_TEXT.length) {
+        options.offset >= TESTING_TEXT_OFFSET + TESTING_TEXT.length) {
       console.error('Reading from a wrong location in the file!');
-      onError('INVALID_FAILED');  // enum ProviderError.
+      onError('FAILED');  // enum ProviderError.
       return;
     }