Disk cache v3: The main index table.

The IndexTable controls what entries are stored in the cache.
This class provides a memory-only view of the underlying structures
that store "the index"

BUG=241277
TEST=net_unittests

Review URL: https://codereview.chromium.org/53313004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@244027 0039d316-1c4b-4281-b951-d872f2087c98

net/disk_cache/addr.h

@@ -54,6 +54,18 @@ const int kFirstAdditionalBlockFileV3 = 7;
 //   0000 0011 0000 0000 0000 0000 0000 0000 : number of contiguous blocks 1-4
 //   0000 0000 1111 1111 0000 0000 0000 0000 : file selector 0 - 255
 //   0000 0000 0000 0000 1111 1111 1111 1111 : block#  0 - 65,535 (2^16)
+//
+// Note that an Addr can be used to "point" to a variety of different objects,
+// from a given type of entry to random blobs of data. Conceptually, an Addr is
+// just a number that someone can inspect to find out how to locate the desired
+// record. Most users will not care about the specific bits inside Addr, for
+// example, what parts of it point to a file number; only the code that has to
+// select a specific file would care about those specific bits.
+//
+// From a general point of view, an Addr has a total capacity of 2^24 entities,
+// in that it has 24 bits that can identify individual records. Note that the
+// address space is bigger for independent files (2^28), but that would not be
+// the general case.
 class NET_EXPORT_PRIVATE Addr {
  public:
   Addr() : value_(0) {}
@@ -108,14 +120,6 @@ class NET_EXPORT_PRIVATE Addr {
     return value_ != other.value_;
   }
 
-  static Addr FromEntryAddress(uint32 value) {
-    return Addr(kInitializedMask + (BLOCK_ENTRIES << kFileTypeOffset) + value);
-  }
-
-  static Addr FromEvictedAddress(uint32 value) {
-    return Addr(kInitializedMask + (BLOCK_EVICTED << kFileTypeOffset) + value);
-  }
-
   static int BlockSizeForFileType(FileType file_type) {
     switch (file_type) {
       case RANKINGS:

net/disk_cache/disk_format_base.h

@@ -33,6 +33,7 @@ const uint32 kBlockCurrentVersion = 0x30000;  // Version 3.0.
 const uint32 kBlockMagic = 0xC104CAC3;
 const int kBlockHeaderSize = 8192;  // Two pages: almost 64k entries
 const int kMaxBlocks = (kBlockHeaderSize - 80) * 8;
+const int kNumExtraBlocks = 1024;  // How fast files grow.
 
 // Bitmap to track used blocks on a block-file.
 typedef uint32 AllocBitmap[kMaxBlocks / 32];

net/disk_cache/v3/disk_format_v3.h

@@ -37,6 +37,10 @@
 // internal structures are modified, so it is possible to detect (most of the
 // time) when the process dies in the middle of an update. There are dedicated
 // backup files for cache bitmaps, used to detect entries out of date.
+//
+// Although cache files are to be consumed on the same machine that creates
+// them, if files are to be moved accross machines, little endian storage is
+// assumed.
 
 #ifndef NET_DISK_CACHE_V3_DISK_FORMAT_V3_H_
 #define NET_DISK_CACHE_V3_DISK_FORMAT_V3_H_
@@ -46,14 +50,15 @@
 
 namespace disk_cache {
 
-const int kBaseTableLen = 0x10000;
+const int kBaseTableLen = 0x400;
 const uint32 kIndexMagicV3 = 0xC103CAC3;
 const uint32 kVersion3 = 0x30000;  // Version 3.0.
 
 // Flags for a given cache.
 enum CacheFlags {
-  CACHE_EVICTION_2 = 1,      // Keep multiple lists for eviction.
-  CACHE_EVICTED = 1 << 1     // Already evicted at least one entry.
+  SMALL_CACHE = 1 << 0,       // See IndexCell.
+  CACHE_EVICTION_2 = 1 << 1,  // Keep multiple lists for eviction.
+  CACHE_EVICTED = 1 << 2      // Already evicted at least one entry.
 };
 
 // Header for the master index file.
@@ -119,30 +124,81 @@ COMPILE_ASSERT(ENTRY_USED <= 7, group_uses_3_bits);
 struct IndexCell {
   void Clear() { memset(this, 0, sizeof(*this)); }
 
-  uint64      address : 22;
-  uint64      hash : 18;
-  uint64      timestamp : 20;
-  uint64      reuse : 4;
-  uint8       state : 3;
-  uint8       group : 3;
-  uint8       sum : 2;
+  // A cell is a 9 byte bit-field that stores 7 values:
+  //   location : 22 bits
+  //   id : 18 bits
+  //   timestamp : 20 bits
+  //   reuse : 4 bits
+  //   state : 3 bits
+  //   group : 3 bits
+  //   sum : 2 bits
+  // The id is derived from the full hash of the entry.
+  //
+  // The actual layout is as follows:
+  //
+  // first_part (low order 32 bits):
+  //   0000 0000 0011 1111 1111 1111 1111 1111 : location
+  //   1111 1111 1100 0000 0000 0000 0000 0000 : id
+  //
+  // first_part (high order 32 bits):
+  //   0000 0000 0000 0000 0000 0000 1111 1111 : id
+  //   0000 1111 1111 1111 1111 1111 0000 0000 : timestamp
+  //   1111 0000 0000 0000 0000 0000 0000 0000 : reuse
+  //
+  // last_part:
+  //   0000 0111 : state
+  //   0011 1000 : group
+  //   1100 0000 : sum
+  //
+  // The small-cache version of the format moves some bits from the location to
+  // the id fileds, like so:
+  //   location : 16 bits
+  //   id : 24 bits
+  //
+  // first_part (low order 32 bits):
+  //   0000 0000 0000 0000 1111 1111 1111 1111 : location
+  //   1111 1111 1111 1111 0000 0000 0000 0000 : id
+  //
+  // The actual bit distribution between location and id is determined by the
+  // table size (IndexHeaderV3.table_len). Tables smaller than 65536 entries
+  // use the small-cache version; after that size, caches should have the
+  // SMALL_CACHE flag cleared.
+  //
+  // To locate a given entry after recovering the location from the cell, the
+  // file type and file number are appended (see disk_cache/addr.h). For a large
+  // table only the file type is implied; for a small table, the file number
+  // is also implied, and it should be the first file for that type of entry,
+  // as determined by the EntryGroup (two files in total, one for active entries
+  // and another one for evicted entries).
+  //
+  // For example, a small table may store something like 0x1234 as the location
+  // field. That means it stores the entry number 0x1234. If that record belongs
+  // to a deleted entry, the regular cache address may look something like
+  //     BLOCK_EVICTED + 1 block + file number 6 + entry number 0x1234
+  //     so Addr = 0xf0061234
+  //
+  // If that same Addr is stored on a large table, the location field would be
+  // 0x61234
+
+  uint64      first_part;
+  uint8       last_part;
 };
 COMPILE_ASSERT(sizeof(IndexCell) == 9, bad_IndexCell);
 
+const int kCellsPerBucket = 4;
 struct IndexBucket {
-  IndexCell   cells[4];
+  IndexCell   cells[kCellsPerBucket];
   int32       next;
-  uint32      hash : 24;      // The last byte is only defined for buckets of
-  uint32      reserved : 8;   // the extra table.
+  uint32      hash;  // The high order byte is reserved (should be zero).
 };
 COMPILE_ASSERT(sizeof(IndexBucket) == 44, bad_IndexBucket);
-const int kBytesPerCell = 44 / 4;
+const int kBytesPerCell = 44 / kCellsPerBucket;
 
 // The main cache index. Backed by a file named index_tb1.
 // The extra table (index_tb2) has a similar format, but different size.
 struct Index {
   // Default size. Actual size controlled by header.table_len.
-  IndexBucket table[kBaseTableLen / 4];
+  IndexBucket table[kBaseTableLen / kCellsPerBucket];
 };
 #pragma pack(pop)
 

net/disk_cache/v3/index_table.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/disk_cache/v3/index_table.h"
+
+#include <algorithm>
+#include <set>
+#include <utility>
+
+#include "base/bits.h"
+#include "net/base/io_buffer.h"
+#include "net/base/net_errors.h"
+#include "net/disk_cache/disk_cache.h"
+
+using base::Time;
+using base::TimeDelta;
+using disk_cache::CellInfo;
+using disk_cache::CellList;
+using disk_cache::IndexCell;
+using disk_cache::IndexIterator;
+
+namespace {
+
+// The following constants describe the bitfields of an IndexCell so they are
+// implicitly synchronized with the descrption of IndexCell on file_format_v3.h.
+const uint64 kCellLocationMask = (1 << 22) - 1;
+const uint64 kCellIdMask = (1 << 18) - 1;
+const uint64 kCellTimestampMask = (1 << 20) - 1;
+const uint64 kCellReuseMask = (1 << 4) - 1;
+const uint8 kCellStateMask = (1 << 3) - 1;
+const uint8 kCellGroupMask = (1 << 3) - 1;
+const uint8 kCellSumMask = (1 << 2) - 1;
+
+const uint64 kCellSmallTableLocationMask = (1 << 16) - 1;
+const uint64 kCellSmallTableIdMask = (1 << 24) - 1;
+
+const int kCellIdOffset = 22;
+const int kCellTimestampOffset = 40;
+const int kCellReuseOffset = 60;
+const int kCellGroupOffset = 3;
+const int kCellSumOffset = 6;
+
+const int kCellSmallTableIdOffset = 16;
+
+// The number of bits that a hash has to be shifted to grab the part that
+// defines the cell id.
+const int kHashShift = 14;
+const int kSmallTableHashShift = 8;
+
+// Unfortunately we have to break the abstaction a little here: the file number
+// where entries are stored is outside of the control of this code, and it is
+// usually part of the stored address. However, for small tables we only store
+// 16 bits of the address so the file number is never stored on a cell. We have
+// to infere the file number from the type of entry (normal vs evicted), and
+// the knowledge that given that the table will not keep more than 64k entries,
+// a single file of each type is enough.
+const int kEntriesFile = disk_cache::BLOCK_ENTRIES - 1;
+const int kEvictedEntriesFile = disk_cache::BLOCK_EVICTED - 1;
+const int kMaxLocation = 1 << 22;
+const int kMinFileNumber = 1 << 16;
+
+uint32 GetCellLocation(const IndexCell& cell) {
+  return cell.first_part & kCellLocationMask;
+}
+
+uint32 GetCellSmallTableLocation(const IndexCell& cell) {
+  return cell.first_part & kCellSmallTableLocationMask;
+}
+
+uint32 GetCellId(const IndexCell& cell) {
+  return (cell.first_part >> kCellIdOffset) & kCellIdMask;
+}
+
+uint32 GetCellSmallTableId(const IndexCell& cell) {
+  return (cell.first_part >> kCellSmallTableIdOffset) &
+         kCellSmallTableIdMask;
+}
+
+int GetCellTimestamp(const IndexCell& cell) {
+  return (cell.first_part >> kCellTimestampOffset) & kCellTimestampMask;
+}
+
+int GetCellReuse(const IndexCell& cell) {
+  return (cell.first_part >> kCellReuseOffset) & kCellReuseMask;
+}
+
+int GetCellState(const IndexCell& cell) {
+  return cell.last_part & kCellStateMask;
+}
+
+int GetCellGroup(const IndexCell& cell) {
+  return (cell.last_part >> kCellGroupOffset) & kCellGroupMask;
+}
+
+int GetCellSum(const IndexCell& cell) {
+  return (cell.last_part >> kCellSumOffset) & kCellSumMask;
+}
+
+void SetCellLocation(IndexCell* cell, uint32 address) {
+  DCHECK_LE(address, static_cast<uint32>(kCellLocationMask));
+  cell->first_part &= ~kCellLocationMask;
+  cell->first_part |= address;
+}
+
+void SetCellSmallTableLocation(IndexCell* cell, uint32 address) {
+  DCHECK_LE(address, static_cast<uint32>(kCellSmallTableLocationMask));
+  cell->first_part &= ~kCellSmallTableLocationMask;
+  cell->first_part |= address;
+}
+
+void SetCellId(IndexCell* cell, uint32 hash) {
+  DCHECK_LE(hash, static_cast<uint32>(kCellIdMask));
+  cell->first_part &= ~(kCellIdMask << kCellIdOffset);
+  cell->first_part |= static_cast<int64>(hash) << kCellIdOffset;
+}
+
+void SetCellSmallTableId(IndexCell* cell, uint32 hash) {
+  DCHECK_LE(hash, static_cast<uint32>(kCellSmallTableIdMask));
+  cell->first_part &= ~(kCellSmallTableIdMask << kCellSmallTableIdOffset);
+  cell->first_part |= static_cast<int64>(hash) << kCellSmallTableIdOffset;
+}
+
+void SetCellTimestamp(IndexCell* cell, int timestamp) {
+  DCHECK_LT(timestamp, 1 << 20);
+  DCHECK_GE(timestamp, 0);
+  cell->first_part &= ~(kCellTimestampMask << kCellTimestampOffset);
+  cell->first_part |= static_cast<int64>(timestamp) << kCellTimestampOffset;
+}
+
+void SetCellReuse(IndexCell* cell, int count) {
+  DCHECK_LT(count, 16);
+  DCHECK_GE(count, 0);
+  cell->first_part &= ~(kCellReuseMask << kCellReuseOffset);
+  cell->first_part |= static_cast<int64>(count) << kCellReuseOffset;
+}
+
+void SetCellState(IndexCell* cell, disk_cache::EntryState state) {
+  cell->last_part &= ~kCellStateMask;
+  cell->last_part |= state;
+}
+
+void SetCellGroup(IndexCell* cell, disk_cache::EntryGroup group) {
+  cell->last_part &= ~(kCellGroupMask << kCellGroupOffset);
+  cell->last_part |= group << kCellGroupOffset;
+}
+
+void SetCellSum(IndexCell* cell, int sum) {
+  DCHECK_LT(sum, 4);
+  DCHECK_GE(sum, 0);
+  cell->last_part &= ~(kCellSumMask << kCellSumOffset);
+  cell->last_part |= sum << kCellSumOffset;
+}
+
+// This is a very particular way to calculate the sum, so it will not match if
+// compared a gainst a pure 2 bit, modulo 2 sum.
+int CalculateCellSum(const IndexCell& cell) {
+  uint32* words = bit_cast<uint32*>(&cell);
+  uint8* bytes = bit_cast<uint8*>(&cell);
+  uint32 result = words[0] + words[1];
+  result += result >> 16;
+  result += (result >> 8) + (bytes[8] & 0x3f);
+  result += result >> 4;
+  result += result >> 2;
+  return result & 3;
+}
+
+bool SanityCheck(const IndexCell& cell) {
+  if (GetCellSum(cell) != CalculateCellSum(cell))
+    return false;
+
+  if (GetCellState(cell) > disk_cache::ENTRY_USED ||
+      GetCellGroup(cell) == disk_cache::ENTRY_RESERVED ||
+      GetCellGroup(cell) > disk_cache::ENTRY_EVICTED) {
+    return false;
+  }
+
+  return true;
+}
+
+int FileNumberFromLocation(int location) {
+  return location / kMinFileNumber;
+}
+
+int StartBlockFromLocation(int location) {
+  return location % kMinFileNumber;
+}
+
+bool IsValidAddress(disk_cache::Addr address) {
+  if (!address.is_initialized() ||
+      (address.file_type() != disk_cache::BLOCK_EVICTED &&
+       address.file_type() != disk_cache::BLOCK_ENTRIES)) {
+    return false;
+  }
+
+  return address.FileNumber() < FileNumberFromLocation(kMaxLocation);
+}
+
+bool IsNormalState(const IndexCell& cell) {
+  disk_cache::EntryState state =
+      static_cast<disk_cache::EntryState>(GetCellState(cell));
+  DCHECK_NE(state, disk_cache::ENTRY_FREE);
+  return state != disk_cache::ENTRY_DELETED &&
+         state != disk_cache::ENTRY_FIXING;
+}
+
+inline int GetNextBucket(int min_bucket_num, int max_bucket_num,
+                         disk_cache::IndexBucket* table,
+                         disk_cache::IndexBucket** bucket) {
+  if (!(*bucket)->next)
+    return 0;
+
+  int bucket_num = (*bucket)->next / disk_cache::kCellsPerBucket;
+  if (bucket_num < min_bucket_num || bucket_num > max_bucket_num) {
+    // The next bucket must fall within the extra table. Note that this is not
+    // an uncommon path as growing the table may not cleanup the link from the
+    // main table to the extra table, and that cleanup is performed here when
+    // accessing that bucket for the first time. This behavior has to change if
+    // the tables are ever shrinked.
+    (*bucket)->next = 0;
+    return 0;
+  }
+  *bucket = &table[bucket_num - min_bucket_num];
+  return bucket_num;
+}
+
+// Updates the |iterator| with the current |cell|. This cell may cause all
+// previous cells to be deleted (when a new target timestamp is found), the cell
+// may be added to the list (if it matches the target timestamp), or may it be
+// ignored.
+void UpdateIterator(const disk_cache::EntryCell& cell,
+                    int limit_time,
+                    IndexIterator* iterator) {
+  int time = cell.GetTimestamp();
+  // Look for not interesting times.
+  if (iterator->forward && time <= limit_time)
+    return;
+  if (!iterator->forward && time >= limit_time)
+    return;
+
+  if ((iterator->forward && time < iterator->timestamp) ||
+      (!iterator->forward && time > iterator->timestamp)) {
+    // This timestamp is better than the one we had.
+    iterator->timestamp = time;
+    iterator->cells.clear();
+  }
+  if (time == iterator->timestamp) {
+    CellInfo cell_info = { cell.hash(), cell.GetAddress() };
+    iterator->cells.push_back(cell_info);
+  }
+}
+
+void InitIterator(IndexIterator* iterator) {
+  iterator->cells.clear();
+  iterator->timestamp = iterator->forward ? kint32max : 0;
+}
+
+}  // namespace
+
+namespace disk_cache {
+
+EntryCell::~EntryCell() {
+}
+
+bool EntryCell::IsValid() const {
+  return GetCellLocation(cell_) != 0;
+}
+
+// This code has to map the cell address (up to 22 bits) to a general cache Addr
+// (up to 24 bits of general addressing). It also set the implied file_number
+// in the case of small tables. See also the comment by the definition of
+// kEntriesFile.
+Addr EntryCell::GetAddress() const {
+  uint32 location = GetLocation();
+  int file_number = FileNumberFromLocation(location);
+  if (small_table_) {
+    DCHECK_EQ(0, file_number);
+    file_number = (GetGroup() == ENTRY_EVICTED) ? kEvictedEntriesFile :
+                                                  kEntriesFile;
+  }
+  DCHECK_NE(0, file_number);
+  FileType file_type = (GetGroup() == ENTRY_EVICTED) ? BLOCK_EVICTED :
+                                                       BLOCK_ENTRIES;
+  return Addr(file_type, 1, file_number, StartBlockFromLocation(location));
+}
+
+EntryState EntryCell::GetState() const {
+  return static_cast<EntryState>(GetCellState(cell_));
+}
+
+EntryGroup EntryCell::GetGroup() const {
+  return static_cast<EntryGroup>(GetCellGroup(cell_));
+}
+
+int EntryCell::GetReuse() const {
+  return GetCellReuse(cell_);
+}
+
+int EntryCell::GetTimestamp() const {
+  return GetCellTimestamp(cell_);
+}
+
+void EntryCell::SetState(EntryState state) {
+  SetCellState(&cell_, state);
+}
+
+void EntryCell::SetGroup(EntryGroup group) {
+  SetCellGroup(&cell_, group);
+}
+
+void EntryCell::SetReuse(int count) {
+  SetCellReuse(&cell_, count);
+}
+
+void EntryCell::SetTimestamp(int timestamp) {
+  SetCellTimestamp(&cell_, timestamp);
+}
+
+// Static.
+EntryCell EntryCell::GetEntryCellForTest(int32 cell_num,
+                                         uint32 hash,
+                                         Addr address,
+                                         IndexCell* cell,
+                                         bool small_table) {
+  if (cell) {
+    EntryCell entry_cell(cell_num, hash, *cell, small_table);
+    return entry_cell;
+  }
+
+  return EntryCell(cell_num, hash, address, small_table);
+}
+
+void EntryCell::SerializaForTest(IndexCell* destination) {
+  FixSum();
+  Serialize(destination);
+}
+
+EntryCell::EntryCell() : cell_num_(0), hash_(0), small_table_(false) {
+  cell_.Clear();
+}
+
+EntryCell::EntryCell(int32 cell_num,
+                     uint32 hash,
+                     Addr address,
+                     bool small_table)
+    : cell_num_(cell_num),
+      hash_(hash),
+      small_table_(small_table) {
+  DCHECK(IsValidAddress(address) || !address.value());
+
+  cell_.Clear();
+  SetCellState(&cell_, ENTRY_NEW);
+  SetCellGroup(&cell_, ENTRY_NO_USE);
+  if (small_table) {
+    DCHECK(address.FileNumber() == kEntriesFile ||
+           address.FileNumber() == kEvictedEntriesFile);
+    SetCellSmallTableLocation(&cell_, address.start_block());
+    SetCellSmallTableId(&cell_, hash >> kSmallTableHashShift);
+  } else {
+    uint32 location = address.FileNumber() << 16 | address.start_block();
+    SetCellLocation(&cell_, location);
+    SetCellId(&cell_, hash >> kHashShift);
+  }
+}
+
+EntryCell::EntryCell(int32 cell_num,
+                     uint32 hash,
+                     const IndexCell& cell,
+                     bool small_table)
+    : cell_num_(cell_num),
+      hash_(hash),
+      cell_(cell),
+      small_table_(small_table) {
+}
+
+void EntryCell::FixSum() {
+  SetCellSum(&cell_, CalculateCellSum(cell_));
+}
+
+uint32 EntryCell::GetLocation() const {
+  if (small_table_)
+    return GetCellSmallTableLocation(cell_);
+
+  return GetCellLocation(cell_);
+}
+
+uint32 EntryCell::RecomputeHash() {
+  if (small_table_) {
+    hash_ &= (1 << kSmallTableHashShift) - 1;
+    hash_ |= GetCellSmallTableId(cell_) << kSmallTableHashShift;
+    return hash_;
+  }
+
+  hash_ &= (1 << kHashShift) - 1;
+  hash_ |= GetCellId(cell_) << kHashShift;
+  return hash_;
+}
+
+void EntryCell::Serialize(IndexCell* destination) const {
+  *destination = cell_;
+}
+
+EntrySet::EntrySet() : evicted_count(0), current(0) {
+}
+
+EntrySet::~EntrySet() {
+}
+
+IndexIterator::IndexIterator() {
+}
+
+IndexIterator::~IndexIterator() {
+}
+
+IndexTableInitData::IndexTableInitData() {
+}
+
+IndexTableInitData::~IndexTableInitData() {
+}
+
+// -----------------------------------------------------------------------
+
+IndexTable::IndexTable(IndexTableBackend* backend)
+    : backend_(backend),
+      header_(NULL),
+      main_table_(NULL),
+      extra_table_(NULL),
+      modified_(false),
+      small_table_(false) {
+}
+
+IndexTable::~IndexTable() {
+}
+
+// For a general description of the index tables see:
+// http://www.chromium.org/developers/design-documents/network-stack/disk-cache/disk-cache-v3#TOC-Index
+//
+// The index is split between two tables: the main_table_ and the extra_table_.
+// The main table can grow only by doubling its number of cells, while the
+// extra table can grow slowly, because it only contain cells that overflow
+// from the main table. In order to locate a given cell, part of the hash is
+// used directly as an index into the main table; once that bucket is located,
+// all cells with that partial hash (i.e., belonging to that bucket) are
+// inspected, and if present, the next bucket (located on the extra table) is
+// then located. For more information on bucket chaining see:
+// http://www.chromium.org/developers/design-documents/network-stack/disk-cache/disk-cache-v3#TOC-Buckets
+//
+// There are two cases when increasing the size:
+//  - Doubling the size of the main table
+//  - Adding more entries to the extra table
+//
+// For example, consider a 64k main table with 8k cells on the extra table (for
+// a total of 72k cells). Init can be called to add another 8k cells at the end
+// (grow to 80k cells). When the size of the extra table approaches 64k, Init
+// can be called to double the main table (to 128k) and go back to a small extra
+// table.
+void IndexTable::Init(IndexTableInitData* params) {
+  bool growing = header_ != NULL;
+  scoped_ptr<IndexBucket[]> old_extra_table;
+  header_ = &params->index_bitmap->header;
+
+  if (params->main_table) {
+    if (main_table_) {
+      // This is doubling the size of main table.
+      DCHECK_EQ(base::bits::Log2Floor(header_->table_len),
+                base::bits::Log2Floor(backup_header_->table_len) + 1);
+      int extra_size = (header()->max_bucket - mask_) * kCellsPerBucket;
+      DCHECK_GE(extra_size, 0);
+
+      // Doubling the size implies deleting the extra table and moving as many
+      // cells as we can to the main table, so we first copy the old one. This
+      // is not required when just growing the extra table because we don't
+      // move any cell in that case.
+      old_extra_table.reset(new IndexBucket[extra_size]);
+      memcpy(old_extra_table.get(), extra_table_,
+             extra_size * sizeof(IndexBucket));
+      memset(params->extra_table, 0, extra_size * sizeof(IndexBucket));
+    }
+    main_table_ = params->main_table;
+  }
+  DCHECK(main_table_);
+  extra_table_ = params->extra_table;
+
+  // extra_bits_ is really measured against table-size specific values.
+  const int kMaxAbsoluteExtraBits = 12;  // From smallest to largest table.
+  const int kMaxExtraBitsSmallTable = 6;  // From smallest to 64K table.
+
+  extra_bits_ = base::bits::Log2Floor(header_->table_len) -
+                base::bits::Log2Floor(kBaseTableLen);
+  DCHECK_GE(extra_bits_, 0);
+  DCHECK_LT(extra_bits_, kMaxAbsoluteExtraBits);
+
+  // Note that following the previous code the constants could be derived as
+  // kMaxAbsoluteExtraBits = base::bits::Log2Floor(max table len) -
+  //                         base::bits::Log2Floor(kBaseTableLen);
+  //                       = 22 - base::bits::Log2Floor(1024) = 22 - 10;
+  // kMaxExtraBitsSmallTable = base::bits::Log2Floor(max 16 bit table) - 10.
+
+  mask_ = ((kBaseTableLen / kCellsPerBucket) << extra_bits_) - 1;
+  small_table_ = extra_bits_ < kMaxExtraBitsSmallTable;
+  if (!small_table_)
+    extra_bits_ -= kMaxExtraBitsSmallTable;
+
+  // table_len keeps the max number of cells stored by the index. We need a
+  // bitmap with 1 bit per cell, and that bitmap has num_words 32-bit words.
+  int num_words = (header_->table_len + 31) / 32;
+
+  if (old_extra_table) {
+    // All the cells from the extra table are moving to the new tables so before
+    // creating the bitmaps, clear the part of the bitmap referring to the extra
+    // table.
+    int old_main_table_bit_words = ((mask_ >> 1) + 1) * kCellsPerBucket / 32;
+    DCHECK_GT(num_words, old_main_table_bit_words);
+    memset(params->index_bitmap->bitmap + old_main_table_bit_words, 0,
+           (num_words - old_main_table_bit_words) * sizeof(int32));
+
+    DCHECK(growing);
+    int old_num_words = (backup_header_.get()->table_len + 31) / 32;
+    DCHECK_GT(old_num_words, old_main_table_bit_words);
+    memset(backup_bitmap_storage_.get() + old_main_table_bit_words, 0,
+           (old_num_words - old_main_table_bit_words) * sizeof(int32));
+  }
+  bitmap_.reset(new Bitmap(params->index_bitmap->bitmap, header_->table_len,
+                           num_words));
+
+  if (growing) {
+    int old_num_words = (backup_header_.get()->table_len + 31) / 32;
+    DCHECK_GE(num_words, old_num_words);
+    scoped_ptr<uint32[]> storage(new uint32[num_words]);
+    memcpy(storage.get(), backup_bitmap_storage_.get(),
+           old_num_words * sizeof(int32));
+    memset(storage.get() + old_num_words, 0,
+           (num_words - old_num_words) * sizeof(int32));
+
+    backup_bitmap_storage_.swap(storage);
+    backup_header_->table_len = header_->table_len;
+  } else {
+    backup_bitmap_storage_.reset(params->backup_bitmap.release());
+    backup_header_.reset(params->backup_header.release());
+  }
+
+  num_words = (backup_header_->table_len + 31) / 32;
+  backup_bitmap_.reset(new Bitmap(backup_bitmap_storage_.get(),
+                                  backup_header_->table_len, num_words));
+  if (old_extra_table)
+    MoveCells(old_extra_table.get());
+
+  if (small_table_)
+    DCHECK(header_->flags & SMALL_CACHE);
+
+  // All tables and backups are needed for operation.
+  DCHECK(main_table_);
+  DCHECK(extra_table_);
+  DCHECK(bitmap_.get());
+}
+
+void IndexTable::Shutdown() {
+  header_ = NULL;
+  main_table_ = NULL;
+  extra_table_ = NULL;
+  bitmap_.reset();
+  backup_bitmap_.reset();
+  backup_header_.reset();
+  backup_bitmap_storage_.reset();
+  modified_ = false;
+}
+
+// The general method for locating cells is to:
+// 1. Get the first bucket. This usually means directly indexing the table (as
+//     this method does), or iterating through all possible buckets.
+// 2. Iterate through all the cells in that first bucket.
+// 3. If there is a linked bucket, locate it directly in the extra table.
+// 4. Go back to 2, as needed.
+//
+// One consequence of this pattern is that we never start looking at buckets in
+// the extra table, unless we are following a link from the main table.
+EntrySet IndexTable::LookupEntries(uint32 hash) {
+  EntrySet entries;
+  int bucket_num = static_cast<int>(hash & mask_);
+  IndexBucket* bucket = &main_table_[bucket_num];
+  do {
+    for (int i = 0; i < kCellsPerBucket; i++) {
+      IndexCell* current_cell = &bucket->cells[i];
+      if (!GetLocation(*current_cell))
+        continue;
+      if (!SanityCheck(*current_cell)) {
+        NOTREACHED();
+        int cell_num = bucket_num * kCellsPerBucket + i;
+        current_cell->Clear();
+        bitmap_->Set(cell_num, false);
+        backup_bitmap_->Set(cell_num, false);
+        modified_ = true;
+        continue;
+      }
+      int cell_num = bucket_num * kCellsPerBucket + i;
+      if (MisplacedHash(*current_cell, hash)) {
+        HandleMisplacedCell(current_cell, cell_num, hash & mask_);
+      } else if (IsHashMatch(*current_cell, hash)) {
+        EntryCell entry_cell(cell_num, hash, *current_cell, small_table_);
+        CheckState(entry_cell);
+        if (entry_cell.GetState() != ENTRY_DELETED) {
+          entries.cells.push_back(entry_cell);
+          if (entry_cell.GetGroup() == ENTRY_EVICTED)
+            entries.evicted_count++;
+        }
+      }
+    }
+    bucket_num = GetNextBucket(mask_ + 1, header()->max_bucket, extra_table_,
+                               &bucket);
+  } while (bucket_num);
+  return entries;
+}
+
+EntryCell IndexTable::CreateEntryCell(uint32 hash, Addr address) {
+  DCHECK(IsValidAddress(address));
+  DCHECK(address.FileNumber() || address.start_block());
+
+  int bucket_num = static_cast<int>(hash & mask_);
+  int cell_num = 0;
+  IndexBucket* bucket = &main_table_[bucket_num];
+  IndexCell* current_cell = NULL;
+  bool found = false;
+  do {
+    for (int i = 0; i < kCellsPerBucket && !found; i++) {
+      current_cell = &bucket->cells[i];
+      if (!GetLocation(*current_cell)) {
+        cell_num = bucket_num * kCellsPerBucket + i;
+        found = true;
+      }
+    }
+    if (found)
+      break;
+    bucket_num = GetNextBucket(mask_ + 1, header()->max_bucket, extra_table_,
+                               &bucket);
+  } while (bucket_num);
+
+  if (!found) {
+    bucket_num = NewExtraBucket();
+    if (bucket_num) {
+      cell_num = bucket_num * kCellsPerBucket;
+      bucket->next = cell_num;
+      bucket = &extra_table_[bucket_num - (mask_ + 1)];
+      bucket->hash = hash & mask_;
+      found = true;
+    } else {
+      // address 0 is a reserved value, and the caller interprets it as invalid.
+      address.set_value(0);
+    }
+  }
+
+  EntryCell entry_cell(cell_num, hash, address, small_table_);
+  if (address.file_type() == BLOCK_EVICTED)
+    entry_cell.SetGroup(ENTRY_EVICTED);
+  else
+    entry_cell.SetGroup(ENTRY_NO_USE);
+  Save(&entry_cell);
+
+  if (found) {
+    bitmap_->Set(cell_num, true);
+    backup_bitmap_->Set(cell_num, true);
+    header()->used_cells++;
+    modified_ = true;
+  }
+
+  return entry_cell;
+}
+
+EntryCell IndexTable::FindEntryCell(uint32 hash, Addr address) {
+  return FindEntryCellImpl(hash, address, false);
+}
+
+int IndexTable::CalculateTimestamp(Time time) {
+  TimeDelta delta = time - Time::FromInternalValue(header_->base_time);
+  return std::max(delta.InMinutes(), 0);
+}
+
+base::Time IndexTable::TimeFromTimestamp(int timestamp) {
+  return Time::FromInternalValue(header_->base_time) +
+         TimeDelta::FromMinutes(timestamp);
+}
+
+void IndexTable::SetSate(uint32 hash, Addr address, EntryState state) {
+  EntryCell cell = FindEntryCellImpl(hash, address, state == ENTRY_FREE);
+  if (!cell.IsValid()) {
+    NOTREACHED();
+    return;
+  }
+
+  EntryState old_state = cell.GetState();
+  switch (state) {
+    case ENTRY_FREE:
+      DCHECK_EQ(old_state, ENTRY_DELETED);
+      break;
+    case ENTRY_NEW:
+      DCHECK_EQ(old_state, ENTRY_FREE);
+      break;
+    case ENTRY_OPEN:
+      DCHECK_EQ(old_state, ENTRY_USED);
+      break;
+    case ENTRY_MODIFIED:
+      DCHECK_EQ(old_state, ENTRY_OPEN);
+      break;
+    case ENTRY_DELETED:
+      DCHECK(old_state == ENTRY_NEW || old_state == ENTRY_OPEN ||
+             old_state == ENTRY_MODIFIED);
+      break;
+    case ENTRY_USED:
+      DCHECK(old_state == ENTRY_NEW || old_state == ENTRY_OPEN ||
+             old_state == ENTRY_MODIFIED);
+      break;
+    case ENTRY_FIXING:
+      break;
+  };
+
+  modified_ = true;
+  if (state == ENTRY_DELETED) {
+    bitmap_->Set(cell.cell_num(), false);
+    backup_bitmap_->Set(cell.cell_num(), false);
+  } else if (state == ENTRY_FREE) {
+    cell.Clear();
+    Write(cell);
+    header()->used_cells--;
+    return;
+  }
+  cell.SetState(state);
+
+  Save(&cell);
+}
+
+void IndexTable::UpdateTime(uint32 hash, Addr address, base::Time current) {
+  EntryCell cell = FindEntryCell(hash, address);
+  if (!cell.IsValid())
+    return;
+
+  int minutes = CalculateTimestamp(current);
+
+  // Keep about 3 months of headroom.
+  const int kMaxTimestamp = (1 << 20) - 60 * 24 * 90;
+  if (minutes > kMaxTimestamp) {
+    // TODO(rvargas):
+    // Update header->old_time and trigger a timer
+    // Rebaseline timestamps and don't update sums
+    // Start a timer (about 2 backups)
+    // fix all ckecksums and trigger another timer
+    // update header->old_time because rebaseline is done.
+    minutes = std::min(minutes, (1 << 20) - 1);
+  }
+
+  cell.SetTimestamp(minutes);
+  Save(&cell);
+}
+
+void IndexTable::Save(EntryCell* cell) {
+  cell->FixSum();
+  Write(*cell);
+}
+
+void IndexTable::GetOldest(IndexIterator* no_use,
+                           IndexIterator* low_use,
+                           IndexIterator* high_use) {
+  no_use->forward = true;
+  low_use->forward = true;
+  high_use->forward = true;
+  InitIterator(no_use);
+  InitIterator(low_use);
+  InitIterator(high_use);
+
+  WalkTables(-1, no_use, low_use, high_use);
+}
+
+bool IndexTable::GetNextCells(IndexIterator* iterator) {
+  int current_time = iterator->timestamp;
+  InitIterator(iterator);
+
+  WalkTables(current_time, iterator, iterator, iterator);
+  return !iterator->cells.empty();
+}
+
+void IndexTable::OnBackupTimer() {
+  if (!modified_)
+    return;
+
+  int num_words = (header_->table_len + 31) / 32;
+  int num_bytes = num_words * 4 + static_cast<int>(sizeof(*header_));
+  scoped_refptr<net::IOBuffer> buffer(new net::IOBuffer(num_bytes));
+  memcpy(buffer->data(), header_, sizeof(*header_));
+  memcpy(buffer->data() + sizeof(*header_), backup_bitmap_storage_.get(),
+         num_words * 4);
+  backend_->SaveIndex(buffer, num_bytes);
+  modified_ = false;
+}
+
+// -----------------------------------------------------------------------
+
+EntryCell IndexTable::FindEntryCellImpl(uint32 hash, Addr address,
+                                        bool allow_deleted) {
+  int bucket_num = static_cast<int>(hash & mask_);
+  IndexBucket* bucket = &main_table_[bucket_num];
+  do {
+    for (int i = 0; i < kCellsPerBucket; i++) {
+      IndexCell* current_cell = &bucket->cells[i];
+      if (!GetLocation(*current_cell))
+        continue;
+      DCHECK(SanityCheck(*current_cell));
+      if (IsHashMatch(*current_cell, hash)) {
+        // We have a match.
+        int cell_num = bucket_num * kCellsPerBucket + i;
+        EntryCell entry_cell(cell_num, hash, *current_cell, small_table_);
+        if (entry_cell.GetAddress() != address)
+          continue;
+
+        if (!allow_deleted && entry_cell.GetState() == ENTRY_DELETED)
+          continue;
+
+        return entry_cell;
+      }
+    }
+    bucket_num = GetNextBucket(mask_ + 1, header()->max_bucket, extra_table_,
+                               &bucket);
+  } while (bucket_num);
+  return EntryCell();
+}
+
+void IndexTable::CheckState(const EntryCell& cell) {
+  int current_state = cell.GetState();
+  if (current_state != ENTRY_FIXING) {
+    bool present = ((current_state & 3) != 0);  // Look at the last two bits.
+    if (present != bitmap_->Get(cell.cell_num()) ||
+        present != backup_bitmap_->Get(cell.cell_num())) {
+      // There's a mismatch.
+      if (current_state == ENTRY_DELETED) {
+        // We were in the process of deleting this entry. Finish now.
+        backend_->DeleteCell(cell);
+      } else {
+        current_state = ENTRY_FIXING;
+        EntryCell bad_cell(cell);
+        bad_cell.SetState(ENTRY_FIXING);
+        Save(&bad_cell);
+      }
+    }
+  }
+
+  if (current_state == ENTRY_FIXING)
+    backend_->FixCell(cell);
+}
+
+void IndexTable::Write(const EntryCell& cell) {
+  IndexBucket* bucket = NULL;
+  int bucket_num = cell.cell_num() / kCellsPerBucket;
+  if (bucket_num < static_cast<int32>(mask_ + 1)) {
+    bucket = &main_table_[bucket_num];
+  } else {
+    DCHECK_LE(bucket_num, header()->max_bucket);
+    bucket = &extra_table_[bucket_num - (mask_ + 1)];
+  }
+
+  int cell_number = cell.cell_num() % kCellsPerBucket;
+  if (GetLocation(bucket->cells[cell_number]) && cell.GetLocation()) {
+    DCHECK_EQ(cell.GetLocation(),
+              GetLocation(bucket->cells[cell_number]));
+  }
+  cell.Serialize(&bucket->cells[cell_number]);
+}
+
+int IndexTable::NewExtraBucket() {
+  int safe_window = (header()->table_len < kNumExtraBlocks * 2) ?
+                    kNumExtraBlocks / 4 : kNumExtraBlocks;
+  if (header()->table_len - header()->max_bucket * kCellsPerBucket <
+      safe_window) {
+    backend_->GrowIndex();
+  }
+
+  if (header()->max_bucket * kCellsPerBucket ==
+      header()->table_len - kCellsPerBucket) {
+    return 0;
+  }
+
+  header()->max_bucket++;
+  return header()->max_bucket;
+}
+
+void IndexTable::WalkTables(int limit_time,
+                            IndexIterator* no_use,
+                            IndexIterator* low_use,
+                            IndexIterator* high_use) {
+  header_->num_no_use_entries = 0;
+  header_->num_low_use_entries = 0;
+  header_->num_high_use_entries = 0;
+  header_->num_evicted_entries = 0;
+
+  for (int i = 0; i < static_cast<int32>(mask_ + 1); i++) {
+    int bucket_num = i;
+    IndexBucket* bucket = &main_table_[i];
+    do {
+      UpdateFromBucket(bucket, i, limit_time, no_use, low_use, high_use);
+
+      bucket_num = GetNextBucket(mask_ + 1, header()->max_bucket, extra_table_,
+                                 &bucket);
+    } while (bucket_num);
+  }
+  header_->num_entries = header_->num_no_use_entries +
+                         header_->num_low_use_entries +
+                         header_->num_high_use_entries +
+                         header_->num_evicted_entries;
+  modified_ = true;
+}
+
+void IndexTable::UpdateFromBucket(IndexBucket* bucket, int bucket_hash,
+                                  int limit_time,
+                                  IndexIterator* no_use,
+                                  IndexIterator* low_use,
+                                  IndexIterator* high_use) {
+  for (int i = 0; i < kCellsPerBucket; i++) {
+    IndexCell& current_cell = bucket->cells[i];
+    if (!GetLocation(current_cell))
+      continue;
+    DCHECK(SanityCheck(current_cell));
+    if (!IsNormalState(current_cell))
+      continue;
+
+    EntryCell entry_cell(0, GetFullHash(current_cell, bucket_hash),
+                         current_cell, small_table_);
+    switch (GetCellGroup(current_cell)) {
+      case ENTRY_NO_USE:
+        UpdateIterator(entry_cell, limit_time, no_use);
+        header_->num_no_use_entries++;
+        break;
+      case ENTRY_LOW_USE:
+        UpdateIterator(entry_cell, limit_time, low_use);
+        header_->num_low_use_entries++;
+        break;
+      case ENTRY_HIGH_USE:
+        UpdateIterator(entry_cell, limit_time, high_use);
+        header_->num_high_use_entries++;
+        break;
+      case ENTRY_EVICTED:
+        header_->num_evicted_entries++;
+        break;
+      default:
+        NOTREACHED();
+    }
+  }
+}
+
+// This code is only called from Init() so the internal state of this object is
+// in flux (this method is performing the last steps of re-initialization). As
+// such, random methods are not supposed to work at this point, so whatever this
+// method calls should be relatively well controlled and it may require some
+// degree of "stable state faking".
+void IndexTable::MoveCells(IndexBucket* old_extra_table) {
+  int max_hash = (mask_ + 1) / 2;
+  int max_bucket = header()->max_bucket;
+  header()->max_bucket = mask_;
+  int used_cells = header()->used_cells;
+
+  // Consider a large cache: a cell stores the upper 18 bits of the hash
+  // (h >> 14). If the table is say 8 times the original size (growing from 4x),
+  // the bit that we are interested in would be the 3rd bit of the stored value,
+  // in other words 'multiplier' >> 1.
+  uint32 new_bit = (1 << extra_bits_) >> 1;
+
+  scoped_ptr<IndexBucket[]> old_main_table;
+  IndexBucket* source_table = main_table_;
+  bool upgrade_format = !extra_bits_;
+  if (upgrade_format) {
+    // This method should deal with migrating a small table to a big one. Given
+    // that the first thing to do is read the old table, set small_table_ for
+    // the size of the old table. Now, when moving a cell, the result cannot be
+    // placed in the old table or we will end up reading it again and attempting
+    // to move it, so we have to copy the whole table at once.
+    DCHECK(!small_table_);
+    small_table_ = true;
+    old_main_table.reset(new IndexBucket[max_hash]);
+    memcpy(old_main_table.get(), main_table_, max_hash * sizeof(IndexBucket));
+    memset(main_table_, 0, max_hash * sizeof(IndexBucket));
+    source_table = old_main_table.get();
+  }
+
+  for (int i = 0; i < max_hash; i++) {
+    int bucket_num = i;
+    IndexBucket* bucket = &source_table[i];
+    do {
+      for (int j = 0; j < kCellsPerBucket; j++) {
+        IndexCell& current_cell = bucket->cells[j];
+        if (!GetLocation(current_cell))
+          continue;
+        DCHECK(SanityCheck(current_cell));
+        if (bucket_num == i) {
+          if (upgrade_format || (GetHashValue(current_cell) & new_bit)) {
+            // Move this cell to the upper half of the table.
+            MoveSingleCell(&current_cell, bucket_num * kCellsPerBucket + j, i,
+                           true);
+          }
+        } else {
+          // All cells on extra buckets have to move.
+          MoveSingleCell(&current_cell, bucket_num * kCellsPerBucket + j, i,
+                         true);
+        }
+      }
+
+      // There is no need to clear the old bucket->next value because if falls
+      // within the main table so it will be fixed when attempting to follow
+      // the link.
+      bucket_num = GetNextBucket(max_hash, max_bucket, old_extra_table,
+                                 &bucket);
+    } while (bucket_num);
+  }
+
+  DCHECK_EQ(header()->used_cells, used_cells);
+
+  if (upgrade_format) {
+    small_table_ = false;
+    header()->flags &= ~SMALL_CACHE;
+  }
+}
+
+void IndexTable::MoveSingleCell(IndexCell* current_cell, int cell_num,
+                                int main_table_index, bool growing) {
+  uint32 hash = GetFullHash(*current_cell, main_table_index);
+  EntryCell old_cell(cell_num, hash, *current_cell, small_table_);
+
+  // This method may be called when moving entries from a small table to a
+  // normal table. In that case, the caller (MoveCells) has to read the old
+  // table, so it needs small_table_ set to true, but this method needs to
+  // write to the new table so small_table_ has to be set to false, and the
+  // value restored to true before returning.
+  bool upgrade_format = !extra_bits_ && growing;
+  if (upgrade_format)
+    small_table_ = false;
+  EntryCell new_cell = CreateEntryCell(hash, old_cell.GetAddress());
+
+  if (!new_cell.IsValid()) {
+    // We'll deal with this entry later.
+    if (upgrade_format)
+      small_table_ = true;
+    return;
+  }
+
+  new_cell.SetState(old_cell.GetState());
+  new_cell.SetGroup(old_cell.GetGroup());
+  new_cell.SetReuse(old_cell.GetReuse());
+  new_cell.SetTimestamp(old_cell.GetTimestamp());
+  Save(&new_cell);
+  modified_ = true;
+  if (upgrade_format)
+    small_table_ = true;
+
+  if (old_cell.GetState() == ENTRY_DELETED) {
+    bitmap_->Set(new_cell.cell_num(), false);
+    backup_bitmap_->Set(new_cell.cell_num(), false);
+  }
+
+  if (!growing || cell_num / kCellsPerBucket == main_table_index) {
+    // Only delete entries that live on the main table.
+    if (!upgrade_format) {
+      old_cell.Clear();
+      Write(old_cell);
+    }
+
+    if (cell_num != new_cell.cell_num()) {
+      bitmap_->Set(old_cell.cell_num(), false);
+      backup_bitmap_->Set(old_cell.cell_num(), false);
+    }
+  }
+  header()->used_cells--;
+}
+
+void IndexTable::HandleMisplacedCell(IndexCell* current_cell, int cell_num,
+                                     int main_table_index) {
+  NOTREACHED();  // No unit tests yet.
+
+  // The cell may be misplaced, or a duplicate cell exists with this data.
+  uint32 hash = GetFullHash(*current_cell, main_table_index);
+  MoveSingleCell(current_cell, cell_num, main_table_index, false);
+
+  // Now look for a duplicate cell.
+  CheckBucketList(hash & mask_);
+}
+
+void IndexTable::CheckBucketList(int bucket_num) {
+  typedef std::pair<int, EntryGroup> AddressAndGroup;
+  std::set<AddressAndGroup> entries;
+  IndexBucket* bucket = &main_table_[bucket_num];
+  int bucket_hash = bucket_num;
+  do {
+    for (int i = 0; i < kCellsPerBucket; i++) {
+      IndexCell* current_cell = &bucket->cells[i];
+      if (!GetLocation(*current_cell))
+        continue;
+      if (!SanityCheck(*current_cell)) {
+        NOTREACHED();
+        current_cell->Clear();
+        continue;
+      }
+      int cell_num = bucket_num * kCellsPerBucket + i;
+      EntryCell cell(cell_num, GetFullHash(*current_cell, bucket_hash),
+                     *current_cell, small_table_);
+      if (!entries.insert(std::make_pair(cell.GetAddress().value(),
+                                         cell.GetGroup())).second) {
+        current_cell->Clear();
+        continue;
+      }
+      CheckState(cell);
+    }
+
+    bucket_num = GetNextBucket(mask_ + 1, header()->max_bucket, extra_table_,
+                              &bucket);
+  } while (bucket_num);
+}
+
+uint32 IndexTable::GetLocation(const IndexCell& cell) {
+  if (small_table_)
+    return GetCellSmallTableLocation(cell);
+
+  return GetCellLocation(cell);
+}
+
+uint32 IndexTable::GetHashValue(const IndexCell& cell) {
+  if (small_table_)
+    return GetCellSmallTableId(cell);
+
+  return GetCellId(cell);
+}
+
+uint32 IndexTable::GetFullHash(const IndexCell& cell, uint32 lower_part) {
+  // It is OK for the high order bits of lower_part to overlap with the stored
+  // part of the hash.
+  if (small_table_)
+    return (GetCellSmallTableId(cell) << kSmallTableHashShift) | lower_part;
+
+  return (GetCellId(cell) << kHashShift) | lower_part;
+}
+
+// All the bits stored in the cell should match the provided hash.
+bool IndexTable::IsHashMatch(const IndexCell& cell, uint32 hash) {
+  hash = small_table_ ? hash >> kSmallTableHashShift : hash >> kHashShift;
+  return GetHashValue(cell) == hash;
+}
+
+bool IndexTable::MisplacedHash(const IndexCell& cell, uint32 hash) {
+  if (!extra_bits_)
+    return false;
+
+  uint32 mask = (1 << extra_bits_) - 1;
+  hash = small_table_ ? hash >> kSmallTableHashShift : hash >> kHashShift;
+  return (GetHashValue(cell) & mask) != (hash & mask);
+}
+
+}  // namespace disk_cache

net/disk_cache/v3/index_table.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_DISK_CACHE_V3_INDEX_TABLE_H_
+#define NET_DISK_CACHE_V3_INDEX_TABLE_H_
+
+// The IndexTable class is in charge of handling all the details about the main
+// index table of the cache. It provides methods to locate entries in the cache,
+// create new entries and modify existing entries. It hides the fact that the
+// table is backed up across multiple physical files, and that the files can
+// grow and be remapped while the cache is in use. However, note that this class
+// doesn't do any direct management of the backing files, and it operates only
+// with the tables in memory.
+//
+// When the current index needs to grow, the backend is notified so that files
+// are extended and remapped as needed. After that, the IndexTable should be
+// re-initialized with the new structures. Note that the IndexTable instance is
+// still functional while the backend performs file IO.
+
+#include <vector>
+
+#include "base/basictypes.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/time/time.h"
+#include "net/base/net_export.h"
+#include "net/disk_cache/addr.h"
+#include "net/disk_cache/bitmap.h"
+#include "net/disk_cache/v3/disk_format_v3.h"
+
+namespace net {
+class IOBuffer;
+}
+
+namespace disk_cache {
+
+class BackendImplV3;
+struct InitResult;
+
+// An EntryCell represents a single entity stored by the index table. Users are
+// expected to handle and store EntryCells on their own to track operations that
+// they are performing with a given entity, as opposed to deal with pointers to
+// individual positions on the table, given that the whole table can be moved to
+// another place, and that would invalidate any pointers to individual cells in
+// the table.
+// However, note that it is also possible for an entity to be moved from one
+// position to another, so an EntryCell may be invalid by the time a long
+// operation completes. In that case, the caller should consult the table again
+// using FindEntryCell().
+class NET_EXPORT_PRIVATE EntryCell {
+ public:
+  ~EntryCell();
+
+  bool IsValid() const;
+
+  int32 cell_num() const { return cell_num_; }
+  uint32 hash() const { return hash_; }
+
+  Addr GetAddress() const;
+  EntryState GetState() const;
+  EntryGroup GetGroup() const;
+  int GetReuse() const;
+  int GetTimestamp() const;
+
+  void SetState(EntryState state);
+  void SetGroup(EntryGroup group);
+  void SetReuse(int count);
+  void SetTimestamp(int timestamp);
+
+  static EntryCell GetEntryCellForTest(int32 cell_num,
+                                       uint32 hash,
+                                       Addr address,
+                                       IndexCell* cell,
+                                       bool small_table);
+  void SerializaForTest(IndexCell* destination);
+
+ private:
+  friend class IndexTable;
+  friend class CacheDumperHelper;
+
+  EntryCell();
+  EntryCell(int32 cell_num, uint32 hash, Addr address, bool small_table);
+  EntryCell(int32 cell_num,
+            uint32 hash,
+            const IndexCell& cell,
+            bool small_table);
+
+  void Clear() { cell_.Clear(); }
+  void FixSum();
+
+  // Returns the raw value stored on the index table.
+  uint32 GetLocation() const;
+
+  // Recalculates hash_ assuming that only the low order bits are valid and the
+  // rest come from cell_.
+  uint32 RecomputeHash();
+
+  void Serialize(IndexCell* destination) const;
+
+  int32 cell_num_;
+  uint32 hash_;
+  IndexCell cell_;
+  bool small_table_;
+};
+
+// Keeps a collection of EntryCells in order to be processed.
+struct NET_EXPORT_PRIVATE EntrySet {
+  EntrySet();
+  ~EntrySet();
+
+  int evicted_count;  // The numebr of evicted entries in this set.
+  size_t current;     // The number of the cell that is being processed.
+  std::vector<EntryCell> cells;
+};
+
+// A given entity referenced by the index table is uniquely identified by the
+// combination of hash and address.
+struct CellInfo { uint32 hash; Addr address; };
+typedef std::vector<CellInfo> CellList;
+
+// An index iterator is used to get a group of cells that share the same
+// timestamp. When this structure is passed to GetNextCells(), the caller sets
+// the initial timestamp and direction; whet it is used with GetOldest, the
+// initial values are ignored.
+struct NET_EXPORT_PRIVATE IndexIterator {
+  IndexIterator();
+  ~IndexIterator();
+
+  CellList cells;
+  int timestamp;  // The current low resolution timestamp for |cells|.
+  bool forward;   // The direction of the iteration, in time.
+};
+
+// Methods that the backend has to implement to support the table. Note that the
+// backend is expected to own all IndexTable instances, so it is expected to
+// outlive the table.
+class NET_EXPORT_PRIVATE IndexTableBackend {
+ public:
+  virtual ~IndexTableBackend() {}
+
+  // The index has to grow.
+  virtual void GrowIndex() = 0;
+
+  // Save the index to the backup file.
+  virtual void SaveIndex(net::IOBuffer* buffer, int buffer_len) = 0;
+
+  // Deletes or fixes an invalid cell from the backend.
+  virtual void DeleteCell(EntryCell cell) = 0;
+  virtual void FixCell(EntryCell cell) = 0;
+};
+
+// The data required to initialize an index. Note that not all fields have to
+// be provided when growing the tables.
+struct NET_EXPORT_PRIVATE IndexTableInitData {
+  IndexTableInitData();
+  ~IndexTableInitData();
+
+  IndexBitmap* index_bitmap;
+  IndexBucket* main_table;
+  IndexBucket* extra_table;
+  scoped_ptr<IndexHeaderV3> backup_header;
+  scoped_ptr<uint32[]> backup_bitmap;
+};
+
+// See the description at the top of this file.
+class NET_EXPORT_PRIVATE IndexTable {
+ public:
+  explicit IndexTable(IndexTableBackend* backend);
+  ~IndexTable();
+
+  // Initializes the object, or re-initializes it when the backing files grow.
+  // Note that the only supported way to initialize this objeect is using
+  // pointers that come from the files being directly mapped in memory. If that
+  // is not the case, it must be emulated in a convincing way, for example
+  // making sure that the tables for re-init look the same as the tables to be
+  // replaced.
+  void Init(IndexTableInitData* params);
+
+  // Releases the resources acquired during Init().
+  void Shutdown();
+
+  // Locates a resouce on the index. Returns a list of all resources that match
+  // the provided hash.
+  EntrySet LookupEntries(uint32 hash);
+
+  // Creates a new cell to store a new resource.
+  EntryCell CreateEntryCell(uint32 hash, Addr address);
+
+  // Locates a particular cell. This method allows a caller to perform slow
+  // operations with some entries while the index evolves, by returning the
+  // current state of a cell. If the desired cell cannot be located, the return
+  // object will be invalid.
+  EntryCell FindEntryCell(uint32 hash, Addr address);
+
+  // Returns an IndexTable timestamp for a given absolute time. The actual
+  // resolution of the timestamp should be considered an implementation detail,
+  // but it certainly is lower than seconds. The important part is that a group
+  // of cells will share the same timestamp (see IndexIterator).
+  int CalculateTimestamp(base::Time time);
+
+  // Returns the equivalent time for a cell timestamp.
+  base::Time TimeFromTimestamp(int timestamp);
+
+  // Updates a particular cell.
+  void SetSate(uint32 hash, Addr address, EntryState state);
+  void UpdateTime(uint32 hash, Addr address, base::Time current);
+
+  // Saves the contents of |cell| to the table.
+  void Save(EntryCell* cell);
+
+  // Returns the oldest entries for each group of entries. The initial values
+  // for the provided iterators are ignored. Entries are assigned to iterators
+  // according to their EntryGroup.
+  void GetOldest(IndexIterator* no_use,
+                 IndexIterator* low_use,
+                 IndexIterator* high_use);
+
+  // Returns the next group of entries for the provided iterator. This method
+  // does not return the cells matching the initial iterator's timestamp,
+  // but rather cells after (or before, depending on the iterator's |forward|
+  // member) that timestamp.
+  bool GetNextCells(IndexIterator* iterator);
+
+  // Called each time the index should save the backup information. The caller
+  // can assume that anything that needs to be saved is saved when this method
+  // is called, and that there is only one source of timming information, and
+  // that source is controlled by the owner of this object.
+  void OnBackupTimer();
+
+  IndexHeaderV3* header() { return header_; }
+  const IndexHeaderV3* header() const { return header_; }
+
+ private:
+  EntryCell FindEntryCellImpl(uint32 hash, Addr address, bool allow_deleted);
+  void CheckState(const EntryCell& cell);
+  void Write(const EntryCell& cell);
+  int NewExtraBucket();
+  void WalkTables(int limit_time,
+                  IndexIterator* no_use,
+                  IndexIterator* low_use,
+                  IndexIterator* high_use);
+  void UpdateFromBucket(IndexBucket* bucket, int bucket_hash,
+                        int limit_time,
+                        IndexIterator* no_use,
+                        IndexIterator* low_use,
+                        IndexIterator* high_use);
+  void MoveCells(IndexBucket* old_extra_table);
+  void MoveSingleCell(IndexCell* current_cell, int cell_num,
+                      int main_table_index, bool growing);
+  void HandleMisplacedCell(IndexCell* current_cell, int cell_num,
+                           int main_table_index);
+  void CheckBucketList(int bucket_id);
+
+  uint32 GetLocation(const IndexCell& cell);
+  uint32 GetHashValue(const IndexCell& cell);
+  uint32 GetFullHash(const IndexCell& cell, uint32 lower_part);
+  bool IsHashMatch(const IndexCell& cell, uint32 hash);
+  bool MisplacedHash(const IndexCell& cell, uint32 hash);
+
+  IndexTableBackend* backend_;
+  IndexHeaderV3* header_;
+  scoped_ptr<Bitmap> bitmap_;
+  scoped_ptr<Bitmap> backup_bitmap_;
+  scoped_ptr<uint32[]> backup_bitmap_storage_;
+  scoped_ptr<IndexHeaderV3> backup_header_;
+  IndexBucket* main_table_;
+  IndexBucket* extra_table_;
+  uint32 mask_;     // Binary mask to map a hash to the hash table.
+  int extra_bits_;  // How many bits are in mask_ above the default value.
+  bool modified_;
+  bool small_table_;
+
+  DISALLOW_COPY_AND_ASSIGN(IndexTable);
+};
+
+}  // namespace disk_cache
+
+#endif  // NET_DISK_CACHE_V3_INDEX_TABLE_H_

net/disk_cache/v3/index_table_unittest.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/basictypes.h"
+#include "base/logging.h"
+#include "net/disk_cache/addr.h"
+#include "net/disk_cache/v3/disk_format_v3.h"
+#include "net/disk_cache/v3/index_table.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using disk_cache::EntryCell;
+using disk_cache::IndexCell;
+using disk_cache::IndexTable;
+using disk_cache::IndexTableInitData;
+
+namespace {
+
+int GetChecksum(const IndexCell& source) {
+  // Only the cell pointer is relevant.
+  disk_cache::Addr addr;
+  IndexCell* cell = const_cast<IndexCell*>(&source);
+  EntryCell entry = EntryCell::GetEntryCellForTest(0, 0, addr, cell, false);
+
+  IndexCell result;
+  entry.SerializaForTest(&result);
+  return result.last_part >> 6;
+}
+
+class MockIndexBackend : public disk_cache::IndexTableBackend {
+ public:
+  MockIndexBackend() : grow_called_(false), buffer_len_(-1) {}
+  virtual ~MockIndexBackend() {}
+
+  bool grow_called() const { return grow_called_; }
+  int buffer_len() const { return buffer_len_; }
+
+  virtual void GrowIndex() OVERRIDE { grow_called_ = true; }
+  virtual void SaveIndex(net::IOBuffer* buffer, int buffer_len) OVERRIDE {
+    buffer_len_ = buffer_len;
+  }
+  virtual void DeleteCell(EntryCell cell) OVERRIDE {}
+  virtual void FixCell(EntryCell cell) OVERRIDE {}
+
+ private:
+  bool grow_called_;
+  int buffer_len_;
+};
+
+class TestCacheTables {
+ public:
+  // |num_entries| is the capacity of the main table. The extra table is half
+  // the size of the main table.
+  explicit TestCacheTables(int num_entries);
+  ~TestCacheTables() {}
+
+  void GetInitData(IndexTableInitData* result);
+  void CopyFrom(const TestCacheTables& other);
+  base::Time start_time() const { return start_time_; }
+
+ private:
+  scoped_ptr<uint64[]> main_bitmap_;
+  scoped_ptr<disk_cache::IndexBucket[]> main_table_;
+  scoped_ptr<disk_cache::IndexBucket[]> extra_table_;
+  base::Time start_time_;
+  int num_bitmap_bytes_;
+
+  DISALLOW_COPY_AND_ASSIGN(TestCacheTables);
+};
+
+TestCacheTables::TestCacheTables(int num_entries) {
+  DCHECK_GE(num_entries, 1024);
+  DCHECK_EQ(num_entries, num_entries / 1024 * 1024);
+  main_table_.reset(new disk_cache::IndexBucket[num_entries]);
+  extra_table_.reset(new disk_cache::IndexBucket[num_entries / 2]);
+  memset(main_table_.get(), 0, num_entries * sizeof(*main_table_.get()));
+  memset(extra_table_.get(), 0, num_entries / 2 * sizeof(*extra_table_.get()));
+
+  // We allow IndexBitmap smaller than a page because the code should not really
+  // depend on that.
+  num_bitmap_bytes_ = (num_entries + num_entries / 2) / 8;
+  size_t required_size = sizeof(disk_cache::IndexHeaderV3) + num_bitmap_bytes_;
+  main_bitmap_.reset(new uint64[required_size / sizeof(uint64)]);
+  memset(main_bitmap_.get(), 0, required_size);
+
+  disk_cache::IndexHeaderV3* header =
+      reinterpret_cast<disk_cache::IndexHeaderV3*>(main_bitmap_.get());
+
+  header->magic = disk_cache::kIndexMagicV3;
+  header->version = disk_cache::kVersion3;
+  header->table_len = num_entries + num_entries / 2;
+  header->max_bucket = num_entries / 4 - 1;
+
+  start_time_ = base::Time::Now();
+  header->create_time = start_time_.ToInternalValue();
+  header->base_time =
+      (start_time_ - base::TimeDelta::FromDays(20)).ToInternalValue();
+
+  if (num_entries < 64 * 1024)
+    header->flags = disk_cache::SMALL_CACHE;
+}
+
+void TestCacheTables::GetInitData(IndexTableInitData* result) {
+  result->index_bitmap =
+      reinterpret_cast<disk_cache::IndexBitmap*>(main_bitmap_.get());
+
+  result->main_table = main_table_.get();
+  result->extra_table = extra_table_.get();
+
+  result->backup_header.reset(new disk_cache::IndexHeaderV3);
+  memcpy(result->backup_header.get(), result->index_bitmap,
+         sizeof(result->index_bitmap->header));
+
+  result->backup_bitmap.reset(new uint32[num_bitmap_bytes_ / sizeof(uint32)]);
+  memcpy(result->backup_bitmap.get(), result->index_bitmap->bitmap,
+         num_bitmap_bytes_);
+}
+
+void TestCacheTables::CopyFrom(const TestCacheTables& other) {
+  disk_cache::IndexBitmap* this_bitmap =
+    reinterpret_cast<disk_cache::IndexBitmap*>(main_bitmap_.get());
+  disk_cache::IndexBitmap* other_bitmap =
+    reinterpret_cast<disk_cache::IndexBitmap*>(other.main_bitmap_.get());
+
+  DCHECK_GE(this_bitmap->header.table_len, other_bitmap->header.table_len);
+  DCHECK_GE(num_bitmap_bytes_, other.num_bitmap_bytes_);
+
+  memcpy(this_bitmap->bitmap, other_bitmap->bitmap, other.num_bitmap_bytes_);
+
+  int main_table_buckets = (other_bitmap->header.table_len * 2 / 3) / 4;
+  int extra_table_buckets = (other_bitmap->header.table_len * 1 / 3) / 4;
+  memcpy(main_table_.get(), other.main_table_.get(),
+         main_table_buckets * sizeof(disk_cache::IndexBucket));
+  memcpy(extra_table_.get(), other.extra_table_.get(),
+         extra_table_buckets * sizeof(disk_cache::IndexBucket));
+
+  this_bitmap->header.num_entries = other_bitmap->header.num_entries;
+  this_bitmap->header.used_cells = other_bitmap->header.used_cells;
+  this_bitmap->header.max_bucket = other_bitmap->header.max_bucket;
+  this_bitmap->header.create_time = other_bitmap->header.create_time;
+  this_bitmap->header.base_time = other_bitmap->header.base_time;
+  this_bitmap->header.flags = other_bitmap->header.flags;
+  start_time_ = other.start_time_;
+}
+
+}  // namespace
+
+TEST(DiskCacheIndexTable, EntryCell) {
+  uint32 hash = 0x55aa6699;
+  disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, 0x4531);
+  bool small_table = true;
+  int cell_num = 88;
+  int reuse = 6;
+  int timestamp = 123456;
+  disk_cache::EntryState state = disk_cache::ENTRY_MODIFIED;
+  disk_cache::EntryGroup group = disk_cache::ENTRY_HIGH_USE;
+
+  for (int i = 0; i < 4; i++) {
+    SCOPED_TRACE(i);
+    EntryCell entry = EntryCell::GetEntryCellForTest(cell_num, hash, addr, NULL,
+                                                     small_table);
+    EXPECT_EQ(disk_cache::ENTRY_NO_USE, entry.GetGroup());
+    EXPECT_EQ(disk_cache::ENTRY_NEW, entry.GetState());
+
+    entry.SetGroup(group);
+    entry.SetState(state);
+    entry.SetReuse(reuse);
+    entry.SetTimestamp(timestamp);
+
+    EXPECT_TRUE(entry.IsValid());
+    EXPECT_EQ(hash, entry.hash());
+    EXPECT_EQ(cell_num, entry.cell_num());
+    EXPECT_EQ(addr.value(), entry.GetAddress().value());
+
+    EXPECT_EQ(group, entry.GetGroup());
+    EXPECT_EQ(state, entry.GetState());
+    EXPECT_EQ(reuse, entry.GetReuse());
+    EXPECT_EQ(timestamp, entry.GetTimestamp());
+
+    // Store the data and read it again.
+    IndexCell cell;
+    entry.SerializaForTest(&cell);
+
+    EntryCell entry2 = EntryCell::GetEntryCellForTest(cell_num, hash, addr,
+                                                      &cell, small_table);
+
+    EXPECT_EQ(addr.value(), entry2.GetAddress().value());
+
+    EXPECT_EQ(group, entry2.GetGroup());
+    EXPECT_EQ(state, entry2.GetState());
+    EXPECT_EQ(reuse, entry2.GetReuse());
+    EXPECT_EQ(timestamp, entry2.GetTimestamp());
+
+    small_table = !small_table;
+    if (i == 1) {
+      hash = ~hash;
+      cell_num *= 5;
+      state = disk_cache::ENTRY_USED;
+      group = disk_cache::ENTRY_EVICTED;
+      addr = disk_cache::Addr(disk_cache::BLOCK_EVICTED, 1, 6, 0x18a5);
+      reuse = 15;  // 4 bits
+      timestamp = 0xfffff;  // 20 bits.
+    }
+  }
+}
+
+// Goes over some significant values for a cell's sum.
+TEST(DiskCacheIndexTable, EntryCellSum) {
+  IndexCell source;
+  source.Clear();
+  EXPECT_EQ(0, GetChecksum(source));
+
+  source.first_part++;
+  EXPECT_EQ(1, GetChecksum(source));
+
+  source.Clear();
+  source.last_part = 0x80;
+  EXPECT_EQ(0, GetChecksum(source));
+
+  source.last_part = 0x55;
+  EXPECT_EQ(3, GetChecksum(source));
+
+  source.first_part = 0x555555;
+  EXPECT_EQ(2, GetChecksum(source));
+
+  source.last_part = 0;
+  EXPECT_EQ(1, GetChecksum(source));
+
+  source.first_part = GG_UINT64_C(0x8000000080000000);
+  EXPECT_EQ(0, GetChecksum(source));
+
+  source.first_part = GG_UINT64_C(0x4000000040000000);
+  EXPECT_EQ(2, GetChecksum(source));
+
+  source.first_part = GG_UINT64_C(0x200000020000000);
+  EXPECT_EQ(1, GetChecksum(source));
+
+  source.first_part = GG_UINT64_C(0x100000010010000);
+  EXPECT_EQ(3, GetChecksum(source));
+
+  source.first_part = 0x80008000;
+  EXPECT_EQ(0, GetChecksum(source));
+
+  source.first_part = GG_UINT64_C(0x800000008000);
+  EXPECT_EQ(1, GetChecksum(source));
+
+  source.first_part = 0x8080;
+  EXPECT_EQ(0, GetChecksum(source));
+
+  source.first_part = 0x800080;
+  EXPECT_EQ(1, GetChecksum(source));
+
+  source.first_part = 0x88;
+  EXPECT_EQ(0, GetChecksum(source));
+
+  source.first_part = 0x808;
+  EXPECT_EQ(1, GetChecksum(source));
+
+  source.first_part = 0xA;
+  EXPECT_EQ(0, GetChecksum(source));
+
+  source.first_part = 0x22;
+  EXPECT_EQ(1, GetChecksum(source));
+}
+
+TEST(DiskCacheIndexTable, Basics) {
+  TestCacheTables cache(1024);
+  IndexTableInitData init_data;
+  cache.GetInitData(&init_data);
+
+  IndexTable index(NULL);
+  index.Init(&init_data);
+
+  // Write some entries.
+  disk_cache::CellList entries;
+  for (int i = 0; i < 250; i++) {
+    SCOPED_TRACE(i);
+    uint32 hash = i * i * 1111 + i * 11;
+    disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, i * 13 + 1);
+    EntryCell entry = index.CreateEntryCell(hash, addr);
+    EXPECT_TRUE(entry.IsValid());
+
+    disk_cache::CellInfo info = { hash, addr };
+    entries.push_back(info);
+  }
+
+  // Read them back.
+  for (size_t i = 0; i < entries.size(); i++) {
+    SCOPED_TRACE(i);
+    uint32 hash = entries[i].hash;
+    disk_cache::Addr addr = entries[i].address;
+
+    disk_cache::EntrySet found_entries = index.LookupEntries(hash);
+    ASSERT_EQ(1u, found_entries.cells.size());
+    EXPECT_TRUE(found_entries.cells[0].IsValid());
+    EXPECT_EQ(hash, found_entries.cells[0].hash());
+    EXPECT_EQ(addr.value(), found_entries.cells[0].GetAddress().value());
+
+    EntryCell entry = index.FindEntryCell(hash, addr);
+    EXPECT_TRUE(entry.IsValid());
+    EXPECT_EQ(hash, entry.hash());
+    EXPECT_EQ(addr.value(), entry.GetAddress().value());
+
+    // Delete the first 100 entries.
+    if (i < 100)
+      index.SetSate(hash, addr, disk_cache::ENTRY_DELETED);
+  }
+
+  // See what we have now.
+  for (size_t i = 0; i < entries.size(); i++) {
+    SCOPED_TRACE(i);
+    uint32 hash = entries[i].hash;
+    disk_cache::Addr addr = entries[i].address;
+
+    disk_cache::EntrySet found_entries = index.LookupEntries(hash);
+    if (i < 100) {
+      EXPECT_EQ(0u, found_entries.cells.size());
+    } else {
+      ASSERT_EQ(1u, found_entries.cells.size());
+      EXPECT_TRUE(found_entries.cells[0].IsValid());
+      EXPECT_EQ(hash, found_entries.cells[0].hash());
+      EXPECT_EQ(addr.value(), found_entries.cells[0].GetAddress().value());
+    }
+  }
+}
+
+// Tests handling of multiple entries with the same hash.
+TEST(DiskCacheIndexTable, SameHash) {
+  TestCacheTables cache(1024);
+  IndexTableInitData init_data;
+  cache.GetInitData(&init_data);
+
+  IndexTable index(NULL);
+  index.Init(&init_data);
+
+  disk_cache::CellList entries;
+  uint32 hash = 0x55aa55bb;
+  for (int i = 0; i < 6; i++) {
+    SCOPED_TRACE(i);
+    disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, i * 13 + 1);
+    EntryCell entry = index.CreateEntryCell(hash, addr);
+    EXPECT_TRUE(entry.IsValid());
+
+    disk_cache::CellInfo info = { hash, addr };
+    entries.push_back(info);
+  }
+
+  disk_cache::EntrySet found_entries = index.LookupEntries(hash);
+  EXPECT_EQ(0, found_entries.evicted_count);
+  ASSERT_EQ(6u, found_entries.cells.size());
+
+  for (size_t i = 0; i < found_entries.cells.size(); i++) {
+    SCOPED_TRACE(i);
+    EXPECT_EQ(entries[i].address, found_entries.cells[i].GetAddress());
+  }
+
+  // Now verify handling of entries on different states.
+  index.SetSate(hash, entries[0].address, disk_cache::ENTRY_DELETED);
+  index.SetSate(hash, entries[1].address, disk_cache::ENTRY_DELETED);
+  index.SetSate(hash, entries[2].address, disk_cache::ENTRY_USED);
+  index.SetSate(hash, entries[3].address, disk_cache::ENTRY_USED);
+  index.SetSate(hash, entries[4].address, disk_cache::ENTRY_USED);
+
+  found_entries = index.LookupEntries(hash);
+  EXPECT_EQ(0, found_entries.evicted_count);
+  ASSERT_EQ(4u, found_entries.cells.size());
+
+  index.SetSate(hash, entries[3].address, disk_cache::ENTRY_OPEN);
+  index.SetSate(hash, entries[4].address, disk_cache::ENTRY_OPEN);
+
+  found_entries = index.LookupEntries(hash);
+  EXPECT_EQ(0, found_entries.evicted_count);
+  ASSERT_EQ(4u, found_entries.cells.size());
+
+  index.SetSate(hash, entries[4].address, disk_cache::ENTRY_MODIFIED);
+
+  found_entries = index.LookupEntries(hash);
+  EXPECT_EQ(0, found_entries.evicted_count);
+  ASSERT_EQ(4u, found_entries.cells.size());
+
+  index.SetSate(hash, entries[1].address, disk_cache::ENTRY_FREE);
+
+  found_entries = index.LookupEntries(hash);
+  EXPECT_EQ(0, found_entries.evicted_count);
+  ASSERT_EQ(4u, found_entries.cells.size());
+
+  // FindEntryCell should not see deleted entries.
+  EntryCell entry = index.FindEntryCell(hash, entries[0].address);
+  EXPECT_FALSE(entry.IsValid());
+
+  // A free entry is gone.
+  entry = index.FindEntryCell(hash, entries[1].address);
+  EXPECT_FALSE(entry.IsValid());
+
+  // Locate a used entry, and evict it. This is not really a correct operation
+  // in that an existing cell doesn't transition to evicted; instead a new cell
+  // for the evicted entry (on a different block file) should be created. Still,
+  // at least evicted_count would be valid.
+  entry = index.FindEntryCell(hash, entries[2].address);
+  EXPECT_TRUE(entry.IsValid());
+  entry.SetGroup(disk_cache::ENTRY_EVICTED);
+  index.Save(&entry);
+
+  found_entries = index.LookupEntries(hash);
+  EXPECT_EQ(1, found_entries.evicted_count);
+  ASSERT_EQ(4u, found_entries.cells.size());
+
+  // Now use the proper way to get an evicted entry.
+  disk_cache::Addr addr2(disk_cache::BLOCK_EVICTED, 1, 6, 6);  // Any address.
+  entry = index.CreateEntryCell(hash, addr2);
+  EXPECT_TRUE(entry.IsValid());
+  EXPECT_EQ(disk_cache::ENTRY_EVICTED, entry.GetGroup());
+
+  found_entries = index.LookupEntries(hash);
+  EXPECT_EQ(2, found_entries.evicted_count);
+  ASSERT_EQ(5u, found_entries.cells.size());
+}
+
+TEST(DiskCacheIndexTable, Timestamps) {
+  TestCacheTables cache(1024);
+  IndexTableInitData init_data;
+  cache.GetInitData(&init_data);
+
+  IndexTable index(NULL);
+  index.Init(&init_data);
+
+  // The granularity should be 1 minute.
+  int timestamp1 = index.CalculateTimestamp(cache.start_time());
+  int timestamp2 = index.CalculateTimestamp(cache.start_time() +
+                                            base::TimeDelta::FromSeconds(59));
+  EXPECT_EQ(timestamp1, timestamp2);
+
+  int timestamp3 = index.CalculateTimestamp(cache.start_time() +
+                                            base::TimeDelta::FromSeconds(61));
+  EXPECT_EQ(timestamp1 + 1, timestamp3);
+
+  int timestamp4 = index.CalculateTimestamp(cache.start_time() +
+                                            base::TimeDelta::FromSeconds(119));
+  EXPECT_EQ(timestamp1 + 1, timestamp4);
+
+  int timestamp5 = index.CalculateTimestamp(cache.start_time() +
+                                            base::TimeDelta::FromSeconds(121));
+  EXPECT_EQ(timestamp1 + 2, timestamp5);
+
+  int timestamp6 = index.CalculateTimestamp(cache.start_time() -
+                                            base::TimeDelta::FromSeconds(30));
+  EXPECT_EQ(timestamp1 - 1, timestamp6);
+
+  // The base should be 20 days in the past.
+  int timestamp7 = index.CalculateTimestamp(cache.start_time() -
+                                            base::TimeDelta::FromDays(20));
+  int timestamp8 = index.CalculateTimestamp(cache.start_time() -
+                                            base::TimeDelta::FromDays(35));
+  EXPECT_EQ(timestamp7, timestamp8);
+  EXPECT_EQ(0, timestamp8);
+
+  int timestamp9 = index.CalculateTimestamp(cache.start_time() -
+                                            base::TimeDelta::FromDays(19));
+  EXPECT_NE(0, timestamp9);
+}
+
+// Tests GetOldest and GetNextCells.
+TEST(DiskCacheIndexTable, Iterations) {
+  TestCacheTables cache(1024);
+  IndexTableInitData init_data;
+  cache.GetInitData(&init_data);
+
+  IndexTable index(NULL);
+  index.Init(&init_data);
+
+  base::Time time = cache.start_time();
+
+  // Write some entries.
+  disk_cache::CellList entries;
+  for (int i = 0; i < 44; i++) {
+    SCOPED_TRACE(i);
+    uint32 hash = i;  // The entries will be ordered on the table.
+    disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, i * 13 + 1);
+    if (i < 10 || i == 40)
+      addr = disk_cache::Addr(disk_cache::BLOCK_EVICTED, 1, 6, i * 13 + 1);
+
+    EntryCell entry = index.CreateEntryCell(hash, addr);
+    EXPECT_TRUE(entry.IsValid());
+
+    disk_cache::CellInfo info = { hash, addr };
+    entries.push_back(info);
+
+    if (i < 10 || i == 40) {
+      // Do nothing. These are ENTRY_EVICTED by default.
+    } else if (i < 20 || i == 41) {
+      entry.SetGroup(disk_cache::ENTRY_HIGH_USE);
+      index.Save(&entry);
+    } else if (i < 30 || i == 42) {
+      entry.SetGroup(disk_cache::ENTRY_LOW_USE);
+      index.Save(&entry);
+    }
+
+    // Entries [30,39] and 43 are marked as ENTRY_NO_USE (the default).
+
+    if (!(i % 10))
+      time += base::TimeDelta::FromMinutes(1);
+
+    index.UpdateTime(hash, addr, time);
+  }
+
+  // Get the oldest entries of each group.
+  disk_cache::IndexIterator no_use, low_use, high_use;
+  index.GetOldest(&no_use, &low_use, &high_use);
+  ASSERT_EQ(10u, no_use.cells.size());
+  ASSERT_EQ(10u, low_use.cells.size());
+  ASSERT_EQ(10u, high_use.cells.size());
+
+  EXPECT_EQ(entries[10].hash, high_use.cells[0].hash);
+  EXPECT_EQ(entries[19].hash, high_use.cells[9].hash);
+  EXPECT_EQ(entries[20].hash, low_use.cells[0].hash);
+  EXPECT_EQ(entries[29].hash, low_use.cells[9].hash);
+  EXPECT_EQ(entries[30].hash, no_use.cells[0].hash);
+  EXPECT_EQ(entries[39].hash, no_use.cells[9].hash);
+
+  // Now start an iteration from the head (most recent entry).
+  disk_cache::IndexIterator iterator;
+  iterator.timestamp = index.CalculateTimestamp(time) + 1;
+  iterator.forward = false;  // Back in time.
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(3u, iterator.cells.size());
+  EXPECT_EQ(entries[41].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[42].hash, iterator.cells[1].hash);
+  EXPECT_EQ(entries[43].hash, iterator.cells[2].hash);
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(10u, iterator.cells.size());
+  EXPECT_EQ(entries[30].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[39].hash, iterator.cells[9].hash);
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(10u, iterator.cells.size());
+  EXPECT_EQ(entries[20].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[29].hash, iterator.cells[9].hash);
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(10u, iterator.cells.size());
+  EXPECT_EQ(entries[10].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[19].hash, iterator.cells[9].hash);
+
+  ASSERT_FALSE(index.GetNextCells(&iterator));
+
+  // Now start an iteration from the tail (oldest entry).
+  iterator.timestamp = 0;
+  iterator.forward = true;
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(10u, iterator.cells.size());
+  EXPECT_EQ(entries[10].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[19].hash, iterator.cells[9].hash);
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(10u, iterator.cells.size());
+  EXPECT_EQ(entries[20].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[29].hash, iterator.cells[9].hash);
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(10u, iterator.cells.size());
+  EXPECT_EQ(entries[30].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[39].hash, iterator.cells[9].hash);
+
+  ASSERT_TRUE(index.GetNextCells(&iterator));
+  ASSERT_EQ(3u, iterator.cells.size());
+  EXPECT_EQ(entries[41].hash, iterator.cells[0].hash);
+  EXPECT_EQ(entries[42].hash, iterator.cells[1].hash);
+  EXPECT_EQ(entries[43].hash, iterator.cells[2].hash);
+}
+
+// Tests doubling of the table.
+TEST(DiskCacheIndexTable, Doubling) {
+  IndexTable index(NULL);
+  int size = 1024;
+  scoped_ptr<TestCacheTables> cache(new TestCacheTables(size));
+  int entry_id = 0;
+  disk_cache::CellList entries;
+
+  // Go from 1024 to 256k cells.
+  for (int resizes = 0; resizes <= 8; resizes++) {
+    scoped_ptr<TestCacheTables> old_cache(cache.Pass());
+    cache.reset(new TestCacheTables(size));
+    cache.get()->CopyFrom(*old_cache.get());
+
+    IndexTableInitData init_data;
+    cache.get()->GetInitData(&init_data);
+    index.Init(&init_data);
+
+    // Write some entries.
+    for (int i = 0; i < 250; i++, entry_id++) {
+      SCOPED_TRACE(entry_id);
+      uint32 hash = entry_id * i * 321 + entry_id * 13;
+      disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, entry_id * 17 + 1);
+      EntryCell entry = index.CreateEntryCell(hash, addr);
+      EXPECT_TRUE(entry.IsValid());
+
+      disk_cache::CellInfo info = { hash, addr };
+      entries.push_back(info);
+    }
+    size *= 2;
+  }
+
+  // Access all the entries.
+  for (size_t i = 0; i < entries.size(); i++) {
+    SCOPED_TRACE(i);
+    disk_cache::EntrySet found_entries = index.LookupEntries(entries[i].hash);
+    ASSERT_EQ(1u, found_entries.cells.size());
+    EXPECT_TRUE(found_entries.cells[0].IsValid());
+  }
+}
+
+// Tests bucket chaining when growing the index.
+TEST(DiskCacheIndexTable, BucketChains) {
+  IndexTable index(NULL);
+  int size = 1024;
+  scoped_ptr<TestCacheTables> cache(new TestCacheTables(size));
+  disk_cache::CellList entries;
+
+  IndexTableInitData init_data;
+  cache.get()->GetInitData(&init_data);
+  index.Init(&init_data);
+
+  // Write some entries.
+  for (int i = 0; i < 8; i++) {
+    SCOPED_TRACE(i);
+    uint32 hash = i * 256;
+    disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, i * 7 + 1);
+    EntryCell entry = index.CreateEntryCell(hash, addr);
+    EXPECT_TRUE(entry.IsValid());
+
+    disk_cache::CellInfo info = { hash, addr };
+    entries.push_back(info);
+  }
+
+  // Double the size.
+  scoped_ptr<TestCacheTables> old_cache(cache.Pass());
+  cache.reset(new TestCacheTables(size * 2));
+  cache.get()->CopyFrom(*old_cache.get());
+
+  cache.get()->GetInitData(&init_data);
+  index.Init(&init_data);
+
+  // Write more entries, starting with the upper half of the table.
+  for (int i = 9; i < 11; i++) {
+    SCOPED_TRACE(i);
+    uint32 hash = i * 256;
+    disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, i * 7 + 1);
+    EntryCell entry = index.CreateEntryCell(hash, addr);
+    EXPECT_TRUE(entry.IsValid());
+
+    disk_cache::CellInfo info = { hash, addr };
+    entries.push_back(info);
+  }
+
+  // Access all the entries.
+  for (size_t i = 0; i < entries.size(); i++) {
+    SCOPED_TRACE(i);
+    disk_cache::EntrySet found_entries = index.LookupEntries(entries[i].hash);
+    ASSERT_EQ(1u, found_entries.cells.size());
+    EXPECT_TRUE(found_entries.cells[0].IsValid());
+  }
+}
+
+// Tests that GrowIndex is called.
+TEST(DiskCacheIndexTable, GrowIndex) {
+  TestCacheTables cache(1024);
+  IndexTableInitData init_data;
+  cache.GetInitData(&init_data);
+  MockIndexBackend backend;
+
+  IndexTable index(&backend);
+  index.Init(&init_data);
+
+  // Write some entries.
+  for (int i = 0; i < 512; i++) {
+    SCOPED_TRACE(i);
+    uint32 hash = 0;
+    disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, i + 1);
+    EntryCell entry = index.CreateEntryCell(hash, addr);
+    EXPECT_TRUE(entry.IsValid());
+  }
+
+  EXPECT_TRUE(backend.grow_called());
+}
+
+TEST(DiskCacheIndexTable, SaveIndex) {
+  TestCacheTables cache(1024);
+  IndexTableInitData init_data;
+  cache.GetInitData(&init_data);
+  MockIndexBackend backend;
+
+  IndexTable index(&backend);
+  index.Init(&init_data);
+
+  uint32 hash = 0;
+  disk_cache::Addr addr(disk_cache::BLOCK_ENTRIES, 1, 5, 6);
+  EntryCell entry = index.CreateEntryCell(hash, addr);
+  EXPECT_TRUE(entry.IsValid());
+
+  index.OnBackupTimer();
+  int expected = (1024 + 512) / 8 + sizeof(disk_cache::IndexHeaderV3);
+  EXPECT_EQ(expected, backend.buffer_len());
+}

net/net.gyp

@@ -460,6 +460,8 @@
         'disk_cache/v3/block_bitmaps.cc',
         'disk_cache/v3/block_bitmaps.h',
         'disk_cache/v3/disk_format_v3.h',
+        'disk_cache/v3/index_table.cc',
+        'disk_cache/v3/index_table.h',
         'dns/address_sorter.h',
         'dns/address_sorter_posix.cc',
         'dns/address_sorter_posix.h',
@@ -1673,6 +1675,7 @@
         'disk_cache/flash/segment_unittest.cc',
         'disk_cache/flash/storage_unittest.cc',
         'disk_cache/v3/block_bitmaps_unittest.cc',
+        'disk_cache/v3/index_table_unittest.cc',
         'dns/address_sorter_posix_unittest.cc',
         'dns/address_sorter_unittest.cc',
         'dns/dns_config_service_posix_unittest.cc',