Disk cache v3: The main index table.

The IndexTable controls what entries are stored in the cache.
This class provides a memory-only view of the underlying structures
that store "the index"

BUG=241277
TEST=net_unittests

Review URL: https://codereview.chromium.org/53313004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@244027 0039d316-1c4b-4281-b951-d872f2087c98

net/disk_cache/addr.h

@@ -54,6 +54,18 @@ const int kFirstAdditionalBlockFileV3 = 7;
 //   0000 0011 0000 0000 0000 0000 0000 0000 : number of contiguous blocks 1-4
 //   0000 0000 1111 1111 0000 0000 0000 0000 : file selector 0 - 255
 //   0000 0000 0000 0000 1111 1111 1111 1111 : block#  0 - 65,535 (2^16)
+//
+// Note that an Addr can be used to "point" to a variety of different objects,
+// from a given type of entry to random blobs of data. Conceptually, an Addr is
+// just a number that someone can inspect to find out how to locate the desired
+// record. Most users will not care about the specific bits inside Addr, for
+// example, what parts of it point to a file number; only the code that has to
+// select a specific file would care about those specific bits.
+//
+// From a general point of view, an Addr has a total capacity of 2^24 entities,
+// in that it has 24 bits that can identify individual records. Note that the
+// address space is bigger for independent files (2^28), but that would not be
+// the general case.
 class NET_EXPORT_PRIVATE Addr {
  public:
   Addr() : value_(0) {}
@@ -108,14 +120,6 @@ class NET_EXPORT_PRIVATE Addr {
     return value_ != other.value_;
   }
 
-  static Addr FromEntryAddress(uint32 value) {
-    return Addr(kInitializedMask + (BLOCK_ENTRIES << kFileTypeOffset) + value);
-  }
-
-  static Addr FromEvictedAddress(uint32 value) {
-    return Addr(kInitializedMask + (BLOCK_EVICTED << kFileTypeOffset) + value);
-  }
-
   static int BlockSizeForFileType(FileType file_type) {
     switch (file_type) {
       case RANKINGS:

net/disk_cache/disk_format_base.h

@@ -33,6 +33,7 @@ const uint32 kBlockCurrentVersion = 0x30000;  // Version 3.0.
 const uint32 kBlockMagic = 0xC104CAC3;
 const int kBlockHeaderSize = 8192;  // Two pages: almost 64k entries
 const int kMaxBlocks = (kBlockHeaderSize - 80) * 8;
+const int kNumExtraBlocks = 1024;  // How fast files grow.
 
 // Bitmap to track used blocks on a block-file.
 typedef uint32 AllocBitmap[kMaxBlocks / 32];

net/disk_cache/v3/disk_format_v3.h

@@ -37,6 +37,10 @@
 // internal structures are modified, so it is possible to detect (most of the
 // time) when the process dies in the middle of an update. There are dedicated
 // backup files for cache bitmaps, used to detect entries out of date.
+//
+// Although cache files are to be consumed on the same machine that creates
+// them, if files are to be moved accross machines, little endian storage is
+// assumed.
 
 #ifndef NET_DISK_CACHE_V3_DISK_FORMAT_V3_H_
 #define NET_DISK_CACHE_V3_DISK_FORMAT_V3_H_
@@ -46,14 +50,15 @@
 
 namespace disk_cache {
 
-const int kBaseTableLen = 0x10000;
+const int kBaseTableLen = 0x400;
 const uint32 kIndexMagicV3 = 0xC103CAC3;
 const uint32 kVersion3 = 0x30000;  // Version 3.0.
 
 // Flags for a given cache.
 enum CacheFlags {
-  CACHE_EVICTION_2 = 1,      // Keep multiple lists for eviction.
-  CACHE_EVICTED = 1 << 1     // Already evicted at least one entry.
+  SMALL_CACHE = 1 << 0,       // See IndexCell.
+  CACHE_EVICTION_2 = 1 << 1,  // Keep multiple lists for eviction.
+  CACHE_EVICTED = 1 << 2      // Already evicted at least one entry.
 };
 
 // Header for the master index file.
@@ -119,30 +124,81 @@ COMPILE_ASSERT(ENTRY_USED <= 7, group_uses_3_bits);
 struct IndexCell {
   void Clear() { memset(this, 0, sizeof(*this)); }
 
-  uint64      address : 22;
-  uint64      hash : 18;
-  uint64      timestamp : 20;
-  uint64      reuse : 4;
-  uint8       state : 3;
-  uint8       group : 3;
-  uint8       sum : 2;
+  // A cell is a 9 byte bit-field that stores 7 values:
+  //   location : 22 bits
+  //   id : 18 bits
+  //   timestamp : 20 bits
+  //   reuse : 4 bits
+  //   state : 3 bits
+  //   group : 3 bits
+  //   sum : 2 bits
+  // The id is derived from the full hash of the entry.
+  //
+  // The actual layout is as follows:
+  //
+  // first_part (low order 32 bits):
+  //   0000 0000 0011 1111 1111 1111 1111 1111 : location
+  //   1111 1111 1100 0000 0000 0000 0000 0000 : id
+  //
+  // first_part (high order 32 bits):
+  //   0000 0000 0000 0000 0000 0000 1111 1111 : id
+  //   0000 1111 1111 1111 1111 1111 0000 0000 : timestamp
+  //   1111 0000 0000 0000 0000 0000 0000 0000 : reuse
+  //
+  // last_part:
+  //   0000 0111 : state
+  //   0011 1000 : group
+  //   1100 0000 : sum
+  //
+  // The small-cache version of the format moves some bits from the location to
+  // the id fileds, like so:
+  //   location : 16 bits
+  //   id : 24 bits
+  //
+  // first_part (low order 32 bits):
+  //   0000 0000 0000 0000 1111 1111 1111 1111 : location
+  //   1111 1111 1111 1111 0000 0000 0000 0000 : id
+  //
+  // The actual bit distribution between location and id is determined by the
+  // table size (IndexHeaderV3.table_len). Tables smaller than 65536 entries
+  // use the small-cache version; after that size, caches should have the
+  // SMALL_CACHE flag cleared.
+  //
+  // To locate a given entry after recovering the location from the cell, the
+  // file type and file number are appended (see disk_cache/addr.h). For a large
+  // table only the file type is implied; for a small table, the file number
+  // is also implied, and it should be the first file for that type of entry,
+  // as determined by the EntryGroup (two files in total, one for active entries
+  // and another one for evicted entries).
+  //
+  // For example, a small table may store something like 0x1234 as the location
+  // field. That means it stores the entry number 0x1234. If that record belongs
+  // to a deleted entry, the regular cache address may look something like
+  //     BLOCK_EVICTED + 1 block + file number 6 + entry number 0x1234
+  //     so Addr = 0xf0061234
+  //
+  // If that same Addr is stored on a large table, the location field would be
+  // 0x61234
+
+  uint64      first_part;
+  uint8       last_part;
 };
 COMPILE_ASSERT(sizeof(IndexCell) == 9, bad_IndexCell);
 
+const int kCellsPerBucket = 4;
 struct IndexBucket {
-  IndexCell   cells[4];
+  IndexCell   cells[kCellsPerBucket];
   int32       next;
-  uint32      hash : 24;      // The last byte is only defined for buckets of
-  uint32      reserved : 8;   // the extra table.
+  uint32      hash;  // The high order byte is reserved (should be zero).
 };
 COMPILE_ASSERT(sizeof(IndexBucket) == 44, bad_IndexBucket);
-const int kBytesPerCell = 44 / 4;
+const int kBytesPerCell = 44 / kCellsPerBucket;
 
 // The main cache index. Backed by a file named index_tb1.
 // The extra table (index_tb2) has a similar format, but different size.
 struct Index {
   // Default size. Actual size controlled by header.table_len.
-  IndexBucket table[kBaseTableLen / 4];
+  IndexBucket table[kBaseTableLen / kCellsPerBucket];
 };
 #pragma pack(pop)
 

net/net.gyp

@@ -460,6 +460,8 @@
         'disk_cache/v3/block_bitmaps.cc',
         'disk_cache/v3/block_bitmaps.h',
         'disk_cache/v3/disk_format_v3.h',
+        'disk_cache/v3/index_table.cc',
+        'disk_cache/v3/index_table.h',
         'dns/address_sorter.h',
         'dns/address_sorter_posix.cc',
         'dns/address_sorter_posix.h',
@@ -1673,6 +1675,7 @@
         'disk_cache/flash/segment_unittest.cc',
         'disk_cache/flash/storage_unittest.cc',
         'disk_cache/v3/block_bitmaps_unittest.cc',
+        'disk_cache/v3/index_table_unittest.cc',
         'dns/address_sorter_posix_unittest.cc',
         'dns/address_sorter_unittest.cc',
         'dns/dns_config_service_posix_unittest.cc',