Store field trial parameters in shared memory

This CL writes field trial parameters into its allocated spot in shared memory. The format of the pickle should now look like:

TrialName
GroupName
ParamKey1
ParamValue1
ParamKey2
ParamValue2
...

BUG=660041

Review-Url: https://codereview.chromium.org/2463223002
Cr-Commit-Position: refs/heads/master@{#435615}

base/metrics/field_trial.cc

@@ -12,6 +12,7 @@
 #include "base/command_line.h"
 #include "base/feature_list.h"
 #include "base/logging.h"
+#include "base/metrics/field_trial_param_associator.h"
 #include "base/pickle.h"
 #include "base/process/memory.h"
 #include "base/rand_util.h"
@@ -80,25 +81,88 @@ struct FieldTrialEntry {
   // Size of the pickled structure, NOT the total size of this entry.
   uint32_t size;
 
-  // Calling this is only valid when the entry is initialized. That is, it
-  // resides in shared memory and has a pickle containing the trial name and
-  // group name following it.
-  bool GetTrialAndGroupName(StringPiece* trial_name,
-                            StringPiece* group_name) const {
+  // Returns an iterator over the data containing names and params.
+  PickleIterator GetPickleIterator() const {
     char* src = reinterpret_cast<char*>(const_cast<FieldTrialEntry*>(this)) +
                 sizeof(FieldTrialEntry);
 
     Pickle pickle(src, size);
-    PickleIterator pickle_iter(pickle);
+    return PickleIterator(pickle);
+  }
 
-    if (!pickle_iter.ReadStringPiece(trial_name))
+  // Takes the iterator and writes out the first two items into |trial_name| and
+  // |group_name|.
+  bool ReadStringPair(PickleIterator* iter,
+                      StringPiece* trial_name,
+                      StringPiece* group_name) const {
+    if (!iter->ReadStringPiece(trial_name))
       return false;
-    if (!pickle_iter.ReadStringPiece(group_name))
+    if (!iter->ReadStringPiece(group_name))
       return false;
     return true;
   }
+
+  // Calling this is only valid when the entry is initialized. That is, it
+  // resides in shared memory and has a pickle containing the trial name and
+  // group name following it.
+  bool GetTrialAndGroupName(StringPiece* trial_name,
+                            StringPiece* group_name) const {
+    PickleIterator iter = GetPickleIterator();
+    return ReadStringPair(&iter, trial_name, group_name);
+  }
+
+  // Calling this is only valid when the entry is initialized as well. Reads the
+  // parameters following the trial and group name and stores them as key-value
+  // mappings in |params|.
+  bool GetParams(std::map<std::string, std::string>* params) const {
+    PickleIterator iter = GetPickleIterator();
+    StringPiece tmp;
+    if (!ReadStringPair(&iter, &tmp, &tmp))
+      return false;
+
+    while (true) {
+      StringPiece key;
+      StringPiece value;
+      if (!ReadStringPair(&iter, &key, &value))
+        return key.empty();  // Non-empty is bad: got one of a pair.
+      (*params)[key.as_string()] = value.as_string();
+    }
+  }
 };
 
+// Writes out string1 and then string2 to pickle.
+bool WriteStringPair(Pickle* pickle,
+                     const StringPiece& string1,
+                     const StringPiece& string2) {
+  if (!pickle->WriteString(string1))
+    return false;
+  if (!pickle->WriteString(string2))
+    return false;
+  return true;
+}
+
+// Writes out the field trial's contents (via trial_state) to the pickle. The
+// format of the pickle looks like:
+// TrialName, GroupName, ParamKey1, ParamValue1, ParamKey2, ParamValue2, ...
+// If there are no parameters, then it just ends at GroupName.
+bool PickleFieldTrial(const FieldTrial::State& trial_state, Pickle* pickle) {
+  if (!WriteStringPair(pickle, trial_state.trial_name, trial_state.group_name))
+    return false;
+
+  // Get field trial params.
+  std::map<std::string, std::string> params;
+  FieldTrialParamAssociator::GetInstance()->GetFieldTrialParamsWithoutFallback(
+      trial_state.trial_name.as_string(), trial_state.group_name.as_string(),
+      &params);
+
+  // Write params to pickle.
+  for (const auto& param : params) {
+    if (!WriteStringPair(pickle, param.first, param.second))
+      return false;
+  }
+  return true;
+}
+
 // Created a time value based on |year|, |month| and |day_of_month| parameters.
 Time CreateTimeFromParams(int year, int month, int day_of_month) {
   DCHECK_GT(year, 1970);
@@ -917,6 +981,106 @@ size_t FieldTrialList::GetFieldTrialCount() {
   return global_->registered_.size();
 }
 
+// static
+bool FieldTrialList::GetParamsFromSharedMemory(
+    FieldTrial* field_trial,
+    std::map<std::string, std::string>* params) {
+  DCHECK(global_);
+  // If the field trial allocator is not set up yet, then there are several
+  // cases:
+  //   - We are in the browser process and the allocator has not been set up
+  //   yet. If we got here, then we couldn't find the params in
+  //   FieldTrialParamAssociator, so it's definitely not here. Return false.
+  //   - Using shared memory for field trials is not enabled. If we got here,
+  //   then there's nothing in shared memory. Return false.
+  //   - We are in the child process and the allocator has not been set up yet.
+  //   If this is the case, then you are calling this too early. The field trial
+  //   allocator should get set up very early in the lifecycle. Try to see if
+  //   you can call it after it's been set up.
+  AutoLock auto_lock(global_->lock_);
+  if (!global_->field_trial_allocator_)
+    return false;
+
+  // If ref_ isn't set, then the field trial data can't be in shared memory.
+  if (!field_trial->ref_)
+    return false;
+
+  const FieldTrialEntry* entry =
+      global_->field_trial_allocator_->GetAsObject<const FieldTrialEntry>(
+          field_trial->ref_, kFieldTrialType);
+
+  size_t allocated_size =
+      global_->field_trial_allocator_->GetAllocSize(field_trial->ref_);
+  size_t actual_size = sizeof(FieldTrialEntry) + entry->size;
+  if (allocated_size < actual_size)
+    return false;
+
+  return entry->GetParams(params);
+}
+
+// static
+void FieldTrialList::ClearParamsFromSharedMemoryForTesting() {
+  if (!global_)
+    return;
+
+  AutoLock auto_lock(global_->lock_);
+  if (!global_->field_trial_allocator_)
+    return;
+
+  // To clear the params, we iterate through every item in the allocator, copy
+  // just the trial and group name into a newly-allocated segment and then clear
+  // the existing item.
+  FieldTrialAllocator* allocator = global_->field_trial_allocator_.get();
+  FieldTrialAllocator::Iterator mem_iter(allocator);
+
+  // List of refs to eventually be made iterable. We can't make it in the loop,
+  // since it would go on forever.
+  std::vector<FieldTrial::FieldTrialRef> new_refs;
+
+  FieldTrial::FieldTrialRef prev_ref;
+  while ((prev_ref = mem_iter.GetNextOfType(kFieldTrialType)) !=
+         FieldTrialAllocator::kReferenceNull) {
+    // Get the existing field trial entry in shared memory.
+    const FieldTrialEntry* prev_entry =
+        allocator->GetAsObject<const FieldTrialEntry>(prev_ref,
+                                                      kFieldTrialType);
+    StringPiece trial_name;
+    StringPiece group_name;
+    if (!prev_entry->GetTrialAndGroupName(&trial_name, &group_name))
+      continue;
+
+    // Write a new entry, minus the params.
+    Pickle pickle;
+    pickle.WriteString(trial_name);
+    pickle.WriteString(group_name);
+    size_t total_size = sizeof(FieldTrialEntry) + pickle.size();
+    FieldTrial::FieldTrialRef new_ref =
+        allocator->Allocate(total_size, kFieldTrialType);
+    FieldTrialEntry* new_entry =
+        allocator->GetAsObject<FieldTrialEntry>(new_ref, kFieldTrialType);
+    new_entry->activated = prev_entry->activated;
+    new_entry->size = pickle.size();
+
+    // TODO(lawrencewu): Modify base::Pickle to be able to write over a section
+    // in memory, so we can avoid this memcpy.
+    char* dst = reinterpret_cast<char*>(new_entry) + sizeof(FieldTrialEntry);
+    memcpy(dst, pickle.data(), pickle.size());
+
+    // Update the ref on the field trial and add it to the list to be made
+    // iterable.
+    FieldTrial* trial = global_->PreLockedFind(trial_name.as_string());
+    trial->ref_ = new_ref;
+    new_refs.push_back(new_ref);
+
+    // Mark the existing entry as unused.
+    allocator->ChangeType(prev_ref, 0, kFieldTrialType);
+  }
+
+  for (const auto& ref : new_refs) {
+    allocator->MakeIterable(ref);
+  }
+}
+
 #if defined(OS_WIN)
 // static
 bool FieldTrialList::CreateTrialsFromHandleSwitch(
@@ -1039,14 +1203,18 @@ void FieldTrialList::AddToAllocatorWhileLocked(FieldTrial* field_trial) {
     return;
 
   Pickle pickle;
-  pickle.WriteString(trial_state.trial_name);
-  pickle.WriteString(trial_state.group_name);
+  if (!PickleFieldTrial(trial_state, &pickle)) {
+    NOTREACHED();
+    return;
+  }
 
   size_t total_size = sizeof(FieldTrialEntry) + pickle.size();
   FieldTrial::FieldTrialRef ref =
       allocator->Allocate(total_size, kFieldTrialType);
-  if (ref == FieldTrialAllocator::kReferenceNull)
+  if (ref == FieldTrialAllocator::kReferenceNull) {
+    NOTREACHED();
     return;
+  }
 
   FieldTrialEntry* entry =
       allocator->GetAsObject<FieldTrialEntry>(ref, kFieldTrialType);

base/metrics/field_trial.h

@@ -224,6 +224,7 @@ class BASE_EXPORT FieldTrial : public RefCounted<FieldTrial> {
   FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DoesNotSurpassTotalProbability);
   FRIEND_TEST_ALL_PREFIXES(FieldTrialListTest,
                            DoNotAddSimulatedFieldTrialsToAllocator);
+  FRIEND_TEST_ALL_PREFIXES(FieldTrialListTest, ClearParamsFromSharedMemory);
 
   friend class base::FieldTrialList;
 
@@ -555,12 +556,24 @@ class BASE_EXPORT FieldTrialList {
   // Return the number of active field trials.
   static size_t GetFieldTrialCount();
 
+  // Gets the parameters for |field_trial| from shared memory and stores them in
+  // |params|. This is only exposed for use by FieldTrialParamAssociator and
+  // shouldn't be used by anything else.
+  static bool GetParamsFromSharedMemory(
+      FieldTrial* field_trial,
+      std::map<std::string, std::string>* params);
+
+  // Clears all the params in the allocator.
+  static void ClearParamsFromSharedMemoryForTesting();
+
  private:
   // Allow tests to access our innards for testing purposes.
   FRIEND_TEST_ALL_PREFIXES(FieldTrialListTest, InstantiateAllocator);
   FRIEND_TEST_ALL_PREFIXES(FieldTrialListTest, AddTrialsToAllocator);
   FRIEND_TEST_ALL_PREFIXES(FieldTrialListTest,
                            DoNotAddSimulatedFieldTrialsToAllocator);
+  FRIEND_TEST_ALL_PREFIXES(FieldTrialListTest, AssociateFieldTrialParams);
+  FRIEND_TEST_ALL_PREFIXES(FieldTrialListTest, ClearParamsFromSharedMemory);
 
 #if defined(OS_WIN)
   // Takes in |handle_switch| from the command line which represents the shared
@@ -625,8 +638,8 @@ class BASE_EXPORT FieldTrialList {
   // FieldTrialList is created after that.
   static bool used_without_global_;
 
-  // Lock for access to registered_.
-  base::Lock lock_;
+  // Lock for access to registered_ and field_trial_allocator_.
+  Lock lock_;
   RegistrationMap registered_;
 
   std::map<std::string, std::string> seen_states_;

base/metrics/field_trial_param_associator.cc

@@ -37,9 +37,26 @@ bool FieldTrialParamAssociator::AssociateFieldTrialParams(
 bool FieldTrialParamAssociator::GetFieldTrialParams(
     const std::string& trial_name,
     FieldTrialParams* params) {
+  FieldTrial* field_trial = FieldTrialList::Find(trial_name);
+  if (!field_trial)
+    return false;
+
+  // First try the local map, falling back to getting it from shared memory.
+  if (GetFieldTrialParamsWithoutFallback(trial_name, field_trial->group_name(),
+                                         params)) {
+    return true;
+  }
+
+  // TODO(lawrencewu): add the params to field_trial_params_ for next time.
+  return FieldTrialList::GetParamsFromSharedMemory(field_trial, params);
+}
+
+bool FieldTrialParamAssociator::GetFieldTrialParamsWithoutFallback(
+    const std::string& trial_name,
+    const std::string& group_name,
+    FieldTrialParams* params) {
   AutoLock scoped_lock(lock_);
 
-  const std::string group_name = FieldTrialList::FindFullName(trial_name);
   const FieldTrialKey key(trial_name, group_name);
   if (!ContainsKey(field_trial_params_, key))
     return false;
@@ -51,6 +68,12 @@ bool FieldTrialParamAssociator::GetFieldTrialParams(
 void FieldTrialParamAssociator::ClearAllParamsForTesting() {
   AutoLock scoped_lock(lock_);
   field_trial_params_.clear();
+  FieldTrialList::ClearParamsFromSharedMemoryForTesting();
+}
+
+void FieldTrialParamAssociator::ClearAllCachedParamsForTesting() {
+  AutoLock scoped_lock(lock_);
+  field_trial_params_.clear();
 }
 
 }  // namespace base

base/metrics/field_trial_param_associator.h

@@ -11,6 +11,7 @@
 
 #include "base/base_export.h"
 #include "base/memory/singleton.h"
+#include "base/metrics/field_trial.h"
 #include "base/synchronization/lock.h"
 
 namespace base {
@@ -33,13 +34,26 @@ class BASE_EXPORT FieldTrialParamAssociator {
                                  const std::string& group_name,
                                  const FieldTrialParams& params);
 
-  // Gets the parameters for a field trial and its chosen group.
+  // Gets the parameters for a field trial and its chosen group. If not found in
+  // field_trial_params_, then tries to looks it up in shared memory.
   bool GetFieldTrialParams(const std::string& trial_name,
                            FieldTrialParams* params);
 
-  // Clears the internal field_trial_params_ mapping.
+  // Gets the parameters for a field trial and its chosen group. Does not
+  // fallback to looking it up in shared memory. This should only be used if you
+  // know for sure the params are in the mapping, like if you're in the browser
+  // process, and even then you should probably just use GetFieldTrialParams().
+  bool GetFieldTrialParamsWithoutFallback(const std::string& trial_name,
+                                          const std::string& group_name,
+                                          FieldTrialParams* params);
+
+  // Clears the internal field_trial_params_ mapping, plus removes all params in
+  // shared memory.
   void ClearAllParamsForTesting();
 
+  // Clears the internal field_trial_params_ mapping.
+  void ClearAllCachedParamsForTesting();
+
  private:
   friend struct DefaultSingletonTraits<FieldTrialParamAssociator>;
 

base/metrics/field_trial_unittest.cc

@@ -12,6 +12,7 @@
 #include "base/macros.h"
 #include "base/memory/ptr_util.h"
 #include "base/message_loop/message_loop.h"
+#include "base/metrics/field_trial_param_associator.h"
 #include "base/rand_util.h"
 #include "base/run_loop.h"
 #include "base/strings/string_number_conversions.h"
@@ -1228,4 +1229,82 @@ TEST(FieldTrialListTest, DoNotAddSimulatedFieldTrialsToAllocator) {
   ASSERT_EQ(check_string.find("Simulated"), std::string::npos);
 }
 
+TEST(FieldTrialListTest, AssociateFieldTrialParams) {
+  std::string trial_name("Trial1");
+  std::string group_name("Group1");
+
+  // Create a field trial with some params.
+  FieldTrialList field_trial_list(nullptr);
+  FieldTrialList::CreateFieldTrial(trial_name, group_name);
+  std::map<std::string, std::string> params;
+  params["key1"] = "value1";
+  params["key2"] = "value2";
+  FieldTrialParamAssociator::GetInstance()->AssociateFieldTrialParams(
+      trial_name, group_name, params);
+  FieldTrialList::InstantiateFieldTrialAllocatorIfNeeded();
+
+  // Clear all cached params from the associator.
+  FieldTrialParamAssociator::GetInstance()->ClearAllCachedParamsForTesting();
+  // Check that the params have been cleared from the cache.
+  std::map<std::string, std::string> cached_params;
+  FieldTrialParamAssociator::GetInstance()->GetFieldTrialParamsWithoutFallback(
+      trial_name, group_name, &cached_params);
+  EXPECT_EQ(0U, cached_params.size());
+
+  // Check that we fetch the param from shared memory properly.
+  std::map<std::string, std::string> new_params;
+  FieldTrialParamAssociator::GetInstance()->GetFieldTrialParams(trial_name,
+                                                                &new_params);
+  EXPECT_EQ("value1", new_params["key1"]);
+  EXPECT_EQ("value2", new_params["key2"]);
+  EXPECT_EQ(2U, new_params.size());
+}
+
+TEST(FieldTrialListTest, ClearParamsFromSharedMemory) {
+  std::string trial_name("Trial1");
+  std::string group_name("Group1");
+
+  base::SharedMemoryHandle handle;
+  {
+    // Create a field trial with some params.
+    FieldTrialList field_trial_list(nullptr);
+    FieldTrial* trial =
+        FieldTrialList::CreateFieldTrial(trial_name, group_name);
+    std::map<std::string, std::string> params;
+    params["key1"] = "value1";
+    params["key2"] = "value2";
+    FieldTrialParamAssociator::GetInstance()->AssociateFieldTrialParams(
+        trial_name, group_name, params);
+    FieldTrialList::InstantiateFieldTrialAllocatorIfNeeded();
+
+    // Clear all params from the associator AND shared memory. The allocated
+    // segments should be different.
+    FieldTrial::FieldTrialRef old_ref = trial->ref_;
+    FieldTrialParamAssociator::GetInstance()->ClearAllParamsForTesting();
+    FieldTrial::FieldTrialRef new_ref = trial->ref_;
+    EXPECT_NE(old_ref, new_ref);
+
+    // Check that there are no params associated with the field trial anymore.
+    std::map<std::string, std::string> new_params;
+    FieldTrialParamAssociator::GetInstance()->GetFieldTrialParams(trial_name,
+                                                                  &new_params);
+    EXPECT_EQ(0U, new_params.size());
+
+    // Now duplicate the handle so we can easily check that the trial is still
+    // in shared memory via AllStatesToString.
+    handle = base::SharedMemory::DuplicateHandle(
+        field_trial_list.field_trial_allocator_->shared_memory()->handle());
+  }
+
+  // Check that we have the trial.
+  FieldTrialList field_trial_list2(nullptr);
+  std::unique_ptr<base::SharedMemory> shm(new SharedMemory(handle, true));
+  // 4 KiB is enough to hold the trials only created for this test.
+  shm.get()->Map(4 << 10);
+  FieldTrialList::CreateTrialsFromSharedMemory(std::move(shm));
+  std::string check_string;
+  FieldTrialList::AllStatesToString(&check_string);
+  EXPECT_EQ("*Trial1/Group1/", check_string);
+}
+
 }  // namespace base

components/variations/variations_seed_processor.cc

@@ -86,6 +86,9 @@ void ForceExperimentState(
   RegisterExperimentParams(study, experiment);
   RegisterVariationIds(experiment, study.name());
   if (study.activation_type() == Study_ActivationType_ACTIVATION_AUTO) {
+    // This call must happen after all params have been registered for the
+    // trial. Otherwise, since we look up params by trial and group name, the
+    // params won't be registered under the correct key.
     trial->group();
     // UI Strings can only be overridden from ACTIVATION_AUTO experiments.
     ApplyUIStringOverrides(experiment, override_callback);
@@ -310,6 +313,9 @@ void VariationsSeedProcessor::CreateTrialFromStudy(
     RegisterFeatureOverrides(processed_study, trial.get(), feature_list);
 
   if (study.activation_type() == Study_ActivationType_ACTIVATION_AUTO) {
+    // This call must happen after all params have been registered for the
+    // trial. Otherwise, since we look up params by trial and group name, the
+    // params won't be registered under the correct key.
     const std::string& group_name = trial->group_name();
 
     // Don't try to apply overrides if none of the experiments in this study had