[Android NTP] Logging histograms for users that are part of a control group.

This adds the possibility of a control group for suggestions, where the user is served local TopSites suggestions, but would have received suggestions from the SuggestionsService. This enables us to avoid a population bias.

Regarding UMA_ prefix for histograms, will rebase once https://codereview.chromium.org/343643002/ gets in.

BUG=385159
TEST=SuggestionsServiceTest*

Review URL: https://codereview.chromium.org/341943002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@278333 0039d316-1c4b-4281-b951-d872f2087c98

chrome/browser/android/most_visited_sites.cc

@@ -58,6 +58,8 @@ const char kNumEmptyTilesHistogramName[] = "NewTabPage.NumberOfGrayTiles";
 const char kNumServerTilesHistogramName[] = "NewTabPage.NumberOfExternalTiles";
 // Client suggestion opened.
 const char kOpenedItemClientHistogramName[] = "NewTabPage.MostVisited.client";
+// Control group suggestion opened.
+const char kOpenedItemControlHistogramName[] = "NewTabPage.MostVisited.client0";
 // Server suggestion opened, no provider.
 const char kOpenedItemServerHistogramName[] = "NewTabPage.MostVisited.server";
 // Server suggestion opened with provider.
@@ -66,6 +68,9 @@ const char kOpenedItemServerProviderHistogramFormat[] =
 // Client impression.
 const char kImpressionClientHistogramName[] =
     "NewTabPage.SuggestionsImpression.client";
+// Control group impression.
+const char kImpressionControlHistogramName[] =
+    "NewTabPage.SuggestionsImpression.client0";
 // Server suggestion impression, no provider.
 const char kImpressionServerHistogramName[] =
     "NewTabPage.SuggestionsImpression.server";
@@ -168,8 +173,9 @@ void LogHistogramEvent(const std::string& histogram, int position,
 }  // namespace
 
 MostVisitedSites::MostVisitedSites(Profile* profile)
-    : profile_(profile), num_sites_(0), num_local_thumbs_(0),
-      num_server_thumbs_(0), num_empty_thumbs_(0), weak_ptr_factory_(this) {
+    : profile_(profile), num_sites_(0), is_control_group_(false),
+      num_local_thumbs_(0), num_server_thumbs_(0), num_empty_thumbs_(0),
+      weak_ptr_factory_(this) {
   // Register the debugging page for the Suggestions Service and the thumbnails
   // debugging page.
   content::URLDataSource::Add(profile_,
@@ -278,7 +284,9 @@ void MostVisitedSites::RecordOpenedMostVisitedItem(JNIEnv* env,
                                                    jint index) {
   switch (mv_source_) {
     case TOP_SITES: {
-      UMA_HISTOGRAM_SPARSE_SLOWLY(kOpenedItemClientHistogramName, index);
+      const std::string histogram = is_control_group_ ?
+          kOpenedItemControlHistogramName : kOpenedItemClientHistogramName;
+      LogHistogramEvent(histogram, index, num_sites_);
       break;
     }
     case SUGGESTIONS_SERVICE: {
@@ -354,8 +362,10 @@ void MostVisitedSites::OnMostVisitedURLsAvailable(
 
   int num_tiles = urls.size();
   UMA_HISTOGRAM_SPARSE_SLOWLY(kNumTilesHistogramName, num_tiles);
+  const std::string histogram = is_control_group_ ?
+      kImpressionControlHistogramName : kImpressionClientHistogramName;
   for (int i = 0; i < num_tiles; ++i) {
-    UMA_HISTOGRAM_SPARSE_SLOWLY(kImpressionClientHistogramName, i);
+    LogHistogramEvent(histogram, i, num_sites_);
   }
 
   JNIEnv* env = AttachCurrentThread();
@@ -370,8 +380,14 @@ void MostVisitedSites::OnSuggestionsProfileAvailable(
     ScopedJavaGlobalRef<jobject>* j_observer,
     const SuggestionsProfile& suggestions_profile) {
   int size = suggestions_profile.suggestions_size();
-  if (size == 0) {
-    // No suggestions data available, initiate Top Sites query.
+
+  // Determine if the user is in a control group (they would have received
+  // suggestions, but are in a group where they shouldn't).
+  is_control_group_ = size && SuggestionsService::IsControlGroup();
+
+  // If no suggestions data is available or the user is in a control group,
+  // initiate Top Sites query.
+  if (is_control_group_ || !size) {
     InitiateTopSitesQuery();
     return;
   }

chrome/browser/android/most_visited_sites.h

@@ -98,6 +98,9 @@ class MostVisitedSites : public content::NotificationObserver {
   // The maximum number of most visited sites to return.
   int num_sites_;
 
+  // Whether the user is in a control group for the purposes of logging.
+  bool is_control_group_;
+
   // Counters for UMA metrics.
 
   // Number of tiles using a local thumbnail image for this NTP session.

chrome/browser/search/suggestions/suggestions_service.cc

@@ -79,6 +79,7 @@ const char kSuggestionsFieldTrialSuggestionsSuffixParam[] =
     "suggestions_suffix";
 const char kSuggestionsFieldTrialBlacklistSuffixParam[] = "blacklist_suffix";
 const char kSuggestionsFieldTrialStateParam[] = "state";
+const char kSuggestionsFieldTrialControlParam[] = "control";
 const char kSuggestionsFieldTrialStateEnabled[] = "enabled";
 const char kSuggestionsFieldTrialTimeoutMs[] = "timeout_ms";
 
@@ -111,6 +112,12 @@ bool SuggestionsService::IsEnabled() {
          kSuggestionsFieldTrialStateEnabled;
 }
 
+// static
+bool SuggestionsService::IsControlGroup() {
+  return GetExperimentParam(kSuggestionsFieldTrialControlParam) ==
+         kSuggestionsFieldTrialStateEnabled;
+}
+
 void SuggestionsService::FetchSuggestionsData(
     SuggestionsService::ResponseCallback callback) {
   DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));

chrome/browser/search/suggestions/suggestions_service.h

@@ -37,6 +37,7 @@ extern const char kSuggestionsFieldTrialURLParam[];
 extern const char kSuggestionsFieldTrialSuggestionsSuffixParam[];
 extern const char kSuggestionsFieldTrialBlacklistSuffixParam[];
 extern const char kSuggestionsFieldTrialStateParam[];
+extern const char kSuggestionsFieldTrialControlParam[];
 extern const char kSuggestionsFieldTrialStateEnabled[];
 
 // An interface to fetch server suggestions asynchronously.
@@ -51,6 +52,9 @@ class SuggestionsService : public KeyedService, public net::URLFetcherDelegate {
   // Whether this service is enabled.
   static bool IsEnabled();
 
+  // Whether the user is part of a control group.
+  static bool IsControlGroup();
+
   // Request suggestions data, which will be passed to |callback|. Initiates a
   // fetch request unless a pending one exists. To prevent multiple requests,
   // we place all |callback|s in a queue and update them simultaneously when

chrome/browser/search/suggestions/suggestions_service_unittest.cc

@@ -122,7 +122,8 @@ class SuggestionsServiceTest : public testing::Test {
   // Enables the "ChromeSuggestions.Group1" field trial.
   void EnableFieldTrial(const std::string& url,
                         const std::string& suggestions_suffix,
-                        const std::string& blacklist_suffix) {
+                        const std::string& blacklist_suffix,
+                        bool control_group) {
     // Clear the existing |field_trial_list_| to avoid firing a DCHECK.
     field_trial_list_.reset(NULL);
     field_trial_list_.reset(
@@ -132,6 +133,10 @@ class SuggestionsServiceTest : public testing::Test {
     std::map<std::string, std::string> params;
     params[kSuggestionsFieldTrialStateParam] =
         kSuggestionsFieldTrialStateEnabled;
+    if (control_group) {
+      params[kSuggestionsFieldTrialControlParam] =
+          kSuggestionsFieldTrialStateEnabled;
+    }
     params[kSuggestionsFieldTrialURLParam] = url;
     params[kSuggestionsFieldTrialSuggestionsSuffixParam] = suggestions_suffix;
     params[kSuggestionsFieldTrialBlacklistSuffixParam] = blacklist_suffix;
@@ -159,7 +164,7 @@ class SuggestionsServiceTest : public testing::Test {
   void FetchSuggestionsDataNoTimeoutHelper(bool interleaved_requests) {
     // Field trial enabled with a specific suggestions URL.
     EnableFieldTrial(kFakeSuggestionsURL, kFakeSuggestionsSuffix,
-                     kFakeBlacklistSuffix);
+                     kFakeBlacklistSuffix, false);
     scoped_ptr<SuggestionsService> suggestions_service(
         CreateSuggestionsServiceWithMockStore());
     EXPECT_TRUE(suggestions_service != NULL);
@@ -223,10 +228,19 @@ TEST_F(SuggestionsServiceTest, ServiceBeingCreated) {
   EXPECT_TRUE(CreateSuggestionsService() == NULL);
 
   // Field trial enabled.
-  EnableFieldTrial("", "", "");
+  EnableFieldTrial("", "", "", false);
   EXPECT_TRUE(CreateSuggestionsService() != NULL);
 }
 
+TEST_F(SuggestionsServiceTest, IsControlGroup) {
+  // Field trial enabled.
+  EnableFieldTrial("", "", "", false);
+  EXPECT_FALSE(SuggestionsService::IsControlGroup());
+
+  EnableFieldTrial("", "", "", true);
+  EXPECT_TRUE(SuggestionsService::IsControlGroup());
+}
+
 TEST_F(SuggestionsServiceTest, FetchSuggestionsDataNoTimeout) {
   FetchSuggestionsDataNoTimeoutHelper(false);
 }
@@ -238,7 +252,7 @@ TEST_F(SuggestionsServiceTest, FetchSuggestionsDataNoTimeoutInterleaved) {
 TEST_F(SuggestionsServiceTest, FetchSuggestionsDataRequestError) {
   // Field trial enabled with a specific suggestions URL.
   EnableFieldTrial(kFakeSuggestionsURL, kFakeSuggestionsSuffix,
-                   kFakeBlacklistSuffix);
+                   kFakeBlacklistSuffix, false);
   scoped_ptr<SuggestionsService> suggestions_service(
       CreateSuggestionsServiceWithMockStore());
   EXPECT_TRUE(suggestions_service != NULL);
@@ -268,7 +282,7 @@ TEST_F(SuggestionsServiceTest, FetchSuggestionsDataRequestError) {
 TEST_F(SuggestionsServiceTest, FetchSuggestionsDataResponseNotOK) {
   // Field trial enabled with a specific suggestions URL.
   EnableFieldTrial(kFakeSuggestionsURL, kFakeSuggestionsSuffix,
-                   kFakeBlacklistSuffix);
+                   kFakeBlacklistSuffix, false);
   scoped_ptr<SuggestionsService> suggestions_service(
       CreateSuggestionsServiceWithMockStore());
   EXPECT_TRUE(suggestions_service != NULL);
@@ -297,7 +311,7 @@ TEST_F(SuggestionsServiceTest, FetchSuggestionsDataResponseNotOK) {
 
 TEST_F(SuggestionsServiceTest, BlacklistURL) {
   EnableFieldTrial(kFakeSuggestionsURL, kFakeSuggestionsSuffix,
-                   kFakeBlacklistSuffix);
+                   kFakeBlacklistSuffix, false);
   scoped_ptr<SuggestionsService> suggestions_service(
       CreateSuggestionsServiceWithMockStore());
   EXPECT_TRUE(suggestions_service != NULL);