[omnibox] [doc|drive] Remember scores for sync, cached matches.

Drive responses have noticeable latency. To avoid matches switching
styling between drive-like (e.g. 'Title - Date - Owner - Google Docs')
and non-drive-like (e.g. 'Title - URL'), the provider caches previous
matches and returns them synchronously before the async matches from the
backend are ready.

Cached docs that no longer match the input, should not be displayed;
e.g. the match 'doc x' should not be displayed for input 'doc y' even
though it may be cached from the previous input 'doc '. Therefore,
cached matches previously had their scores set to 0; if still relevant,
they will inherit a non-0 score from a duplicate history or bookmark
match; otherwise, they'll keep a score of 0 and be hidden.

This had 2 problems that resulted in matches either appearing &
disappearing or swapping places during the sync (on keystroke) and async
(on backend response) phases: 1) There isn't always a duplicate match to
inherit a score from. 2) Ranking per history/bookmark scoring could
differ from the ranking per the max of the history/bookmark & document
scoring.

To address the above 2 issues, cached matches no longer have their
scores set to 0 during the async phase. To preserve the original intent,
cached matches will continue to have their scores set to 0 during the
sync phase.

Also groups the code controlling whether to run the document provider
into |IsDocumentProviderAllowed()|, should have no effect.

Also updates doc |stripped_destination_url| to use
|AutocompleteMatch::GURLToStrippedGURL()| (which invokes
|DocumentProvider::GetURLForDeduping()|) instead of calling the latter
directly. The former resorts to a generic striped URL computation if the
latter doc-specific computation doesn't succeed. This ensures a
non-empty |stripped_destination_url| which avoids overriding cache
entries.

Change-Id: I7f8b5c4935df40c0c8a937725f4fb99d93bdd637
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2634131
Commit-Queue: manuk hovanesian <manukh@chromium.org>
Reviewed-by: Justin Donnelly <jdonnelly@chromium.org>
Cr-Commit-Position: refs/heads/master@{#845704}

components/omnibox/browser/document_provider.cc

@@ -365,8 +365,10 @@ bool DocumentProvider::IsDocumentProviderAllowed(
       template_url_service->GetDefaultSearchProvider();
   if (default_provider == nullptr ||
       default_provider->GetEngineType(
-          template_url_service->search_terms_data()) != SEARCH_ENGINE_GOOGLE)
+          template_url_service->search_terms_data()) != SEARCH_ENGINE_GOOGLE) {
     return false;
+  }
+
   if (OmniboxFieldTrial::IsExperimentalKeywordModeEnabled() &&
       input.prefer_keyword()) {
     // If a keyword provider matches, and we're explicitly in keyword mode,
@@ -376,14 +378,31 @@ bool DocumentProvider::IsDocumentProviderAllowed(
     const TemplateURL* keyword_provider =
         KeywordProvider::GetSubstitutingTemplateURLForInput(
             template_url_service, &keyword_input);
-    if (keyword_provider == nullptr)
-      return true;
-    // True if not explicitly in keyword mode, or a Drive suggestion.
-    return !IsExplicitlyInKeywordMode(input, keyword_provider->keyword()) ||
-           base::StartsWith(input.text(),
-                            base::ASCIIToUTF16("drive.google.com"),
-                            base::CompareCase::SENSITIVE);
+    if (keyword_provider &&
+        IsExplicitlyInKeywordMode(input, keyword_provider->keyword()) &&
+        !base::StartsWith(input.text(), base::ASCIIToUTF16("drive.google.com"),
+                          base::CompareCase::SENSITIVE)) {
+      return false;
+    }
+  }
+
+  // There should be no document suggestions fetched for on-focus suggestion
+  // requests, or if the input is empty.
+  if (input.focus_type() != OmniboxFocusType::DEFAULT ||
+      input.type() == metrics::OmniboxInputType::EMPTY) {
+    return false;
   }
+
+  // Experiment: don't issue queries for inputs under some length.
+  if (!WithinBounds(input.text().length(), min_query_length_,
+                    max_query_length_)) {
+    return false;
+  }
+
+  // Don't issue queries for input likely to be a URL.
+  if (IsInputLikelyURL(input))
+    return false;
+
   return true;
 }
 
@@ -417,31 +436,15 @@ void DocumentProvider::Start(const AutocompleteInput& input,
 
   // Perform various checks - feature is enabled, user is allowed to use the
   // feature, we're not under backoff, etc.
-  if (!IsDocumentProviderAllowed(client_, input)) {
+  if (!IsDocumentProviderAllowed(client_, input))
     return;
-  }
-
-  // There should be no document suggestions fetched for on-focus suggestion
-  // requests, or if the input is empty.
-  if (input.focus_type() != OmniboxFocusType::DEFAULT ||
-      input.type() == metrics::OmniboxInputType::EMPTY) {
-    return;
-  }
-
-  // Experiment: don't issue queries for inputs under some length.
-  if (!WithinBounds(input.text().length(), min_query_length_,
-                    max_query_length_))
-    return;
-
-  // Don't issue queries for input likely to be a URL.
-  if (IsInputLikelyURL(input)) {
-    return;
-  }
 
   input_ = input;
 
-  // Return cached suggestions synchronously.
+  // Return cached suggestions synchronously after setting the relevance of any
+  // beyond |provider_max_matches_| to 0.
   CopyCachedMatchesToMatches();
+  DemoteMatchesBeyondMax();
 
   if (!input.want_asynchronous_matches()) {
     return;
@@ -600,11 +603,30 @@ bool DocumentProvider::UpdateResults(const std::string& json_data) {
   if (!response)
     return false;
 
+  // 1) Fill |matches_| with <N> new server matches.
   matches_ = ParseDocumentSearchResults(*response);
+  // 2) Clear cached matches' scores to ensure cached matches for all but the
+  // previous input can only be shown if deduped. E.g., this allows matches for
+  // the input 'pari' to be displayed synchronously for the input 'paris', but
+  // be hidden if the user clears their input and starts anew 'london'.
+  SetCachedMatchesScoresTo0();
+  // 3) Push the <N> new matches to the cache.
   for (auto it = matches_.rbegin(); it != matches_.rend(); ++it)
     matches_cache_.Put(it->stripped_destination_url, *it);
+  // 4) Copy the cached matches to |matches_|, skipping the most recent <N>
+  // cached matches since they were already added in step (1). Pass
+  // |set_scores_to_0| as true as we don't trust cached scores since they may no
+  // longer match the current input; if the cached matches were still relevant,
+  // they would have been returned from the server again.
   CopyCachedMatchesToMatches(matches_.size());
+  // 5) Only now can we shrink the cache to |cache_size_|. Doing this
+  // automatically when pushing the new matches to the cache would reduce it's
+  // effective size, especially if the server returns close to |cache_size_|
+  // matches.
   matches_cache_.ShrinkToSize(cache_size_);
+  // 6) Limit matches to |provider_max_matches_| unless used for deduping; i.e.
+  // set the scores of matches beyond the limit to 0.
+  DemoteMatchesBeyondMax();
 
   return !matches_.empty();
 }
@@ -779,18 +801,14 @@ ACMatches DocumentProvider::ParseDocumentSearchResults(
     int server_score = 0;
     result->GetInteger("score", &server_score);
     int score = 0;
-    // Set |score| only if we haven't surpassed |provider_max_matches_| yet.
-    // Otherwise, score the remaining matches 0 to avoid displaying them except
-    // when deduped with history, shortcut, or bookmark matches.
-    if (matches.size() < provider_max_matches_) {
+
     if (use_client_score && use_server_score)
       score = std::min(client_score, server_score);
     else
       score = use_client_score ? client_score : server_score;
 
     if (cap_score_per_rank) {
-        int score_cap =
-            i < score_caps.size() ? score_caps[i] : score_caps.back();
+      int score_cap = i < score_caps.size() ? score_caps[i] : score_caps.back();
       score = std::min(score, score_cap);
     }
 
@@ -803,7 +821,6 @@ ACMatches DocumentProvider::ParseDocumentSearchResults(
     if (!use_client_score && score >= previous_score)
       score = std::max(previous_score - 1, 0);
     previous_score = score;
-    }
 
     AutocompleteMatch match(this, score, false,
                             AutocompleteMatchType::DOCUMENT_SUGGESTION);
@@ -813,10 +830,17 @@ ACMatches DocumentProvider::ParseDocumentSearchResults(
     match.destination_url = GURL(url);
     base::string16 original_url;
     if (result->GetString("originalUrl", &original_url)) {
-      GURL stripped_url = GetURLForDeduping(GURL(original_url));
-      if (stripped_url.is_valid())
-        match.stripped_destination_url = stripped_url;
+      // |AutocompleteMatch::GURLToStrippedGURL()| will try to use
+      // |GetURLForDeduping()| to extract a doc ID and generate a canonical doc
+      // URL; this is ideal as it handles different URL formats pointing to the
+      // same doc. Otherwise, it'll resort to the typical stripped URL
+      // generation that can still be used for generic deduping and as a key to
+      // |matches_cache_|.
+      match.stripped_destination_url = AutocompleteMatch::GURLToStrippedGURL(
+          GURL(original_url), input_, client_->GetTemplateURLService(),
+          base::string16());
     }
+
     match.contents = AutocompleteMatch::SanitizeString(title);
     match.contents_class = Classify(match.contents, input_.text());
     const base::DictionaryValue* metadata = nullptr;
@@ -874,9 +898,8 @@ ACMatches DocumentProvider::ParseDocumentSearchResults(
 void DocumentProvider::CopyCachedMatchesToMatches(
     size_t skip_n_most_recent_matches) {
   std::for_each(std::next(matches_cache_.begin(), skip_n_most_recent_matches),
-                matches_cache_.end(), [this](const auto& cache_key_match_pair) {
+                matches_cache_.end(), [&](const auto& cache_key_match_pair) {
                   auto match = cache_key_match_pair.second;
-                  match.relevance = 0;
                   match.allowed_to_be_default_match = false;
                   match.TryRichAutocompletion(
                       base::UTF8ToUTF16(match.destination_url.spec()),
@@ -888,6 +911,18 @@ void DocumentProvider::CopyCachedMatchesToMatches(
                 });
 }
 
+void DocumentProvider::SetCachedMatchesScoresTo0() {
+  std::for_each(matches_cache_.begin(), matches_cache_.end(),
+                [&](auto& cache_key_match_pair) {
+                  cache_key_match_pair.second.relevance = 0;
+                });
+}
+
+void DocumentProvider::DemoteMatchesBeyondMax() {
+  for (size_t i = provider_max_matches_; i < matches_.size(); ++i)
+    matches_[i].relevance = 0;
+}
+
 // static
 ACMatchClassifications DocumentProvider::Classify(
     const base::string16& text,

components/omnibox/browser/document_provider.h

@@ -76,19 +76,7 @@ class DocumentProvider : public AutocompleteProvider {
   static const GURL GetURLForDeduping(const GURL& url);
 
  private:
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, CheckFeatureBehindFlag);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest,
-                           CheckFeaturePrerequisiteNoIncognito);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest,
-                           CheckFeaturePrerequisiteNoSync);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest,
-                           CheckFeaturePrerequisiteClientSettingOff);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest,
-                           CheckFeaturePrerequisiteDefaultSearch);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest,
-                           CheckFeatureNotInExplicitKeywordMode);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest,
-                           CheckFeaturePrerequisiteServerBackoff);
+  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, IsDocumentProviderAllowed);
   FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, IsInputLikelyURL);
   FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, ParseDocumentSearchResults);
   FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest,
@@ -104,8 +92,8 @@ class DocumentProvider : public AutocompleteProvider {
                            ParseDocumentSearchResultsWithBadResponse);
   FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, GenerateLastModifiedString);
   FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, Scoring);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, Caching);
-  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, MinQueryLength);
+  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, CachingForAsyncMatches);
+  FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, CachingForSyncMatches);
   FRIEND_TEST_ALL_PREFIXES(DocumentProviderTest, StartCallsStop);
 
   using MatchesCache = base::MRUCache<GURL, AutocompleteMatch>;
@@ -151,11 +139,25 @@ class DocumentProvider : public AutocompleteProvider {
   // Appends |matches_cache_| to |matches_|. Updates their classifications
   // according to |input_.text()| and sets their relevance to 0.
   // |skip_n_most_recent_matches| indicates the number of cached matches already
-  // in |matches_|. E.g. if the drive server responded with 3 docs, these 3 docs
-  // are added both to |matches_| and |matches_cache| prior to invoking
-  // |AddCachedMatches| in order to avoid duplicate matches.
+  //   in |matches_|. E.g. if the drive server responded with 3 docs, these 3
+  //   docs are added both to |matches_| and |matches_cache| prior to invoking
+  //   |CopyCachedMatchesToMatches()| in order to avoid duplicate matches.
   void CopyCachedMatchesToMatches(size_t skip_n_most_recent_matches = 0);
 
+  // Sets the scores of all cached matches to 0. This is invoked before pushing
+  // the latest async response returns so that the scores aren't preserved for
+  // further inputs. E.g., the input 'london' shouldn't display cached docs from
+  // a previous input 'paris'. This can't be done by automatically (i.e. set
+  // scores to 0 before pushing to the cache), as the scores are needed for the
+  // async pass if the user continued their input.
+  void SetCachedMatchesScoresTo0();
+
+  // Sets the scores of matches beyond the first |provider_max_matches_| to 0.
+  // This ensures the doc provider doesn't exceed it's allocated suggestions
+  // while also allowing docs from other providers to be deduped and styled like
+  // docs from the doc provider.
+  void DemoteMatchesBeyondMax();
+
   // Generates the localized last-modified timestamp to present to the user.
   // Full date for old files, mm/dd within the same calendar year, or time-of-
   // day if a file was modified on the same date.

components/omnibox/browser/document_provider_unittest.cc

@@ -78,6 +78,27 @@ class DocumentProviderTest : public testing::Test,
   // AutocompleteProviderListener:
   void OnProviderUpdate(bool updated_matches) override;
 
+  // Set's up |client_| call expectations to enable the doc suggestions; i.e. so
+  // that |IsDocumentProviderAllowed()| returns true. This is not necessary when
+  // invoking helper methods directly, but is required when invoking |Start()|.
+  void InitClient();
+
+  // Return a mock server response containing 1 doc per ID in |doc_ids|.
+  static std::string MakeTestResponse(const std::vector<std::string>& doc_ids,
+                                      int scores) {
+    std::string results = "";
+    for (auto doc_id : doc_ids)
+      results += base::StringPrintf(
+          R"({
+              "title": "Document %s longer title",
+              "score": %d,
+              "url": "https://drive.google.com/open?id=%s",
+              "originalUrl": "https://drive.google.com/open?id=%s",
+            },)",
+          doc_id.c_str(), scores, doc_id.c_str(), doc_id.c_str());
+    return base::StringPrintf(R"({"results": [%s]})", results.c_str());
+  }
+
   std::unique_ptr<FakeAutocompleteProviderClient> client_;
   scoped_refptr<DocumentProvider> provider_;
   TemplateURL* default_template_url_;
@@ -121,86 +142,71 @@ void DocumentProviderTest::OnProviderUpdate(bool updated_matches) {
   // No action required.
 }
 
-TEST_F(DocumentProviderTest, CheckFeatureBehindFlag) {
-  base::test::ScopedFeatureList feature_list;
-  feature_list.InitAndDisableFeature(omnibox::kDocumentProvider);
-  EXPECT_FALSE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
-}
-
-TEST_F(DocumentProviderTest, CheckFeaturePrerequisiteNoIncognito) {
-  base::test::ScopedFeatureList feature_list;
-  feature_list.InitAndEnableFeature(omnibox::kDocumentProvider);
+void DocumentProviderTest::InitClient() {
   EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
       .WillRepeatedly(Return(true));
   EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
   EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
   EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
-
-  // Feature starts enabled.
-  EXPECT_TRUE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
-
-  // Feature should be disabled in incognito.
-  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(true));
-  EXPECT_FALSE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
 }
 
-TEST_F(DocumentProviderTest, CheckFeaturePrerequisiteNoSync) {
+TEST_F(DocumentProviderTest, IsDocumentProviderAllowed) {
+  // Setup so that all checks pass.
   base::test::ScopedFeatureList feature_list;
   feature_list.InitAndEnableFeature(omnibox::kDocumentProvider);
-  EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
-      .WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
+  InitClient();
+  AutocompleteInput input = AutocompleteInput(base::ASCIIToUTF16("text text"),
+                                              metrics::OmniboxEventProto::OTHER,
+                                              TestSchemeClassifier());
 
-  // Feature starts enabled.
-  EXPECT_TRUE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
+  // Check |IsDocumentProviderAllowed()| returns true when all conditions pass.
+  EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
 
-  // Feature should be disabled without active sync.
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillOnce(Return(false));
-  EXPECT_FALSE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
-}
+  // Fail each condition individually and ensure |IsDocumentProviderAllowed()|
+  // returns false.
 
-TEST_F(DocumentProviderTest, CheckFeaturePrerequisiteClientSettingOff) {
+  // Feature must be enabled.
+  {
     base::test::ScopedFeatureList feature_list;
-  feature_list.InitAndEnableFeature(omnibox::kDocumentProvider);
-  EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
-      .WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
+    feature_list.InitAndDisableFeature(omnibox::kDocumentProvider);
+    EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  }
 
-  // Feature starts enabled.
-  EXPECT_TRUE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
+  // Search suggestions must be enabled.
+  EXPECT_CALL(*client_.get(), IsSyncActive()).WillOnce(Return(false));
+  EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
+  EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
 
-  // Disabling toggle in chrome://settings should be respected.
+  // Client-side toggle must be enabled. This should be enabled by default; i.e.
+  // we didn't explicitly enable this above.
   PrefService* fake_prefs = client_->GetPrefs();
   fake_prefs->SetBoolean(omnibox::kDocumentSuggestEnabled, false);
-  EXPECT_FALSE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
+  EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
   fake_prefs->SetBoolean(omnibox::kDocumentSuggestEnabled, true);
-}
+  EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
 
-TEST_F(DocumentProviderTest, CheckFeaturePrerequisiteDefaultSearch) {
-  base::test::ScopedFeatureList feature_list;
-  feature_list.InitAndEnableFeature(omnibox::kDocumentProvider);
-  EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
-      .WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
+  // Should not be an incognito window.
+  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillOnce(Return(true));
+  EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
   EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
+  EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
 
-  // Feature starts enabled.
-  EXPECT_TRUE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
+  // Sync should be enabled.
+  EXPECT_CALL(*client_.get(), IsSyncActive()).WillOnce(Return(false));
+  EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
+  EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
 
-  // Switching default search disables it.
+  // |backoff_for_session_| should be false. This should be the case by default;
+  // i.e. we didn't explicitly set this to false above.
+  provider_->backoff_for_session_ = true;
+  EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  provider_->backoff_for_session_ = false;
+  EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+
+  // Google should be the default search provider. This should be the case by
+  // default; i.e. we didn't explicitly set this above.
   TemplateURLService* template_url_service = client_->GetTemplateURLService();
   TemplateURLData data;
   data.SetShortName(base::ASCIIToUTF16("t"));
@@ -210,67 +216,76 @@ TEST_F(DocumentProviderTest, CheckFeaturePrerequisiteDefaultSearch) {
       template_url_service->Add(std::make_unique<TemplateURL>(data));
   template_url_service->SetUserSelectedDefaultSearchProvider(
       new_default_provider);
-  EXPECT_FALSE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
+  EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
   template_url_service->SetUserSelectedDefaultSearchProvider(
       default_template_url_);
   template_url_service->Remove(new_default_provider);
-}
+  EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
 
-TEST_F(DocumentProviderTest, CheckFeatureNotInExplicitKeywordMode) {
+  // Should not be in explicit keyword mode unless the keyword is the default or
+  // drive.google.com.
+  {
     base::test::ScopedFeatureList feature_list;
     feature_list.InitWithFeatures(
         {omnibox::kDocumentProvider, omnibox::kExperimentalKeywordMode}, {});
-  EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
-      .WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
-
-  // Prevent document search results in explicit keyword mode.
     {
       AutocompleteInput input(base::ASCIIToUTF16("wikipedia.org soup"),
-                            metrics::OmniboxEventProto::NTP,
+                              metrics::OmniboxEventProto::OTHER,
                               TestSchemeClassifier());
       input.set_prefer_keyword(true);
-
       EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
     }
     {
+      // Amazon is not registered as a keyword in |SetUp()|.
       AutocompleteInput input(base::ASCIIToUTF16("amazon.com soup"),
-                            metrics::OmniboxEventProto::NTP,
+                              metrics::OmniboxEventProto::OTHER,
                               TestSchemeClassifier());
       input.set_prefer_keyword(true);
-
       EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
     }
     {
       AutocompleteInput input(base::ASCIIToUTF16("drive.google.com soup"),
-                            metrics::OmniboxEventProto::NTP,
+                              metrics::OmniboxEventProto::OTHER,
                               TestSchemeClassifier());
       input.set_prefer_keyword(true);
-
       EXPECT_TRUE(provider_->IsDocumentProviderAllowed(client_.get(), input));
     }
-}
+  }
 
-TEST_F(DocumentProviderTest, CheckFeaturePrerequisiteServerBackoff) {
-  base::test::ScopedFeatureList feature_list;
-  feature_list.InitAndEnableFeature(omnibox::kDocumentProvider);
-  EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
-      .WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
+  // Input should not be on-focus.
+  {
+    AutocompleteInput input(base::ASCIIToUTF16("text text"),
+                            metrics::OmniboxEventProto::OTHER,
+                            TestSchemeClassifier());
+    input.set_focus_type(OmniboxFocusType::ON_FOCUS);
+    EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  }
 
-  // Feature starts enabled.
-  EXPECT_TRUE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
+  // Input should not be empty.
+  {
+    AutocompleteInput input(base::ASCIIToUTF16("                           "),
+                            metrics::OmniboxEventProto::OTHER,
+                            TestSchemeClassifier());
+    EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  }
 
-  // Server setting backoff flag disables it.
-  provider_->backoff_for_session_ = true;
-  EXPECT_FALSE(
-      provider_->IsDocumentProviderAllowed(client_.get(), AutocompleteInput()));
+  // Input should be of sufficient length. The default limit is 4, which can't
+  // be set here since it's read when the doc provider is constructed.
+  {
+    AutocompleteInput input(base::ASCIIToUTF16("12"),
+                            metrics::OmniboxEventProto::OTHER,
+                            TestSchemeClassifier());
+    EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  }
+
+  // Input should not look like a URL.
+  {
+    AutocompleteInput input(base::ASCIIToUTF16("www.x.com"),
+                            metrics::OmniboxEventProto::OTHER,
+                            TestSchemeClassifier());
+    input.set_focus_type(OmniboxFocusType::ON_FOCUS);
+    EXPECT_FALSE(provider_->IsDocumentProviderAllowed(client_.get(), input));
+  }
 }
 
 TEST_F(DocumentProviderTest, IsInputLikelyURL) {
@@ -313,6 +328,11 @@ TEST_F(DocumentProviderTest, ParseDocumentSearchResults) {
         {
           "title": "Document 2 longer title",
           "url": "https://documentprovider.tld/doc?id=2"
+        },
+        {
+          "title": "Document 3 longer title",
+          "url": "https://documentprovider.tld/doc?id=3",
+          "originalUrl": "http://sites.google.com/google.com/abc/def"
         }
       ]
      })",
@@ -329,7 +349,7 @@ TEST_F(DocumentProviderTest, ParseDocumentSearchResults) {
   provider_->input_.UpdateText(base::UTF8ToUTF16("document longer title"), 0,
                                {});
   ACMatches matches = provider_->ParseDocumentSearchResults(*response);
-  EXPECT_EQ(matches.size(), 2u);
+  EXPECT_EQ(matches.size(), 3u);
 
   EXPECT_EQ(matches[0].contents, base::ASCIIToUTF16("Document 1 longer title"));
   EXPECT_EQ(matches[0].destination_url,
@@ -343,6 +363,15 @@ TEST_F(DocumentProviderTest, ParseDocumentSearchResults) {
   EXPECT_EQ(matches[1].relevance, 0);
   EXPECT_TRUE(matches[1].stripped_destination_url.is_empty());
 
+  EXPECT_EQ(matches[2].contents, base::ASCIIToUTF16("Document 3 longer title"));
+  EXPECT_EQ(matches[2].destination_url,
+            GURL("https://documentprovider.tld/doc?id=3"));
+  EXPECT_EQ(matches[2].relevance, 0);
+  // Matches with an original URL that doesn't contain a doc ID should resort to
+  // using |AutocompleteMatch::GURLToStrippedGURL()|.
+  EXPECT_EQ(matches[2].stripped_destination_url,
+            "http://sites.google.com/google.com/abc/def");
+
   EXPECT_FALSE(provider_->backoff_for_session_);
 }
 
@@ -794,7 +823,8 @@ TEST_F(DocumentProviderTest, GenerateLastModifiedString) {
 
 TEST_F(DocumentProviderTest, GetURLForDeduping) {
   // Checks that |url_string| is a URL for opening |expected_id|. An empty ID
-  // signifies |url_string| is not a Drive document.
+  // signifies |url_string| is not a Drive document and |GetURLForDeduping()| is
+  // expected to simply return an empty (invalid) GURL.
   auto CheckDeduper = [](const std::string& url_string,
                          const std::string& expected_id) {
     const GURL url(url_string);
@@ -806,7 +836,7 @@ TEST_F(DocumentProviderTest, GetURLForDeduping) {
                 GURL("https://drive.google.com/open?id=" + expected_id))
           << url_string;
     } else {
-      EXPECT_EQ(got_output, GURL()) << url_string;
+      EXPECT_FALSE(got_output.is_valid()) << url_string;
     }
   };
 
@@ -878,12 +908,13 @@ TEST_F(DocumentProviderTest, GetURLForDeduping) {
   CheckDeduper("https://drive.google.com/a/google.com/accounts?continueUrl=https%3A%2F%2Fdocs.google.com%2Fa%2Fgoogle.com%2Fdocument%2Fd%2FtH3_d0C-1d%2Fedit", "tH3_d0C-1d");
   CheckDeduper("https://drive.google.com/accounts?continueUrl=https%3A%2F%2Fdocs.google.com%2Fa%2Fgoogle.com%2Fdocument%2Fd%2FtH3_d0C-1d%2Fedit", "tH3_d0C-1d");
 
-  // URLs that do not represent docs and shouldn't be deduped with doc URLs:
+  // URLs that do not represent docs should return an empty (invalid) URL.
   CheckDeduper("https://support.google.com/a/users/answer/1?id=2", "");
   CheckDeduper("https://www.google.com", "");
   CheckDeduper("https://www.google.com/url?url=https://drive.google.com/homepage", "");
   CheckDeduper("https://www.google.com/url?url=https://www.youtube.com/view", "");
   CheckDeduper("https://notdrive.google.com/?x=https%3A%2F%2Fdocs.google.com%2Fa%2Fgoogle.com%2Fdocument%2Fd%2FtH3_d0C-1d%2Fedit", "");
+  CheckDeduper("https://sites.google.com/google.com/abc/def", "");
 
   // clang-format on
 }
@@ -991,21 +1022,7 @@ TEST_F(DocumentProviderTest, Scoring) {
       "rain bow", {669, 669, 793});
 }
 
-TEST_F(DocumentProviderTest, Caching) {
-  auto MakeTestResponse = [](const std::vector<std::string>& doc_ids) {
-    std::string results = "";
-    for (auto doc_id : doc_ids)
-      results += base::StringPrintf(
-          R"({
-              "title": "Document %s longer title",
-              "score": 1150,
-              "url": "https://drive.google.com/open?id=%s",
-              "originalUrl": "https://drive.google.com/open?id=%s",
-            },)",
-          doc_id.c_str(), doc_id.c_str(), doc_id.c_str());
-    return base::StringPrintf(R"({"results": [%s]})", results.c_str());
-  };
-
+TEST_F(DocumentProviderTest, CachingForAsyncMatches) {
   auto GetTestProviderMatches = [this](const std::string& input_text,
                                        const std::string& response_str) {
     provider_->input_.UpdateText(base::UTF8ToUTF16(input_text), 0, {});
@@ -1015,14 +1032,15 @@ TEST_F(DocumentProviderTest, Caching) {
 
   // Partially fill the cache as setup for following tests.
   auto matches =
-      GetTestProviderMatches("input", MakeTestResponse({"0", "1", "2"}));
+      GetTestProviderMatches("input", MakeTestResponse({"0", "1", "2"}, 1150));
   EXPECT_EQ(matches.size(), size_t(3));
   EXPECT_EQ(matches[0].contents, base::UTF8ToUTF16("Document 0 longer title"));
   EXPECT_EQ(matches[1].contents, base::UTF8ToUTF16("Document 1 longer title"));
   EXPECT_EQ(matches[2].contents, base::UTF8ToUTF16("Document 2 longer title"));
 
   // Cache should remove duplicates.
-  matches = GetTestProviderMatches("input", MakeTestResponse({"1", "2", "3"}));
+  matches =
+      GetTestProviderMatches("input", MakeTestResponse({"1", "2", "3"}, 1150));
   EXPECT_EQ(matches.size(), size_t(4));
   EXPECT_EQ(matches[0].contents, base::UTF8ToUTF16("Document 1 longer title"));
   EXPECT_EQ(matches[1].contents, base::UTF8ToUTF16("Document 2 longer title"));
@@ -1031,7 +1049,8 @@ TEST_F(DocumentProviderTest, Caching) {
 
   // Cache size (4) should not restrict number of matches from the current
   // response.
-  matches = GetTestProviderMatches("input", MakeTestResponse({"3", "4", "5"}));
+  matches =
+      GetTestProviderMatches("input", MakeTestResponse({"3", "4", "5"}, 1150));
   EXPECT_EQ(matches.size(), size_t(6));
   EXPECT_EQ(matches[0].contents, base::UTF8ToUTF16("Document 3 longer title"));
   EXPECT_EQ(matches[1].contents, base::UTF8ToUTF16("Document 4 longer title"));
@@ -1041,7 +1060,8 @@ TEST_F(DocumentProviderTest, Caching) {
   EXPECT_EQ(matches[5].contents, base::UTF8ToUTF16("Document 0 longer title"));
 
   // Cache size (4) should restrict number of cached matches appended.
-  matches = GetTestProviderMatches("input", MakeTestResponse({"0", "4", "6"}));
+  matches =
+      GetTestProviderMatches("input", MakeTestResponse({"0", "4", "6"}, 1150));
   EXPECT_EQ(matches.size(), size_t(6));
   EXPECT_EQ(matches[0].contents, base::UTF8ToUTF16("Document 0 longer title"));
   EXPECT_EQ(matches[1].contents, base::UTF8ToUTF16("Document 4 longer title"));
@@ -1056,11 +1076,11 @@ TEST_F(DocumentProviderTest, Caching) {
   // scores coming into play in this test, set the input to match the title
   // similarly enough that the client score will surpass the server score.
   matches = GetTestProviderMatches("docum longer title",
-                                   MakeTestResponse({"5", "4", "7"}));
+                                   MakeTestResponse({"5", "4", "7"}, 1140));
   EXPECT_EQ(matches.size(), size_t(6));
   EXPECT_EQ(matches[0].contents, base::UTF8ToUTF16("Document 5 longer title"));
   EXPECT_EQ(matches[0].GetAdditionalInfo("from cache"), "");
-  EXPECT_EQ(matches[0].relevance, 1150);
+  EXPECT_EQ(matches[0].relevance, 1140);
   EXPECT_THAT(matches[0].contents_class,
               testing::ElementsAre(
                   ACMatchClassification{0, 2}, ACMatchClassification{5, 0},
@@ -1068,7 +1088,7 @@ TEST_F(DocumentProviderTest, Caching) {
                   ACMatchClassification{18, 2}));
   EXPECT_EQ(matches[1].contents, base::UTF8ToUTF16("Document 4 longer title"));
   EXPECT_EQ(matches[1].GetAdditionalInfo("from cache"), "");
-  EXPECT_EQ(matches[1].relevance, 1150);
+  EXPECT_EQ(matches[1].relevance, 1140);
   EXPECT_THAT(matches[1].contents_class,
               testing::ElementsAre(
                   ACMatchClassification{0, 2}, ACMatchClassification{5, 0},
@@ -1076,7 +1096,7 @@ TEST_F(DocumentProviderTest, Caching) {
                   ACMatchClassification{18, 2}));
   EXPECT_EQ(matches[2].contents, base::UTF8ToUTF16("Document 7 longer title"));
   EXPECT_EQ(matches[2].GetAdditionalInfo("from cache"), "");
-  EXPECT_EQ(matches[2].relevance, 1150);
+  EXPECT_EQ(matches[2].relevance, 1140);
   EXPECT_THAT(matches[2].contents_class,
               testing::ElementsAre(
                   ACMatchClassification{0, 2}, ACMatchClassification{5, 0},
@@ -1108,30 +1128,51 @@ TEST_F(DocumentProviderTest, Caching) {
                   ACMatchClassification{18, 2}));
 }
 
-TEST_F(DocumentProviderTest, MinQueryLength) {
+TEST_F(DocumentProviderTest, CachingForSyncMatches) {
   base::test::ScopedFeatureList feature_list;
   feature_list.InitAndEnableFeature(omnibox::kDocumentProvider);
-  EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
-      .WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
+  InitClient();
 
-  // Expect document provider to ignore inputs shorter than min_query_length_.
-  AutocompleteInput short_input(base::ASCIIToUTF16("12"),
+  AutocompleteInput input(base::ASCIIToUTF16("document"),
                           metrics::OmniboxEventProto::OTHER,
                           TestSchemeClassifier());
-  short_input.set_want_asynchronous_matches(false);
-  provider_->Start(short_input, false);
-  EXPECT_NE(short_input.text(), provider_->input_.text());
-
-  // Expect document provider to process inputs longer than min_query_length_.
-  AutocompleteInput long_input(base::ASCIIToUTF16("123456"),
-                               metrics::OmniboxEventProto::OTHER,
-                               TestSchemeClassifier());
-  long_input.set_want_asynchronous_matches(false);
-  provider_->Start(long_input, false);
-  EXPECT_EQ(long_input.text(), provider_->input_.text());
+  input.set_want_asynchronous_matches(false);
+
+  // Expect sync matches to be scored.
+  // Fill cache.
+  provider_->input_ = input;
+  provider_->UpdateResults(MakeTestResponse({"0", "1", "2", "3", "4"}, 1000));
+  // Retrieve sync matches.
+  provider_->Start(input, false);
+  EXPECT_EQ(provider_->matches_.size(), size_t(4));
+  // Sync matches should have scores.
+  EXPECT_EQ(provider_->matches_[0].relevance, 1000);
+  EXPECT_EQ(provider_->matches_[1].relevance, 1000);
+  EXPECT_EQ(provider_->matches_[2].relevance, 1000);
+  // Sync matches beyond |provider_max_matches_| should have scores set to 0.
+  EXPECT_EQ(provider_->matches_[3].relevance, 0);
+
+  // Expect sync match scores to clear scores when receiving new async results.
+  // Fill cache.
+  provider_->UpdateResults(MakeTestResponse({"4", "5"}, 600));
+  // Retrieve sync matches.
+  provider_->Start(input, false);
+  EXPECT_EQ(provider_->matches_.size(), size_t(4));
+  // Sync matches from the latest response should have scores.
+  EXPECT_EQ(provider_->matches_[0].contents,
+            base::UTF8ToUTF16("Document 4 longer title"));
+  EXPECT_EQ(provider_->matches_[0].relevance, 600);
+  EXPECT_EQ(provider_->matches_[1].contents,
+            base::UTF8ToUTF16("Document 5 longer title"));
+  EXPECT_EQ(provider_->matches_[1].relevance, 600);
+  // Sync matches from previous responses should not have scores.
+  EXPECT_EQ(provider_->matches_[2].contents,
+            base::UTF8ToUTF16("Document 0 longer title"));
+  EXPECT_EQ(provider_->matches_[2].relevance, 0);
+  // Sync matches beyond |provider_max_matches_| should have scores set to 0.
+  EXPECT_EQ(provider_->matches_[3].contents,
+            base::UTF8ToUTF16("Document 1 longer title"));
+  EXPECT_EQ(provider_->matches_[3].relevance, 0);
 }
 
 TEST_F(DocumentProviderTest, StartCallsStop) {
@@ -1139,11 +1180,7 @@ TEST_F(DocumentProviderTest, StartCallsStop) {
   // from appearing with the new input
   base::test::ScopedFeatureList feature_list;
   feature_list.InitAndEnableFeature(omnibox::kDocumentProvider);
-  EXPECT_CALL(*client_.get(), SearchSuggestEnabled())
-      .WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsAuthenticated()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsSyncActive()).WillRepeatedly(Return(true));
-  EXPECT_CALL(*client_.get(), IsOffTheRecord()).WillRepeatedly(Return(false));
+  InitClient();
 
   AutocompleteInput invalid_input(base::ASCIIToUTF16("12"),
                                   metrics::OmniboxEventProto::OTHER,