Extracting page shingle hashes for similarity detection.

BUG=

Review URL: https://codereview.chromium.org/268673007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@269976 0039d316-1c4b-4281-b951-d872f2087c98

chrome/common/safe_browsing/client_model.proto

@@ -88,4 +88,10 @@ message ClientSideModel {
 
   // Murmur hash seed that was used to hash the page words.
   optional fixed32 murmur_hash_seed = 8;
+
+  // Maximum number of unique shingle hashes per page.
+  optional int32 max_shingles_per_page = 9 [default = 200];
+
+  // The number of words in a shingle.
+  optional int32 shingle_size = 10 [default = 4];
 }

chrome/common/safe_browsing/csd.proto

@@ -68,6 +68,9 @@ message ClientPhishingRequest {
   optional string OBSOLETE_referrer_url = 9;
 
   // Field 11 is only used on the server.
+
+  // List of shingle hashes we extracted.
+  repeated uint32 shingle_hashes = 12 [packed = true];
 }
 
 message ClientPhishingResponse {

chrome/renderer/safe_browsing/phishing_classifier.cc

@@ -63,6 +63,8 @@ void PhishingClassifier::set_phishing_scorer(const Scorer* scorer) {
         &scorer_->page_words(),
         scorer_->max_words_per_term(),
         scorer_->murmurhash3_seed(),
+        scorer_->max_shingles_per_page(),
+        scorer_->shingle_size(),
         clock_.get()));
   } else {
     // We're disabling client-side phishing detection, so tear down all
@@ -154,12 +156,14 @@ void PhishingClassifier::CancelPendingClassification() {
 }
 
 void PhishingClassifier::DOMExtractionFinished(bool success) {
+  shingle_hashes_.reset(new std::set<uint32>);
   if (success) {
     // Term feature extraction can take awhile, so it runs asynchronously
     // in several chunks of work and invokes the callback when finished.
     term_extractor_->ExtractFeatures(
         page_text_,
         features_.get(),
+        shingle_hashes_.get(),
         base::Bind(&PhishingClassifier::TermExtractionFinished,
                    base::Unretained(this)));
   } else {
@@ -197,6 +201,10 @@ void PhishingClassifier::TermExtractionFinished(bool success) {
       feature->set_name(it->first);
       feature->set_value(it->second);
     }
+    for (std::set<uint32>::const_iterator it = shingle_hashes_->begin();
+         it != shingle_hashes_->end(); ++it) {
+      verdict.add_shingle_hashes(*it);
+    }
     float score = static_cast<float>(scorer_->ComputeScore(hashed_features));
     verdict.set_client_score(score);
     verdict.set_is_phishing(score >= kPhishyThreshold);
@@ -236,6 +244,7 @@ void PhishingClassifier::Clear() {
   page_text_ = NULL;
   done_callback_.Reset();
   features_.reset(NULL);
+  shingle_hashes_.reset(NULL);
 }
 
 }  // namespace safe_browsing

chrome/renderer/safe_browsing/phishing_classifier.h

@@ -18,6 +18,8 @@
 #ifndef CHROME_RENDERER_SAFE_BROWSING_PHISHING_CLASSIFIER_H_
 #define CHROME_RENDERER_SAFE_BROWSING_PHISHING_CLASSIFIER_H_
 
+#include <set>
+
 #include "base/basictypes.h"
 #include "base/callback.h"
 #include "base/memory/scoped_ptr.h"
@@ -135,6 +137,7 @@ class PhishingClassifier {
 
   // State for any in-progress extraction.
   scoped_ptr<FeatureMap> features_;
+  scoped_ptr<std::set<uint32> > shingle_hashes_;
   const base::string16* page_text_;  // owned by the caller
   DoneCallback done_callback_;
 

chrome/renderer/safe_browsing/phishing_classifier_browsertest.cc

@@ -93,6 +93,8 @@ class PhishingClassifierTest : public InProcessBrowserTest {
     model.set_murmur_hash_seed(2777808611U);
     model.add_page_word(MurmurHash3String("login", model.murmur_hash_seed()));
     model.set_max_words_per_term(1);
+    model.set_max_shingles_per_page(100);
+    model.set_shingle_size(3);
 
     clock_ = new MockFeatureExtractorClock;
     scorer_.reset(Scorer::Create(model.SerializeAsString()));

chrome/renderer/safe_browsing/phishing_term_feature_extractor.cc

@@ -38,14 +38,19 @@ const int PhishingTermFeatureExtractor::kClockCheckGranularity = 5;
 // actual phishing page.
 const int PhishingTermFeatureExtractor::kMaxTotalTimeMs = 500;
 
-// The maximum size of the negative word cache.
-const int PhishingTermFeatureExtractor::kMaxNegativeWordCacheSize = 1000;
-
 // All of the state pertaining to the current feature extraction.
 struct PhishingTermFeatureExtractor::ExtractionState {
   // Stores up to max_words_per_term_ previous words separated by spaces.
   std::string previous_words;
 
+  // Stores the current shingle after a new word is processed and added in.
+  std::string current_shingle;
+
+  // Stores the sizes of the words in current_shingle. Note: the size includes
+  // the space after each word. In other words, the sum of all sizes in this
+  // list is equal to the length of current_shingle.
+  std::list<size_t> shingle_word_sizes;
+
   // Stores the sizes of the words in previous_words.  Note: the size includes
   // the space after each word.  In other words, the sum of all sizes in this
   // list is equal to the length of previous_words.
@@ -81,12 +86,15 @@ PhishingTermFeatureExtractor::PhishingTermFeatureExtractor(
     const base::hash_set<uint32>* page_word_hashes,
     size_t max_words_per_term,
     uint32 murmurhash3_seed,
+    size_t max_shingles_per_page,
+    size_t shingle_size,
     FeatureExtractorClock* clock)
     : page_term_hashes_(page_term_hashes),
       page_word_hashes_(page_word_hashes),
       max_words_per_term_(max_words_per_term),
       murmurhash3_seed_(murmurhash3_seed),
-      negative_word_cache_(kMaxNegativeWordCacheSize),
+      max_shingles_per_page_(max_shingles_per_page),
+      shingle_size_(shingle_size),
       clock_(clock),
       weak_factory_(this) {
   Clear();
@@ -101,6 +109,7 @@ PhishingTermFeatureExtractor::~PhishingTermFeatureExtractor() {
 void PhishingTermFeatureExtractor::ExtractFeatures(
     const base::string16* page_text,
     FeatureMap* features,
+    std::set<uint32>* shingle_hashes,
     const DoneCallback& done_callback) {
   // The RenderView should have called CancelPendingExtraction() before
   // starting a new extraction, so DCHECK this.
@@ -111,6 +120,7 @@ void PhishingTermFeatureExtractor::ExtractFeatures(
 
   page_text_ = page_text;
   features_ = features;
+  shingle_hashes_ = shingle_hashes,
   done_callback_ = done_callback;
 
   state_.reset(new ExtractionState(*page_text_, clock_->Now()));
@@ -184,18 +194,24 @@ void PhishingTermFeatureExtractor::ExtractFeaturesWithTimeout() {
 
 void PhishingTermFeatureExtractor::HandleWord(
     const base::StringPiece16& word) {
-  // Quickest out if we have seen this word before and know that it's not
-  // part of any term. This avoids the lowercasing and UTF conversion, both of
-  // which are relatively expensive.
-  if (negative_word_cache_.Get(word) != negative_word_cache_.end()) {
-    // We know we're no longer in a possible n-gram, so clear the previous word
-    // state.
-    state_->previous_words.clear();
-    state_->previous_word_sizes.clear();
-    return;
+  // First, extract shingle hashes.
+  const std::string& word_lower = base::UTF16ToUTF8(base::i18n::ToLower(word));
+  state_->current_shingle.append(word_lower + " ");
+  state_->shingle_word_sizes.push_back(word_lower.size() + 1);
+  if (state_->shingle_word_sizes.size() == shingle_size_) {
+    shingle_hashes_->insert(
+        MurmurHash3String(state_->current_shingle, murmurhash3_seed_));
+    state_->current_shingle.erase(0, state_->shingle_word_sizes.front());
+    state_->shingle_word_sizes.pop_front();
+  }
+  // Check if the size of shingle hashes is over the limit.
+  if (shingle_hashes_->size() > max_shingles_per_page_) {
+    // Pop the largest one.
+    std::set<uint32>::iterator it = shingle_hashes_->end();
+    shingle_hashes_->erase(--it);
   }
 
-  std::string word_lower = base::UTF16ToUTF8(base::i18n::ToLower(word));
+  // Next, extract page terms.
   uint32 word_hash = MurmurHash3String(word_lower, murmurhash3_seed_);
 
   // Quick out if the word is not part of any term, which is the common case.
@@ -203,8 +219,6 @@ void PhishingTermFeatureExtractor::HandleWord(
     // Word doesn't exist in our terms so we can clear the n-gram state.
     state_->previous_words.clear();
     state_->previous_word_sizes.clear();
-    // Insert into negative cache so that we don't try this again.
-    negative_word_cache_.Put(word, true);
     return;
   }
 
@@ -276,9 +290,9 @@ void PhishingTermFeatureExtractor::RunCallback(bool success) {
 void PhishingTermFeatureExtractor::Clear() {
   page_text_ = NULL;
   features_ = NULL;
+  shingle_hashes_ = NULL;
   done_callback_.Reset();
   state_.reset(NULL);
-  negative_word_cache_.Clear();
 }
 
 }  // namespace safe_browsing

chrome/renderer/safe_browsing/phishing_term_feature_extractor.h

@@ -16,12 +16,12 @@
 #ifndef CHROME_RENDERER_SAFE_BROWSING_PHISHING_TERM_FEATURE_EXTRACTOR_H_
 #define CHROME_RENDERER_SAFE_BROWSING_PHISHING_TERM_FEATURE_EXTRACTOR_H_
 
+#include <set>
 #include <string>
 
 #include "base/basictypes.h"
 #include "base/callback.h"
 #include "base/containers/hash_tables.h"
-#include "base/containers/mru_cache.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/weak_ptr.h"
 #include "base/strings/string16.h"
@@ -47,6 +47,11 @@ class PhishingTermFeatureExtractor {
   // must ensure that they are valid until the PhishingTermFeatureExtractor is
   // destroyed.
   //
+  // In addition to extracting page terms, we will also extract text shingling
+  // sketch, which consists of hashes of N-gram-words (referred to as shingles)
+  // in the page. |shingle_size| defines N, and |max_shingles_per_page| defines
+  // the maximum number of unique shingle hashes we extracted per page.
+  //
   // |clock| is used for timing feature extractor operations, and may be mocked
   // for testing.  The caller keeps ownership of the clock.
   PhishingTermFeatureExtractor(
@@ -54,6 +59,8 @@ class PhishingTermFeatureExtractor {
       const base::hash_set<uint32>* page_word_hashes,
       size_t max_words_per_term,
       uint32 murmurhash3_seed,
+      size_t max_shingles_per_page,
+      size_t shingle_size,
       FeatureExtractorClock* clock);
   ~PhishingTermFeatureExtractor();
 
@@ -67,11 +74,12 @@ class PhishingTermFeatureExtractor {
   // |done_callback| is run on the current thread.
   // PhishingTermFeatureExtractor takes ownership of the callback.
   //
-  // |page_text| and |features| are owned by the caller, and must not be
-  // destroyed until either |done_callback| is run or
+  // |page_text|, |features|, and |shingle_hashes| are owned by the caller,
+  // and must not be destroyed until either |done_callback| is run or
   // CancelPendingExtraction() is called.
   void ExtractFeatures(const base::string16* page_text,
                        FeatureMap* features,
+                       std::set<uint32>* shingle_hashes,
                        const DoneCallback& done_callback);
 
   // Cancels any pending feature extraction.  The DoneCallback will not be run.
@@ -95,10 +103,6 @@ class PhishingTermFeatureExtractor {
   // before giving up on the current page.
   static const int kMaxTotalTimeMs;
 
-  // The size of the cache that we use to determine if we can avoid lower
-  // casing, hashing, and UTF conversion.
-  static const int kMaxNegativeWordCacheSize;
-
   // Does the actual work of ExtractFeatures.  ExtractFeaturesWithTimeout runs
   // until a predefined maximum amount of time has elapsed, then posts a task
   // to the current MessageLoop to continue extraction.  When extraction
@@ -135,12 +139,11 @@ class PhishingTermFeatureExtractor {
   // The seed for murmurhash3.
   const uint32 murmurhash3_seed_;
 
-  // This cache is used to see if we need to check the word at all, as
-  // converting to UTF8, lowercasing, and hashing are all relatively expensive
-  // operations. Though this is called an MRU cache, it seems to behave like
-  // an LRU cache (i.e. it evicts the oldest accesses first).
-  typedef base::HashingMRUCache<base::StringPiece16, bool> WordCache;
-  WordCache negative_word_cache_;
+  // The maximum number of unique shingle hashes we extract in a page.
+  const size_t max_shingles_per_page_;
+
+  // The number of words in a shingle.
+  const size_t shingle_size_;
 
   // Non-owned pointer to our clock.
   FeatureExtractorClock* clock_;
@@ -148,6 +151,7 @@ class PhishingTermFeatureExtractor {
   // The output parameters from the most recent call to ExtractFeatures().
   const base::string16* page_text_;  // The caller keeps ownership of this.
   FeatureMap* features_;  // The caller keeps ownership of this.
+  std::set<uint32>* shingle_hashes_;
   DoneCallback done_callback_;
 
   // Stores the current state of term extraction from |page_text_|.

chrome/renderer/safe_browsing/scorer.cc

@@ -103,6 +103,14 @@ uint32 Scorer::murmurhash3_seed() const {
   return model_.murmur_hash_seed();
 }
 
+size_t Scorer::max_shingles_per_page() const {
+  return model_.max_shingles_per_page();
+}
+
+size_t Scorer::shingle_size() const {
+  return model_.shingle_size();
+}
+
 double Scorer::ComputeRuleScore(const ClientSideModel::Rule& rule,
                                 const FeatureMap& features) const {
   const base::hash_map<std::string, double>& feature_map = features.features();

chrome/renderer/safe_browsing/scorer.h

@@ -57,6 +57,12 @@ class Scorer {
   // Returns the murmurhash3 seed for the loaded model.
   uint32 murmurhash3_seed() const;
 
+  // Return the maximum number of unique shingle hashes per page.
+  size_t max_shingles_per_page() const;
+
+  // Return the number of words in a shingle.
+  size_t shingle_size() const;
+
  protected:
   // Most clients should use the factory method.  This constructor is public
   // to allow for mock implementations.
@@ -79,6 +85,6 @@ class Scorer {
 
   DISALLOW_COPY_AND_ASSIGN(Scorer);
 };
-}  // namepsace safe_browsing
+}  // namespace safe_browsing
 
 #endif  // CHROME_RENDERER_SAFE_BROWSING_SCORER_H_

chrome/renderer/safe_browsing/scorer_unittest.cc

@@ -55,6 +55,8 @@ class PhishingScorerTest : public ::testing::Test {
 
     model_.set_max_words_per_term(2);
     model_.set_murmur_hash_seed(12345U);
+    model_.set_max_shingles_per_page(10);
+    model_.set_shingle_size(3);
   }
 
   ClientSideModel model_;
@@ -96,6 +98,8 @@ TEST_F(PhishingScorerTest, PageWords) {
               ::testing::ContainerEq(expected_page_words));
   EXPECT_EQ(2U, scorer->max_words_per_term());
   EXPECT_EQ(12345U, scorer->murmurhash3_seed());
+  EXPECT_EQ(10U, scorer->max_shingles_per_page());
+  EXPECT_EQ(3U, scorer->shingle_size());
 }
 
 TEST_F(PhishingScorerTest, ComputeScore) {