Media Engagement: Add DAFSA generator and reader

Add a tool to generate preload data in DAFSA format and
add a MediaEngagementPreloadedList class to read it.

BUG=787464

TBR=dpranke@chromium.org

Change-Id: If0d99a2e1091d0b23699f2173d8146582173ee54
Reviewed-on: https://chromium-review.googlesource.com/803994
Commit-Queue: Becca Hughes <beccahughes@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Mounir Lamouri <mlamouri@chromium.org>
Cr-Commit-Position: refs/heads/master@{#522068}

chrome/browser/BUILD.gn

@@ -621,6 +621,8 @@ split_static_library("browser") {
     "media/media_device_id_salt.h",
     "media/media_engagement_contents_observer.cc",
     "media/media_engagement_contents_observer.h",
+    "media/media_engagement_preloaded_list.cc",
+    "media/media_engagement_preloaded_list.h",
     "media/media_engagement_score.cc",
     "media/media_engagement_score.h",
     "media/media_engagement_service.cc",
@@ -1561,6 +1563,7 @@ split_static_library("browser") {
     "//chrome/app/theme:theme_resources",
     "//chrome/browser/budget_service:budget_proto",
     "//chrome/browser/devtools",
+    "//chrome/browser/media:media_engagement_preload_proto",
     "//chrome/browser/media:mojo_bindings",
     "//chrome/browser/media/router",
     "//chrome/browser/metrics:expired_histograms_array",

chrome/browser/media/BUILD.gn

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import("//mojo/public/tools/bindings/mojom.gni")
+import("//third_party/protobuf/proto_library.gni")
 
 mojom("mojo_bindings") {
   sources = [
@@ -14,3 +15,9 @@ mojom("mojo_bindings") {
     "//url/mojo:url_mojom_gurl",
   ]
 }
+
+proto_library("media_engagement_preload_proto") {
+  sources = [
+    "media_engagement_preload.proto",
+  ]
+}

chrome/browser/media/media_engagement_preload.proto

+syntax = "proto2";
+
+package chrome_browser_media;
+
+// Chrome requires this.
+option optimize_for = LITE_RUNTIME;
+
+// Next available id: 2
+message PreloadedData {
+  // The data in DAFSA format.
+  optional bytes dafsa = 1;
+}

chrome/browser/media/media_engagement_preloaded_list.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/media/media_engagement_preloaded_list.h"
+
+#include "base/files/file_util.h"
+#include "base/path_service.h"
+#include "chrome/browser/media/media_engagement_preload.pb.h"
+#include "net/base/lookup_string_in_fixed_set.h"
+#include "url/origin.h"
+
+MediaEngagementPreloadedList::MediaEngagementPreloadedList() = default;
+
+MediaEngagementPreloadedList::~MediaEngagementPreloadedList() = default;
+
+bool MediaEngagementPreloadedList::LoadFromFile(const base::FilePath& path) {
+  // Check the file exists.
+  if (!base::PathExists(path))
+    return false;
+
+  // Read the file to a string.
+  std::string file_data;
+  if (!base::ReadFileToString(path, &file_data))
+    return false;
+
+  // Load the preloaded list into a proto message.
+  chrome_browser_media::PreloadedData message;
+  if (!message.ParseFromString(file_data))
+    return false;
+
+  // Copy data from the protobuf message.
+  dafsa_ = std::vector<unsigned char>(
+      message.dafsa().c_str(),
+      message.dafsa().c_str() + message.dafsa().length());
+
+  is_loaded_ = true;
+  return true;
+}
+
+bool MediaEngagementPreloadedList::CheckOriginIsPresent(
+    const url::Origin& origin) const {
+  return CheckStringIsPresent(origin.Serialize());
+}
+
+bool MediaEngagementPreloadedList::CheckStringIsPresent(
+    const std::string& input) const {
+  return net::LookupStringInFixedSet(dafsa_.data(), dafsa_.size(),
+                                     input.c_str(),
+                                     input.size()) == net::kDafsaFound;
+}

chrome/browser/media/media_engagement_preloaded_list.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_MEDIA_MEDIA_ENGAGEMENT_PRELOADED_LIST_H_
+#define CHROME_BROWSER_MEDIA_MEDIA_ENGAGEMENT_PRELOADED_LIST_H_
+
+#include <string>
+#include <vector>
+
+#include "base/macros.h"
+
+namespace base {
+class FilePath;
+}  // namespace base
+
+namespace url {
+class Origin;
+}  // namespace url
+
+class MediaEngagementPreloadedList {
+ public:
+  MediaEngagementPreloadedList();
+  ~MediaEngagementPreloadedList();
+
+  // Load the contents from |path|.
+  bool LoadFromFile(const base::FilePath& path);
+
+  // Checks whether |origin| has a high global engagement and is present in the
+  // preloaded list.
+  bool CheckOriginIsPresent(const url::Origin& origin) const;
+
+  // Check whether we have loaded a list.
+  bool loaded() const { return is_loaded_; }
+
+  // Check whether the list we have loaded is empty.
+  bool empty() const { return dafsa_.empty(); }
+
+ protected:
+  friend class MediaEngagementPreloadedListTest;
+
+  // Checks if |input| is present in the preloaded data.
+  bool CheckStringIsPresent(const std::string& input) const;
+
+ private:
+  // The preloaded data in dafsa format.
+  std::vector<unsigned char> dafsa_;
+
+  // If a list has been successfully loaded.
+  bool is_loaded_ = false;
+
+  DISALLOW_COPY_AND_ASSIGN(MediaEngagementPreloadedList);
+};
+
+#endif  // CHROME_BROWSER_MEDIA_MEDIA_ENGAGEMENT_PRELOADED_LIST_H_

chrome/browser/media/media_engagement_preloaded_list_unittest.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/media/media_engagement_preloaded_list.h"
+
+#include <utility>
+
+#include "base/files/file_path.h"
+#include "base/macros.h"
+#include "base/path_service.h"
+#include "build/build_config.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/test/base/testing_profile.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "url/gurl.h"
+#include "url/origin.h"
+
+namespace {
+
+const base::FilePath kTestDataPath = base::FilePath(
+    FILE_PATH_LITERAL("gen/chrome/test/data/media/engagement/preload"));
+
+// This sample data is auto generated at build time.
+const base::FilePath kSampleDataPath = kTestDataPath.AppendASCII("test.pb");
+
+const base::FilePath kMissingFilePath = kTestDataPath.AppendASCII("missing.pb");
+
+const base::FilePath kBadFormatFilePath =
+    kTestDataPath.AppendASCII("bad_format.pb");
+
+const base::FilePath kEmptyFilePath = kTestDataPath.AppendASCII("empty.pb");
+
+base::FilePath GetModulePath() {
+  base::FilePath module_dir;
+#if defined(OS_ANDROID)
+  EXPECT_TRUE(PathService::Get(base::DIR_SOURCE_ROOT, &module_dir));
+#else
+  EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &module_dir));
+#endif
+  return module_dir;
+}
+
+}  // namespace
+
+class MediaEngagementPreloadedListTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    preloaded_list_ = std::make_unique<MediaEngagementPreloadedList>();
+    ASSERT_FALSE(IsLoaded());
+    ASSERT_TRUE(IsEmpty());
+  }
+
+  bool LoadFromFile(base::FilePath path) {
+    return preloaded_list_->LoadFromFile(path);
+  }
+
+  bool CheckOriginIsPresent(GURL url) {
+    return preloaded_list_->CheckOriginIsPresent(url::Origin::Create(url));
+  }
+
+  bool CheckStringIsPresent(const std::string& input) {
+    return preloaded_list_->CheckStringIsPresent(input);
+  }
+
+  base::FilePath GetFilePathRelativeToModule(base::FilePath path) {
+    return GetModulePath().Append(path);
+  }
+
+  bool IsLoaded() { return preloaded_list_->loaded(); }
+
+  bool IsEmpty() { return preloaded_list_->empty(); }
+
+ protected:
+  std::unique_ptr<MediaEngagementPreloadedList> preloaded_list_;
+};
+
+TEST_F(MediaEngagementPreloadedListTest, CheckOriginIsPresent) {
+  ASSERT_TRUE(LoadFromFile(GetFilePathRelativeToModule(kSampleDataPath)));
+  EXPECT_TRUE(IsLoaded());
+  EXPECT_FALSE(IsEmpty());
+
+  EXPECT_TRUE(CheckOriginIsPresent(GURL("https://example.com")));
+  EXPECT_TRUE(CheckOriginIsPresent(GURL("https://example.org:1234")));
+  EXPECT_TRUE(CheckOriginIsPresent(GURL("https://test--3ya.com")));
+  EXPECT_TRUE(CheckOriginIsPresent(GURL("http://123.123.123.123")));
+
+  EXPECT_FALSE(CheckOriginIsPresent(GURL("https://example.org")));
+  EXPECT_FALSE(CheckOriginIsPresent(GURL("http://example.com")));
+  EXPECT_FALSE(CheckOriginIsPresent(GURL("http://123.123.123.124")));
+
+  // Make sure only the full origin matches.
+  EXPECT_FALSE(CheckStringIsPresent("123"));
+  EXPECT_FALSE(CheckStringIsPresent("http"));
+  EXPECT_FALSE(CheckStringIsPresent("example.com"));
+}
+
+TEST_F(MediaEngagementPreloadedListTest, LoadMissingFile) {
+  ASSERT_FALSE(LoadFromFile(GetFilePathRelativeToModule(kMissingFilePath)));
+  EXPECT_FALSE(IsLoaded());
+  EXPECT_TRUE(IsEmpty());
+  EXPECT_FALSE(CheckOriginIsPresent(GURL("https://example.com")));
+}
+
+TEST_F(MediaEngagementPreloadedListTest, LoadBadFormatFile) {
+  ASSERT_FALSE(LoadFromFile(GetFilePathRelativeToModule(kBadFormatFilePath)));
+  EXPECT_FALSE(IsLoaded());
+  EXPECT_TRUE(IsEmpty());
+  EXPECT_FALSE(CheckOriginIsPresent(GURL("https://example.com")));
+}
+
+TEST_F(MediaEngagementPreloadedListTest, LoadEmptyFile) {
+  ASSERT_TRUE(LoadFromFile(GetFilePathRelativeToModule(kEmptyFilePath)));
+  EXPECT_TRUE(IsLoaded());
+  EXPECT_TRUE(IsEmpty());
+  EXPECT_FALSE(CheckOriginIsPresent(GURL("https://example.com")));
+}

chrome/test/BUILD.gn

@@ -2244,6 +2244,7 @@ test("unit_tests") {
     "../browser/media/android/router/media_router_android_unittest.cc",
     "../browser/media/cast_remoting_connector_unittest.cc",
     "../browser/media/media_engagement_contents_observer_unittest.cc",
+    "../browser/media/media_engagement_preloaded_list_unittest.cc",
     "../browser/media/media_engagement_score_unittest.cc",
     "../browser/media/media_engagement_service_unittest.cc",
     "../browser/media/media_engagement_session_unittest.cc",
@@ -2545,6 +2546,8 @@ test("unit_tests") {
 
   data_deps = [
     "//testing/buildbot/filters:unit_tests_filters",
+    "//chrome/test/data/media/engagement/preload:generate_preload_list",
+    "//chrome/test/data/media/engagement/preload:test_data",
   ]
 
   data = [

chrome/test/data/media/engagement/preload/BUILD.gn

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/compiled_action.gni")
+
+# Generates a proto file based on the real list.
+action_foreach("generate_preload_list") {
+  script = "//tools/media_engagement_preload/make_dafsa.py"
+
+  sources = [
+    "test.json",
+  ]
+  outputs = [
+    "$target_gen_dir/{{source_name_part}}.pb",
+  ]
+  args = [
+    "{{source}}",
+    rebase_path("${target_gen_dir}/{{source_name_part}}.pb", root_build_dir),
+  ]
+}
+
+# Copy over protobuf files to output directory.
+copy("test_data") {
+  sources = [
+    "bad_format.pb",
+    "empty.pb",
+  ]
+
+  outputs = [
+    "$target_gen_dir/{{source_file_part}}",
+  ]
+}

chrome/test/data/media/engagement/preload/bad_format.pb

+test

chrome/test/data/media/engagement/preload/test.json

+[
+  "https://example.com",
+  "https://example.org:1234",
+  "https://test--3ya.com",
+  "http://123.123.123.123"
+]

tools/media_engagement_preload/OWNERS

+beccahughes@chromium.org
+mlamouri@chromium.org
+
+# COMPONENT: Internals>Media

tools/media_engagement_preload/PRESUBMIT.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Chromium presubmit script for src/tools/media_engagement_preload."""
+
+
+def _RunMakeDafsaTests(input_api, output_api):
+  """Runs unittest for make_dafsa if any related file has been modified."""
+  files = ('tools/media_engagement_preload/make_dafsa.py',
+           'tools/media_engagement_preload/make_dafsa_unittest.py')
+  if not any(f in input_api.LocalPaths() for f in files):
+    return []
+  test_path = input_api.os_path.join(input_api.PresubmitLocalPath(),
+                                     'make_dafsa_unittest.py')
+  cmd_name = 'make_dafsa_unittest'
+  cmd = [input_api.python_executable, test_path]
+  test_cmd = input_api.Command(
+    name=cmd_name,
+    cmd=cmd,
+    kwargs={},
+    message=output_api.PresubmitPromptWarning)
+  return input_api.RunTests([test_cmd])
+
+
+def CheckChangeOnUpload(input_api, output_api):
+  return _RunMakeDafsaTests(input_api, output_api)
+
+
+def CheckChangeOnCommit(input_api, output_api):
+  return _RunMakeDafsaTests(input_api, output_api)

tools/media_engagement_preload/build.sh

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+protoc --python_out . \
+  ../../chrome/browser/media/media_engagement_preload.proto \
+  --proto_path ../../chrome/browser/media

tools/media_engagement_preload/media_engagement_preload_pb2.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: media_engagement_preload.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='media_engagement_preload.proto',
+  package='chrome_browser_media',
+  serialized_pb='\n\x1emedia_engagement_preload.proto\x12\x14\x63hrome_browser_media\"\x1e\n\rPreloadedData\x12\r\n\x05\x64\x61\x66sa\x18\x01 \x01(\x0c\x42\x02H\x03')
+
+
+
+
+_PRELOADEDDATA = _descriptor.Descriptor(
+  name='PreloadedData',
+  full_name='chrome_browser_media.PreloadedData',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='dafsa', full_name='chrome_browser_media.PreloadedData.dafsa', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value="",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  serialized_start=56,
+  serialized_end=86,
+)
+
+DESCRIPTOR.message_types_by_name['PreloadedData'] = _PRELOADEDDATA
+
+class PreloadedData(_message.Message):
+  __metaclass__ = _reflection.GeneratedProtocolMessageType
+  DESCRIPTOR = _PRELOADEDDATA
+
+  # @@protoc_insertion_point(class_scope:chrome_browser_media.PreloadedData)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), 'H\003')
+# @@protoc_insertion_point(module_scope)