Add data reduction proxy integration tests for video compression.

BUG=474397

Review URL: https://codereview.chromium.org/1065763002

Cr-Commit-Position: refs/heads/master@{#330856}

tools/chrome_proxy/integration_tests/chrome_proxy_benchmark.py

@@ -7,12 +7,16 @@ from integration_tests import chrome_proxy_pagesets as pagesets
 from integration_tests import chrome_proxy_measurements as measurements
 from telemetry import benchmark
 from telemetry.core.backends.chrome import android_browser_finder
+from telemetry.core.backends.chrome import cros_browser_finder
+from telemetry.core.backends.chrome import desktop_browser_finder
 
 
 ANDROID_CHROME_BROWSERS = [
   browser for browser in android_browser_finder.CHROME_PACKAGE_NAMES
     if 'webview' not in browser]
 
+DESKTOP_CHROME_BROWSERS = (desktop_browser_finder.FindAllBrowserTypes(None) +
+  cros_browser_finder.FindAllBrowserTypes(None))
 
 class ChromeProxyClientVersion(ChromeProxyBenchmark):
   tag = 'client_version'
@@ -112,6 +116,7 @@ class ChromeProxySafeBrowsingOff(ChromeProxyBenchmark):
   def Name(cls):
     return 'chrome_proxy_benchmark.safebrowsing_off.safebrowsing'
 
+
 class ChromeProxyHTTPFallbackProbeURL(ChromeProxyBenchmark):
   tag = 'fallback_probe'
   test = measurements.ChromeProxyHTTPFallbackProbeURL
@@ -161,3 +166,42 @@ class ChromeProxySmoke(ChromeProxyBenchmark):
   @classmethod
   def Name(cls):
     return 'chrome_proxy_benchmark.smoke.smoke'
+
+
+@benchmark.Enabled(*DESKTOP_CHROME_BROWSERS)
+class ChromeProxyVideoDirect(benchmark.Benchmark):
+  tag = 'video'
+  test = measurements.ChromeProxyVideoValidation
+  page_set = pagesets.VideoDirectPageSet
+
+  @classmethod
+  def Name(cls):
+    return 'chrome_proxy_benchmark.video.direct'
+
+
+@benchmark.Enabled(*DESKTOP_CHROME_BROWSERS)
+class ChromeProxyVideoProxied(benchmark.Benchmark):
+  tag = 'video'
+  test = measurements.ChromeProxyVideoValidation
+  page_set = pagesets.VideoProxiedPageSet
+
+  @classmethod
+  def Name(cls):
+    return 'chrome_proxy_benchmark.video.proxied'
+
+
+@benchmark.Enabled(*DESKTOP_CHROME_BROWSERS)
+class ChromeProxyVideoCompare(benchmark.Benchmark):
+  """Comparison of direct and proxied video fetches.
+
+  This benchmark runs the ChromeProxyVideoDirect and ChromeProxyVideoProxied
+  benchmarks, then compares their results.
+  """
+
+  tag = 'video'
+  test = measurements.ChromeProxyVideoValidation
+  page_set = pagesets.VideoComparePageSet
+
+  @classmethod
+  def Name(cls):
+    return 'chrome_proxy_benchmark.video.compare'

tools/chrome_proxy/integration_tests/chrome_proxy_measurements.py

@@ -12,33 +12,17 @@ from metrics import loading
 from telemetry.core import exceptions
 from telemetry.page import page_test
 
-class ChromeProxyLatency(page_test.PageTest):
-  """Chrome proxy latency measurement."""
-
-  def __init__(self, *args, **kwargs):
-    super(ChromeProxyLatency, self).__init__(*args, **kwargs)
-    self._metrics = metrics.ChromeProxyMetric()
-
-  def CustomizeBrowserOptions(self, options):
-    options.AppendExtraBrowserArgs('--enable-spdy-proxy-auth')
-
-  def WillNavigateToPage(self, page, tab):
-    tab.ClearCache(force=True)
-
-  def ValidateAndMeasurePage(self, page, tab, results):
-    # Wait for the load event.
-    tab.WaitForJavaScriptExpression('performance.timing.loadEventStart', 300)
-    self._metrics.AddResultsForLatency(tab, results)
-
 
 class ChromeProxyDataSaving(page_test.PageTest):
   """Chrome proxy data saving measurement."""
   def __init__(self, *args, **kwargs):
     super(ChromeProxyDataSaving, self).__init__(*args, **kwargs)
     self._metrics = metrics.ChromeProxyMetric()
+    self._enable_proxy = True
 
   def CustomizeBrowserOptions(self, options):
-    options.AppendExtraBrowserArgs('--enable-spdy-proxy-auth')
+    if self._enable_proxy:
+      options.AppendExtraBrowserArgs('--enable-spdy-proxy-auth')
 
   def WillNavigateToPage(self, page, tab):
     tab.ClearCache(force=True)
@@ -372,3 +356,92 @@ class ChromeProxySmoke(ChromeProxyValidation):
           self._page.name, page_to_metrics.keys()))
     for add_result in page_to_metrics[self._page.name]:
       add_result(tab, results)
+
+
+PROXIED = metrics.PROXIED
+DIRECT = metrics.DIRECT
+
+class ChromeProxyVideoValidation(page_test.PageTest):
+  """Validation for video pages.
+
+  Measures pages using metrics.ChromeProxyVideoMetric. Pages can be fetched
+  either direct from the origin server or via the proxy. If a page is fetched
+  both ways, then the PROXIED and DIRECT measurements are compared to ensure
+  the same video was loaded in both cases.
+  """
+
+  def __init__(self):
+    super(ChromeProxyVideoValidation, self).__init__(
+        needs_browser_restart_after_each_page=True,
+        clear_cache_before_each_run=True)
+    # The type is _allMetrics[url][PROXIED,DIRECT][metricName] = value,
+    # where (metricName,value) is a metric computed by videowrapper.js.
+    self._allMetrics = {}
+
+  def CustomizeBrowserOptionsForSinglePage(self, page, options):
+    if page.use_chrome_proxy:
+      options.AppendExtraBrowserArgs('--enable-spdy-proxy-auth')
+
+  def DidNavigateToPage(self, page, tab):
+    self._currMetrics = metrics.ChromeProxyVideoMetric(tab)
+    self._currMetrics.Start(page, tab)
+
+  def ValidateAndMeasurePage(self, page, tab, results):
+    assert self._currMetrics
+    self._currMetrics.Stop(page, tab)
+    if page.url not in self._allMetrics:
+      self._allMetrics[page.url] = {}
+
+    # Verify this page.
+    if page.use_chrome_proxy:
+      self._currMetrics.AddResultsForProxied(tab, results)
+      self._allMetrics[page.url][PROXIED] = self._currMetrics.videoMetrics
+    else:
+      self._currMetrics.AddResultsForDirect(tab, results)
+      self._allMetrics[page.url][DIRECT] = self._currMetrics.videoMetrics
+    self._currMetrics = None
+
+    # Compare proxied and direct results for this url, if they exist.
+    m = self._allMetrics[page.url]
+    if PROXIED in m and DIRECT in m:
+      self._CompareProxiedAndDirectMetrics(page.url, m[PROXIED], m[DIRECT])
+
+  def _CompareProxiedAndDirectMetrics(self, url, pm, dm):
+    """Compare metrics from PROXIED and DIRECT fetches.
+
+    Compares video metrics computed by videowrapper.js for pages that were
+    fetch both PROXIED and DIRECT.
+
+    Args:
+        url: The url for the page being tested.
+        pm: Metrics when loaded by the Flywheel proxy.
+        dm: Metrics when loaded directly from the origin server.
+
+    Raises:
+        ChromeProxyMetricException on failure.
+    """
+    def err(s):
+      raise ChromeProxyMetricException, s
+
+    if not pm['ready']:
+      err('Proxied page did not load video: %s' % page.url)
+    if not dm['ready']:
+      err('Direct page did not load video: %s' % page.url)
+
+    # Compare metrics that should match for PROXIED and DIRECT.
+    for x in ('video_height', 'video_width', 'video_duration',
+              'decoded_frames'):
+      if x not in pm:
+        err('Proxied page has no %s: %s' % (x, page.url))
+      if x not in dm:
+        err('Direct page has no %s: %s' % (x, page.url))
+      if pm[x] != dm[x]:
+        err('Mismatch for %s (proxied=%s direct=%s): %s' %
+            (x, str(pm[x]), str(dm[x]), page.url))
+
+    # Proxied XOCL should match direct CL.
+    pxocl = pm['x_original_content_length_header']
+    dcl = dm['content_length_header']
+    if pxocl != dcl:
+      err('Mismatch for content length (proxied=%s direct=%s): %s' %
+          (str(pxocl), str(dcl), page.url))

tools/chrome_proxy/integration_tests/chrome_proxy_metrics.py

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import logging
+import os
 import time
 
 from common import chrome_proxy_metrics
@@ -526,3 +527,102 @@ class ChromeProxyMetric(network_metrics.NetworkMetric):
         results.current_page, 'bypass', 'count', bypass_count))
     results.AddValue(scalar.ScalarValue(
         results.current_page, 'via', 'count', via_count))
+
+
+PROXIED = 'proxied'
+DIRECT = 'direct'
+
+class ChromeProxyVideoMetric(network_metrics.NetworkMetric):
+  """Metrics for video pages.
+
+  Wraps the video metrics produced by videowrapper.js, such as the video
+  duration and size in pixels. Also checks a few basic HTTP response headers
+  such as Content-Type and Content-Length in the video responses.
+  """
+
+  def __init__(self, tab):
+    super(ChromeProxyVideoMetric, self).__init__()
+    with open(os.path.join(os.path.dirname(__file__), 'videowrapper.js')) as f:
+      js = f.read()
+      tab.ExecuteJavaScript(js)
+
+  def Start(self, page, tab):
+    tab.ExecuteJavaScript('window.__chromeProxyCreateVideoWrappers()')
+    self.videoMetrics = None
+    super(ChromeProxyVideoMetric, self).Start(page, tab)
+
+  def Stop(self, page, tab):
+    tab.WaitForJavaScriptExpression('window.__chromeProxyVideoLoaded', 30)
+    m = tab.EvaluateJavaScript('window.__chromeProxyVideoMetrics')
+
+    # Now wait for the video to stop playing.
+    # Give it 2x the total duration to account for buffering.
+    waitTime = 2 * m['video_duration']
+    tab.WaitForJavaScriptExpression('window.__chromeProxyVideoEnded', waitTime)
+
+    # Load the final metrics.
+    m = tab.EvaluateJavaScript('window.__chromeProxyVideoMetrics')
+    self.videoMetrics = m
+    # Cast this to an integer as it is often approximate (for an unknown reason)
+    m['video_duration'] = int(m['video_duration'])
+    super(ChromeProxyVideoMetric, self).Stop(page, tab)
+
+  def ResponseFromEvent(self, event):
+    return chrome_proxy_metrics.ChromeProxyResponse(event)
+
+  def AddResults(self, tab, results):
+    raise NotImplementedError
+
+  def AddResultsForProxied(self, tab, results):
+    return self._AddResultsShared(PROXIED, tab, results)
+
+  def AddResultsForDirect(self, tab, results):
+    return self._AddResultsShared(DIRECT, tab, results)
+
+  def _AddResultsShared(self, kind, tab, results):
+    def err(s):
+      raise ChromeProxyMetricException, s
+
+    # Should have played the video.
+    if not self.videoMetrics['ready']:
+      err('%s: video not played' % kind)
+
+    # Should have an HTTP response for the video.
+    wantContentType = 'video/webm' if kind == PROXIED else 'video/mp4'
+    found = False
+    for r in self.IterResponses(tab):
+      resp = r.response
+      if kind == DIRECT and r.HasChromeProxyViaHeader():
+        err('%s: page has proxied Via header' % kind)
+      if resp.GetHeader('Content-Type') != wantContentType:
+        continue
+      if found:
+        err('%s: multiple video responses' % kind)
+      found = True
+
+      cl = resp.GetHeader('Content-Length')
+      xocl = resp.GetHeader('X-Original-Content-Length')
+      if cl != None:
+        self.videoMetrics['content_length_header'] = int(cl)
+      if xocl != None:
+        self.videoMetrics['x_original_content_length_header'] = int(xocl)
+
+      # Should have CL always.
+      if cl == None:
+        err('%s: missing ContentLength' % kind)
+      # Proxied: should have CL < XOCL
+      # Direct: should not have XOCL
+      if kind == PROXIED:
+        if xocl == None or int(cl) >= int(xocl):
+          err('%s: bigger response (%s > %s)' % (kind, str(cl), str(xocl)))
+      else:
+        if xocl != None:
+          err('%s: has XOriginalContentLength' % kind)
+
+    if not found:
+      err('%s: missing video response' % kind)
+
+    # Finally, add all the metrics to the results.
+    for (k,v) in self.videoMetrics.iteritems():
+      k = "%s_%s" % (k, kind)
+      results.AddValue(scalar.ScalarValue(results.current_page, k, "", v))

tools/chrome_proxy/integration_tests/chrome_proxy_pagesets/__init__.py

@@ -8,6 +8,7 @@ import sys
 
 from telemetry.core import discover
 from telemetry.page import page_set
+import video
 
 
 # Import all submodules' PageSet classes.
@@ -17,3 +18,10 @@ base_class = page_set.PageSet
 for cls in discover.DiscoverClasses(
     start_dir, top_level_dir, base_class).values():
   setattr(sys.modules[__name__], cls.__name__, cls)
+
+# DiscoverClasses makes the assumption that there is exactly one matching
+# class per file, however the following are declared in the same file.
+for cls in (video.VideoDirectPageSet,
+            video.VideoProxiedPageSet,
+            video.VideoComparePageSet):
+  setattr(sys.modules[__name__], cls.__name__, cls)

tools/chrome_proxy/integration_tests/chrome_proxy_pagesets/video.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from telemetry.page import page as page_module
+from telemetry.page import page_set as page_set_module
+
+
+class VideoPage(page_module.Page):
+  """A test page containing a video.
+
+  Attributes:
+      use_chrome_proxy: If true, fetches use the data reduction proxy.
+          Otherwise, fetches are sent directly to the origin.
+  """
+
+  def __init__(self, url, page_set, use_chrome_proxy):
+    super(VideoPage, self).__init__(url=url, page_set=page_set)
+    self.use_chrome_proxy = use_chrome_proxy
+
+
+class VideoPageSet(page_set_module.PageSet):
+  """Base class for Chrome proxy video tests."""
+
+  def __init__(self, mode):
+    super(VideoPageSet, self).__init__()
+    urls_list = [
+        'http://check.googlezip.net/cacheable/video/buck_bunny_tiny.html',
+    ]
+    for url in urls_list:
+      self._AddUserStoryForURL(url)
+
+  def _AddUserStoryForURL(self, url):
+    raise NotImplementedError
+
+
+class VideoDirectPageSet(VideoPageSet):
+  """Chrome proxy video tests: direct fetch."""
+  def __init__(self):
+    super(VideoDirectPageSet, self).__init__('direct')
+
+  def _AddUserStoryForURL(self, url):
+      self.AddUserStory(VideoPage(url, self, False))
+
+
+class VideoProxiedPageSet(VideoPageSet):
+  """Chrome proxy video tests: proxied fetch."""
+  def __init__(self):
+    super(VideoProxiedPageSet, self).__init__('proxied')
+
+  def _AddUserStoryForURL(self, url):
+      self.AddUserStory(VideoPage(url, self, True))
+
+
+class VideoComparePageSet(VideoPageSet):
+  """Chrome proxy video tests: compare direct and proxied fetches."""
+  def __init__(self):
+    super(VideoComparePageSet, self).__init__('compare')
+
+  def _AddUserStoryForURL(self, url):
+      self.AddUserStory(VideoPage(url, self, False))
+      self.AddUserStory(VideoPage(url, self, True))

tools/chrome_proxy/integration_tests/videowrapper.js

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This script finds the first video element on a page and collect metrics
+// for that element. This is based on src/tools/perf/metrics/media.js.
+
+(function() {
+  // VideoWrapper attaches event listeners to collect metrics.
+  // The constructor starts playing the video.
+  function VideoWrapper(element) {
+    if (!(element instanceof HTMLVideoElement))
+      throw new Error('Unrecognized video element type ' + element);
+    metrics['ready'] = false;
+    this.element = element;
+    element.loop = false;
+    // Set the basic event handlers for this HTML5 video element.
+    this.element.addEventListener('loadedmetadata', this.onLoaded.bind(this));
+    this.element.addEventListener('canplay', this.onCanplay.bind(this));
+    this.element.addEventListener('ended', this.onEnded.bind(this));
+    this.playbackTimer = new Timer();
+    element.play()
+  }
+
+  VideoWrapper.prototype.onLoaded = function(e) {
+    if (this.element.readyState == HTMLMediaElement.HAVE_NOTHING) {
+      return
+    }
+    metrics['ready'] = true;
+    metrics['video_height'] = this.element.videoHeight;
+    metrics['video_width'] = this.element.videoWidth;
+    metrics['video_duration'] = this.element.duration;
+    window.__chromeProxyVideoLoaded = true;
+  };
+
+  VideoWrapper.prototype.onCanplay = function(event) {
+    metrics['time_to_play_ms'] = this.playbackTimer.stop();
+  };
+
+  VideoWrapper.prototype.onEnded = function(event) {
+    var time_to_end = this.playbackTimer.stop() - metrics['time_to_play_ms'];
+    metrics['buffering_time_ms'] = time_to_end - this.element.duration * 1000;
+    metrics['decoded_audio_bytes'] = this.element.webkitAudioDecodedByteCount;
+    metrics['decoded_video_bytes'] = this.element.webkitVideoDecodedByteCount;
+    metrics['decoded_frames'] = this.element.webkitDecodedFrameCount;
+    metrics['dropped_frames'] = this.element.webkitDroppedFrameCount;
+    window.__chromeProxyVideoEnded = true;
+  };
+
+  function MediaMetric(element) {
+    if (element instanceof HTMLMediaElement)
+      return new VideoWrapper(element);
+    throw new Error('Unrecognized media element type.');
+  }
+
+  function Timer() {
+    this.start();
+  }
+
+  Timer.prototype = {
+    start: function() {
+      this.start_ = getCurrentTime();
+    },
+
+    stop: function() {
+      // Return delta time since start in millisecs.
+      return Math.round((getCurrentTime() - this.start_) * 1000) / 1000;
+    }
+  };
+
+  function getCurrentTime() {
+    if (window.performance)
+      return (performance.now ||
+              performance.mozNow ||
+              performance.msNow ||
+              performance.oNow ||
+              performance.webkitNow).call(window.performance);
+    else
+      return Date.now();
+  }
+
+  function createVideoWrappersForDocument() {
+    var videos = document.querySelectorAll('video');
+    switch (videos.length) {
+    case 0:
+      throw new Error('Page has no videos.');
+    case 1:
+      break;
+    default:
+      throw new Error('Page too many videos: ' + videos.length.toString());
+    }
+    new VideoWrapper(videos[0])
+  }
+
+  metrics = {};
+  window.__chromeProxyCreateVideoWrappers = createVideoWrappersForDocument;
+  window.__chromeProxyVideoMetrics = metrics;
+  window.__chromeProxyVideoLoaded = false;
+  window.__chromeProxyVideoEnded = false;
+})();