CNS seek tests for <video>.

We record seek performance using the product of: - Video formats: webm and ogv. - Network constraints: cable, wifi, and no constraints. - Seek cases: long, short, and buffered seeks. - Video location: cached and un-cached videos. BUG=122749 TEST=manual run Review URL: http://codereview.chromium.org/9960063 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@133585 0039d316-1c4b-4281-b951-d872f2087c98

CNS seek tests for <video>.
We record seek performance using the product of: - Video formats: webm and ogv. - Network constraints: cable, wifi, and no constraints. - Seek cases: long, short, and buffered seeks. - Video location: cached and un-cached videos. BUG=122749 TEST=manual run Review URL: http://codereview.chromium.org/9960063 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@133585 0039d316-1c4b-4281-b951-d872f2087c98
5f13770d · shadi@chromium.org · cb34e2d4 · 5f13770d · 5f13770d · 5f13770d
Commit 5f13770d authored Apr 24, 2012 by shadi@chromium.org
4 changed files
--- a/chrome/test/data/media/html/media_seek.html
+++ b/chrome/test/data/media/html/media_seek.html
+<!-- Used by media_seek_perf to record seek perf metrics. -->
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <title>CNS Seek Tests</title>
+     <script src="utils.js" type="text/javascript"></script>
+  </head>
+  <body>
+    <video controls></video>
+    <div></div>
+  </body>
+  <script type="text/javascript">
+    var video = document.querySelector("video");
+    var logDiv = document.querySelector("div");
+    var ITERATIONS = 3;
+    var SeekTestCase = {
+      SHORT_SEEK: 0,
+      LONG_SEEK: 1,
+      BUFFERED_SEEK: 2
+    }
+    var CachedState = {
+      UNCACHED: 0,
+      CACHED: 1
+    }
+    function log(text) {
+      logDiv.innerText += text + "\n";
+    }
+    function resetSeekRecords() {
+      seekRecords = [];
+      for (cache_index in Object.keys(CachedState)) {
+        seekRecords[cache_index] = [];
+        for (seek_index in Object.keys(SeekTestCase)) {
+           seekRecords[cache_index][seek_index] = [];
+        }
+      }
+    }
+    // Called by the PyAuto controller to initiate testing.
+    function startTest(src) {
+      if (window.domAutomationController)
+        window.domAutomationController.send(true);
+      resetSeekRecords();
+      endTest = false;
+      errorMsg = "";
+      timer = new Timer();
+      video.addEventListener("playing", playing);
+      video.addEventListener("seeked",  seeked);
+      video.addEventListener("error",
+                             function() { end("Error loading media"); });
+      originalSrc = src;
+      log("Running tests on " + originalSrc);
+      log("Starting seek tests without browser caching:");
+      cacheState = CachedState.UNCACHED;
+      iteration = 0;
+      IterationTest();
+    }
+    function IterationTest() {
+      if (iteration < ITERATIONS) {
+        iteration++;
+        log("Test iteration " + iteration);
+        seekState = SeekTestCase.SHORT_SEEK;
+        video.src = getVideoSrc();
+        video.play();
+      } else if (cacheState == CachedState.UNCACHED) {
+        log("Starting seek tests with browser caching:");
+        cacheState = CachedState.CACHED;
+        iteration = 0;
+        IterationTest();
+      } else {
+        endTest = true;
+      }
+    }
+    function getVideoSrc() {
+      if (cacheState == CachedState.UNCACHED) {
+        return GenerateUniqueURL(originalSrc);
+      } else {
+        return video.src;
+      }
+    }
+    function playing() {
+      if (seekState == SeekTestCase.SHORT_SEEK) {
+        timer.start();
+        video.currentTime = 1;
+      }
+    }
+    function seeked() {
+      delta = timer.stop();
+      switch (seekState) {
+        case SeekTestCase.SHORT_SEEK:
+          seekRecords[cacheState][SeekTestCase.SHORT_SEEK].push(delta);
+          log ("short seek in " + delta + "ms.")
+          seekState = SeekTestCase.LONG_SEEK;
+          timer.start();
+          video.currentTime = video.duration - 1;
+          break;
+        // Seek to almost end of file (unbuffered area).
+        case SeekTestCase.LONG_SEEK:
+          seekRecords[cacheState][SeekTestCase.LONG_SEEK].push(delta);
+          log("long seek in " + delta + "ms.")
+          seekState = SeekTestCase.BUFFERED_SEEK;
+          timer.start();
+          video.currentTime = 1;
+          break;
+        case SeekTestCase.BUFFERED_SEEK:
+          seekRecords[cacheState][SeekTestCase.BUFFERED_SEEK].push(delta);
+          log("buffered seek in " + delta + "ms.")
+          IterationTest();
+          break;
+        default:
+          end("An un-expected seek occured.");
+      }
+    }
+    function end(msg) {
+      errorMsg = msg;
+      endTest = true;
+      log(msg);
+    }
+  </script>
+</html>
--- a/chrome/test/data/media/html/utils.js
+++ b/chrome/test/data/media/html/utils.js
@@ -27,3 +27,30 @@ var QueryString = function () {
  return params;
 } ();
+function Timer() {
+  this.start_ = 0;
+  this.times_ = [];
+}
+Timer.prototype = {
+  start: function() {
+    this.start_ = new Date().getTime();
+  },
+  stop: function() {
+    var delta = new Date().getTime() - this.start_;
+    this.times_.push(delta);
+    return delta;
+  },
+  reset: function() {
+    this.start_ = 0;
+    this.times_ = [];
+  }
+}
+function GenerateUniqueURL(src) {
+  var ch = src.indexOf('?') >= 0 ? '&' : '?';
+  return src + ch + 't=' + (new Date()).getTime();
+}
\ No newline at end of file
--- a/chrome/test/functional/media/media_seek_perf.py
+++ b/chrome/test/functional/media/media_seek_perf.py
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Seek performance testing for <video>.
+Calculates the short and long seek times for different video formats on
+different network constraints.
+"""
+import logging
+import os
+import pyauto_media
+import pyauto_utils
+import cns_test_base
+import worker_thread
+# Number of threads to use during testing.
+_TEST_THREADS = 3
+# HTML test path; relative to src/chrome/test/data.
+_TEST_HTML_PATH = os.path.join('media', 'html', 'media_seek.html')
+# The media files used for testing.
+# Path under CNS root folder (pyauto_private/media).
+_TEST_VIDEOS = [os.path.join('dartmoor', 'dartmoor.ogg')]
+_TEST_VIDEOS.extend(os.path.join('crowd', name) for name in
+                    ['crowd2160.webm', 'crowd1080.webm', 'crowd360.webm',
+                     'crowd2160.ogv', 'crowd1080.ogv', 'crowd360.ogv',
+                     'crowd.wav'])
+# Constraints to run tests on.
+_TESTS_TO_RUN = [
+    cns_test_base.Cable,
+    cns_test_base.Wifi,
+    cns_test_base.NoConstraints]
+class SeekWorkerThread(worker_thread.WorkerThread):
+  """Worker thread.  Runs a test for each task in the queue."""
+  def RunTask(self, unique_url, task):
+    """Runs the specific task on the url given.
+    It is assumed that a tab with the unique_url is already loaded.
+    Args:
+      unique_url: A unique identifier of the test page.
+      task: A (series_name, settings, file_name) tuple to run the test on.
+    """
+    series_name, settings, file_name = task
+    video_url = cns_test_base.GetFileURL(
+        file_name, bandwidth=settings[0], latency=settings[1],
+        loss=settings[2])
+    # Start the test!
+    self.CallJavascriptFunc('startTest', [video_url], unique_url)
+    logging.debug('Running perf test for %s.', video_url)
+    # Time out is dependent on (seeking time * iterations).  For 3 iterations
+    # per seek we get total of 18 seeks per test.  We expect buffered and
+    # cached seeks to be fast.  Through experimentation an average of 10 secs
+    # per seek was found to be adequate.
+    if not self.WaitUntil(self.GetDOMValue, args=['endTest', unique_url],
+                          retry_sleep=5, timeout=180, debug=False):
+      error_msg = 'Seek tests timed out.'
+    else:
+      error_msg = self.GetDOMValue('errorMsg', unique_url)
+    if error_msg:
+      logging.error('Error while running the tests: %s.', error_msg)
+    cached_states = self.GetDOMValue(
+        "Object.keys(CachedState).join(',')", unique_url).split(',')
+    seek_test_cases = self.GetDOMValue(
+        "Object.keys(SeekTestCase).join(',')", unique_url).split(',')
+    graph_name = series_name + '_' + os.path.basename(file_name)
+    for state in cached_states:
+      for seek_case in seek_test_cases:
+        if error_msg:
+          results = [-1]
+        else:
+          results = [float(value) for value in self.GetDOMValue(
+              "seekRecords[CachedState.%s][SeekTestCase.%s].join(',')" %
+              (state, seek_case), unique_url).split(',')]
+        pyauto_utils.PrintPerfResult('seek', '%s_%s_%s' %
+                                     (state, seek_case, graph_name),
+                                     results, 'ms')
+class MediaSeekPerfTest(cns_test_base.CNSTestBase):
+  """PyAuto test container.  See file doc string for more information."""
+  def testMediaSeekPerformance(self):
+    """Launches HTML test which plays each video and records seek stats."""
+    tasks = cns_test_base.CreateCNSPerfTasks(_TESTS_TO_RUN, _TEST_VIDEOS)
+    worker_thread.RunWorkerThreads(self, SeekWorkerThread, tasks, _TEST_THREADS,
+                                   _TEST_HTML_PATH)
+if __name__ == '__main__':
+  pyauto_media.Main()
--- a/media/tools/constrained_network_server/cns.py
+++ b/media/tools/constrained_network_server/cns.py
@@ -185,7 +185,7 @@ class ConstrainedNetworkServer(object):
  @cherrypy.expose
  def ServeConstrained(self, f=None, bandwidth=None, latency=None, loss=None,
-                       new_port=False):
+                       new_port=False, no_cache=False, **kwargs):
    """Serves the requested file with the requested constraints.
    Subsequent requests for the same constraints from the same IP will share the
@@ -199,9 +199,16 @@ class ConstrainedNetworkServer(object):
      latency: time to add to each packet (integer in ms).
      loss: percentage of packets to drop (integer, 0-100).
      new_port: whether to use a new port for this request or not.
+      no_cache: Set reponse's cache-control to no-cache.
    """
    cherrypy.log('Got request for %s, bandwidth=%s, latency=%s, loss=%s, '
-                 'new_port=%s' % (f, bandwidth, latency, loss, new_port))
+                 'new_port=%s, no_cache=%s, kwargs=%s' %
+                 (f, bandwidth, latency, loss, new_port, no_cache, kwargs))
+    if no_cache:
+      response = cherrypy.response
+      response.headers['Pragma'] = 'no-cache'
+      response.headers['Cache-Control'] = 'no-cache'
    # CherryPy is a bit wonky at detecting parameters, so just make them all
    # optional and validate them ourselves.
    if not f:
@@ -246,11 +253,14 @@ class ConstrainedNetworkServer(object):
    cherrypy.log('Time to set up port %d = %ssec.' %
                 (constrained_port, end_time - start_time))
-    # Build constrained URL. Only pass on the file parameter.
+    # Build constrained URL using the constrained port and original URL
-    constrained_url = '%s?f=%s' % (
+    # parameters except the network constraints (bandwidth, latency, and loss).
+    constrained_url = '%s?f=%s&no_cache=%s&%s' % (
        cherrypy.url().replace(
            ':%d' % self._options.port, ':%d' % constrained_port),
-        f)
+        f,
+        no_cache,
+        '&'.join(['%s=%s' % (key, kwargs[key]) for key in kwargs]))
    # Redirect request to the constrained port.
    cherrypy.lib.cptools.redirect(constrained_url, internal=False)