[Telemetry] Add MSE tests to media perf tests.

Currently these tests only provide audio_time_to_play
and video_time_to_play media metrics (along with generic
cpu and memory metrics). Adding new metrics after these
tests are committed simply means adding trace events to
the product code and turning those into metrics in
media_metric.html. In this way, the code I have added
is just the scenarios.

These scenarios are meant to be working examples
that can be edited significantly in the future.
That said, once this is committed, we're probably
fine dropping the legacy mse_cases (since they are
really flaky and haven't been finding any bugs.)
These tests will be enough to cover them.

Bug: 772971,713335
Change-Id: Ib2c32e4ab97738cc8d8e4b745b11820761213ad3
Reviewed-on: https://chromium-review.googlesource.com/733985
Commit-Queue: Caleb Rouleau <crouleau@chromium.org>
Reviewed-by: John Chen <johnchen@chromium.org>
Reviewed-by: Matthew Wolenetz <wolenetz@chromium.org>
Reviewed-by: Ned Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#513002}

tools/perf/page_sets/tough_video_cases.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 from telemetry.page import page as page_module
 from telemetry import story
 
 _PAGE_TAGS_LIST = [
-    # Audio codecs:
+    # Audio codecs.
     'pcm',
     'mp3',
     'aac',
     'vorbis',
     'opus',
-    # Video codecs:
+    # Video codecs.
     'h264',
     'vp8',
     'vp9',
-    # Test types:
+    # Test types.
     'audio_video',
     'audio_only',
     'video_only',
-    # Other filter tags:
+    # Other filter tags.
     'is_50fps',
     'is_4k',
-    # Play action:
+    # Play action.
     'seek',
     'beginning_to_end',
     'background',
-    # Add javascript load
-    'busyjs'
+    # Add javascript load.
+    'busyjs',
+    # VideoStack API.
+    'src',
+    'mse'
 ]
 
+#
+# The following section contains base classes for pages.
+#
 
-class ToughVideoCasesPage(page_module.Page):
+class MediaPage(page_module.Page):
 
   def __init__(self, url, page_set, tags, extra_browser_args=None):
     if tags:
       for t in tags:
         assert t in _PAGE_TAGS_LIST
-    super(ToughVideoCasesPage, self).__init__(
+    assert not ('src' in tags and 'mse' in tags)
+    super(MediaPage, self).__init__(
         url=url, page_set=page_set, tags=tags, name=url.split('/')[-1],
         extra_browser_args=extra_browser_args)
 
 
-class BeginningToEndPlayPage(ToughVideoCasesPage):
+class BeginningToEndPlayPage(MediaPage):
   """A normal play page simply plays the given media until the end."""
 
   def __init__(self, url, page_set, tags, extra_browser_args=None):
     tags.append('beginning_to_end')
+    tags.append('src')
     self.add_browser_metrics = True
     super(BeginningToEndPlayPage, self).__init__(
         url, page_set, tags, extra_browser_args)
@@ -59,16 +68,16 @@ class BeginningToEndPlayPage(ToughVideoCasesPage):
     if self.page_set.measure_memory:
       action_runner.MeasureMemory()
 
-class SeekPage(ToughVideoCasesPage):
+class SeekPage(MediaPage):
   """A seek page seeks twice in the video and measures the seek time."""
 
   def __init__(self, url, page_set, tags, extra_browser_args=None,
                action_timeout_in_seconds=60):
     tags.append('seek')
+    tags.append('src')
     self.skip_basic_metrics = True
     self._action_timeout = action_timeout_in_seconds
-    super(SeekPage, self).__init__(
-        url, page_set, tags, extra_browser_args)
+    super(SeekPage, self).__init__(url, page_set, tags, extra_browser_args)
 
   def RunPageInteractions(self, action_runner):
     timeout = self._action_timeout
@@ -87,7 +96,7 @@ class SeekPage(ToughVideoCasesPage):
     if self.page_set.measure_memory:
       action_runner.MeasureMemory()
 
-class BackgroundPlaybackPage(ToughVideoCasesPage):
+class BackgroundPlaybackPage(MediaPage):
   """A Background playback page plays the given media in a background tab.
 
   The motivation for this test case is crbug.com/678663.
@@ -98,6 +107,7 @@ class BackgroundPlaybackPage(ToughVideoCasesPage):
     self._background_time = background_time
     self.skip_basic_metrics = True
     tags.append('background')
+    tags.append('src')
     # disable-media-suspend is required since for Android background playback
     # gets suspended. This flag makes Android work the same way as desktop and
     # not turn off video playback in the background.
@@ -124,6 +134,25 @@ class BackgroundPlaybackPage(ToughVideoCasesPage):
       action_runner.MeasureMemory()
 
 
+class MSEPage(MediaPage):
+
+  def __init__(self, url, page_set, tags, extra_browser_args=None):
+    tags.append('mse')
+    super(MSEPage, self).__init__(
+        url, page_set, tags, extra_browser_args)
+
+  def RunPageInteractions(self, action_runner):
+    # The page automatically runs the test at load time.
+    action_runner.WaitForJavaScriptCondition('window.__testDone == true')
+    test_failed = action_runner.EvaluateJavaScript('window.__testFailed')
+    if test_failed:
+      raise RuntimeError(action_runner.EvaluateJavaScript('window.__testError'))
+
+
+#
+# The following section contains concrete test page definitions.
+#
+
 class Page2(BeginningToEndPlayPage):
 
   def __init__(self, page_set):
@@ -351,10 +380,50 @@ class Page38(BeginningToEndPlayPage):
       tags=['h264', 'aac', 'audio_video', 'busyjs'])
 
 
+class Page39(MSEPage):
+
+  def __init__(self, page_set):
+    super(Page39, self).__init__(
+      url='file://tough_video_cases/mse.html?media=aac_audio.mp4,h264_video.mp4',
+      page_set=page_set,
+      tags=['h264', 'aac', 'audio_video'])
+
+
+class Page40(MSEPage):
+
+  def __init__(self, page_set):
+    super(Page40, self).__init__(
+      url=('file://tough_video_cases/mse.html?'
+           'media=aac_audio.mp4,h264_video.mp4&waitForPageLoaded=true'),
+      page_set=page_set,
+      tags=['h264', 'aac', 'audio_video'])
+
+
+class Page41(MSEPage):
+
+  def __init__(self, page_set):
+    super(Page41, self).__init__(
+      url='file://tough_video_cases/mse.html?media=aac_audio.mp4',
+      page_set=page_set,
+      tags=['aac', 'audio_only'])
+
+
+class Page42(MSEPage):
+
+  def __init__(self, page_set):
+    super(Page42, self).__init__(
+      url='file://tough_video_cases/mse.html?media=h264_video.mp4',
+      page_set=page_set,
+      tags=['h264', 'video_only'])
+
+
+
 class ToughVideoCasesPageSet(story.StorySet):
   """
   Description: Video Stack Perf pages that report time_to_play, seek time and
-  many other media-specific and generic metrics.
+  many other media-specific and generic metric.
+
+  TODO(crbug/713335): Rename this to MediaStorySet.
   """
   def __init__(self, measure_memory=False):
     super(ToughVideoCasesPageSet, self).__init__(
@@ -362,7 +431,16 @@ class ToughVideoCasesPageSet(story.StorySet):
 
     self.measure_memory = measure_memory
 
-    # Normal play tests:
+    # TODO(crouleau): Clean up this PageX stuff. Either
+    # 1. Add magic that will add all pages to the StorySet
+    # automatically so that the following is unnecessary. See
+    # https://stackoverflow.com/questions/1796180. Just add all classes with
+    # name like "PageX" where X is a number.
+    # or
+    # 2. Don't use classes at all and instead just have a list containing
+    # configuration for each one.
+
+    # Normal play tests.
     self.AddStory(Page2(self))
     self.AddStory(Page4(self))
     self.AddStory(Page7(self))
@@ -378,7 +456,7 @@ class ToughVideoCasesPageSet(story.StorySet):
     self.AddStory(Page32(self))
     self.AddStory(Page34(self))
 
-    # Seek tests:
+    # Seek tests.
     self.AddStory(Page19(self))
     self.AddStory(Page20(self))
     self.AddStory(Page23(self))
@@ -389,8 +467,21 @@ class ToughVideoCasesPageSet(story.StorySet):
     self.AddStory(Page33(self))
     self.AddStory(Page36(self))
 
-    # Background playback tests:
+    # Background playback tests.
     self.AddStory(Page37(self))
 
-    # Tests with high JS load
+    # Tests with high JS load.
     self.AddStory(Page38(self))
+
+    # MSE tests.
+    # TODO(crouleau): Figure out a way to make MSE pages provide consistent
+    # data. Currently the data haa a lot of outliers for time_to_play and other
+    # startup metrics. To do this, we must either:
+    # 1. Tell Telemetry to repeat these pages (this requires Telemetry's
+    # providing this option in the API.)
+    # 2. Edit the page to reload and run multiple times (you can clear the cache
+    # with tab.CleanCache). This requires crbug/775264.
+    self.AddStory(Page39(self))
+    self.AddStory(Page40(self))
+    self.AddStory(Page41(self))
+    self.AddStory(Page42(self))

tools/perf/page_sets/tough_video_cases/aac_audio.mp4.sha1

+2d0fc9b7d1db36c94eadad0c231bdae7a11b0d0d
\ No newline at end of file

tools/perf/page_sets/tough_video_cases/h264_video.mp4.sha1

+0dccf2486fa7d35503c574b31a0c9377aea2501b
\ No newline at end of file

tools/perf/page_sets/tough_video_cases/mse.html

+<!doctype html>
+<!-- This page is used to test Media Source Extensions implementation. -->
+<html>
+  <head>
+    <title>Media Source Extensions (MSE) Performance Test</title>
+    <script src="mse.js"> </script>
+  </head>
+  <body>
+    <div>
+      <video controls style="width: 640px; height: 360px;" id="video_id"></video>
+    </div>
+    <script>
+      __test();
+    </script>
+  </body>
+</html>

tools/perf/page_sets/tough_video_cases/mse.js

+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// The file runs a series of Media Source Entensions (MSE) operations on a
+// video tag to set up a media file for playback. The test takes several URL
+// parameters described in the loadTestParams() function.
+
+(() => {
+  // Map from media content to MIME type. All test content must be added to this
+  // map. (Feel free to extend it for your test case!)
+  const MEDIA_MIMES = {
+    "aac_audio.mp4": "audio/mp4; codecs=\"mp4a.40.2\"",
+    "h264_video.mp4": "video/mp4; codecs=\"avc1.640028\"",
+  };
+  const testParams = {}
+
+  function test() {
+    loadTestParams();
+    if (testParams.waitForPageLoaded) {
+      document.body.onload = () => {
+        runTest();
+      }
+    } else {
+      runTest();
+    }
+  }
+
+  function loadTestParams() {
+    var queryParameters = parseQueryParameters();
+    // waitForPageLoaded determines whether to wait for body.onload event or
+    // to start right away.
+    testParams.waitForPageLoaded =
+        (queryParameters["waitForPageLoaded"] === "true");
+    // startOffset is used to start the media at an offset instead of at the
+    // beginning of the file.
+    testParams.startOffset = parseInt(queryParameters["startOffset"] || "0");
+    // appendSize determines how large a chuck of the media file to append.
+    testParams.appendSize = parseInt(queryParameters["appendSize"] || "65536");
+    // media argument lists the media files to play.
+    testParams.media = queryParameters["media"];
+    if (!testParams.media)
+      throw Error("media parameter must be defined to provide test content");
+    if (!Array.isArray(testParams.media))
+      testParams.media = [testParams.media];
+  }
+
+  function parseQueryParameters() {
+    var params = {};
+    var r = /([^&=]+)=([^&]*)/g;
+    var match;
+    while (match = r.exec(window.location.search.substring(1))) {
+      key = decodeURIComponent(match[1])
+      value = decodeURIComponent(match[2]);
+      if (value.includes(',')) {
+        value = value.split(",");
+      }
+      params[key] = value;
+    }
+    return params;
+  }
+
+  function runTest() {
+    let appenders = [];
+    let mediaElement = document.getElementById("video_id");
+    let mediaSource = new window.MediaSource();
+    window.__mediaSource = mediaSource;
+
+    // Pass the test if currentTime of the media increases since that means that
+    // the file has started playing.
+    // This code can be modified in the future for full playback tests.
+    mediaElement.addEventListener("timeupdate", () => {
+      window.clearTimeout(timeout);
+      PassTest("Test completed after timeupdate event was received.")
+    }, {once: true});
+
+    // Fail the test if we time out.
+    var timeout = setTimeout(function() {
+      FailTest("Test timed out waiting for a mediaElement timeupdate event.");
+    }, 10000);
+
+    for (const media_file of testParams.media) {
+      appenders.push(new BufferAppender(media_file, MEDIA_MIMES[media_file]));
+    }
+    mediaSource.addEventListener('sourceopen', (open_event) => {
+      let mediaSource = open_event.target;
+      for (let i = 0; i < appenders.length; ++i) {
+        appenders[i].onSourceOpen(mediaSource);
+      }
+      for (let i = 0; i < appenders.length; ++i) {
+        appenders[i].attemptAppend(mediaSource);
+      }
+      mediaElement.play();
+    });
+
+    // Fire off XHRs to get media data.
+    for (var i = 0; i < appenders.length; ++i) {
+      appenders[i].requestMediaBytes();
+    }
+
+    // This attaches the mediaSource object to the mediaElement. Once this
+    // operation has completed internally, the mediaSource object readyState
+    // will transition from closed to open, and the sourceopen event will fire.
+    mediaElement.src = URL.createObjectURL(mediaSource);
+  }
+
+  class BufferAppender {
+    constructor(media_file, mimetype) {
+      this.media_file = media_file;
+      this.mimetype = mimetype;
+      this.xhr = new XMLHttpRequest();
+      this.sourceBuffer = null;
+    }
+    requestMediaBytes() {
+      this.xhr.addEventListener('loadend', this.attemptAppend.bind(this));
+      this.xhr.open('GET', this.media_file);
+      this.xhr.setRequestHeader(
+          'Range', 'bytes=' + testParams.startOffset + '-' +
+          (testParams.startOffset + testParams.appendSize - 1));
+      this.xhr.responseType = 'arraybuffer';
+      this.xhr.send();
+    }
+    onSourceOpen(mediaSource) {
+      if (this.sourceBuffer)
+        return;
+      this.sourceBuffer = mediaSource.addSourceBuffer(this.mimetype);
+    }
+    attemptAppend() {
+      if (!this.xhr.response || !this.sourceBuffer)
+        return;
+      this.sourceBuffer.appendBuffer(this.xhr.response);
+      this.xhr = null;
+    }
+  } // End BufferAppender
+
+  function PassTest(message) {
+    console.log("Test passed: " + message);
+    window.__testDone = true;
+  }
+
+  function FailTest(error_message) {
+    console.error("Test failed: " + error_message);
+    window.__testFailed = true;
+    window.__testError = error_message;
+    window.__testDone = true;
+  }
+
+  window.onerror = (messageOrEvent, source, lineno, colno, error) => {
+    // Fail the test if there are any errors. See crbug/777489.
+    // Note that error.stack already contains error.message.
+    FailTest(error.stack);
+  }
+
+  window.__test = test;
+  // These are outputs to be consumed by media Telemetry test driver code.
+  window.__testDone = false;
+  window.__testFailed = false;
+  window.__testError = "";
+})();

tools/perf/page_sets/tough_video_cases/video.html

 <!DOCTYPE html>
+<!-- This page is used to test media src= (video and audio) performance.
+
+  This can be used for src= (but not for MSE or EME) for content of any format
+  that Chrome supports playing.
+
+  TODO(crouleau): Rename file to contrast with mse.html. "video.html" doesn't
+  make sense because sometimes we play audio-only content. Note that this will
+  require a data migration on the chromeperf dashboard since "video.html" is
+  included in the metrics' page names.
+-->
 <html>
   <body>
   </body>
@@ -27,7 +37,8 @@
     netConfig['dsl'] = [1536, 50];
     netConfig['wifi'] = [1024, 60];
     netConfig['none'] = null;
-    // Constrained network server URL.
+    // Constrained network server URL. Note that this is currently broken. A fix
+    // is forthcoming. See crbug.com/676345.
     var CNS_BASE_URL = 'http://cns.chrome:9000/ServeConstrained?';
 
     function getNetsimURL(net) {