Add RAF tests to blink_perf.canvas benchmark.

https://chromium-review.googlesource.com/c/chromium/src/+/1351501 has
reverted some perf changes due to high-end mac laptops crashing. Re-add
that CL piece by piece. This CL adds no new tests, only the RAF
functionality.

Current canvas benchmarks only measure CPU time.
Using requestAnimationFrame (raf) in javascript we can
measure CPU + raster + GPU time. This CL adds a mode to
blink_perf.py that runs canvas all canvas test in a way
that takes raster and GPU time into account.

Roughly doubles the duration of `blink_perf.canvas`
benchmark from ~2 minutes to ~4 minutes on a linux desktop.

Bug: 894043
Change-Id: I50eeacbe47f3f3f0b09f1cb44806cabcad97dc2e
Reviewed-on: https://chromium-review.googlesource.com/c/1352475
Commit-Queue: Caleb Rouleau <crouleau@chromium.org>
Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#611518}

docs/speed/benchmark/harnesses/blink_perf.md

@@ -147,6 +147,28 @@ Example of tracing asynchronous tests:
 
 [simple-blob-measure-async.html](https://chromium.googlesource.com/chromium/src/+/master/third_party/blink/perf_tests/test_data/simple-blob-measure-async.html)
 
+## Canvas Tests
+
+The sub-framework [canvas_runner.js](https://chromium.googlesource.com/chromium/src/+/master/third_party/blink/perf_tests/canvas/resources/canvas_runner.js) is used for
+tests in the `canvas` directory. This can measure rasterization and GPU time
+using requestAnimationFrame (RAF) and contains a callback framework for video.
+
+Normal tests using `runTest()` work similarly to the asynchronous test above,
+but crucially wait for RAF after completing a single trial of
+`MEASURE_DRAW_TIMES` runs.
+
+RAF tests are triggered by appending the query string `raf` (case insensitive)
+to the test's url. These tests wait for RAF to return before making a
+measurement. This way rasterization and GPU time are included in the
+measurement.
+
+For example:
+
+The test [gpu-bound-shader.html](https://chromium.googlesource.com/chromium/src/+/master/third_party/blink/perf_tests/canvas/gpu-bound-shader.html) is just measuring
+CPU, and thus looks extremely fast as the test is just one slow shader.
+
+The url `gpu-bound-shader.html?raf` will measure rasterization and GPU time as
+well, thus giving a more realistic measurement of performance.
 
 ## Running Tests
 

third_party/blink/perf_tests/canvas/resources/canvas_runner.js

-// CanvasRunner is a wrapper of PerformanceTests/resources/runner.js for canvas tests.
+/*
+Runs canvas performance tests to calculate runs/sec for test.doRun().
+This module works in two different ways, depending on requestAnimationFrame
+(RAF). The query string `RAF` in the url determines which test is run.
+*/
 (function () {
   var MEASURE_DRAW_TIMES = 50;
   var MAX_MEASURE_DRAW_TIMES = 1000;
   var MAX_MEASURE_TIME_PER_FRAME = 1000; // 1 sec
+  var IS_RAF_TEST = (
+    document.location.search.substr(1,).toUpperCase() === "RAF");
   var currentTest = null;
   var isTestDone = false;
 
@@ -19,10 +25,15 @@
           return;
         }
         currentTest = test;
+        if (IS_RAF_TEST === true) {
+          runTestRAF();
+        } else {
           runTest();
+        }
       }});
   }
 
+  // Times the CPU on the main thread
   function runTest() {
     try {
       if (currentTest.preRun)
@@ -30,7 +41,8 @@
 
       var start = PerfTestRunner.now();
       var count = 0;
-            while ((PerfTestRunner.now() - start <= MAX_MEASURE_TIME_PER_FRAME) && (count * MEASURE_DRAW_TIMES < MAX_MEASURE_DRAW_TIMES)) {
+      while ((PerfTestRunner.now() - start <= MAX_MEASURE_TIME_PER_FRAME) &&
+        (count * MEASURE_DRAW_TIMES < MAX_MEASURE_DRAW_TIMES)) {
         for (var i = 0; i < MEASURE_DRAW_TIMES; i++) {
           currentTest.doRun();
         }
@@ -42,16 +54,77 @@
       if (currentTest.postRun)
         currentTest.postRun();
 
-            PerfTestRunner.measureValueAsync(MEASURE_DRAW_TIMES * count * 1000 / elapsedTime);
+      let runsPerSecond = MEASURE_DRAW_TIMES * count * 1000 / elapsedTime;
+      PerfTestRunner.measureValueAsync(runsPerSecond);
     } catch(err) {
       CanvasRunner.logFatalError("test fails due to GPU issue. " + err);
-            return;
+      throw err;
     }
 
     if (!isTestDone)
       requestAnimationFrame(runTest);
   }
 
+  // Times CPU + raster + GPU for draw calls, invoked with the ?RAF query string
+  // All times in milliseconds
+  function runTestRAF() {
+    // How long in ms we want each trial to take
+    // Must be much greater than 16 (16ms is the v-sync rate)
+    const GOAL_TIME = 200;
+
+    function runTrial(numRuns) {
+      if (currentTest.preRun) currentTest.preRun();
+      let startTime = PerfTestRunner.now();
+      for (var i = 0; i < numRuns; i++) {
+        currentTest.doRun();
+      }
+      requestAnimationFrame(() => {
+        let elapsedTime = PerfTestRunner.now() - startTime;
+        let runsPerSecond = numRuns * 1000 / elapsedTime;
+        PerfTestRunner.measureValueAsync(runsPerSecond);
+        if (!isTestDone) runTrial(numRuns, startTime);
+      });
+      if (currentTest.ensureComplete) currentTest.ensureComplete();
+      if (currentTest.postRun) currentTest.postRun();
+    }
+
+    // Figure out how many times currentTest.doRun() + RAF will be required
+    // to last GOAL_TIME
+    function calculateNumberOfRuns(resolve, numRuns) {
+      numRuns = numRuns || 1;
+      if (currentTest.preRun) currentTest.preRun();
+      const startTime = PerfTestRunner.now();
+
+      for (var i = 0; i < numRuns; i++) {
+        currentTest.doRun();
+      }
+
+      requestAnimationFrame(() => {
+        let elapsedTime = PerfTestRunner.now() - startTime;
+        if (elapsedTime >= GOAL_TIME) {
+          const timePerRun = elapsedTime / numRuns;
+          const numRunsFinal = Math.round(GOAL_TIME / timePerRun);
+          if (currentTest.ensureComplete) currentTest.ensureComplete();
+          if (currentTest.postRun) currentTest.postRun();
+          resolve(numRunsFinal);
+        } else {
+          calculateNumberOfRuns(resolve, numRuns * 2);
+        }
+      });
+    }
+
+    try {
+      new Promise(function(resolve, reject) {
+        calculateNumberOfRuns(resolve);
+      }).then(function(numberOfRuns) {
+        runTrial(numberOfRuns);
+      });
+    } catch(err) {
+      CanvasRunner.logFatalError("test fails due to GPU issue. " + err);
+      throw err;
+    }
+  }
+
   function testDone() {
     isTestDone = true;
   }

third_party/blink/perf_tests/canvas/upload-webgl-to-texture.html

@@ -12,6 +12,8 @@ var destCtx = destCanvas3D.getContext('webgl');
 if (!sourceCtx || !destCtx)
     CanvasRunner.logFatalError("WebGL is not supported or enabled on this platform!");
 var tex = null;
+const width = 1024;
+const height = 1024;
 
 function setSize(width, height) {
     sourceCanvas3D.width = width;
@@ -28,7 +30,7 @@ function renderWebGL(gl) {
     gl.disable(gl.SCISSOR_TEST);
     gl.clear(gl.COLOR_BUFER_BIT);
     gl.enable(gl.SCISSOR_TEST);
-    gl.scissor(rand(1024), rand(1024), rand(1024), rand(1024));
+    gl.scissor(rand(width), rand(height), rand(width), rand(height));
     gl.clearColor(Math.random(), Math.random(), Math.random(), 1);
     gl.clear(gl.COLOR_BUFFER_BIT);
 }
@@ -51,7 +53,7 @@ function postRun() {
 }
 
 window.onload = function () {
-    setSize(1024, 1024);
+    setSize(width, height);
     renderWebGL(sourceCtx);
     CanvasRunner.start({
         description: "This benchmark checks the speed on uploading WebGL(1024x1024) to WebGL Texture(1024x1024).",

tools/perf/benchmark.csv

@@ -5,7 +5,7 @@ angle_perftests,"jmadill@chromium.org, chrome-gpu-perf-owners@chromium.org",Inte
 base_perftests,"skyostil@chromium.org, gab@chromium.org",Internals>SequenceManager,https://chromium.googlesource.com/chromium/src/+/HEAD/base/README.md#performance-testing,
 blink_perf.accessibility,dmazzoni@chromium.org,Blink>Accessibility,https://bit.ly/blink-perf-benchmarks,
 blink_perf.bindings,"jbroman@chromium.org, yukishiino@chromium.org, haraken@chromium.org",Blink>Bindings,https://bit.ly/blink-perf-benchmarks,
-blink_perf.canvas,fserb@chromium.org,Blink>Canvas,https://bit.ly/blink-perf-benchmarks,
+blink_perf.canvas,"aaronhk@chromium.org, fserb@chromium.org",Blink>Canvas,https://bit.ly/blink-perf-benchmarks,
 blink_perf.css,"futhark@chromium.org, andruud@chromium.org",Blink>CSS,https://bit.ly/blink-perf-benchmarks,
 blink_perf.dom,"hayato@chromium.org, tkent@chromium.org",Blink>DOM,https://bit.ly/blink-perf-benchmarks,
 blink_perf.events,hayato@chromium.org,Blink>DOM,https://bit.ly/blink-perf-benchmarks,

tools/perf/benchmarks/blink_perf.py

@@ -39,10 +39,18 @@ class _BlinkPerfPage(page_module.Page):
     action_runner.ExecuteJavaScript('testRunner.scheduleTestRun()')
     action_runner.WaitForJavaScriptCondition('testRunner.isDone', timeout=600)
 
+def StoryNameFromUrl(url, prefix):
+  filename = url[len(prefix):].strip('/')
+  baseName, extension = filename.split('.')
+  if extension.find('?') != -1:
+    query = extension.split('?')[1]
+    baseName += "_" + query # So that queried page-names don't collide
+  return "{b}.{e}".format(b=baseName, e=extension)
 
 def CreateStorySetFromPath(path, skipped_file,
                            shared_page_state_class=(
-                               shared_page_state.SharedPageState)):
+                               shared_page_state.SharedPageState),
+                           append_query=None):
   assert os.path.exists(path)
 
   page_urls = []
@@ -54,7 +62,11 @@ def CreateStorySetFromPath(path, skipped_file,
     if '../' in open(path, 'r').read():
       # If the page looks like it references its parent dir, include it.
       serving_dirs.add(os.path.dirname(os.path.dirname(path)))
-    page_urls.append('file://' + path.replace('\\', '/'))
+    page_url = 'file://' + path.replace('\\', '/')
+    if append_query:
+      page_url += '?' + append_query
+    page_urls.append(page_url)
+
 
   def _AddDir(dir_path, skipped):
     for candidate_path in os.listdir(dir_path):
@@ -85,7 +97,7 @@ def CreateStorySetFromPath(path, skipped_file,
   all_urls = [p.rstrip('/') for p in page_urls]
   common_prefix = os.path.dirname(os.path.commonprefix(all_urls))
   for url in sorted(page_urls):
-    name = url[len(common_prefix):].strip('/')
+    name = StoryNameFromUrl(url, common_prefix)
     ps.AddStory(_BlinkPerfPage(
         url, ps, ps.base_dir,
         shared_page_state_class=shared_page_state_class,
@@ -349,7 +361,7 @@ class _BlinkPerfBenchmark(perf_benchmark.PerfBenchmark):
   test = _BlinkPerfMeasurement
 
   def CreateStorySet(self, options):
-    path = os.path.join(BLINK_PERF_BASE_DIR, self.subdir)
+    path = os.path.join(BLINK_PERF_BASE_DIR, self.SUBDIR)
     return CreateStorySetFromPath(path, SKIPPED_FILE)
 
 
@@ -357,8 +369,7 @@ class _BlinkPerfBenchmark(perf_benchmark.PerfBenchmark):
                 component='Blink>Accessibility',
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfAccessibility(_BlinkPerfBenchmark):
-  tag = 'accessibility'
-  subdir = 'accessibility'
+  SUBDIR = 'accessibility'
 
   @classmethod
   def Name(cls):
@@ -376,7 +387,7 @@ class BlinkPerfAccessibility(_BlinkPerfBenchmark):
             'haraken@chromium.org'],
     documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfBindings(_BlinkPerfBenchmark):
-  subdir = 'bindings'
+  SUBDIR = 'bindings'
 
   @classmethod
   def Name(cls):
@@ -387,43 +398,47 @@ class BlinkPerfBindings(_BlinkPerfBenchmark):
                 documentation_url='https://bit.ly/blink-perf-benchmarks',
                 component='Blink>CSS')
 class BlinkPerfCSS(_BlinkPerfBenchmark):
-  subdir = 'css'
+  SUBDIR = 'css'
 
   @classmethod
   def Name(cls):
     return 'blink_perf.css'
 
-
-
-@benchmark.Info(emails=['fserb@chromium.org'],
+@benchmark.Info(emails=['aaronhk@chromium.org', 'fserb@chromium.org'],
                 documentation_url='https://bit.ly/blink-perf-benchmarks',
                 component='Blink>Canvas')
 class BlinkPerfCanvas(_BlinkPerfBenchmark):
-  subdir = 'canvas'
+  SUBDIR = 'canvas'
 
   @classmethod
   def Name(cls):
     return 'blink_perf.canvas'
 
   def CreateStorySet(self, options):
-    path = os.path.join(BLINK_PERF_BASE_DIR, self.subdir)
+    path = os.path.join(BLINK_PERF_BASE_DIR, self.SUBDIR)
     story_set = CreateStorySetFromPath(
         path, SKIPPED_FILE,
         shared_page_state_class=(
             webgl_supported_shared_state.WebGLSupportedSharedState))
+    raf_story_set = CreateStorySetFromPath(
+        path, SKIPPED_FILE,
+        shared_page_state_class=(
+            webgl_supported_shared_state.WebGLSupportedSharedState),
+        append_query="RAF")
+    for raf_story in raf_story_set:
+      story_set.AddStory(raf_story)
     # WebGLSupportedSharedState requires the skipped_gpus property to
     # be set on each page.
     for page in story_set:
       page.skipped_gpus = []
     return story_set
 
-
 @benchmark.Info(emails=['hayato@chromium.org',
                         'tkent@chromium.org'],
                 component='Blink>DOM',
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfDOM(_BlinkPerfBenchmark):
-  subdir = 'dom'
+  SUBDIR = 'dom'
 
   @classmethod
   def Name(cls):
@@ -434,7 +449,7 @@ class BlinkPerfDOM(_BlinkPerfBenchmark):
                 component='Blink>DOM',
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfEvents(_BlinkPerfBenchmark):
-  subdir = 'events'
+  SUBDIR = 'events'
 
   @classmethod
   def Name(cls):
@@ -445,8 +460,7 @@ class BlinkPerfEvents(_BlinkPerfBenchmark):
                 component='Internals>Images>Codecs',
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfImageDecoder(_BlinkPerfBenchmark):
-  tag = 'image_decoder'
-  subdir = 'image_decoder'
+  SUBDIR = 'image_decoder'
 
   @classmethod
   def Name(cls):
@@ -461,7 +475,7 @@ class BlinkPerfImageDecoder(_BlinkPerfBenchmark):
 @benchmark.Info(emails=['eae@chromium.org'],
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfLayout(_BlinkPerfBenchmark):
-  subdir = 'layout'
+  SUBDIR = 'layout'
 
   @classmethod
   def Name(cls):
@@ -471,7 +485,7 @@ class BlinkPerfLayout(_BlinkPerfBenchmark):
 @benchmark.Info(emails=['dmurph@chromium.org'],
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfOWPStorage(_BlinkPerfBenchmark):
-  subdir = 'owp_storage'
+  SUBDIR = 'owp_storage'
 
   @classmethod
   def Name(cls):
@@ -491,7 +505,7 @@ class BlinkPerfOWPStorage(_BlinkPerfBenchmark):
                 component='Blink>Paint',
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfPaint(_BlinkPerfBenchmark):
-  subdir = 'paint'
+  SUBDIR = 'paint'
 
   @classmethod
   def Name(cls):
@@ -504,7 +518,7 @@ class BlinkPerfPaint(_BlinkPerfBenchmark):
                          'haraken@chromium.org'],
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfParser(_BlinkPerfBenchmark):
-  subdir = 'parser'
+  SUBDIR = 'parser'
 
   @classmethod
   def Name(cls):
@@ -515,7 +529,7 @@ class BlinkPerfParser(_BlinkPerfBenchmark):
                 component='Blink>SVG',
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfSVG(_BlinkPerfBenchmark):
-  subdir = 'svg'
+  SUBDIR = 'svg'
 
   @classmethod
   def Name(cls):
@@ -526,7 +540,7 @@ class BlinkPerfSVG(_BlinkPerfBenchmark):
                 component='Blink>DOM>ShadowDOM',
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfShadowDOM(_BlinkPerfBenchmark):
-  subdir = 'shadow_dom'
+  SUBDIR = 'shadow_dom'
 
   @classmethod
   def Name(cls):

tools/perf/contrib/blink_layoutng_perf/blink_layoutng_perf.py

@@ -8,7 +8,7 @@ from telemetry import benchmark
 @benchmark.Info(emails=['cbiesinger@chromium.org'],
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfLayoutNg(blink_perf._BlinkPerfBenchmark):
-  subdir = 'layout'
+  SUBDIR = 'layout'
 
   def SetExtraBrowserOptions(self, options):
     super(BlinkPerfLayoutNg, self).SetExtraBrowserOptions(options)
@@ -22,7 +22,7 @@ class BlinkPerfLayoutNg(blink_perf._BlinkPerfBenchmark):
 @benchmark.Info(emails=['cbiesinger@chromium.org'],
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfParserLayoutNg(blink_perf._BlinkPerfBenchmark):
-  subdir = 'parser'
+  SUBDIR = 'parser'
 
   def SetExtraBrowserOptions(self, options):
     super(BlinkPerfParserLayoutNg, self).SetExtraBrowserOptions(options)
@@ -36,7 +36,7 @@ class BlinkPerfParserLayoutNg(blink_perf._BlinkPerfBenchmark):
 @benchmark.Info(emails=['cbiesinger@chromium.org'],
                 documentation_url='https://bit.ly/blink-perf-benchmarks')
 class BlinkPerfPaintLayoutNg(blink_perf._BlinkPerfBenchmark):
-  subdir = 'paint'
+  SUBDIR = 'paint'
 
   def SetExtraBrowserOptions(self, options):
     super(BlinkPerfPaintLayoutNg, self).SetExtraBrowserOptions(options)

tools/perf/contrib/blink_perf_cmdline/blink_perf_cmdline.py

@@ -19,8 +19,8 @@ class BlinkPerfAll(blink_perf._BlinkPerfBenchmark):
                       default=blink_perf.BLINK_PERF_BASE_DIR,
                       help=('Path to blink perf tests. Could be an absolute '
                             'path, a relative path with respect to your '
-                            'current directory, or a relative path with '
-                            'respect to third_party/WebKit/PerformanceTest/)'))
+                            'current directory or a relative path with '
+                            'respect to third_party/blink/perf_tests)'))
 
   def CreateStorySet(self, options):
     if os.path.exists(options.test_path):

tools/perf/contrib/blink_perf_xml_http_request/blink_perf_xml_http_request.py

@@ -6,5 +6,4 @@ from benchmarks import blink_perf
 
 # pylint: disable=protected-access
 class BlinkPerfXMLHttpRequest(blink_perf._BlinkPerfBenchmark):
-  tag = 'xml_http_request'
-  subdir = 'xml_http_request'
+  SUBDIR = 'xml_http_request'