Integrate power measurement test with gpu_integration_test harness.

So we can easily take advantage of devtools integration in tests. For example,
the ability to automatically turn into fullscreen video playing mode.

BUG=867155
TEST=manual
R=kbr@chromium.org

Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel;luci.chromium.try:linux_optional_gpu_tests_rel;luci.chromium.try:mac_optional_gpu_tests_rel;luci.chromium.try:win_optional_gpu_tests_rel
Change-Id: I2ed4d91ea82c3fc1308663c5d70e0f37bd50cd78
Reviewed-on: https://chromium-review.googlesource.com/1159690Reviewed-by: Kenneth Russell <kbr@chromium.org>
Commit-Queue: Zhenyao Mo <zmo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#589965}

content/test/gpu/ipg_utils.py → content/test/gpu/gpu_tests/ipg_utils.py

@@ -19,6 +19,7 @@ An easy way to use the APIs are:
    5 seconds, call AnalyzeIPGLogFile(skip_in_sec=5).
 """
 
+import json
 import logging
 import os
 import subprocess
@@ -36,17 +37,17 @@ def LocateIPG():
     return gadget_path
   return None
 
-def GenerateIPGLogFilename(prefix='PowerLog', dir=None, current_run=1,
+def GenerateIPGLogFilename(log_prefix='PowerLog', log_dir=None, current_run=1,
                            total_runs=1, timestamp=False):
   # If all args take default value, it is the IPG's default log path.
-  dir = dir or os.getcwd()
-  dir = os.path.abspath(dir)
+  log_dir = log_dir or os.getcwd()
+  log_dir = os.path.abspath(log_dir)
   if total_runs > 1:
-    prefix = "%s_%d_%d" % (prefix, current_run, total_runs)
+    log_prefix = "%s_%d_%d" % (log_prefix, current_run, total_runs)
   if timestamp:
     now = datetime.datetime.now()
-    prefix = "%s_%s" % (prefix, now.strftime('%Y%m%d%H%M%S'))
-  return os.path.join(dir, prefix + '.csv')
+    log_prefix = "%s_%s" % (log_prefix, now.strftime('%Y%m%d%H%M%S'))
+  return os.path.join(log_dir, log_prefix + '.csv')
 
 def RunIPG(duration_in_s=60, resolution_in_ms=100, logfile=None):
   intel_power_gadget_path = LocateIPG()
@@ -57,7 +58,7 @@ def RunIPG(duration_in_s=60, resolution_in_ms=100, logfile=None):
              (intel_power_gadget_path, duration_in_s, resolution_in_ms))
   if not logfile:
     # It is not necessary but allows to print out the log path for debugging.
-    logfile = GenerateIPGLogFilename();
+    logfile = GenerateIPGLogFilename()
   command = command + (' -file %s' %logfile)
   logging.debug("Running: " + command)
   try:
@@ -87,7 +88,7 @@ def AnalyzeIPGLogFile(logfile=None, skip_in_sec=0):
       cols = len(tokens)
       for ii in range(0, cols):
         if tokens[ii].startswith('Elapsed Time'):
-          col_time = ii;
+          col_time = ii
         elif tokens[ii].endswith('(Watt)'):
           indices.append(ii)
           labels.append(tokens[ii][:-len('(Watt)')])
@@ -109,3 +110,52 @@ def AnalyzeIPGLogFile(logfile=None, skip_in_sec=0):
     for ii in range(0, len(indices)):
       results[labels[ii]] = sums[ii] / samples
   return results
+
+def ProcessResultsFromMultipleIPGRuns(logfiles, skip_in_seconds=0,
+                                      outliers=0, output_json=None):
+  assert len(logfiles) > 1
+  output = {}
+  summary = {}
+  for logfile in logfiles:
+    results = AnalyzeIPGLogFile(logfile, skip_in_seconds)
+    results['log'] = logfile
+    (_, filename) = os.path.split(logfile)
+    (core, _) = os.path.splitext(filename)
+    prefix = 'PowerLog_'
+    if core.startswith(prefix):
+      core = core[len(prefix):]
+    output[core] = results
+
+    for key in results:
+      if key == 'samples' or key == 'log':
+        continue
+      if not key in summary:
+        summary[key] = [results[key]]
+      else:
+        summary[key].append(results[key])
+
+  for key in summary:
+    data = summary[key]
+    assert data and len(data) > 1
+    n = len(data)
+    if outliers > 0:
+      assert outliers * 2 < n
+      data.sort()
+      data = data[outliers:(n - outliers)]
+      n = len(data)
+    logging.debug('%s: valid samples = %d', key, n)
+    mean = sum(data) / float(n)
+    ss = sum((x - mean) ** 2 for x in data)
+    stdev = (ss / float(n)) ** 0.5
+    summary[key] = {
+      'mean': mean,
+      'stdev': stdev,
+    }
+  output['summary'] = summary
+
+  if output_json:
+    json_file = open(output_json, 'w')
+    json_file.write(json.dumps(output, indent=4))
+    json_file.close()
+
+  return summary

content/test/gpu/measure_power_win_intel.py → content/test/gpu/gpu_tests/measure_power_win_intel.py

@@ -16,7 +16,7 @@ python measure_power_win_intel.py --browser=canary --duration=10 --delay=5
   --extra-browser-args="--no-sandbox --disable-features=UseSurfaceLayerForVideo"
 """
 
-import ipg_utils
+from  gpu_tests import ipg_utils
 import logging
 import os
 import shutil
@@ -27,14 +27,14 @@ import time
 import optparse
 
 CHROME_STABLE_PATH = (
-  "C:\Program Files (x86)\Google\Chrome\Application\chrome.exe")
+  r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe")
 CHROME_BETA_PATH = (
-  "C:\Program Files (x86)\Google\Chrome Beta\Application\chrome.exe")
+  r"C:\Program Files (x86)\Google\Chrome Beta\Application\chrome.exe")
 CHROME_DEV_PATH = (
-  "C:\Program Files (x86)\Google\Chrome Dev\Application\chrome.exe")
+  r"C:\Program Files (x86)\Google\Chrome Dev\Application\chrome.exe")
 # The following two paths are relative to the LOCALAPPDATA
-CHROME_CANARY_PATH = "Google\Chrome SxS\Application\chrome.exe"
-CHROMIUM_PATH = "Chromium\Application\chrome.exe"
+CHROME_CANARY_PATH = r"Google\Chrome SxS\Application\chrome.exe"
+CHROMIUM_PATH = r"Chromium\Application\chrome.exe"
 
 SUPPORTED_BROWSERS = ['stable', 'beta', 'dev', 'canary', 'chromium']
 
@@ -55,7 +55,7 @@ def LocateBrowser(options_browser):
   else:
     logging.warning("Invalid value for --browser")
     logging.warning(
-      "Supported values: %s, or a full path to a browser executable." %
+      "Supported values: %s, or a full path to a browser executable.",
       ", ".join(SUPPORTED_BROWSERS))
     return None
   if not os.path.exists(browser):
@@ -94,7 +94,7 @@ def MeasurePowerOnce(browser, logfile, duration, delay, resolution, url,
     time.sleep(0.05)
   try:
     shutil.rmtree(user_data_dir)
-  except Exception as err:
+  except Exception:
     logging.warning("Failed to remove temporary folder: " + user_data_dir)
     logging.warning("Please kill browser and remove it manually to avoid leak")
   results = ipg_utils.AnalyzeIPGLogFile(logfile, delay)
@@ -151,7 +151,7 @@ def main(argv):
   for run in range(0, options.repeat):
     logfile = ipg_utils.GenerateIPGLogFilename(log_prefix, options.logdir,
                                                run, options.repeat, True)
-    logging.info("Iteration #%d out of %d" % (run, options.repeat))
+    logging.info("Iteration #%d out of %d", run, options.repeat)
     results = MeasurePowerOnce(browser, logfile, options.duration,
                                options.delay, options.resolution, options.url,
                                options.extra_browser_args)

content/test/gpu/gpu_tests/power_measurement_integration_test.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This script only works on Windows with Intel CPU. Intel Power Gadget needs
+to be installed on the machine before this script works. The software can be
+downloaded from:
+  https://software.intel.com/en-us/articles/intel-power-gadget-20
+"""
+
+from gpu_tests import gpu_integration_test
+from gpu_tests import ipg_utils
+from gpu_tests.gpu_test_expectations import GpuTestExpectations
+
+import os
+import sys
+
+# There are no expectations for power_measurement
+class PowerMeasurementExpectations(GpuTestExpectations):
+  def SetExpectations(self):
+    pass
+
+class PowerMeasurementIntegrationTest(gpu_integration_test.GpuIntegrationTest):
+  @classmethod
+  def Name(cls):
+    return 'power'
+
+  @classmethod
+  def AddCommandlineArgs(cls, parser):
+    parser.add_option("--duration", default=60, type="int",
+                      help="specify how many seconds Intel Power Gadget "
+                      "measures. By default, 60 seconds is selected.")
+    parser.add_option("--delay", default=10, type="int",
+                      help="specify how many seconds we skip in the data "
+                      "Intel Power Gadget collects. This time is for starting "
+                      "video play, switching to fullscreen mode, etc. "
+                      "By default, 10 seconds is selected.")
+    parser.add_option("--resolution", default=100, type="int",
+                      help="specify how often Intel Power Gadget samples "
+                      "data in milliseconds. By default, 100 ms is selected.")
+    parser.add_option("--url",
+                      help="specify the webpage URL the browser launches with.")
+    parser.add_option("--fullscreen", action="store_true", default=False,
+                      help="specify if the browser goes to fullscreen mode "
+                      "automatically, specifically if there is a single video "
+                      "element in the page, switch it to fullsrceen mode.")
+    parser.add_option("--logdir",
+                      help="Speficy where the Intel Power Gadget log file "
+                      "should be stored. If specified, the log file name will "
+                      "include a timestamp. If not specified, the log file "
+                      "will be PowerLog.csv at the current dir and will be "
+                      "overwritten at next run.")
+    parser.add_option("--repeat", default=1, type="int",
+                      help="specify how many times to repreat the measurement. "
+                      "By default, measure only once. If measure more than "
+                      "once, between each measurement, browser restarts.")
+    parser.add_option("--outliers", default=0, type="int",
+                      help="if a test is repeated multiples and outliers is "
+                      "set to N, then N smallest results and N largest results "
+                      "are discarded before computing mean and stdev.")
+
+  @classmethod
+  def GenerateGpuTests(cls, options):
+    yield ('url', options.url, (options.repeat, options.outliers,
+                                options.fullscreen, options.logdir,
+                                options.duration, options.delay,
+                                options.resolution))
+
+  @classmethod
+  def SetUpProcess(cls):
+    super(cls, PowerMeasurementIntegrationTest).SetUpProcess()
+    cls.CustomizeBrowserArgs([
+      '--autoplay-policy=no-user-gesture-required'
+    ])
+    cls.StartBrowser()
+
+  def RunActualGpuTest(self, test_path, *args):
+    ipg_path = ipg_utils.LocateIPG()
+    if not ipg_path:
+      self.fail("Fail to locate Intel Power Gadget")
+
+    repeat = args[0]
+    outliers = args[1]
+    fullscreen = args[2]
+    ipg_logdir = args[3]
+    ipg_duration = args[4]
+    ipg_delay = args[5]
+    ipg_resolution = args[6]
+
+    print ""
+    print "Total iterations: ", repeat
+    logfiles = []
+    for iteration in range(repeat):
+      run_label = "Iteration_%d" % iteration
+      print run_label
+      if test_path:
+        self.tab.action_runner.Navigate(test_path)
+
+      if fullscreen:
+        # TODO(zmo): implement fullscreen mode.
+        pass
+
+      logfile = None
+      if ipg_logdir:
+        if not os.path.isdir(ipg_logdir):
+          self.fail("Folder " + ipg_logdir + " doesn't exist")
+        logfile = ipg_utils.GenerateIPGLogFilename(log_dir=ipg_logdir,
+                                                   timestamp=True)
+      ipg_utils.RunIPG(ipg_duration + ipg_delay, ipg_resolution, logfile)
+      logfiles.append(logfile)
+
+      if repeat > 1 and iteration < repeat - 1:
+        self.StopBrowser()
+        self.StartBrowser()
+
+    if repeat == 1:
+      results = ipg_utils.AnalyzeIPGLogFile(logfiles[0], ipg_delay)
+      print "Results: ", results
+    else:
+      json_path = None
+      if ipg_logdir:
+        json_path = os.path.join(ipg_logdir, "output.json")
+        print "Results saved in ", json_path
+
+      summary = ipg_utils.ProcessResultsFromMultipleIPGRuns(
+        logfiles, ipg_delay, outliers, json_path)
+      print 'Summary: ', summary
+
+  @classmethod
+  def _CreateExpectations(cls):
+    return PowerMeasurementExpectations()
+
+def load_tests(loader, tests, pattern):
+  del loader, tests, pattern  # Unused.
+  return gpu_integration_test.LoadAllTestsInModule(sys.modules[__name__])

content/test/gpu/gpu_tests/verify_all_expectations_unittest.py

@@ -146,6 +146,7 @@ class VerifyTestExpectations(unittest.TestCase):
       'info_collection_test.InfoCollectionExpectations',
       'maps_expectations.MapsExpectations',
       'pixel_expectations.PixelExpectations',
+      'power_measurement_integration_test.PowerMeasurementExpectations',
       'screenshot_sync_expectations.ScreenshotSyncExpectations',
       'trace_test_expectations.TraceTestExpectations',
       'webgl_conformance_expectations.WebGLConformanceExpectations',