Change for preparation for reference-build for PerfBot. PerfBot will be...

Change for preparation for reference-build for PerfBot. PerfBot will be downloading binaries (pyauto.py and its library) for particular build and compare it with current build. This change is preparation for that. It enables media_perf tests to use different binary if reference build option (-a) is used. When the option is set, media_perf_runner runs the test with current build and then reference build (specified by REFERENCE_BUILD_DIR in pyauto_media.py), which I set bogus directory for now.



Review URL: http://codereview.chromium.org/7031020

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@86361 0039d316-1c4b-4281-b951-d872f2087c98

chrome/test/data/media/csv/media_matrix_data.csv

-"The file contains matrix (video type (columns) and its codec (rows)) and each entry has its specification (or "not available"),
-"exts","video/audio","bbee","mobile","bear","sync","color","scout3d","buck","gromit","ironman","trek","dartmoor","still","tulip","chicken","3dtv","avatar","avatar3d","wembly","pitivi","qjoe","birds","red","crowd","parkrun","sonyh","garden"
+"The file contains matrix (video type (columns) and its codec (rows)) and each entry has its specification (or 'not available') - exts","video/audio","bbee","mobile","bear","sync","color","scout3d","buck","gromit","ironman","trek","dartmoor","still","tulip","chicken","3dtv","avatar","avatar3d","wembly","pitivi","qjoe","birds","red","crowd","parkrun","sonyh","garden"
 "0.webm","video","   '320x240 50 fps VP8 / no audio      1 MBit/s PSNR 36.70',","   '352x240 30 fps VP8 / no audio      1 MBit/s PSNR 31.65',","   '640x360 30 fps VP8 / no audio      800 KBit/s PSNR 39.41',","   '720x480 29.97 fps VP8 / no audio      1 MBit/s PSNR 48.07',","  '720x480 29.97 fps VP8 / no audio            1 MBit/s PSNR 40.19',","   '720x576 25.00 fps VP8 / no audio            1 MBit/s PSNR 37.63',","   '848x480 23.976 fps VP8 / no audio      1 MBit/s PSNR 39.97',","   '960x540 25.000 fps VP8 / no audio      1.2 MBit/s PSNR 39.97',","   '1280x532 29.970 fps VP8 / no audio      2 MBit/s PSNR 50.56',","   '1248x544 23.976 fps VP8 / no audio      2 MBit/s PSNR 46.80',","   '1280x720 25.00 fps VP8 / no audio      2 MBit/s PSNR 40.89',","   '1248x720 1 fps VP8 / no audio      2 MBit/s PSNR 36.54',","   '1280x720 29.970 fps VP8 / no audio      2 MBit/s PSNR 37.45',","   '1280x720 23.976 fps VP8 / no audio      2 MBit/s PSNR 46.50',","   '1280x720 25.00 fps VP8 / no audio      2 MBit/s PSNR 45.68',","   '1280x720 23.98 fps VP8 / no audio      2 MBit/s PSNR 48.60',","   '1280x720 23.98 fps VP8 / no audio      2 MBit/s PSNR 43.65',","   '1280x720 25 fps VP8 / no audio      2 MBit/s PSNR 49.10',","   '1280x720 29.97 fps VP8 / no audio      2 MBit/s PSNR 44.77',","   '1920x800 29.97 fps VP8 / no audio      3 MBit/s PSNR 47.72',","   '1920x1080 29.97 fps VP8 / no audio      3 MBit/s PSNR 40.01',","   '1920x1080 29.97 fps VP8 / no audio      3 MBit/s PSNR 41.33',","   '3840x2160 50.00 fps VP8 / no audio      8 MBit/s PSNR 36.01',","   '1280x720 50.00 fps VP8 / no audio      8 MBit/s PSNR 30.42',","   '1920x800 23.976 fps VP8 / no audio      3 MBit/s PSNR 43.92',","   '4096x2304 23.976 fps VP8 / no audio      16 MBit/s Profile 0 PSNR 42.80',"
 "1.webm","video","   '320x240 50 fps VP8 / Vorbis mono   1 MBit/s PSNR 36.70',","   '352x240 30 fps VP8 / Vorbis mono   1 MBit/s PSNR 31.65',","   '640x360 30 fps VP8 / Vorbis mono   800 KBit/s PSNR 39.41',","   '720x480 29.97 fps VP8 / Vorbis mono   1 MBit/s PSNR 48.07',","   '720x480 29.97 fps VP8 / Vorbis 44KHz mono   1 MBit/s PSNR 40.19',","   '720x576 25.00 fps VP8 / Vorbis 44KHz mono   1 MBit/s PSNR 37.63',","   '848x480 23.976 fps VP8 / Vorbis mono   1 MBit/s PSNR 39.97',","   '960x540 25.000 fps VP8 / Vorbis mono   1.2 MBit/s PSNR 39.97',","   '1280x532 29.970 fps VP8 / Vorbis mono   2 MBit/s PSNR 50.56',","   '1248x544 23.976 fps VP8 / Vorbis mono   2 MBit/s PSNR 46.80',","   '1280x720 25.00 fps VP8 / Vorbis mono   2 MBit/s PSNR 40.89',","   '1248x720 1 fps VP8 / Vorbis mono   2 MBit/s PSNR 36.54',","   '1280x720 29.970 fps VP8 / Vorbis mono   2 MBit/s PSNR 37.45',","   '1280x720 23.976 fps VP8 / Vorbis mono   2 MBit/s PSNR 46.50',","   '1280x720 25.00 fps VP8 / Vorbis mono   2 MBit/s PSNR 45.68',","   '1280x720 23.98 fps VP8 / Vorbis mono   2 MBit/s PSNR 48.60',","   '1280x720 23.98 fps VP8 / Vorbis mono   2 MBit/s PSNR 43.65',","   '1280x720 25 fps VP8 / Vorbis mono   2 MBit/s PSNR 49.10',","   '1280x720 29.97 fps VP8 / Vorbis mono   2 MBit/s PSNR 44.77',","   '1920x800 29.97 fps VP8 / Vorbis mono   3 MBit/s PSNR 47.72',","   '1920x1080 29.97 fps VP8 / Vorbis mono   3 MBit/s PSNR 40.01',","   '1920x1080 29.97 fps VP8 / Vorbis mono   3 MBit/s PSNR 41.33',","   '3840x2160 50.00 fps VP8 / Vorbis mono   8 MBit/s PSNR 36.01',","   '1280x720 50.00 fps VP8 / Vorbis mono   8 MBit/s PSNR 30.42',","   '1920x800 23.976 fps VP8 / Vorbis mono   3 MBit/s PSNR 43.92',","   '4096x2304 23.976 fps VP8 / Vorbis mono   16 MBit/s Profile 0 PSNR 42.80',"
 "2.webm","video","   '320x240 50 fps VP8 / Vorbis stereo 1 MBit/s PSNR 36.70',","   '352x240 30 fps VP8 / Vorbis stereo 1 MBit/s PSNR 31.65',","   '640x360 30 fps VP8 / Vorbis stereo 800 KBit/s PSNR 39.41',","   '720x480 29.97 fps VP8 / Vorbis stereo 1 MBit/s PSNR 48.07',","   '720x480 29.97 fps VP8 / Vorbis 44KHz stereo 1 MBit/s PSNR 40.19',","   '720x576 25.00 fps VP8 / Vorbis 44KHz stereo 1 MBit/s PSNR 37.63',","   '848x480 23.976 fps VP8 / Vorbis stereo 1 MBit/s PSNR 39.97',","   '960x540 25.000 fps VP8 / Vorbis stereo 1.2 MBit/s PSNR 39.97',","   '1280x532 29.970 fps VP8 / Vorbis stereo 2 MBit/s PSNR 50.56',","   '1248x544 23.976 fps VP8 / Vorbis stereo 2 MBit/s PSNR 46.80',","   '1280x720 25.00 fps VP8 / Vorbis stereo 2 MBit/s PSNR 40.89',","   '1248x720 1 fps VP8 / Vorbis stereo 2 MBit/s PSNR 36.54',","   '1280x720 29.970 fps VP8 / Vorbis stereo 2 MBit/s PSNR 37.45',","   '1280x720 23.976 fps VP8 / Vorbis stereo 2 MBit/s PSNR 46.50',","   '1280x720 25.00 fps VP8 / Vorbis stereo 2 MBit/s PSNR 45.68',","   '1280x720 23.98 fps VP8 / Vorbis stereo 2 MBit/s PSNR 48.60',","   '1280x720 23.98 fps VP8 / Vorbis stereo 2 MBit/s PSNR 43.65',","   '1280x720 25 fps VP8 / Vorbis stereo 2 MBit/s PSNR 49.10',","   '1280x720 29.97 fps VP8 / Vorbis stereo 2 MBit/s PSNR 44.77',","   '1920x800 29.97 fps VP8 / Vorbis stereo 3 MBit/s PSNR 47.72',","   '1920x1080 29.97 fps VP8 / Vorbis stereo 3 MBit/s PSNR 40.01',","   '1920x1080 29.97 fps VP8 / Vorbis stereo 3 MBit/s PSNR 41.33',","   '3840x2160 50.00 fps VP8 / Vorbis stereo 8 MBit/s PSNR 36.01',","   '1280x720 50.00 fps VP8 / Vorbis stereo 8 MBit/s PSNR 30.42',","   '1920x800 23.976 fps VP8 / Vorbis stereo 3 MBit/s PSNR 43.92',","   '4096x2304 23.976 fps VP8 / Vorbis stereo 16 MBit/s Profile 0 PSNR 42.80',"

chrome/test/functional/media/media_test_base.py

@@ -74,19 +74,15 @@ class MediaTestBase(pyauto.PyUITest):
         MediaTestEnvNames.PLAYER_HTML_URL_NICKNAME_ENV_NAME,
         self.DEFAULT_PLAYER_HTML_URL_NICKNAME)
     extra_nickname = os.getenv(MediaTestEnvNames.EXTRA_NICKNAME_ENV_NAME, '')
+
+    tag = os.getenv(MediaTestEnvNames.MEDIA_TAG_ENV_NAME,
+                    self.DEFAULT_MEDIA_TAG_NAME)
+    query_dictionary = {'tag': tag, 'media': media_filename}
     # This parameter tricks the media cache into thinking
     # it's a new file every time.
     # However, it looks like does not make much difference in
     # performance.
-    add_t_parameter = os.getenv(
-        MediaTestEnvNames.ADD_T_PARAMETER_ENV_NAME) in ('Y', 'y')
-    # Print only playback time data.
-    print_only_time = os.getenv(
-        MediaTestEnvNames.PRINT_ONLY_TIME_ENV_NAME) in ('Y', 'y')
-    tag = os.getenv(MediaTestEnvNames.MEDIA_TAG_ENV_NAME,
-                    self.DEFAULT_MEDIA_TAG_NAME)
-    query_dictionary = {'tag': tag, 'media': media_filename}
-    if add_t_parameter:
+    if os.getenv(MediaTestEnvNames.ADD_T_PARAMETER_ENV_NAME):
       query_dictionary['t'] = 'dummy'
     query_str = '&'.join(
         [k + '=' + str(v) for (k, v) in query_dictionary.items()])
@@ -168,9 +164,8 @@ class MediaTestBase(pyauto.PyUITest):
     """
     self.media_filename = os.getenv(MediaTestEnvNames.MEDIA_FILENAME_ENV_NAME,
                                     self.DEFAULT_MEDIA_FILENAME)
-    self.remove_first_result = (
-        os.getenv(MediaTestEnvNames.REMOVE_FIRST_RESULT_ENV_NAME)
-        in ('Y', 'y'))
+    self.remove_first_result = os.getenv(
+        MediaTestEnvNames.REMOVE_FIRST_RESULT_ENV_NAME)
     self.number_of_runs = int(os.getenv(MediaTestEnvNames.N_RUNS_ENV_NAME,
                                         self.DEFAULT_NUMBER_OF_RUNS))
     self.url, self.parameter_str = self._GetMediaURLAndParameterString(
@@ -193,6 +188,8 @@ class MediaTestBase(pyauto.PyUITest):
       run_counter: counter for each run.
     """
     self.start = time.time()
+    if os.getenv(MediaTestEnvNames.REFERENCE_BUILD_ENV_NAME):
+      self.current_trace_type = 't_ref'
 
   def PostEachRunProcess(self, run_counter):
     """A method to execute after each run.

chrome/test/functional/media/media_test_env_names.py

@@ -25,16 +25,14 @@ class MediaTestEnvNames:
   # Use this when you want to add extra information in the result output.
   EXTRA_NICKNAME_ENV_NAME = 'EXTRA_NICKNAME'
 
-  # Use this when you do not want to report the first result output.
-  # First result includes time to start up the browser.
+  # Define this environment variable when you do not want to report
+  # the first result output. First result includes time to start up the browser.
   REMOVE_FIRST_RESULT_ENV_NAME = 'REMOVE_FIRST_RESULT'
 
-  # Add t=Data() parameter in query string to disable media cache.
+  # Add t=Data() parameter in query string to disable media cache
+  # if this environment is defined.
   ADD_T_PARAMETER_ENV_NAME = 'ADD_T_PARAMETER'
 
-  # Print out only playback time information.
-  PRINT_ONLY_TIME_ENV_NAME = 'PRINT_ONLY_TIME'
-
   # Define the number of tries.
   N_RUNS_ENV_NAME = 'N_RUNS'
 
@@ -55,3 +53,10 @@ class MediaTestEnvNames:
 
   # Define the test scenario, which contains operations during tests.
   TEST_SCENARIO_ENV_NAME = 'TEST_SCENARIO'
+
+  # Define this environment variable if we want to run test using binaries of
+  # reference build, otherwise do not define this variable.
+  REFERENCE_BUILD_ENV_NAME = 'REFERENCE_BUILD'
+
+  # Define the path to the directory that contains binaries of reference build.
+  REFERENCE_BUILD_DIR_ENV_NAME = 'REFERENCE_BUILD_DIR'
\ No newline at end of file

chrome/test/functional/media/media_test_runner.py

@@ -36,7 +36,7 @@ def main():
   CHROME_FLAGS = ['--chrome-flags=\'--media-cache-size=1\'', '']
   # The 't' parameter is passed to player.html to disable/enable the media
   # cache (refer to data/media/html/player.js).
-  ADD_T_PARAMETERS = ['Y', 'N']
+  ADD_T_PARAMETERS = [False, True]
   # Player.html should contain all the HTML and Javascript that is
   # necessary to run these tests.
   DEFAULT_PLAYER_HTML_URL = 'DEFAULT'
@@ -44,8 +44,7 @@ def main():
   # Default base url nickname used to display the result in case it is not
   # specified by the environment variable.
   DEFAULT_PLAYER_HTML_URL_NICKNAME = 'local'
-  PRINT_ONLY_TIME = 'Y'
-  REMOVE_FIRST_RESULT = 'Y'
+  REMOVE_FIRST_RESULT = True
   # The number of runs for each test. This is used to compute average values
   # from among all runs.
   DEFAULT_NUMBER_OF_RUNS = 3
@@ -112,6 +111,14 @@ def main():
                     help=('Media file to be played using player.html. '
                           'The relative path needs to be specified starting '
                           'from data/html/ directory.'))
+  parser.add_option('-a', '--reference_build', dest='reference_build',
+                    help='Include reference build run', default=False,
+                    action='store_true')
+  parser.add_option('-k', '--reference_build_dir', dest='reference_build_dir',
+                    help=('A absolute path to the directory that contains'
+                          'binaries of reference build.'))
+  parser.add_option('-v', '--verbose', dest='verbose', help='Verbose mode.',
+                    default=False, action='store_true')
 
   options, args = parser.parse_args()
   if args:
@@ -143,6 +150,13 @@ def main():
           all_data_list, options.input_matrix_testcase_name)
       if media_info is not None:
         test_data_list.append(media_info)
+
+  # Determine whether we need to repeat a test using a reference build.
+  # The default is not to include a test using a reference build.
+  if options.reference_build:
+    reference_build_list = [False, True]
+  else:
+    reference_build_list = [False]
   # This is a loop for iterating through all videos defined above (list
   # or matrix). Each video has associated tag and nickname for display
   # purpose.
@@ -156,51 +170,63 @@ def main():
       # media cache). We are doing both here. Please note the length of
       # CHROME_FLAGS and ADD_T_PARAMETERS should be the same.
       for j in range(len(CHROME_FLAGS)):
-        parent_envs = copy.deepcopy(os.environ)
-        if options.input_matrix_filename is None:
-          filename = os.path.join(os.pardir, filename)
-        envs = {
-          MediaTestEnvNames.MEDIA_TAG_ENV_NAME: tag,
-          MediaTestEnvNames.MEDIA_FILENAME_ENV_NAME: filename,
-          MediaTestEnvNames.MEDIA_FILENAME_NICKNAME_ENV_NAME: nickname,
-          MediaTestEnvNames.PLAYER_HTML_URL_ENV_NAME:
-            options.player_html_url,
-          MediaTestEnvNames.PLAYER_HTML_URL_NICKNAME_ENV_NAME:
-            options.player_html_url_nickname,
-          MediaTestEnvNames.EXTRA_NICKNAME_ENV_NAME:
-            EXTRA_NICKNAMES[j],
-          # Enables or disables the media cache.
-          # (refer to data/media/html/player.js)
-          MediaTestEnvNames.ADD_T_PARAMETER_ENV_NAME: ADD_T_PARAMETERS[j],
-          MediaTestEnvNames.PRINT_ONLY_TIME_ENV_NAME: PRINT_ONLY_TIME,
-          MediaTestEnvNames.N_RUNS_ENV_NAME: str(options.number_of_runs),
-          MediaTestEnvNames.REMOVE_FIRST_RESULT_ENV_NAME:
-            REMOVE_FIRST_RESULT,
-          MediaTestEnvNames.MEASURE_INTERVAL_ENV_NAME:
-            str(options.measure_intervals),
-          MediaTestEnvNames.TEST_SCENARIO_FILE_ENV_NAME:
-            options.test_scenario_input_filename,
-          MediaTestEnvNames.TEST_SCENARIO_ENV_NAME:
-            options.test_scenario,
-        }
-        envs.update(parent_envs)
-        if options.suite is None and options.test_prog_name is not None:
-          # Suite is not used - run test program directly.
-          test_prog_name = options.test_prog_name
-          suite_string = ''
-        else:
-          # Suite is used.
-          # The test script names are in the PYAUTO_TEST file.
-          test_prog_name = pyauto_functional_script_name
-          if options.suite is None:
-            suite_name = DEFAULT_SUITE_NAME
+        for reference_build in reference_build_list:
+          parent_envs = copy.deepcopy(os.environ)
+          if options.input_matrix_filename is None:
+            par_filename = os.path.join(os.pardir, filename)
+          envs = {
+            MediaTestEnvNames.MEDIA_TAG_ENV_NAME: tag,
+            MediaTestEnvNames.MEDIA_FILENAME_ENV_NAME: par_filename,
+            MediaTestEnvNames.MEDIA_FILENAME_NICKNAME_ENV_NAME: nickname,
+            MediaTestEnvNames.PLAYER_HTML_URL_ENV_NAME:
+              options.player_html_url,
+            MediaTestEnvNames.PLAYER_HTML_URL_NICKNAME_ENV_NAME:
+              options.player_html_url_nickname,
+            MediaTestEnvNames.EXTRA_NICKNAME_ENV_NAME:
+              EXTRA_NICKNAMES[j],
+            # Enables or disables the media cache.
+            # (refer to data/media/html/player.js)
+            MediaTestEnvNames.N_RUNS_ENV_NAME: str(options.number_of_runs),
+            MediaTestEnvNames.MEASURE_INTERVAL_ENV_NAME:
+              str(options.measure_intervals),
+            MediaTestEnvNames.TEST_SCENARIO_FILE_ENV_NAME:
+              options.test_scenario_input_filename,
+            MediaTestEnvNames.TEST_SCENARIO_ENV_NAME:
+              options.test_scenario,
+          }
+          # Boolean variables and their related variables.
+          if ADD_T_PARAMETERS[j]:
+            envs[MediaTestEnvNames.ADD_T_PARAMETER_ENV_NAME] = str(
+                ADD_T_PARAMETERS[j])
+          if reference_build:
+            envs[MediaTestEnvNames.REFERENCE_BUILD_ENV_NAME] = str(
+                reference_build)
+          if REMOVE_FIRST_RESULT:
+            envs[MediaTestEnvNames.REMOVE_FIRST_RESULT_ENV_NAME] = str(
+                REMOVE_FIRST_RESULT)
+          if options.reference_build_dir:
+            envs[MediaTestEnvNames.REFERENCE_BUILD_DIR_ENV_NAME] = (
+                options.reference_build_dir)
+          envs.update(parent_envs)
+          if options.suite is None and options.test_prog_name is not None:
+            # Suite is not used - run test program directly.
+            test_prog_name = options.test_prog_name
+            suite_string = ''
           else:
-            suite_name = options.suite
-          suite_string = ' --suite=%s' % suite_name
-        test_prog_name = sys.executable + ' ' + test_prog_name
-        cmd = test_prog_name + suite_string + ' ' + CHROME_FLAGS[j]
-        proc = Popen(cmd, env=envs, shell=True)
-        proc.communicate()
+            # Suite is used.
+            # The test script names are in the PYAUTO_TEST file.
+            test_prog_name = pyauto_functional_script_name
+            if options.suite is None:
+              suite_name = DEFAULT_SUITE_NAME
+            else:
+              suite_name = options.suite
+            suite_string = ' --suite=%s' % suite_name
+          test_prog_name = sys.executable + ' ' + test_prog_name
+          cmd = test_prog_name + suite_string + ' ' + CHROME_FLAGS[j]
+          if options.verbose:
+            cmd += ' -v'
+          proc = Popen(cmd, env=envs, shell=True)
+          proc.communicate()
         if options.one_combination:
           sys.exit(0)
 

chrome/test/functional/media/pyauto_media.py

@@ -24,6 +24,9 @@ idea).
 
 import os
 import sys
+import tempfile
+
+from media_test_env_names import MediaTestEnvNames
 
 
 def _SetupPaths():
@@ -34,6 +37,15 @@ def _SetupPaths():
   sys.path.append(os.path.normpath(os.path.join(
       media_dir, os.pardir, os.pardir, os.pardir, os.pardir,
       'third_party', 'psutil')))
+  # Setting PYTHONPATH for reference build.
+  if os.getenv(MediaTestEnvNames.REFERENCE_BUILD_ENV_NAME):
+    reference_build_dir = os.getenv(
+        MediaTestEnvNames.REFERENCE_BUILD_DIR_ENV_NAME,
+        # TODO(imasaki@): Change the following default value.
+        # Default directory is just for testing so the correct directory
+        # must be set in the build script.
+        os.path.join(tempfile.gettempdir(), 'chrome-media-test'))
+    sys.path.insert(0, reference_build_dir)
 
 _SetupPaths()
 

chrome/test/functional/media/ui_perf_test_utils.py

@@ -61,7 +61,7 @@ class UIPerfTestUtils:
     Args:
       measurement: measurement string (such as a parameter list).
       modifier: modifier string (such as a file name).
-      trace: trace string (not currently used).
+      trace: trace string used for PerfBot graph name (such as 't' or 't_ref').
       values: list of values that displayed as "[value1,value2....]".
       units: units of values such as "sec" or "msec".