rebaseline_cl.py: Set step_name when retrieving results

Layout test results are now uploaded into a directory indicating which
step name they were run as. This is to allow for without patch runs to
be recorded for analytics. This CL changes rebaseline_cl to pass the
step name when retrieving resuls.

Bug: 882946
Change-Id: Iad18ca54cc8804fff76f059ceab49d8c69fadd3e
Reviewed-on: https://chromium-review.googlesource.com/1244598
Commit-Queue: Stephen Martinis <martiniss@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Reviewed-by: Quinten Yearsley <qyearsley@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#595949}

third_party/blink/tools/blinkpy/common/net/buildbot.py

@@ -39,6 +39,8 @@ _log = logging.getLogger(__name__)
 
 RESULTS_URL_BASE = 'https://test-results.appspot.com/data/layout_results'
 
+DEFAULT_STEP_NAME = 'webkit_layout_tests (with patch)'
+
 
 class Build(collections.namedtuple('Build', ('builder_name', 'build_number'))):
     """Represents a combination of builder and build number.
@@ -58,7 +60,8 @@ class BuildBot(object):
         https://www.chromium.org/developers/the-json-test-results-format
     """
 
-    def results_url(self, builder_name, build_number=None):
+    def results_url(self, builder_name, build_number=None,
+                    step_name=DEFAULT_STEP_NAME):
         """Returns a URL for one set of archived layout test results.
 
         If a build number is given, this will be results for a particular run;
@@ -66,9 +69,11 @@ class BuildBot(object):
         the latest results.
         """
         if build_number:
-            assert str(build_number).isdigit(), 'expected numeric build number, got %s' % build_number
+            assert str(build_number).isdigit(), (
+                'expected numeric build number, got %s' % build_number)
             url_base = self.builder_results_url_base(builder_name)
-            return '%s/%s/layout-test-results' % (url_base, build_number)
+            return '%s/%s/%s/layout-test-results' % (
+                url_base, build_number, step_name)
         return self.accumulated_results_url_base(builder_name)
 
     def builder_results_url_base(self, builder_name):
@@ -81,7 +86,7 @@ class BuildBot(object):
         return '%s/%s' % (RESULTS_URL_BASE, re.sub('[ .()]', '_', builder_name))
 
     @memoized
-    def fetch_retry_summary_json(self, build):
+    def fetch_retry_summary_json(self, build, step_name=DEFAULT_STEP_NAME):
         """Fetches and returns the text of the archived retry_summary file.
 
         This file is expected to contain the results of retrying layout tests
@@ -89,7 +94,9 @@ class BuildBot(object):
         that failed only with the patch ("failures"), and tests that failed
         both with and without ("ignored").
         """
-        url_base = '%s/%s' % (self.builder_results_url_base(build.builder_name), build.build_number)
+        url_base = '%s/%s/%s' % (
+            self.builder_results_url_base(build.builder_name),
+            build.build_number, step_name)
         return NetworkTransaction(return_none_on_404=True).run(
             lambda: self.fetch_file(url_base, 'retry_summary.json'))
 

third_party/blink/tools/blinkpy/common/net/buildbot_unittest.py

@@ -46,7 +46,8 @@ class BuilderTest(LoggingTestCase):
     def test_results_url_with_build_number(self):
         self.assertEqual(
             BuildBot().results_url('Test Builder', 10),
-            'https://test-results.appspot.com/data/layout_results/Test_Builder/10/layout-test-results')
+            'https://test-results.appspot.com/data/layout_results/Test_Builder/'
+            '10/webkit_layout_tests (with patch)/layout-test-results')
 
     def test_results_url_with_non_numeric_build_number(self):
         with self.assertRaisesRegexp(AssertionError, 'expected numeric build number'):

third_party/blink/tools/blinkpy/tool/commands/rebaseline_cl.py

@@ -60,6 +60,10 @@ class RebaselineCL(AbstractParallelRebaselineCommand):
                 '--builders', default=None, action='append',
                 help=('Comma-separated-list of builders to pull new baselines '
                       'from (can also be provided multiple times).')),
+            optparse.make_option(
+                '--step-name', default='webkit_layout_tests (with patch)',
+                help=('Layout test step name to fetch results from. Defaults '
+                      ' to \'webkit_layout_tests (with patch)\'.')),
             optparse.make_option(
                 '--patchset', default=None,
                 help='Patchset number to fetch new baselines from.'),
@@ -106,7 +110,7 @@ class RebaselineCL(AbstractParallelRebaselineCommand):
             self.trigger_try_jobs(builders_with_no_jobs)
             return 1
 
-        jobs_to_results = self._fetch_results(jobs)
+        jobs_to_results = self._fetch_results(jobs, options.step_name)
 
         builders_with_results = {b.builder_name for b in jobs_to_results}
         builders_without_results = set(self.selected_try_bots) - builders_with_results
@@ -210,7 +214,7 @@ class RebaselineCL(AbstractParallelRebaselineCommand):
         for builder in sorted(builders):
             _log.info('  %s', builder)
 
-    def _fetch_results(self, jobs):
+    def _fetch_results(self, jobs, step_name):
         """Fetches results for all of the given builds.
 
         There should be a one-to-one correspondence between Builds, supported
@@ -221,6 +225,7 @@ class RebaselineCL(AbstractParallelRebaselineCommand):
 
         Args:
             jobs: A dict mapping Build objects to TryJobStatus objects.
+            step_name: The step name to fetch when retrieving test results.
 
         Returns:
             A dict mapping Build to LayoutTestResults for all completed jobs.
@@ -237,7 +242,8 @@ class RebaselineCL(AbstractParallelRebaselineCommand):
                 # Only completed failed builds will contain actual failed
                 # layout tests to download baselines for.
                 continue
-            results_url = buildbot.results_url(build.builder_name, build.build_number)
+            results_url = buildbot.results_url(
+                build.builder_name, build.build_number, step_name)
             layout_test_results = buildbot.fetch_results(build)
             if layout_test_results is None:
                 _log.info('Failed to fetch results for "%s".', build.builder_name)

third_party/blink/tools/blinkpy/tool/commands/rebaseline_cl_unittest.py

@@ -119,6 +119,7 @@ class RebaselineCLTest(BaseTestCase, LoggingTestCase):
             'verbose': False,
             'builders': [],
             'patchset': None,
+            'step_name': 'webkit_layout_tests (with patch)',
         }
         options.update(kwargs)
         return optparse.Values(dict(**options))
@@ -411,7 +412,7 @@ class RebaselineCLTest(BaseTestCase, LoggingTestCase):
             'INFO: Finished try jobs found for all try bots.\n',
             'INFO: Failed to fetch results for "MOCK Try Win".\n',
             ('INFO: Results URL: https://test-results.appspot.com/data/layout_results'
-             '/MOCK_Try_Win/5000/layout-test-results/results.html\n'),
+             '/MOCK_Try_Win/5000/webkit_layout_tests (with patch)/layout-test-results/results.html\n'),
             'INFO: There are some builders with no results:\n',
             'INFO:   MOCK Try Win\n',
             'INFO: Would you like to continue?\n',
@@ -428,7 +429,7 @@ class RebaselineCLTest(BaseTestCase, LoggingTestCase):
             'INFO: Finished try jobs found for all try bots.\n',
             'INFO: Failed to fetch results for "MOCK Try Win".\n',
             ('INFO: Results URL: https://test-results.appspot.com/data/layout_results'
-             '/MOCK_Try_Win/5000/layout-test-results/results.html\n'),
+             '/MOCK_Try_Win/5000/webkit_layout_tests (with patch)/layout-test-results/results.html\n'),
             'INFO: There are some builders with no results:\n',
             'INFO:   MOCK Try Win\n',
             'INFO: For one/flaky-fail.html:\n',

third_party/blink/tools/blinkpy/web_tests/try_flag_unittest.py

@@ -134,7 +134,7 @@ class TryFlagTest(unittest.TestCase):
         TryFlag(cmd, host, MockGitCL(host, self.mock_try_results)).run()
 
         def results_url(build):
-            return '%s/%s/%s/layout-test-results/results.html' % (
+            return '%s/%s/%s/webkit_layout_tests (with patch)/layout-test-results/results.html' % (
                 'https://test-results.appspot.com/data/layout_results',
                 build.builder_name,
                 build.build_number