Switch buildershandler over to using the json mirror.

The json mirror is apparently better able to scale to load
and is less dependent on the buildbot master being in
good state.

As part of this, we how have a way of querying the step
information for the last build on a builder, so we can
skip the request and code related to finding out the
build number of the latest run.

Change the masters data structure to store a url name
instead of the whole URL. The url name is the master name
to use for constructing URLs. This way buildershandler
can use the mirror, but builders.js can continue pointing
to the actual master for it's non-json queries.

Also, get rid of the force_update code path in buildershandler.
This seemed like a good idea at the time, but in practice it
just masks errors by making it seem like things are working.
The downside of deleting this is that persistent errors loading
the JSON won't be noticed until the memcache is flushed for
some reason. We should add monitoring of 500s to catch these
things earlier.

Review URL: https://codereview.chromium.org/337433003

git-svn-id: svn://svn.chromium.org/blink/trunk@176147 bbb929c8-8fbe-4397-9dbb-9b2b20218538

third_party/WebKit/Tools/TestResultServer/handlers/buildershandler.py

@@ -38,15 +38,15 @@ import webapp2
 from google.appengine.api import memcache
 
 MASTERS = [
-    {'name': 'ChromiumWin', 'url': 'http://build.chromium.org/p/chromium.win', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumMac', 'url': 'http://build.chromium.org/p/chromium.mac', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumLinux', 'url': 'http://build.chromium.org/p/chromium.linux', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumChromiumOS', 'url': 'http://build.chromium.org/p/chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
-    {'name': 'ChromiumGPU', 'url': 'http://build.chromium.org/p/chromium.gpu', 'groups': ['@ToT Chromium']},
-    {'name': 'ChromiumGPUFYI', 'url': 'http://build.chromium.org/p/chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
-    {'name': 'ChromiumWebkit', 'url': 'http://build.chromium.org/p/chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
-    {'name': 'ChromiumFYI', 'url': 'http://build.chromium.org/p/chromium.fyi', 'groups': ['@ToT Chromium FYI']},
-    {'name': 'V8', 'url': 'http://build.chromium.org/p/client.v8', 'groups': ['@ToT V8']},
+    {'name': 'ChromiumWin', 'url_name': 'chromium.win', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumMac', 'url_name': 'chromium.mac', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumLinux', 'url_name': 'chromium.linux', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumChromiumOS', 'url_name': 'chromium.chromiumos', 'groups': ['@ToT ChromeOS']},
+    {'name': 'ChromiumGPU', 'url_name': 'chromium.gpu', 'groups': ['@ToT Chromium']},
+    {'name': 'ChromiumGPUFYI', 'url_name': 'chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']},
+    {'name': 'ChromiumWebkit', 'url_name': 'chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']},
+    {'name': 'ChromiumFYI', 'url_name': 'chromium.fyi', 'groups': ['@ToT Chromium FYI']},
+    {'name': 'V8', 'url_name': 'client.v8', 'groups': ['@ToT V8']},
 ]
 
 # Buildbot steps that have test in the name, but don't run tests.
@@ -77,19 +77,12 @@ TEST_STEPS_THAT_DO_NOT_UPLOAD_YET = [
     'webkit_unit_tests',
 ]
 
-class FetchBuildersException(Exception): pass
+BUILDS_URL = 'http://chrome-build-extract.appspot.com/get_builds?builder=%s&master=%s&num_builds=1'
+MASTER_URL = 'http://chrome-build-extract.appspot.com/get_master/%s'
 
 
-def master_json_url(master_url):
-    return master_url + '/json/builders'
-
-
-def builder_json_url(master_url, builder):
-    return master_json_url(master_url) + '/' + urllib2.quote(builder)
-
-
-def cached_build_json_url(master_url, builder, build_number):
-    return builder_json_url(master_url, builder) + '/builds/' + str(build_number)
+class FetchBuildersException(Exception):
+    pass
 
 
 def fetch_json(url):
@@ -111,64 +104,34 @@ def fetch_json(url):
     return fetched_json
 
 
-def get_latest_build(build_data):
-    cached_builds = []
-    if 'cachedBuilds' in build_data:
-        cached_builds = build_data['cachedBuilds']
-
-    current_builds = build_data['currentBuilds']
-
-    latest_cached_builds = set(cached_builds) - set(current_builds)
-    if len(latest_cached_builds) != 0:
-        latest_cached_builds = sorted(list(latest_cached_builds))
-        latest_build = latest_cached_builds[-1]
-    elif len(current_builds) != 0:
-        latest_build = current_builds[0]
-    else:
-        basedir = build_data['basedir'] if 'basedir' in build_data else 'current builder'
-        logging.info('No cached or current builds for %s', basedir)
-        return None
-
-    return latest_build
-
-
 def dump_json(data):
     return json.dumps(data, separators=(',', ':'), sort_keys=True)
 
 
-def fetch_buildbot_data(masters, force_update=False):
-    if force_update:
-        logging.info('Starting a forced buildbot update. Failure to fetch a master\'s data will not abort the fetch.')
-
+def fetch_buildbot_data(masters):
     start_time = datetime.datetime.now()
     master_data = masters[:]
     for master in master_data:
-        master_url = master['url']
-        tests_object = master.setdefault('tests', {})
-        master['tests'] = tests_object
-
-        builders = fetch_json(master_json_url(master_url))
+        master_url = MASTER_URL % master['url_name']
+        builders = fetch_json(master_url)
         if not builders:
-            msg = 'Could not fetch builders from master "%s": %s.' % (master['name'], master_url)
+            msg = 'Aborting fetch. Could not fetch builders from master "%s": %s.' % (master['name'], master_url)
             logging.warning(msg)
-            if force_update:
-                continue
-            else:
-                logging.warning('Aborting fetch.')
-                raise FetchBuildersException(msg)
+            raise FetchBuildersException(msg)
+
+        tests_object = master.setdefault('tests', {})
 
-        for builder in builders:
-            build_data = fetch_json(builder_json_url(master_url, builder))
+        for builder in builders['builders'].keys():
+            build = fetch_json(BUILDS_URL % (urllib2.quote(builder), master['url_name']))
+            if not build:
+                logging.info('Skipping builder %s on master %s due to empty data.', builder, master['url_name'])
+                continue
 
-            latest_build = get_latest_build(build_data)
-            if not latest_build:
-                logging.info('Skipping builder %s because it lacked cached or current builds.', builder)
+            if not build['builds']:
+                logging.info('Skipping builder %s on master %s due to empty builds list.', builder, master['url_name'])
                 continue
 
-            build = fetch_json(cached_build_json_url(master_url, builder, latest_build))
-            if not build:
-                logging.info('Skipping build %s on builder %s due to empty data', latest_build, builder)
-            for step in build['steps']:
+            for step in build['builds'][0]['steps']:
                 step_name = step['name']
 
                 if not 'test' in step_name:
@@ -201,9 +164,8 @@ def fetch_buildbot_data(masters, force_update=False):
 class UpdateBuilders(webapp2.RequestHandler):
     """Fetch and update the cached buildbot data."""
     def get(self):
-        force_update = True if self.request.get('force') else False
         try:
-            buildbot_data = fetch_buildbot_data(MASTERS, force_update)
+            buildbot_data = fetch_buildbot_data(MASTERS)
             memcache.set('buildbot_data', buildbot_data)
             self.response.set_status(200)
             self.response.out.write("ok")
@@ -213,24 +175,23 @@ class UpdateBuilders(webapp2.RequestHandler):
             self.response.out.write(ex.message)
 
 
-
 class GetBuilders(webapp2.RequestHandler):
     """Return a list of masters mapped to their respective builders, possibly using cached data."""
     def get(self):
-        callback = self.request.get('callback')
-
         buildbot_data = memcache.get('buildbot_data')
 
         if not buildbot_data:
             logging.warning('No buildbot data in memcache. If this message repeats, something is probably wrong with memcache.')
-
-            # Since we have no cached buildbot data, we would rather have missing masters than no data at all.
-            buildbot_data = fetch_buildbot_data(MASTERS, True)
             try:
+                buildbot_data = fetch_buildbot_data(MASTERS)
                 memcache.set('buildbot_data', buildbot_data)
-            except ValueError, err:
-                logging.error(str(err))
+            except FetchBuildersException, ex:
+                logging.error('Builders fetch failed: %s', str(ex))
+                self.response.set_status(500)
+                self.response.out.write(ex.message)
+                return
 
+        callback = self.request.get('callback')
         if callback:
             buildbot_data = callback + '(' + buildbot_data + ');'
 

third_party/WebKit/Tools/TestResultServer/static-dashboards/aggregate_results_unittest.js

@@ -43,12 +43,12 @@ function setupAggregateResultsData(includeRevisonNumbers)
             {
                 "groups": [ "@ToT Blink" ],
                 "name": "ChromiumWebkit",
+                "url_name": "chromium.webkit",
                 "tests": {
                     "layout-tests": {
                         "builders": [builderName]
                     }
                 },
-                "url": "http://build.chromium.org/p/chromium.win"
             }
         ]
     });

third_party/WebKit/Tools/TestResultServer/static-dashboards/builders.js

@@ -34,7 +34,7 @@ function LOAD_BUILDBOT_DATA(builderData)
     var testTypesThatDoNotUpload = {};
     builders.noUploadTestTypes = builderData['no_upload_test_types']
     builderData['masters'].forEach(function(master) {
-        builders.masters[master.name] = new builders.BuilderMaster(master.name, master.url, master.tests, master.groups);
+        builders.masters[master.name] = new builders.BuilderMaster(master);
 
         master.groups.forEach(function(group) { groups[group] = true; });
 
@@ -152,12 +152,12 @@ builders.getAllGroupNames = function()
     return builders.groups;
 }
 
-builders.BuilderMaster = function(name, basePath, tests, groups)
+builders.BuilderMaster = function(master_data)
 {
-    this.name = name;
-    this.basePath = basePath;
-    this.tests = tests;
-    this.groups = groups;
+    this.name = master_data.name;
+    this.basePath = 'http://build.chromium.org/p/' + master_data.url_name;
+    this.tests = master_data.tests;
+    this.groups = master_data.groups;
 }
 
 builders.BuilderMaster.prototype = {

third_party/WebKit/Tools/TestResultServer/static-dashboards/builders_unittest.jsonp

@@ -10,6 +10,7 @@ LOAD_BUILDBOT_DATA({
                 "@ToT Chromium"
             ],
             "name": "ChromiumWin",
+            "url_name": "chromium.win",
             "tests": {
                 "ash_unittests": {
                     "builders": [
@@ -42,13 +43,13 @@ LOAD_BUILDBOT_DATA({
                     ]
                 },
             },
-            "url": "http://build.chromium.org/p/chromium.win"
         },
         {
             "groups": [
                 "@ToT Chromium"
             ],
             "name": "ChromiumMac",
+            "url_name": "chromium.mac",
             "tests": {
                 "base_unittests": {
                     "builders": [
@@ -73,7 +74,6 @@ LOAD_BUILDBOT_DATA({
                     ]
                 },
             },
-            "url": "http://build.chromium.org/p/chromium.mac"
         },
         {
             "groups": [
@@ -81,6 +81,7 @@ LOAD_BUILDBOT_DATA({
                 "@ToT Blink"
             ],
             "name": "ChromiumWebkit",
+            "url_name": "chromium.webkit",
             "tests": {
                 "base_unittests": {
                     "builders": [
@@ -108,7 +109,6 @@ LOAD_BUILDBOT_DATA({
                     ]
                 },
             },
-            "url": "http://build.chromium.org/p/chromium.webkit"
         },
     ]
 });

third_party/WebKit/Tools/TestResultServer/static-dashboards/builders_unittests.js

@@ -32,7 +32,7 @@ test('loading steps', 4, function() {
     var tests = {}
     var baseUrl = 'http://dummyurl';
     var name = 'dummyname';
-    var master = new builders.BuilderMaster(name, baseUrl, tests);
+    var master = new builders.BuilderMaster({name: name, url_name: name, tests: tests);
 
     var builder = 'dummybuilder';
     var buildNumber = 12345;

third_party/WebKit/Tools/TestResultServer/static-dashboards/flakiness_dashboard_unittests.js

@@ -48,7 +48,7 @@ function resetGlobals()
         ],
         'masters': [{
             name: 'ChromiumWebkit',
-            url: 'dummyurl',
+            url_name: "chromium.webkit",
             tests: {
                 'layout-tests': {'builders': ['WebKit Linux', 'WebKit Linux (dbg)', 'WebKit Linux (deps)', 'WebKit Mac10.7', 'WebKit Win', 'WebKit Win (dbg)']},
                 'unit_tests': {'builders': ['Linux Tests']},
@@ -56,7 +56,7 @@ function resetGlobals()
             groups: ['@ToT Chromium', '@ToT Blink'],
         },{
             name :'ChromiumWin',
-            url: 'dummyurl2',
+            url_name: "chromium.win",
             tests: {
                 'ash_unittests': {'builders': ['XP Tests (1)', 'Win7 Tests (1)']},
                 'unit_tests': {'builders': ['Linux Tests']},

third_party/WebKit/Tools/TestResultServer/static-dashboards/load_failures_unittests.js

@@ -6,7 +6,7 @@ module('loadfailures');
 
 test('htmlForBuilder', 1, function() {
     var html = loadfailures._htmlForBuilder('MockBuilder', 'MockTestType', {
-        'MockBuilder': new builders.BuilderMaster('MockMaster', 'http://mockbasepath', [], []),
+        'MockBuilder': new builders.BuilderMaster({name: 'MockMaster', url_name: 'mock.master', tests: [], groups: []}),
     });
 
     equal(html, '<tr class="builder">' +
@@ -17,7 +17,7 @@ test('htmlForBuilder', 1, function() {
 });
 
 test('html', 5, function() {
-    var mockBuilderMaster = new builders.BuilderMaster('MockMaster', 'http://mockbasepath', [], []);
+    var mockBuilderMaster = new builders.BuilderMaster({name: 'MockMaster', url_name: 'mock.master', tests: [], groups: []}),
     var failureData = {
         '@ToT Chromium': {
             failingBuilders: {