Exclude test types that don't upload from the flakiness dashboard UI.

Moves some logic from the JS code in to the code that generates
builders.jsonp for steps with test in the name that don't actually
run tests. Also expands this list a bit.

Also, creates a new list for test steps that do run tests, but
don't upload to the dashboard. In a future patch, I'll expand this
list and give it a nice UI in the flakiness dashboard that points
people to a way to get the test step to upload results.

Review URL: https://codereview.chromium.org/309783002

git-svn-id: svn://svn.chromium.org/blink/trunk@175829 bbb929c8-8fbe-4397-9dbb-9b2b20218538

third_party/WebKit/Tools/TestResultServer/handlers/buildershandler.py

@@ -29,6 +29,7 @@
 import datetime
 import json
 import logging
+import re
 import sys
 import traceback
 import urllib2
@@ -48,6 +49,33 @@ MASTERS = [
     {'name': 'V8', 'url': 'http://build.chromium.org/p/client.v8', 'groups': ['@ToT V8']},
 ]
 
+# Buildbot steps that have test in the name, but don't run tests.
+NON_TEST_STEP_NAMES = [
+    'archive',
+    'Run tests',
+    'find isolated tests',
+    'read test spec',
+    'Download latest chromedriver',
+    'compile tests',
+    'create_coverage_',
+    'update test result log',
+    'memory test:',
+    'install_',
+]
+
+# Buildbot steps that run tests but don't upload results to the flakiness dashboard server.
+# FIXME: These should be fixed to upload and then removed from this list.
+TEST_STEPS_THAT_DO_NOT_UPLOAD_YET = [
+    'java_tests(chrome',
+    'python_tests(chrome',
+    'run_all_tests.py',
+    'test_report',
+    'test CronetSample',
+    'test_mini_installer',
+    'telemetry_unittests',
+    'webkit_python_tests',
+    'webkit_unit_tests',
+]
 
 class FetchBuildersException(Exception): pass
 
@@ -142,8 +170,14 @@ def fetch_buildbot_data(masters, force_update=False):
                 logging.info('Skipping build %s on builder %s due to empty data', latest_build, builder)
             for step in build['steps']:
                 step_name = step['name']
-                is_test_step = 'test' in step_name and 'archive' not in step_name and 'Run tests' not in step_name
-                if not is_test_step:
+
+                if not 'test' in step_name:
+                    continue
+
+                if any(name in step_name for name in NON_TEST_STEP_NAMES):
+                    continue
+
+                if re.search('_only|_ignore|_perf$', step_name):
                     continue
 
                 if step_name == 'webkit_tests':
@@ -155,7 +189,7 @@ def fetch_buildbot_data(masters, force_update=False):
         for builders in tests_object.values():
             builders['builders'].sort()
 
-    output_data = {'masters': master_data}
+    output_data = {'masters': master_data, 'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET}
 
     delta = datetime.datetime.now() - start_time
 

third_party/WebKit/Tools/TestResultServer/handlers/buildershandler_unittest.py

-#!/usr/bin/env python
 # Copyright (C) 2012 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -96,11 +95,11 @@ class BuildersHandlerTest(unittest.TestCase):
                     return {'cachedBuilds': [], 'currentBuilds': []}
 
                 if url == 'http://build.chromium.org/p/chromium.webkit/json/builders/WebKit%20Linux/builds/2':
-                    return {'steps': [{'name': 'webkit_tests'}, {'name': 'browser_tests'}, {'name': 'mini_installer_test'}, {'name': 'archive_test_results'}, {'name': 'compile'}]}
+                    return {'steps': [{'name': 'foo_tests_only'}, {'name': 'webkit_tests'}, {'name': 'browser_tests'}, {'name': 'mini_installer_test'}, {'name': 'archive_test_results'}, {'name': 'compile'}]}
                 if url == 'http://build.chromium.org/p/chromium.webkit/json/builders/WebKit%20Win/builds/2':
-                    return {'steps': [{'name': 'webkit_tests'}, {'name': 'mini_installer_test'}, {'name': 'archive_test_results'}, {'name': 'compile'}]}
+                    return {'steps': [{'name': 'foo_tests_ignore'}, {'name': 'webkit_tests'}, {'name': 'mini_installer_test'}, {'name': 'archive_test_results'}, {'name': 'compile'}]}
                 if url == 'http://build.chromium.org/p/chromium.webkit/json/builders/WebKit%20Mac/builds/2':
-                    return {'steps': [{'name': 'browser_tests'}, {'name': 'mini_installer_test'}, {'name': 'archive_test_results'}, {'name': 'compile'}]}
+                    return {'steps': [{'name': 'foo_tests_perf'}, {'name': 'browser_tests'}, {'name': 'mini_installer_test'}, {'name': 'archive_test_results'}, {'name': 'compile'}]}
 
                 logging.error('Cannot fetch fake url: %s' % url)
 
@@ -132,7 +131,9 @@ class BuildersHandlerTest(unittest.TestCase):
                         'browser_tests': {'builders': ['WebKit Linux', 'WebKit Mac']},
                         'mini_installer_test': {'builders': ['WebKit Linux', 'WebKit Mac', 'WebKit Win']},
                         'layout-tests': {'builders': ['WebKit Linux', 'WebKit Win']}},
-                    'name': 'ChromiumWebkit'}]}
+                    'name': 'ChromiumWebkit'}],
+                "no_upload_test_types": buildershandler.TEST_STEPS_THAT_DO_NOT_UPLOAD_YET,
+            }
             expected_json = buildershandler.dump_json(expected_masters)
 
             self.assertEqual(buildbot_data, expected_json)
@@ -205,6 +206,5 @@ class BuildersHandlerTest(unittest.TestCase):
             buildershandler.fetch_json = old_fetch_json
 
 
-
 if __name__ == '__main__':
     unittest.main()

third_party/WebKit/Tools/TestResultServer/static-dashboards/aggregate_results_unittest.js

@@ -38,6 +38,7 @@ function setupAggregateResultsData(includeRevisonNumbers)
 
     var builderName = 'Blink Linux';
     LOAD_BUILDBOT_DATA({
+        "no_upload_test_types": [],
         "masters": [
             {
                 "groups": [ "@ToT Blink" ],

third_party/WebKit/Tools/TestResultServer/static-dashboards/builders.js

@@ -31,6 +31,8 @@ function LOAD_BUILDBOT_DATA(builderData)
     builders.masters = {};
     var groups = {};
     var testTypes = {};
+    var testTypesThatDoNotUpload = {};
+    builders.noUploadTestTypes = builderData['no_upload_test_types']
     builderData['masters'].forEach(function(master) {
         builders.masters[master.name] = new builders.BuilderMaster(master.name, master.url, master.tests, master.groups);
 
@@ -39,12 +41,18 @@ function LOAD_BUILDBOT_DATA(builderData)
         Object.keys(master.tests).forEach(function(testType) {
             if (builders.testTypeUploadsToFlakinessDashboardServer(testType))
                 testTypes[testType] = true;
+            else
+                testTypesThatDoNotUpload[testType] = true;
         });
     });
     builders.groups = Object.keys(groups);
     builders.groups.sort();
     builders.testTypes = Object.keys(testTypes);
     builders.testTypes.sort();
+    // FIXME: Expose this in the flakiness dashboard UI and give a clear error message
+    // pointing to a bug about getting the test type in question uploading results.
+    builders.testTypesThatDoNotUpload = Object.keys(testTypesThatDoNotUpload);
+    builders.testTypesThatDoNotUpload.sort();
 }
 
 var builders = builders || {};
@@ -53,11 +61,11 @@ var builders = builders || {};
 
 builders.testTypeUploadsToFlakinessDashboardServer = function(testType)
 {
-    // FIXME: Encode whether the test uploads to the server in the buildbot json so
-    // we can include that data in buildbot.jsonp and not need to do ugly heuristics
-    // based off the name of the test suite. This code both has some false positives
-    // and some false negatives.
-    return !testType.match(/_only|_ignore|_perf$/) && !testType.match(/^memory test:|install_/) && testType != 'Run tests';
+    for (var i = 0; i < builders.noUploadTestTypes.length; i++) {
+        if (string.contains(testType, builders.noUploadTestTypes[i]))
+            return false;
+    }
+    return true;
 }
 
 var currentBuilderGroup = {};

third_party/WebKit/Tools/TestResultServer/static-dashboards/builders_unittest.jsonp

 // This file is auto-generated by Tools/TestResultServer/generate_builders_json.py. It should not be manually modified.
 // It uses jsonp instead of proper json because we want to be able to load it from a file URL in Chrome for local testing.
 LOAD_BUILDBOT_DATA({
+    "no_upload_test_types": [
+        "webkit_unit_tests"
+    ],
     "masters": [
         {
             "groups": [

third_party/WebKit/Tools/TestResultServer/static-dashboards/flakiness_dashboard_unittests.js

@@ -43,6 +43,9 @@ function resetGlobals()
         historyInstance.crossDashboardState[key] = history.DEFAULT_CROSS_DASHBOARD_STATE_VALUES[key];
 
     LOAD_BUILDBOT_DATA({
+        "no_upload_test_types": [
+            "webkit_unit_tests"
+        ],
         'masters': [{
             name: 'ChromiumWebkit',
             url: 'dummyurl',

third_party/WebKit/Tools/TestResultServer/static-dashboards/load_failures.js

@@ -90,9 +90,6 @@ loadfailures._html = function(failureData)
 {
     var html = '';
     Object.keys(failureData).forEach(function(group) {
-        html += '<h1>' + group + '</h1>' +
-            '<table><tr><th>Test type</th><th>>1 week stale</th><th>>1 day stale, <1 week stale</th></tr>';
-
         var failingBuildersByTestType = failureData[group].failingBuilders;
         var staleBuildersByTestType = failureData[group].staleBuilders;
         var testTypesWithNoSuccessfullLoads = failureData[group].testTypesWithNoSuccessfullLoads;
@@ -102,6 +99,13 @@ loadfailures._html = function(failureData)
         var uniqueTestTypes = testTypes.sort().filter(function(value, index, array) {
             return array.indexOf(value) === index;
         });
+
+        if (!uniqueTestTypes.length)
+            return;
+
+        html += '<h1>' + group + '</h1>' +
+            '<table><tr><th>Test type</th><th>>1 week stale</th><th>>1 day stale, <1 week stale</th></tr>';
+
         uniqueTestTypes.forEach(function(testType) {
             var failures = failingBuildersByTestType[testType] || [];
             var failureHtml = '';