Support CSV output format for analyzer results in layout test analyzer.

CSV output format is used for integration with Google spreadsheet.

Plus several minor issue fixes and adding minor features.

Review URL: http://codereview.chromium.org/8469017

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@109673 0039d316-1c4b-4281-b951-d872f2087c98

media/tools/layout_tests/layouttest_analyzer.py

@@ -21,6 +21,8 @@ from trend_graph import TrendGraph
 DEFAULT_RESULT_DIR = 'result'
 DEFAULT_ANNO_FILE = os.path.join('anno', 'anno.csv')
 DEFAULT_GRAPH_FILE = os.path.join('graph', 'graph.html')
+DEFAULT_STATS_CSV_FILENAME = 'stats.csv'
+DEFAULT_ISSUES_CSV_FILENAME = 'issues.csv'
 # Predefined result files for debug.
 CUR_TIME_FOR_DEBUG = '2011-09-11-19'
 CURRENT_RESULT_FILE_FOR_DEBUG = os.path.join(DEFAULT_RESULT_DIR,
@@ -196,7 +198,8 @@ def ReadEmailInformation(bug_annotation_file_location,
   else:
     data = csv.reader(file_object)
     for row in data:
-      anno_map[row[0]] = row[1]
+      if len(row) > 1:
+        anno_map[row[0]] = row[1]
     file_object.close()
 
   appended_text_to_email = ''
@@ -273,7 +276,8 @@ def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map,
         layouttest_analyzer_helpers.SendStatusEmail(
             prev_time, analyzer_result_map, diff_map, anno_map,
             receiver_email_address, test_group_name,
-            appended_text_to_email, email_content, rev_str)
+            appended_text_to_email, email_content, rev_str,
+            email_only_change_mode)
     if simple_rev_str:
       simple_rev_str = '\'' + simple_rev_str + '\''
     else:
@@ -284,6 +288,8 @@ def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map,
     result_change = True
     diff_map = None
     simple_rev_str = 'undefined'
+    email_content = analyzer_result_map.ConvertToString(None, diff_map,
+                                                        anno_map)
   return (result_change, diff_map, simple_rev_str, rev, rev_date,
           email_content)
 
@@ -454,6 +460,19 @@ def main():
                 anno_map, appended_text_to_email,
                 options.email_only_change_mode, options.debug,
                 options.receiver_email_address, options.test_group_name))
+
+  # Create CSV texts and save them for bug spreadsheet.
+  (stats, issues_txt) = analyzer_result_map.ConvertToCSVText(
+      start_time.strftime('%Y-%m-%d-%H'))
+  file_object = open(os.path.join(options.result_directory_location,
+                                  DEFAULT_STATS_CSV_FILENAME), 'wb')
+  file_object.write(stats)
+  file_object.close()
+  file_object = open(os.path.join(options.result_directory_location,
+                                  DEFAULT_ISSUES_CSV_FILENAME), 'wb')
+  file_object.write(issues_txt)
+  file_object.close()
+
   if not options.debug and (result_change or not prev_analyzer_result_map):
     # Save the current result when result is changed or the script is
     # executed for the first time.

media/tools/layout_tests/layouttest_analyzer_helpers.py

@@ -11,6 +11,7 @@ from email.mime.text import MIMEText
 import fileinput
 import os
 import pickle
+import re
 import smtplib
 import socket
 import sys
@@ -132,6 +133,48 @@ class AnalyzerResultMap:
                        'less than that of "skip"')
     return 100 - len(self.result_map['nonskip'].keys()) * 100 / delta
 
+  def ConvertToCSVText(self, current_time):
+   """Convert |self.result_map| into stats and issues text in CSV format.
+
+   Both are used as inputs for Google spreadsheet.
+
+   Args:
+     current_time: a string depicting a time in year-month-day-hour
+        format (e.g., 2011-11-08-16).
+
+   Returns:
+     a tuple of stats and issues_txt
+        stats: analyzer result in CSV format that shows:
+          (current_time, the number of tests, the number of skipped tests,
+           the number of failing tests)
+          For example,
+            "2011-11-10-15,204,22,12"
+        issues_txt: issues listed in CSV format that shows:
+          (BUGWK or BUGCR, bug number, the test expectation entry,
+           the name of the test)
+          For example,
+            "BUGWK,71543,TIMEOUT PASS,media/media-element-play-after-eos.html,
+             BUGCR,97657,IMAGE CPU MAC TIMEOUT PASS,media/audio-repaint.html,"
+   """
+   stats = ','.join([current_time, str(len(self.result_map['whole'].keys())),
+                     str(len(self.result_map['skip'].keys())),
+                     str(len(self.result_map['nonskip'].keys()))])
+   issues_txt = ''
+   for bug_txt, test_info_list in (
+       self.GetListOfBugsForNonSkippedTests().iteritems()):
+     matches = re.match(r'(BUG(CR|WK))(\d+)', bug_txt)
+     bug_suffix = ''
+     bug_no = ''
+     if matches:
+       bug_suffix = matches.group(1)
+       bug_no = matches.group(3)
+     issues_txt += bug_suffix + ',' + bug_no + ','
+     for test_info in test_info_list:
+       test_name, te_info = test_info
+       issues_txt += ' '.join(te_info.keys()) + ',' + test_name + ','
+     issues_txt += '\n'
+   return stats, issues_txt
+
   def ConvertToString(self, prev_time, diff_map, bug_anno_map):
     """Convert this result to HTML display for email.
 
@@ -144,19 +187,21 @@ class AnalyzerResultMap:
     Returns:
       a analyzer result string in HTML format.
     """
-    return_str = ('<b>Statistics (Diff Compared to %s):</b><ul>'
-           '<li>The number of tests: %d (%s)</li>'
-           '<li>The number of failing skipped tests: %d (%s)</li>'
-           '<li>The number of failing non-skipped tests: %d (%s)</li>'
-           '<li>Passing rate: %d %%</li></ul>') % (
-               prev_time, len(self.result_map['whole'].keys()),
-               AnalyzerResultMap.GetDiffString(diff_map['whole'], 'whole'),
-               len(self.result_map['skip'].keys()),
-               AnalyzerResultMap.GetDiffString(diff_map['skip'], 'skip'),
-               len(self.result_map['nonskip'].keys()),
-               AnalyzerResultMap.GetDiffString(diff_map['nonskip'],
-                                               'nonskip'),
-               self.GetPassingRate())
+    return_str = ''
+    if diff_map:
+      return_str += ('<b>Statistics (Diff Compared to %s):</b><ul>'
+             '<li>The number of tests: %d (%s)</li>'
+             '<li>The number of failing skipped tests: %d (%s)</li>'
+             '<li>The number of failing non-skipped tests: %d (%s)</li>'
+             '<li>Passing rate: %d %%</li></ul>') % (
+                 prev_time, len(self.result_map['whole'].keys()),
+                 AnalyzerResultMap.GetDiffString(diff_map['whole'], 'whole'),
+                 len(self.result_map['skip'].keys()),
+                 AnalyzerResultMap.GetDiffString(diff_map['skip'], 'skip'),
+                 len(self.result_map['nonskip'].keys()),
+                 AnalyzerResultMap.GetDiffString(diff_map['nonskip'],
+                                                 'nonskip'),
+                 self.GetPassingRate())
     return_str += '<b>Current issues about failing non-skipped tests:</b>'
     for (bug_txt, test_info_list) in (
         self.GetListOfBugsForNonSkippedTests().iteritems()):
@@ -255,7 +300,8 @@ class AnalyzerResultMap:
 
 def SendStatusEmail(prev_time, analyzer_result_map, diff_map,
                     bug_anno_map, receiver_email_address, test_group_name,
-                    appended_text_to_email, email_content, rev_str):
+                    appended_text_to_email, email_content, rev_str,
+                    email_only_change_mode):
   """Send status email.
 
   Args:
@@ -285,14 +331,19 @@ def SendStatusEmail(prev_time, analyzer_result_map, diff_map,
     rev_str: a revision string that contains revision information that is sent
         out in the status email. It is obtained by calling
         |GetRevisionString()|.
+    email_only_change_mode: please refer to |options|.
   """
   if rev_str:
     email_content += '<br><b>Revision Information:</b>'
     email_content += rev_str
   localtime = time.asctime(time.localtime(time.time()))
+  change_str = ''
+  if email_only_change_mode:
+    change_str = 'Status Change '
+  subject = 'Layout Test Analyzer Result %s(%s): %s' % (change_str,
+                                                        test_group_name,
+                                                        localtime)
   # TODO(imasaki): remove my name from here.
-  subject = 'Layout Test Analyzer Result (%s): %s' % (test_group_name,
-                                                      localtime)
   SendEmail('imasaki@chromium.org', [receiver_email_address],
             subject, email_content + appended_text_to_email)
 

media/tools/layout_tests/layouttest_analyzer_helpers_unittest.py

@@ -183,6 +183,28 @@ class TestLayoutTestAnalyzerHelpers(unittest.TestCase):
     self.assertFalse(
         layouttest_analyzer_helpers.FindLatestResult('test_data'))
 
+  def testConvertToCSVText(self):
+    file_path = os.path.join('test_data', 'base')
+    analyzerResultMapBase = (
+        layouttest_analyzer_helpers.AnalyzerResultMap.Load(file_path))
+    data, issues_txt = analyzerResultMapBase.ConvertToCSVText('11-10-10-2011')
+    self.assertEquals(data, '11-10-10-2011,204,36,10')
+    expected_issues_txt = """\
+BUGWK,66310,TEXT PASS,media/media-blocked-by-beforeload.html,DEBUG TEXT PASS,\
+media/video-source-error.html,
+BUGCR,86714,GPU IMAGE CRASH MAC,media/video-zoom.html,GPU IMAGE CRASH MAC,\
+media/video-controls-rendering.html,
+BUGCR,74102,GPU IMAGE PASS LINUX,media/video-controls-rendering.html,
+BUGWK,55718,TEXT IMAGE IMAGE+TEXT,media/media-document-audio-repaint.html,
+BUGCR,78376,TIMEOUT,http/tests/media/video-play-stall-seek.html,
+BUGCR,59415,WIN TEXT TIMEOUT PASS,media/video-loop.html,
+BUGCR,72223,IMAGE PASS,media/video-frame-accurate-seek.html,
+BUGCR,75354,TEXT IMAGE IMAGE+TEXT,media/media-document-audio-repaint.html,
+BUGCR,73609,TEXT,http/tests/media/video-play-stall.html,
+BUGWK,64003,DEBUG TEXT MAC PASS,media/video-delay-load-event.html,
+"""
+    self.assertEquals(issues_txt, expected_issues_txt)
+
 
 if __name__ == '__main__':
   unittest.main()

media/tools/layout_tests/layouttest_analyzer_runner.py

@@ -68,6 +68,13 @@ def ParseOption():
                                  '(default to %default and no text is '
                                  'appended in that case.)'),
                            default=None)
+  option_parser.add_option('-e', '--email-only-change-mode',
+                           dest='email_only_change_mode',
+                           help=('With this mode, email is sent out '
+                                 'only when there is a change in the '
+                                 'analyzer result compared to the previous '
+                                 'result (off by default)'),
+                           action='store_true', default=False)
   return option_parser.parse_args()[0]
 
 
@@ -176,7 +183,7 @@ def main():
     anno_file = os.path.join(options.annotation_directory_location,
                              test_group_name_for_data + '.csv')
     cmd = ('python layouttest_analyzer.py -x %s -d %s -t %s'
-           ' -q %s -a %s -c ') % (
+           ' -q %s -a %s ') % (
                test_group, result_dir, graph_file, dashboard_file_location,
                anno_file)
     if run_config_map[test_group][0]:
@@ -185,6 +192,8 @@ def main():
       cmd += '-r ' + run_config_map[test_group][1] + ' '
     if options.email_appended_text_file_location:
       cmd += ' -b ' + options.email_appended_text_file_location
+    if options.email_only_change_mode:
+      cmd += ' -c '
     print 'Running ' + cmd
     proc = Popen(cmd, shell=True)
     proc.communicate()