[results_processor] Simplify artifact creation in processor_test.py

Make it slightly easier to write integration tests for results
processor involving artifacts.

Also fix a bug where creating multiple artifacts of the same type
(e.g. many histogram artifacts for different test runs) would
write all of them to the same file name, causing them to get
overwritten.

Bug: 981349
Change-Id: I7eaa8793d09fe0afc869dd8154df628d9ece7774
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1880034
Commit-Queue: Juan Antonio Navarro Pérez <perezju@chromium.org>
Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#710278}

tools/perf/core/results_processor/processor_test.py

@@ -49,25 +49,24 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
   def CreateHistogramsArtifact(self, hist):
     """Create an artifact with histograms."""
     histogram_dicts = [hist.AsDict()]
-    hist_file = os.path.join(self.output_dir,
-                             compute_metrics.HISTOGRAM_DICTS_FILE)
-    with open(hist_file, 'w') as f:
-      json.dump(histogram_dicts, f)
-    return testing.Artifact(hist_file)
+    with tempfile.NamedTemporaryFile(
+        dir=self.intermediate_dir, delete=False) as artifact_file:
+      json.dump(histogram_dicts, artifact_file)
+    return (compute_metrics.HISTOGRAM_DICTS_FILE,
+            testing.Artifact(artifact_file.name))
 
   def CreateDiagnosticsArtifact(self, **diagnostics):
     """Create an artifact with diagnostics."""
-    diag_file = os.path.join(self.output_dir,
-                             processor.DIAGNOSTICS_NAME)
-    with open(diag_file, 'w') as f:
-      json.dump({'diagnostics': diagnostics}, f)
-    return testing.Artifact(diag_file)
+    with tempfile.NamedTemporaryFile(
+        dir=self.intermediate_dir, delete=False) as artifact_file:
+      json.dump({'diagnostics': diagnostics}, artifact_file)
+    return processor.DIAGNOSTICS_NAME, testing.Artifact(artifact_file.name)
 
   def CreateMeasurementsArtifact(self, measurements):
     with tempfile.NamedTemporaryFile(
         dir=self.intermediate_dir, delete=False) as artifact_file:
       json.dump({'measurements': measurements}, artifact_file)
-    return testing.Artifact(artifact_file.name)
+    return processor.MEASUREMENTS_NAME, testing.Artifact(artifact_file.name)
 
   def testJson3Output(self):
     self.SerializeIntermediateResults(
@@ -135,21 +134,17 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
 
   def testMaxValuesPerTestCase(self):
     def SomeMeasurements(num):
-      return (
-          processor.MEASUREMENTS_NAME,
-          self.CreateMeasurementsArtifact({
-              'n%d' % i: {'unit': 'count', 'samples': [i]}
-              for i in range(num)
-          })
-      )
+      return self.CreateMeasurementsArtifact({
+          'n%d' % i: {'unit': 'count', 'samples': [i]}
+          for i in range(num)})
 
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story1', status='PASS',
-            output_artifacts=dict([SomeMeasurements(3)])),
+            output_artifacts=[SomeMeasurements(3)]),
         testing.TestResult(
             'benchmark/story2', status='PASS',
-            output_artifacts=dict([SomeMeasurements(7)])),
+            output_artifacts=[SomeMeasurements(7)]),
     )
 
     exit_code = processor.main([
@@ -172,16 +167,14 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-                processor.DIAGNOSTICS_NAME:
-                    self.CreateDiagnosticsArtifact(
-                        benchmarks=['benchmark'],
-                        osNames=['linux'],
-                        documentationUrls=[['documentation', 'url']]),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+                self.CreateDiagnosticsArtifact(
+                    benchmarks=['benchmark'],
+                    osNames=['linux'],
+                    documentationUrls=[['documentation', 'url']])
+            ],
             start_time='2009-02-13T23:31:30.987000Z',
         ),
     )
@@ -220,11 +213,10 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+            ],
         ),
     )
 
@@ -259,11 +251,10 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+            ],
         ),
     )
 
@@ -361,23 +352,19 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertIn('traceUrls', hist.diagnostics)
 
   def testHistogramsOutputMeasurements(self):
-    measure_file = os.path.join(self.output_dir,
-                                processor.MEASUREMENTS_NAME)
-    with open(measure_file, 'w') as f:
-      json.dump({'measurements': {
-          'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'},
-          'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'},
-      }}, f)
-
+    measurements = {
+        'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'},
+        'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'},
+    }
     start_ts = 1500000000
     start_iso = datetime.datetime.utcfromtimestamp(start_ts).isoformat() + 'Z'
 
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                processor.MEASUREMENTS_NAME: testing.Artifact(measure_file)
-            },
+            output_artifacts=[
+                self.CreateMeasurementsArtifact(measurements),
+            ],
             tags=['story_tag:test'],
             start_time=start_iso,
         ),
@@ -429,16 +416,14 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-                processor.DIAGNOSTICS_NAME:
-                    self.CreateDiagnosticsArtifact(
-                        benchmarks=['benchmark'],
-                        osNames=['linux'],
-                        documentationUrls=[['documentation', 'url']]),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+                self.CreateDiagnosticsArtifact(
+                    benchmarks=['benchmark'],
+                    osNames=['linux'],
+                    documentationUrls=[['documentation', 'url']]),
+            ],
             start_time='2009-02-13T23:31:30.987000Z',
         ),
     )
@@ -477,11 +462,10 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+            ],
         ),
     )
 
@@ -516,11 +500,10 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+            ],
         ),
     )
 
@@ -557,15 +540,13 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(test_hist),
-                processor.DIAGNOSTICS_NAME:
-                    self.CreateDiagnosticsArtifact(
-                        benchmarks=['benchmark'],
-                        osNames=['linux'],
-                        documentationUrls=[['documentation', 'url']]),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(test_hist),
+                self.CreateDiagnosticsArtifact(
+                    benchmarks=['benchmark'],
+                    osNames=['linux'],
+                    documentationUrls=[['documentation', 'url']]),
+            ],
             start_time='2009-02-13T23:31:30.987000Z',
         ),
     )
@@ -598,11 +579,10 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+            ],
         ),
     )
 
@@ -631,11 +611,10 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.SerializeIntermediateResults(
         testing.TestResult(
             'benchmark/story',
-            output_artifacts={
-                compute_metrics.HISTOGRAM_DICTS_FILE:
-                    self.CreateHistogramsArtifact(
-                        histogram.Histogram('a', 'unitless')),
-            },
+            output_artifacts=[
+                self.CreateHistogramsArtifact(
+                    histogram.Histogram('a', 'unitless')),
+            ],
         ),
     )
 

tools/perf/core/results_processor/testing.py

@@ -25,7 +25,8 @@ def TestResult(test_path, status='PASS', is_expected=None,
     start_time: An optional UTC timestamp recording when the test run started.
     run_duration: An optional duration string recording the amount of time
       that the test run lasted.
-    artifcats: An optional dict mapping artifact names to Artifact dicts.
+    output_artifacts: An optional mapping of artifact names to Artifact dicts,
+      may be given as a dict or a sequence of pairs.
     tags: An optional sequence of tags associated with this test run; each
       tag is given as a '{key}:{value}' string. Keys are not unique, the same
       key may appear multiple times.
@@ -43,7 +44,7 @@ def TestResult(test_path, status='PASS', is_expected=None,
       'runDuration': run_duration
   }
   if output_artifacts is not None:
-    test_result['outputArtifacts'] = output_artifacts
+    test_result['outputArtifacts'] = dict(output_artifacts)
   if tags is not None:
     test_result['tags'] = [_SplitTag(tag) for tag in tags]