[flakiness_cli] Keep cached copies from test result server responses

Prevent us from making too many calls to test results server by keeping
cached copies of the responses we get.

For this we implement a frames.GetWithCache which will store the
dataframes created from test result reponses, and reuse them for
a set amount of time.

Bug: 875251
Change-Id: Ieb87ef4c0a52635287b634e2bd233d248fb878e2
Reviewed-on: https://chromium-review.googlesource.com/1188578Reviewed-by: Ned Nguyen <nednguyen@google.com>
Commit-Queue: Juan Antonio Navarro Pérez <perezju@chromium.org>
Cr-Commit-Position: refs/heads/master@{#586629}

tools/perf/flakiness_cli

@@ -8,14 +8,10 @@
 import argparse
 import sys
 
-from test_results import api
-from test_results import frames
+from test_results import core
 
 
 def main():
-  if frames.pandas is None:
-    return 'ERROR: This tool requires pandas to run, try: pip install pandas'
-
   parser = argparse.ArgumentParser()
   subparsers = parser.add_subparsers(dest='action')
   subparsers.required = True
@@ -27,14 +23,13 @@ def main():
   args = parser.parse_args()
 
   if args.action == 'builders':
-    data = api.GetBuilders()
-    print frames.BuildersDataFrame(data)
+    print core.GetBuilders()
   elif args.action == 'results':
-    data = api.GetTestResults(args.master, args.builder, args.test_type)
-    print frames.TestResultsDataFrame(data)
+    print core.GetTestResults(args.master, args.builder, args.test_type)
   else:
     raise NotImplementedError(args.action)
 
 
 if __name__ == '__main__':
+  core.CheckDependencies()
   sys.exit(main())

tools/perf/test_results/core.py

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import datetime
+import hashlib
+import sys
+
+from test_results import api
+from test_results import frames
+
+
+def CheckDependencies():
+  """Check that module dependencies are satisfied, otherwise exit with error."""
+  if frames.pandas is None:
+    sys.exit('ERROR: This tool requires pandas to run, try: pip install pandas')
+
+
+def GetBuilders():
+  """Get the builders data frame and keep a cached copy."""
+  def make_frame():
+    data = api.GetBuilders()
+    return frames.BuildersDataFrame(data)
+
+  return frames.GetWithCache(
+      'builders.pkl', make_frame, expires_after=datetime.timedelta(hours=12))
+
+
+def GetTestResults(master, builder, test_type):
+  """Get a test results data frame and keep a cached copy."""
+  def make_frame():
+    data = api.GetTestResults(master, builder, test_type)
+    return frames.TestResultsDataFrame(data)
+
+  basename = hashlib.md5('/'.join([master, builder, test_type])).hexdigest()
+  return frames.GetWithCache(
+      basename + '.pkl', make_frame, expires_after=datetime.timedelta(hours=3))

tools/perf/test_results/frames.py

@@ -4,12 +4,18 @@
 
 """Module to convert json responses from test-results into data frames."""
 
+import datetime
+import os
+
 try:
   import pandas
 except ImportError:
   pandas = None
 
 
+CACHE_DIR = os.path.normpath(
+    os.path.join(os.path.dirname(__file__), '..', 'data', 'test_results'))
+
 TEST_RESULTS_COLUMNS = (
     'timestamp', 'builder', 'build_number', 'commit_pos', 'test_suite',
     'test_case', 'result', 'time')
@@ -156,3 +162,33 @@ def TestResultsDataFrame(data):
       df = pandas.DataFrame(columns=TEST_RESULTS_COLUMNS)
 
     return df
+
+
+def GetWithCache(filename, frame_maker, expires_after):
+  """Get a data frame from cache or, if necessary, create and cache it.
+
+  Args:
+    filename: The name of a file for the cached copy of the data frame,
+      it will be stored in the CACHE_DIR.
+    frame_maker: A function that takes no arguments and returns a data frame,
+      only called to create the data frame if the cached copy does not exist
+      or is too old.
+    expires_after: A datetime.timedelta object, the cached copy will not be
+      used if it was created longer that this time ago.
+  """
+  filepath = os.path.join(CACHE_DIR, filename)
+  try:
+    timestamp = os.path.getmtime(filepath)
+    last_modified = datetime.datetime.utcfromtimestamp(timestamp)
+    expired = datetime.datetime.utcnow() > last_modified + expires_after
+  except OSError:  # If the file does not exist.
+    expired = True
+
+  if expired:
+    df = frame_maker()
+    if not os.path.exists(CACHE_DIR):
+      os.makedirs(CACHE_DIR)
+    df.to_pickle(filepath)
+  else:
+    df = pandas.read_pickle(filepath)
+  return df

tools/perf/test_results/frames_unittest.py

@@ -2,8 +2,14 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import datetime
+import os
+import shutil
+import tempfile
 import unittest
 
+import mock
+
 from test_results import frames
 
 
@@ -171,3 +177,46 @@ class TestDataFrames(unittest.TestCase):
     }
     with self.assertRaises(AssertionError):
       frames.TestResultsDataFrame(data)
+
+  def testGetWithCache(self):
+    def make_frame_1():
+      # test_2 was failing.
+      return frames.pandas.DataFrame.from_records(
+          [['test_1', 'P'], ['test_2', 'Q']], columns=('test_name', 'result'))
+
+    def make_frame_2():
+      # test_2 is now passing.
+      return frames.pandas.DataFrame.from_records(
+          [['test_1', 'P'], ['test_2', 'P']], columns=('test_name', 'result'))
+
+    def make_frame_fail():
+      self.fail('make_frame should not be called')
+
+    expected_1 = make_frame_1()
+    expected_2 = make_frame_2()
+    filename = 'example_frame.pkl'
+    one_hour = datetime.timedelta(hours=1)
+    temp_dir = tempfile.mkdtemp()
+    try:
+      with mock.patch.object(
+          frames, 'CACHE_DIR', os.path.join(temp_dir, 'cache')):
+        # Cache is empty, so the frame is created from our function.
+        df = frames.GetWithCache(filename, make_frame_1, one_hour)
+        self.assertTrue(df.equals(expected_1))
+
+        # On the second try, the frame can be retrieved from cache; the
+        # make_frame function should not be called.
+        df = frames.GetWithCache(filename, make_frame_fail, one_hour)
+        self.assertTrue(df.equals(expected_1))
+
+        # Pretend two hours have elapsed, we should now get a new data frame.
+        last_update = datetime.datetime(2018, 8, 24, 15)
+        pretend_now = datetime.datetime(2018, 8, 24, 17)
+        with mock.patch.object(datetime, 'datetime') as dt:
+          dt.utcfromtimestamp.return_value = last_update
+          dt.utcnow.return_value = pretend_now
+          df = frames.GetWithCache(filename, make_frame_2, one_hour)
+        self.assertFalse(df.equals(expected_1))
+        self.assertTrue(df.equals(expected_2))
+    finally:
+      shutil.rmtree(temp_dir)