Refactor of record_wpr.py

* Breaks major steps out into functions
* Adds additional tests
* Adds support for command line flags defined in benchmarks
* Calls the PageTest RunPage directly, rather than kinda sorta reimplementing parts ourselves
* Fixes running benchmarks with record_wpr.py (crbug.com/378064)
* Page sets are referenced via class name on the command line, rather than by filename.

Other minor changes:
* Output will be different for failed pages at the end, using the default results PrintSummary() rather than a separate implementation (in this case GTestTestResults).

BUG=378064,367292

Review URL: https://codereview.chromium.org/374793002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@285490 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/page/record_wpr_unittest.py

@@ -2,56 +2,189 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import unittest
+import os
+import sys
 
+from telemetry import benchmark
+from telemetry.core import util
+from telemetry.core import wpr_modes
 from telemetry.page import page as page_module
+from telemetry.page import page_set as page_set_module
+from telemetry.page import page_test
 from telemetry.page import record_wpr
+from telemetry.unittest import tab_test_case
 
 
-class TestPage(page_module.Page):
-  def __init__(self):
-    super(TestPage, self).__init__(url='file://foo.html',
-                                   page_set=None,
-                                   base_dir=None)
-    self.run_navigate = False
-    self.run_foo = False
-    self.run_bar = False
+class MockPage(page_module.Page):
+  def __init__(self, page_set, url):
+    super(MockPage, self).__init__(url=url,
+                                   page_set=page_set,
+                                   base_dir=util.GetUnittestDataDir())
+    self.func_calls = []
 
-  def RunNavigateSteps(self, _):
-    self.run_navigate = True
+  def RunNavigateSteps(self, action_runner):
+    self.func_calls.append('RunNavigateSteps')
+    super(MockPage, self).RunNavigateSteps(action_runner)
 
   def RunFoo(self, _):
-    self.run_foo = True
+    self.func_calls.append('RunFoo')
 
   def RunBar(self, _):
-    self.run_bar = True
-
+    self.func_calls.append('RunBar')
 
-class FakeFooMeasurement(object):
-  def __init__(self):
-    self.action_name_to_run = "RunFoo"
+  def RunBaz(self, _):
+    self.func_calls.append('RunBaz')
 
 
-class FakeBarMeasurement(object):
-  def __init__(self):
-    self.action_name_to_run = "RunBar"
-
+class MockPageSet(page_set_module.PageSet):
+  def __init__(self, url=''):
+    super(MockPageSet, self).__init__(archive_data_file='data/test.json')
+    self.AddPage(MockPage(self, url))
 
-class FakeTab(object):
-  def WaitForDocumentReadyStateToBeComplete(self):
-    pass
-
-
-class RecordWprUnitTest(unittest.TestCase):
-  def setUp(self):
-    super(RecordWprUnitTest, self).setUp()
-
-  def testRunActions(self):
-    page = TestPage()
-    record_runner = record_wpr.RecordPage({1 : FakeFooMeasurement,
-                                           2 : FakeBarMeasurement})
-    record_runner.RunPage(page, tab=FakeTab(), results=None)
-    self.assertTrue(page.run_navigate)
-    self.assertTrue(page.run_foo)
-    self.assertTrue(page.run_bar)
 
+class MockPageTest(page_test.PageTest):
+  def __init__(self):
+    super(MockPageTest, self).__init__()
+    self._action_name_to_run = "RunBaz"
+    self.func_calls = []
+
+  @classmethod
+  def AddCommandLineArgs(cls, parser):
+    parser.add_option('--mock-page-test-option', action="store_true")
+
+  def WillNavigateToPage(self, page, tab):
+    self.func_calls.append('WillNavigateToPage')
+
+  def DidNavigateToPage(self, page, tab):
+    self.func_calls.append('DidNavigateToPage')
+
+  def WillRunActions(self, page, tab):
+    self.func_calls.append('WillRunActions')
+
+  def DidRunActions(self, page, tab):
+    self.func_calls.append('DidRunActions')
+
+  def ValidatePage(self, page, tab, results):
+    self.func_calls.append('ValidatePage')
+
+
+class MockBenchmark(benchmark.Benchmark):
+  test = MockPageTest
+
+  @classmethod
+  def AddTestCommandLineArgs(cls, group):
+    group.add_option('', '--mock-benchmark-url', action='store', type='string')
+
+  def CreatePageSet(self, options):
+    kwargs = {}
+    if (options.mock_benchmark_url):
+      kwargs['url'] = options.mock_benchmark_url
+    return MockPageSet(**kwargs)
+
+
+class RecordWprUnitTests(tab_test_case.TabTestCase):
+
+  _base_dir = util.GetUnittestDataDir()
+  _test_data_dir = os.path.join(util.GetUnittestDataDir(), 'page_measurements')
+
+  @classmethod
+  def setUpClass(cls):
+    sys.path.extend([cls._base_dir, cls._test_data_dir])
+    super(RecordWprUnitTests, cls).setUpClass()
+    cls._browser.SetHTTPServerDirectories(util.GetUnittestDataDir())
+    blank_html_path = os.path.join(util.GetUnittestDataDir(), 'blank.html')
+    cls._url = cls._browser.http_server.UrlOf(blank_html_path)
+
+  # When the RecorderPageTest is created from a PageSet, we do not have a
+  # PageTest to use. In this case, we will record every available action.
+  def testRunPage_AllActions(self):
+    record_page_test = record_wpr.RecorderPageTest(["RunFoo", "RunBar"])
+    page = MockPage(page_set=MockPageSet(url=self._url), url=self._url)
+    record_page_test.RunPage(page, self._tab, results=None)
+    self.assertTrue('RunFoo' in page.func_calls)
+    self.assertTrue('RunBar' in page.func_calls)
+    self.assertFalse('RunBaz' in page.func_calls)
+
+  def testRunPage_DontReloadSingleActions(self):
+    record_page_test = record_wpr.RecorderPageTest(["RunFoo"])
+    page = MockPage(page_set=MockPageSet(url=self._url), url=self._url)
+    record_page_test.RunPage(page, self._tab, results=None)
+    self.assertFalse('RunNavigateSteps' in page.func_calls)
+
+  def testRunPage_ReloadPageBetweenActions(self):
+    record_page_test = record_wpr.RecorderPageTest(["RunFoo", "RunBar"])
+    page = MockPage(page_set=MockPageSet(url=self._url), url=self._url)
+    record_page_test.RunPage(page, self._tab, results=None)
+    self.assertTrue('RunNavigateSteps' in page.func_calls)
+
+  # When the RecorderPageTest is created from a Benchmark, the benchmark will
+  # have a PageTest, specified by its test attribute.
+  def testRunPage_OnlyRunBenchmarkAction(self):
+    record_page_test = record_wpr.RecorderPageTest(["RunFoo"])
+    record_page_test.page_test = MockBenchmark().test()
+    page = MockPage(page_set=MockPageSet(url=self._url), url=self._url)
+    record_page_test.RunPage(page, self._tab, results=None)
+    self.assertFalse('RunFoo' in page.func_calls)
+    self.assertTrue('RunBaz' in page.func_calls)
+
+  def testRunPage_CallBenchmarksPageTestsFunctions(self):
+    record_page_test = record_wpr.RecorderPageTest([])
+    record_page_test.page_test = MockBenchmark().test()
+    page = MockPage(page_set=MockPageSet(url=self._url), url=self._url)
+    record_page_test.RunPage(page, self._tab, results=None)
+    self.assertEqual(3, len(record_page_test.page_test.func_calls))
+    self.assertEqual('WillRunActions', record_page_test.page_test.func_calls[0])
+    self.assertEqual('DidRunActions', record_page_test.page_test.func_calls[1])
+    self.assertEqual('ValidatePage', record_page_test.page_test.func_calls[2])
+
+  def testWprRecorderWithPageSet(self):
+    wpr_recorder = record_wpr.WprRecorder(self._test_data_dir,
+                                          MockPageSet(url=self._url))
+    results = wpr_recorder.Record()
+    self.assertEquals(1, len(results.successes))
+    mock_page = results.successes.pop()
+    self.assertTrue('RunFoo' in mock_page.func_calls)
+    self.assertFalse('RunBaz' in mock_page.func_calls)
+
+  def testWprRecorderWithBenchmark(self):
+    flags = ['--mock-benchmark-url', self._url]
+    wpr_recorder = record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(),
+                                          flags)
+    results = wpr_recorder.Record()
+    self.assertEquals(1, len(results.successes))
+    mock_page = results.successes.pop()
+    self.assertFalse('RunFoo' in mock_page.func_calls)
+    self.assertTrue('RunBaz' in mock_page.func_calls)
+
+  def testCommandLineFlags(self):
+    flags = [
+        '--page-repeat', '2',
+        '--mock-benchmark-url', self._url,
+        '--mock-page-test-option',
+    ]
+    wpr_recorder = record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(),
+                                          flags)
+    # page_runner command-line args
+    self.assertEquals(2, wpr_recorder.options.page_repeat)
+    # benchmark command-line args
+    self.assertEquals(self._url, wpr_recorder.options.mock_benchmark_url)
+    # benchmark's page_test command-line args
+    self.assertTrue(wpr_recorder.options.mock_page_test_option)
+    # invalid command-line args
+    self.assertFalse(hasattr(wpr_recorder.options, 'not_a_real_option'))
+
+  def testRecordingEnabled(self):
+    flags = ['--mock-benchmark-url', self._url]
+    wpr_recorder = record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(),
+                                          flags)
+    self.assertEqual(wpr_modes.WPR_RECORD,
+                     wpr_recorder.options.browser_options.wpr_mode)
+
+  def testFindAllActionNames(self):
+    # The src/tools/telemetry/unittest_data/page_measurements/ has been
+    # populated with three simple Page Measurement classes, the first two of
+    # which have action_name_to_run defined.
+    action_names_to_run = record_wpr.FindAllActionNames(self._test_data_dir)
+    self.assertTrue('RunFoo' in action_names_to_run)
+    self.assertTrue('RunBar' in action_names_to_run)
+    self.assertFalse('RunBaz' in action_names_to_run)

tools/telemetry/unittest_data/page_measurements/page_measurement1.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A simple PageMeasurement used by page/record_wpr.py's unit tests."""
+
+from telemetry.page import page_measurement
+
+class MockPageMeasurementOne(page_measurement.PageMeasurement):
+  def __init__(self):
+    super(MockPageMeasurementOne, self).__init__(action_name_to_run="RunFoo")
+
+  def MeasurePage(self, page, tab, results):
+    pass

tools/telemetry/unittest_data/page_measurements/page_measurement2.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A simple PageMeasurement used by page/record_wpr.py's unit tests."""
+
+from telemetry.page import page_measurement
+
+class MockPageMeasurementTwo(page_measurement.PageMeasurement):
+  def __init__(self):
+    super(MockPageMeasurementTwo, self).__init__(action_name_to_run="RunBar")
+
+  def MeasurePage(self, page, tab, results):
+    pass

tools/telemetry/unittest_data/page_measurements/page_measurement3.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A simple PageMeasurement used by page/record_wpr.py's unit tests."""
+
+from telemetry.page import page_measurement
+
+class MockPageMeasurementThree(page_measurement.PageMeasurement):
+  def __init__(self):
+    super(MockPageMeasurementThree, self).__init__(action_name_to_run=None)
+
+  def MeasurePage(self, page, tab, results):
+    pass