Revert "Orderfile: training and testing benchmarks."

This reverts commit 3c6537d8.

Reason for revert: block other CL from landing with PRESUBMIT error:

https://logs.chromium.org/v/?s=chromium%2Fbuildbucket%2Fcr-buildbucket.appspot.com%2F8942003279127111280%2F%2B%2Fsteps%2Fpresubmit%2F0%2Fstdout

** Presubmit ERRORS **
Pylint (261 files using ['--disable=cyclic-import'] on 8 cores) (8.11s) failed
************* Module contrib.orderfile.orderfile
E:109,20: story_class is not callable (not-callable)
W:113,19: Access to a protected member _IterAllSystemHealthStoryClasses of a client class (protected-access)
R:144, 0: Too many ancestors (8/7) (too-many-ancestors)
R:155, 0: Too many ancestors (8/7) (too-many-ancestors)
R:189, 0: Too many ancestors (8/7) (too-many-ancestors)
R:194, 0: Too many ancestors (8/7) (too-many-ancestors)
R:200, 0: Too many ancestors (8/7) (too-many-ancestors)
R:206, 0: Too many ancestors (8/7) (too-many-ancestors)
R:212, 0: Too many ancestors (8/7) (too-many-ancestors)
R:218, 0: Too many ancestors (8/7) (too-many-ancestors)
Presubmit checks took 15.4s to calculate.

Original change's description:
> Orderfile: training and testing benchmarks.
> 
> These benchmarks will be used to train the new production android orderfile.
> 
> Bug: 843561
> Change-Id: Iea097d784773c98ec3076f297aac16558c2a8104
> Reviewed-on: https://chromium-review.googlesource.com/1124561
> Reviewed-by: Benoit L <lizeb@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#572219}

TBR=pasko@chromium.org,lizeb@chromium.org,mattcary@chromium.org

Change-Id: I0f22ff64f7532aaa913bb4dbd844be6cfdb7b5ef
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 843561
Reviewed-on: https://chromium-review.googlesource.com/1124702Reviewed-by: Ned Nguyen <nednguyen@google.com>
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#572266}

tools/perf/contrib/orderfile/orderfile.py

-# Copyright 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-""" Orderfile Generation and Testing
-
-This provides a suite of benchmarks for orderfile generation and testing based
-on the mobile system health benchmarks.
-
-The orderfile_generation.training benchmark is meant to be run with an
-orderfile instrumented build to produce function usage profiles. It can be
-invoked as follows:
-    (CHROMIUM_OUTPUT_DIR=out/Instrumented; \
-     ./tools/perf/run_benchmark --device=${YOUR_DEVICE_SN} --browser=exact \
-        --browser-executable=${CHROMIUM_OUTPUT_DIR}/apks/Monochrome.apk \
-        orderfile_generation.training)
-
-The orderfile_generation.testing benchmark has a smaller test set whose
-benchmarks are distinct from those in orderfile_generation.training. Run it as
-follows, note the --orderfile-memory-optimization flag is necessary only with
-legacy orderfile builds of chrome.
-    (CHROMIUM_OUTPUT_DIR=out/Official; \
-     ./tools/perf/run_benchmark --device=${YOUR_DEVICE_SN} --browser=exact \
-        --browser-executable=${CHROMIUM_OUTPUT_DIR}/apks/Monochrome.apk \
-        --extra-browser-args=--orderfile-memory-optimization \
-        --results-label=${LABEL_CORRESPONDING_TO_YOUR_BUILD} \
-        orderfile_generation.training)
-
-The orderfile_generation.variation.* benchmarks use a smaller training set so
-that several test set variations can be produced. They are run as above but
-using the following benchmark names.
-    orderfile_generation.variation.training
-    orderfile_generation.variation.testing0
-    orderfile_generation.variation.testing1
-    orderfile_generation.variation.testing2
-    orderfile_generation.variation.testing3
-
-The orderfile_generation.debugging benchmark is a short benchmark of 3 stories
-that is useful for debugging hardware and test setup problems.
-"""
-
-import random
-
-from benchmarks import system_health
-from page_sets.system_health import platforms
-from page_sets.system_health import system_health_stories
-
-from telemetry import benchmark
-from telemetry import story
-
-
-class OrderfileStorySet(story.StorySet):
-  """User stories for orderfile generation.
-
-  The run set specified in the constructor splits the stories into training and
-  testing sets (or debugging, see the code for details).
-  """
-  # Run set names.
-  TRAINING = 'training'
-  TESTING = 'testing'
-  DEBUGGING = 'debugging'
-
-  _BLACKLIST = set([
-      'browse:chrome:newtab',
-      'browse:shopping:flipkart',
-  ])
-
-  # The random seed used for reproducible runs.
-  SEED = 8675309
-
-  # These defaults are current best practice for production orderfiles.
-  DEFAULT_TRAINING = 50
-  DEFAULT_TESTING = 8
-  DEFAULT_VARIATIONS = 1
-
-  def __init__(self, platform, run_set, num_training=DEFAULT_TRAINING,
-               num_testing=DEFAULT_TESTING, num_variations=DEFAULT_VARIATIONS,
-               test_variation=0):
-    """Create an orderfile training or testing benchmark set.
-
-    Args:
-      platform: from system_health.MobileMemorySystemHealth.
-      run_set: one of TRAINING, TESTING or DEBUGGING.
-      num_training: the number of benchmarks to use for training.
-      num_testing: the number of benchmarks to use in each test set.
-      num_variations: the number of test set variations.
-      test_variation: the test set variation to use.
-    """
-    super(OrderfileStorySet, self).__init__(
-        archive_data_file=('../../page_sets/data/system_health_%s.json' %
-                           platform),
-        cloud_storage_bucket=story.PARTNER_BUCKET)
-
-    assert platform in platforms.ALL_PLATFORMS, '{} not in {}'.format(
-        platform, str(platforms.ALL_PLATFORMS))
-    assert run_set in (self.TRAINING, self.TESTING, self.DEBUGGING)
-    assert test_variation >= 0 and test_variation < num_variations
-
-    self._platform = platform
-    self._run_set = run_set
-    self._num_training = num_training
-    self._num_testing = num_testing
-    self._num_variations = num_variations
-    self._test_variation = test_variation
-
-    # We want the story selection to be consistent across runs.
-    random.seed(self.SEED)
-
-    for story_class in self.RunSetStories():
-      self.AddStory(story_class(self, take_memory_measurement=True))
-
-  def RunSetStories(self):
-    possible_stories = [
-        s for s in system_health_stories._IterAllSystemHealthStoryClasses()
-        if (s.NAME not in self._BLACKLIST and
-            not s.ABSTRACT_STORY and
-            self._platform in s.SUPPORTED_PLATFORMS)]
-    assert (self._num_training + self._num_variations * self._num_testing
-            <= len(possible_stories)), \
-        'We only have {} stories to work with, but want {} + {}*{}'.format(
-            len(possible_stories), self._num_training, self._num_variations,
-            self._num_testing)
-
-    if self._run_set == self.DEBUGGING:
-      return random.sample(possible_stories, 3)
-
-    random.shuffle(possible_stories)
-    if self._run_set == self.TRAINING:
-      return possible_stories[:self._num_training]
-    elif self._run_set == self.TESTING:
-      return possible_stories[
-          (self._num_training + self._test_variation * self._num_testing):
-          (self._num_training + (self._test_variation + 1) * self._num_testing)]
-    assert False, 'Bad run set {}'.format(self._run_set)
-
-
-class _OrderfileBenchmark(system_health.MobileMemorySystemHealth):
-  """Base benchmark for orderfile generation."""
-  STORY_RUN_SET = None  # Must be set in subclasses.
-  def CreateStorySet(self, options):
-    return OrderfileStorySet(
-        platform=self.PLATFORM, run_set=self.STORY_RUN_SET)
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileTraining(_OrderfileBenchmark):
-  STORY_RUN_SET = OrderfileStorySet.TRAINING
-
-  options = {'pageset_repeat': 2}
-
-  @classmethod
-  def Name(cls):
-    return 'orderfile_generation.training'
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileTesting(_OrderfileBenchmark):
-  STORY_RUN_SET = OrderfileStorySet.TESTING
-
-  options = {'pageset_repeat': 7}
-
-  @classmethod
-  def Name(cls):
-    return 'orderfile_generation.testing'
-
-
-class _OrderfileVariation(system_health.MobileMemorySystemHealth):
-  """Like _OrderfileBenchmark, but with orderfile test sets variations."""
-  STORY_RUN_SET = None   # Must be set in all subclasses.
-  TEST_VARIATION = 0     # Can be overridden testing subclasses.
-
-  options = {'pageset_repeat': 7}
-
-  def CreateStorySet(self, options):
-    return OrderfileStorySet(
-        platform=self.PLATFORM, run_set=self.STORY_RUN_SET,
-        num_training=25, num_testing=8, num_variations=4,
-        test_variation=self.TEST_VARIATION)
-
-  @classmethod
-  def Name(cls):
-    if cls.STORY_RUN_SET == OrderfileStorySet.TESTING:
-      return 'orderfile_generation.variation.testing{}'.format(
-          cls.TEST_VARIATION)
-    elif cls.STORY_RUN_SET == OrderfileStorySet.TRAINING:
-      return 'orderfile_generation.variation.training'
-    assert False, 'Bad STORY_RUN_SET {}'.format(cls.STORY_RUN_SET)
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileVariationTraining(_OrderfileVariation):
-  STORY_RUN_SET = OrderfileStorySet.TRAINING
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileVariationTesting0(_OrderfileVariation):
-  STORY_RUN_SET = OrderfileStorySet.TESTING
-  TEST_VARIATION = 0
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileTesting1(_OrderfileVariation):
-  STORY_RUN_SET = OrderfileStorySet.TESTING
-  TEST_VARIATION = 1
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileTesting2(_OrderfileVariation):
-  STORY_RUN_SET = OrderfileStorySet.TESTING
-  TEST_VARIATION = 2
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileTesting3(_OrderfileVariation):
-  STORY_RUN_SET = OrderfileStorySet.TESTING
-  TEST_VARIATION = 3
-
-
-@benchmark.Owner(emails=['mattcary@chromium.org'])
-class OrderfileDebugging(_OrderfileBenchmark):
-  """A very short benchmark for debugging metrics collection."""
-  STORY_RUN_SET = OrderfileStorySet.DEBUGGING
-
-  options = {'pageset_repeat': 1}
-
-  @classmethod
-  def Name(cls):
-    return 'orderfile_generation.debugging'

tools/perf/contrib/orderfile/orderfile_unittest.py

-#!/usr/bin/env vpython
-# Copyright 2015 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import os
-import sys
-import unittest
-
-sys.path.append(os.path.abspath(os.path.join(
-    os.path.dirname(__file__), os.pardir, os.pardir)))
-from core import path_util
-
-path_util.AddTelemetryToPath()
-
-from benchmarks import system_health
-from contrib.orderfile import orderfile
-
-
-class Orderfile(unittest.TestCase):
-  def setUp(self):
-    # Increase failed test output to make updating easier.
-    self.maxDiff = None
-
-  def testDefaults(self):
-    training = [s.NAME for s in orderfile.OrderfileStorySet(
-        system_health.MobileMemorySystemHealth.PLATFORM,
-        orderfile.OrderfileStorySet.TRAINING).RunSetStories()]
-    self.assertListEqual(
-        ['background:social:facebook',
-         'load:media:soundcloud',
-         'load:news:wikipedia',
-         'browse:media:imgur',
-         'browse:tech:discourse_infinite_scroll',
-         'browse:news:cricbuzz',
-         'load:games:lazors',
-         'load:tools:drive',
-         'load:search:google',
-         'load:tools:stackoverflow',
-         'load:news:washingtonpost',
-         'load:news:reddit',
-         'browse:shopping:avito',
-         'load:news:cnn',
-         'browse:news:qq',
-         'load:search:baidu',
-         'load:search:ebay',
-         'long_running:tools:gmail-foreground',
-         'load:media:imgur',
-         'background:news:nytimes',
-         'load:tools:dropbox',
-         'background:search:google',
-         'load:chrome:blank',
-         'browse:social:tumblr_infinite_scroll',
-         'load:news:qq',
-         'load:search:yandex',
-         'load:media:dailymotion',
-         'browse:tools:maps',
-         'load:games:bubbles',
-         'browse:shopping:amazon',
-         'browse:social:instagram',
-         'background:tools:gmail',
-         'load:media:youtube',
-         'load:media:facebook_photos',
-         'browse:media:facebook_photos',
-         'browse:social:facebook',
-         'browse:news:reddit',
-         'load:media:google_images',
-         'load:tools:weather',
-         'load:social:twitter',
-         'browse:news:cnn',
-         'browse:media:flickr_infinite_scroll',
-         'load:games:spychase',
-         'load:tools:docs',
-         'load:news:nytimes',
-         'browse:news:washingtonpost',
-         'browse:social:pinterest_infinite_scroll',
-         'load:news:irctc',
-         'browse:media:youtube',
-         'load:search:yahoo'], training)
-    testing = [s.NAME for s in orderfile.OrderfileStorySet(
-        system_health.MobileMemorySystemHealth.PLATFORM,
-        orderfile.OrderfileStorySet.TESTING).RunSetStories()]
-    self.assertListEqual(
-        ['browse:shopping:lazada',
-         'load:tools:gmail',
-         'browse:news:toi',
-         'browse:chrome:omnibox',
-         'browse:news:globo',
-         'browse:social:facebook_infinite_scroll',
-         'load:search:taobao',
-         'background:media:imgur'], testing)
-
-  def test25TrainingStories(self):
-    training = [s.NAME for s in orderfile.OrderfileStorySet(
-        system_health.MobileMemorySystemHealth.PLATFORM,
-        orderfile.OrderfileStorySet.TRAINING, num_training=25).RunSetStories()]
-    self.assertListEqual(
-        ['background:social:facebook',
-         'load:media:soundcloud',
-         'load:news:wikipedia',
-         'browse:media:imgur',
-         'browse:tech:discourse_infinite_scroll',
-         'browse:news:cricbuzz',
-         'load:games:lazors',
-         'load:tools:drive',
-         'load:search:google',
-         'load:tools:stackoverflow',
-         'load:news:washingtonpost',
-         'load:news:reddit',
-         'browse:shopping:avito',
-         'load:news:cnn',
-         'browse:news:qq',
-         'load:search:baidu',
-         'load:search:ebay',
-         'long_running:tools:gmail-foreground',
-         'load:media:imgur',
-         'background:news:nytimes',
-         'load:tools:dropbox',
-         'background:search:google',
-         'load:chrome:blank',
-         'browse:social:tumblr_infinite_scroll',
-         'load:news:qq'],
-        training)
-
-  def testTestingStories(self):
-    testing = [s.NAME for s in orderfile.OrderfileStorySet(
-        system_health.MobileMemorySystemHealth.PLATFORM,
-        orderfile.OrderfileStorySet.TESTING,
-        num_training=25).RunSetStories()]
-    self.assertListEqual(
-        ['load:search:yandex',
-         'load:media:dailymotion',
-         'browse:tools:maps',
-         'load:games:bubbles',
-         'browse:shopping:amazon',
-         'browse:social:instagram',
-         'background:tools:gmail',
-         'load:media:youtube'],
-        testing)
-
-  def testTestingVariationStories(self):
-    testing = [s.NAME for s in orderfile.OrderfileStorySet(
-        system_health.MobileMemorySystemHealth.PLATFORM,
-        orderfile.OrderfileStorySet.TESTING, num_training=25,
-        num_variations=4, test_variation=0).RunSetStories()]
-    self.assertListEqual(
-        ['load:search:yandex',
-         'load:media:dailymotion',
-         'browse:tools:maps',
-         'load:games:bubbles',
-         'browse:shopping:amazon',
-         'browse:social:instagram',
-         'background:tools:gmail',
-         'load:media:youtube'],
-        testing)
-
-    testing = [s.NAME for s in orderfile.OrderfileStorySet(
-        system_health.MobileMemorySystemHealth.PLATFORM,
-        orderfile.OrderfileStorySet.TESTING, num_training=25,
-        num_variations=4, test_variation=1).RunSetStories()]
-    self.assertListEqual(
-        ['load:media:facebook_photos',
-         'browse:media:facebook_photos',
-         'browse:social:facebook',
-         'browse:news:reddit',
-         'load:media:google_images',
-         'load:tools:weather',
-         'load:social:twitter',
-         'browse:news:cnn'],
-        testing)
-
-    testing = [s.NAME for s in orderfile.OrderfileStorySet(
-        system_health.MobileMemorySystemHealth.PLATFORM,
-        orderfile.OrderfileStorySet.TESTING, num_training=25,
-        num_variations=4, test_variation=3).RunSetStories()]
-    self.assertListEqual(
-        ['load:search:yahoo',
-         'browse:shopping:lazada',
-         'load:tools:gmail',
-         'browse:news:toi',
-         'browse:chrome:omnibox',
-         'browse:news:globo',
-         'browse:social:facebook_infinite_scroll',
-         'load:search:taobao'],
-        testing)
-
-
-if __name__ == '__main__':
-  unittest.main()