Add a script to split variations command line switches into multiple ones.

So we could do "bisecting" and pin-point which experiment/feature triggers a
specific chrome behavior (say, a regression, a bug).

BUG=895142, 932197
TEST=unittest
R=mpearson@chromium.org

Change-Id: I2c76d73c72f093c8d884cc088495a04066e10687
Reviewed-on: https://chromium-review.googlesource.com/c/1335272
Commit-Queue: Zhenyao Mo <zmo@chromium.org>
Reviewed-by: Alexei Svitkine <asvitkine@chromium.org>
Cr-Commit-Position: refs/heads/master@{#636232}

tools/metrics/BUILD.gn

@@ -118,9 +118,12 @@ group("metrics_python_tests") {
     "//tools/variations/fieldtrial_to_struct.py",
     "//tools/variations/fieldtrial_util_unittest.py",
     "//tools/variations/fieldtrial_util.py",
+    "//tools/variations/split_variations_cmd_unittest.py",
+    "//tools/variations/split_variations_cmd.py",
     "//tools/variations/unittest_data/expected_output.cc",
     "//tools/variations/unittest_data/expected_output.h",
     "//tools/variations/unittest_data/test_config.json",
+    "//tools/variations/unittest_data/variations_cmd.txt",
     "//components/variations/field_trial_config/field_trial_testing_config_schema.json",
     "//components/variations/service/generate_ui_string_overrider.py",
     "//components/variations/service/generate_ui_string_overrider_unittest.py",

tools/metrics/metrics_python_tests.py

@@ -30,6 +30,7 @@ sys.exit(typ.main(tests=resolve(
    "../json_to_struct/struct_generator_test.py",
    '../variations/fieldtrial_to_struct_unittest.py',
    '../variations/fieldtrial_util_unittest.py',
+   '../variations/split_variations_cmd_unittest.py',
    '../../components/variations/service/'
        'generate_ui_string_overrider_unittest.py',
    )))

tools/variations/split_variations_cmd_unittest.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+import os
+import split_variations_cmd
+
+_ENABLE_FEATURES_SWITCH_NAME = 'enable-features'
+_DISABLE_FEATURES_SWITCH_NAME = 'disable-features'
+_FORCE_FIELD_TRIALS_SWITCH_NAME = 'force-fieldtrials'
+_FORCE_FIELD_TRIAL_PARAMS_SWITCH_NAME = 'force-fieldtrial-params'
+
+class SplitVariationsCmdUnittest(unittest.TestCase):
+
+  def _CompareCommandLineSwitches(self, filename, cmd_list):
+    """Compares two sets of command line switches.
+
+    Args:
+        filename: Name to a file that contains a set of commandline switches.
+        cmd_list: A list of strings in the form of '--switch_name=switch_value'.
+
+    Return True if they contain the same switches and each switch's values
+    are the same.
+    """
+    assert os.path.isfile(filename)
+    data = None
+    with open(filename, 'r') as f:
+      data = f.read().replace('\n', ' ')
+    switches = split_variations_cmd.ParseCommandLineSwitchesString(data)
+    if len(switches) != len(cmd_list):
+      return False
+    for switch_name, switch_value in switches.items():
+      switch_string = '--%s="%s"' % (switch_name, switch_value)
+      if switch_string not in cmd_list:
+        return False
+    return True
+
+  def _GetUnittestDataDir(self):
+    return os.path.join(os.path.dirname(__file__), 'unittest_data')
+
+  def _VerifySplits(self, switch_name, splits, ref_switch_data):
+    """Verifies splitting commandline switches works correctly.
+
+    Compare that when we combine switch data from all |splits| into one,
+    it's exactly the same as the |ref_switch_data|. Also check the splits are
+    almost evenly distributed, that is, their data size are almost the same.
+
+    Args:
+        switch_name: The name of the switch that is verified.
+        splits: A list of {switch_name: [items]} dictionaries.
+                Each list element represents one of the split switch sets.
+                |items| is a list of items representing switch value.
+        ref_switch_data: A {switch_name: [items]} dictionary.
+                         This is the switch set before splitting.
+                         |items| is a list of items representing switch value.
+    """
+    data_lists = [
+        split[switch_name] for split in splits if switch_name in split]
+    if len(data_lists) == 0:
+      return
+    max_size = max(len(data) for data in data_lists)
+    min_size = min(len(data) for data in data_lists)
+    if switch_name != _FORCE_FIELD_TRIAL_PARAMS_SWITCH_NAME:
+      self.assertTrue(max_size - min_size <= 1)
+    joined_switch_data = []
+    for data in data_lists:
+      joined_switch_data.extend(data)
+    self.assertEqual(ref_switch_data, joined_switch_data)
+
+  def testLoadFromFileAndSaveToStrings(self):
+    # Verifies we load data from the file and save it to a list of strings,
+    # the two data sets contain the same command line switches.
+    data_file = os.path.join(self._GetUnittestDataDir(), 'variations_cmd.txt')
+    assert os.path.isfile(data_file)
+    data = split_variations_cmd.ParseVariationsCmdFromFile(data_file)
+    cmd_list = split_variations_cmd.VariationsCmdToStrings(data)
+    self.assertTrue(self._CompareCommandLineSwitches(data_file, cmd_list))
+
+  def testSplitVariationsCmd(self):
+    # Verifies we correctly and (almost) evenly split one set of command line
+    # switches into two sets.
+    data_file = os.path.join(self._GetUnittestDataDir(), 'variations_cmd.txt')
+    assert os.path.isfile(data_file)
+    data = split_variations_cmd.ParseVariationsCmdFromFile(data_file)
+    splits = split_variations_cmd.SplitVariationsCmd(data)
+    switches = [_ENABLE_FEATURES_SWITCH_NAME,
+                _DISABLE_FEATURES_SWITCH_NAME,
+                _FORCE_FIELD_TRIALS_SWITCH_NAME,
+                _FORCE_FIELD_TRIAL_PARAMS_SWITCH_NAME]
+    for switch in switches:
+      self._VerifySplits(switch, splits, data[switch])
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/variations/unittest_data/variations_cmd.txt

+--force-fieldtrials="*TestTrial1/Default/*TestTrial2/Enabled/TestTrial3/Default/*TestTrial4/Default/TestTrial5/Enabled_Dogfood/TestTrial7/Disable7/*TestTrial8/Default/*TestTrial9/Default_V3/TestTrial10/Whatever/XYZ1.0/Group1.0/" --force-fieldtrial-params="TestTrial2.Enabled:age/unknown/gender/male/hair/blond,TestTrial4.Default:mom/alive/dad/alive/grandma/dead,TestTrial9.Default_V3:apple/fruit/monkey/animal/tomato/vegetable_or_fruit,XYZ1%2E0.Group1%2E0:p0/v0/p1/v1" --enable-features="*Feature1<Feature1,Feature-2<Feature2Rollout,*Feature3<Feature3,Feature4,Feature5<DoNoEvil,Feature6<DoTheRightThing,Feature7<LongLiveChrome,*Feature-8<ByeBye" --disable-features="FeatureA<FeatureA,FeatureB,FeatureCOutOfProcess<FeatureC,FeatureD<HappyNewYear"