[telemetry] Ensure page sets reference the right WPR archives.

There are some page sets that point to the old src-internal tools/perf/data dir.
Also remove the back-pointer from the archive info file to the page set file. It's unused and confusing when the page set is generated by a script instead.
Slight behavior modification: new archives are now named based on the archive info file instead of the page set.


BUG=223667
TEST=tools/telemetry/run_tests PageSetArchiveInfo

Review URL: https://codereview.chromium.org/99483007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@238903 0039d316-1c4b-4281-b951-d872f2087c98

tools/perf/benchmarks/jsgamebench.py

@@ -27,7 +27,7 @@ class Jsgamebench(test.Test):
 
   def CreatePageSet(self, options):
     return page_set.PageSet.FromDict({
-        'archive_data_file': '../data/jsgamebench.json',
+        'archive_data_file': '../page_sets/data/jsgamebench.json',
         'pages': [
           { 'url': 'http://localhost/' }
           ]

tools/perf/benchmarks/kraken.py

@@ -41,7 +41,7 @@ class Kraken(test.Test):
 
   def CreatePageSet(self, options):
     return page_set.PageSet.FromDict({
-        'archive_data_file': '../data/kraken.json',
+        'archive_data_file': '../page_sets/data/kraken.json',
         'pages': [
           { 'url': 'http://krakenbenchmark.mozilla.org/kraken-1.1/driver.html' }
           ]

tools/perf/benchmarks/robohornet_pro.py

@@ -26,7 +26,7 @@ class RobohornetPro(test.Test):
 
   def CreatePageSet(self, options):
     return page_set.PageSet.FromDict({
-        'archive_data_file': '../data/robohornetpro.json',
+        'archive_data_file': '../page_sets/data/robohornet_pro.json',
         # Measurement require use of real Date.now() for measurement.
         'make_javascript_deterministic': False,
         'pages': [

tools/perf/page_sets/data/2012Q3.json

 {
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.",
-    "page_set": "../2012Q3.json",
     "archives": {
         "2012Q3_000.wpr" : [
             "http://www.facebook.com/barackobama",

tools/perf/page_sets/data/browsermark.json

 {
-    "page_set": "../../benchmarks/browsermark.py", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "browsermark_000.wpr": [
             "http://browsermark.rightware.com/tests/"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/calendar_forward_backward.json

 {
-    "page_set": "../calendar_forward_backward.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "calendar_forward_backward_000.wpr": [

tools/perf/page_sets/data/canvasmark.json

 {
-    "page_set": "../canvasmark.json",
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.",
     "archives": {
         "canvasmark_000.wpr": [

tools/perf/page_sets/data/gmail_alt_threadlist_conversation.json

 {
-    "page_set": "../gmail_alt_threadlist_conversation.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "gmail_alt_threadlist_conversation_000.wpr": [

tools/perf/page_sets/data/gmail_alt_two_labels.json

 {
-    "page_set": "../gmail_alt_two_labels.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "gmail_alt_two_labels_000.wpr": [

tools/perf/page_sets/data/gmail_expand_collapse_conversation.json

 {
-    "page_set": "../gmail_expand_collapse_conversation.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "gmail_expand_collapse_conversation_000.wpr": [

tools/perf/page_sets/data/html5gaming.json

 {
-    "page_set": "../html5gaming.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "html5gaming_000.wpr": [
             "http://html5-benchmark.com/"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/intl_ar_fa_he.json

 {
-    "page_set": "../intl_ar_fa_he.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "intl_ar_fa_he_000.wpr": [

tools/perf/page_sets/data/intl_es_fr_pt-BR.json

 {
-    "page_set": "../intl_es_fr_pt-BR.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "intl_es_fr_pt-BR_000.wpr": [

tools/perf/page_sets/data/intl_hi_ru.json

 {
-    "page_set": "../intl_hi_ru.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "intl_hi_ru_000.wpr": [

tools/perf/page_sets/data/intl_ja_zh.json

 {
-    "page_set": "../intl_ja_zh.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "intl_ja_zh_000.wpr": [

tools/perf/page_sets/data/intl_ko_th_vi.json

 {
-    "page_set": "../intl_ko_th_vi.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "intl_ko_th_vi_000.wpr": [

tools/perf/page_sets/data/jsgamebench.json

 {
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.",
-    "page_set": "../jsgamebench.json",
     "archives": {
         "jsgamebench_000.wpr" : [
             "http://localhost/"

tools/perf/page_sets/data/key_desktop_sites.json

 {
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.",
-    "page_set": "../key_desktop_sites.json",
     "archives": {
         "key_desktop_sites_000.wpr" : [
             "http://facebook.com",

tools/perf/page_sets/data/key_mobile_sites.json

 {
-    "page_set": "../key_mobile_sites.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "key_mobile_sites_000.wpr": [

tools/perf/page_sets/data/key_silk_cases.json

 {
-    "page_set": "../key_silk_cases.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "key_silk_cases_000.wpr": [
@@ -15,4 +14,4 @@
             "http://jsfiddle.net/3yDKh/3/show"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/kraken.json

 {
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.",
-    "page_set": "../kraken.json",
     "archives": {
         "kraken_000.wpr" : [
             "http://krakenbenchmark.mozilla.org/kraken-1.1/driver.html"

tools/perf/page_sets/data/maps.json

 {
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.",
-    "page_set": "../maps_tests.json",
     "archives": {
         "maps_001.wpr": [
             "http://localhost:10020/tracker.html"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/octane.json

 {
-    "page_set": "../octane.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "octane_000.wpr": [
             "http://octane-benchmark.googlecode.com/svn/latest/index.html?auto=1"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/peacekeeper_array.json

 {
-    "page_set": "../peacekeeper_array.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "peacekeeper_array_000.wpr": [
@@ -7,4 +6,4 @@
             "http://peacekeeper.futuremark.com/run.action?debug=true&repeat=false&forceSuiteName=array&forceTestName=arrayWeighted"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/peacekeeper_dom.json

 {
-    "page_set": "../peacekeeper_dom.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "peacekeeper_dom_000.wpr": [
@@ -14,4 +13,4 @@
             "http://peacekeeper.futuremark.com/run.action?debug=true&repeat=false&forceSuiteName=dom&forceTestName=domQueryselector"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/peacekeeper_experimental.json

 {
-    "page_set": "../peacekeeper_experimental.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "peacekeeper_experimental_000.wpr": [
@@ -7,4 +6,4 @@
             "http://peacekeeper.futuremark.com/run.action?debug=true&repeat=false&forceSuiteName=experimental&forceTestName=experimentalRipple02"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/peacekeeper_html5.json

 {
-    "page_set": "../peacekeeper_html5.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "peacekeeper_html5_000.wpr": [
@@ -13,4 +12,4 @@
             "http://peacekeeper.futuremark.com/run.action?debug=true&repeat=false&forceSuiteName=html5&forceTestName=workerContrast02"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/peacekeeper_render.json

 {
-    "page_set": "../peacekeeper_render.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "peacekeeper_render_000.wpr": [
@@ -9,4 +8,4 @@
             "http://peacekeeper.futuremark.com/run.action?debug=true&repeat=false&forceSuiteName=render&forceTestName=renderPhysics"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/peacekeeper_string.json

 {
-    "page_set": "../peacekeeper_string.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "peacekeeper_string_000.wpr": [
@@ -10,4 +9,4 @@
             "http://peacekeeper.futuremark.com/run.action?debug=true&repeat=false&forceSuiteName=string&forceTestName=stringValidateForm"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/pica.json

 {
-    "page_set": "../pica.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "pica_002.wpr": [
             "http://localhost/polymer/projects/pica/"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/plus_alt_posts_photos.json

 {
-    "page_set": "../plus_alt_posts_photos.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "plus_alt_posts_photos_000.wpr": [

tools/perf/page_sets/data/robohornetpro.json → tools/perf/page_sets/data/robohornet_pro.json

 {
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.",
-    "page_set": "../robohornetpro.json",
     "archives": {
-        "robohornetpro_000.wpr" : [
+        "robohornet_pro_000.wpr" : [
             "http://ie.microsoft.com/testdrive/performance/robohornetpro/"
         ]
     }

tools/perf/page_sets/data/scirra.json

 {
-    "page_set": "../scirra.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "scirra_000.wpr": [
@@ -8,4 +7,4 @@
             "http://www.scirra.com/labs/renderperf3/"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/sunspider.json

 {
-    "page_set": "../../benchmarks/sunspider.py", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "sunspider_000.wpr": [
             "http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/test.json

 {
-    "page_set": "../test.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "test_000.wpr": [

tools/perf/page_sets/data/top_10.json

 {
-    "page_set": "../top_10.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "top_10_000.wpr": [

tools/perf/page_sets/data/top_10_mobile.json

 {
-    "page_set": "../top_10_mobile.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "top_10_mobile_002.wpr": [
@@ -18,4 +17,4 @@
             "https://mobile.twitter.com/justinbieber&"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/top_25.json

 {
-    "page_set": "../top_25.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "top_25_001.wpr": [
@@ -35,4 +34,4 @@
             "http://techcrunch.com"
         ]
     }
-}
\ No newline at end of file
+}

tools/perf/page_sets/data/tough_canvas_cases.json

 {
-    "page_set": "../tough_canvas_cases.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "tough_canvas_cases_001.wpr": [

tools/perf/page_sets/data/tough_layout_cases.json

 {
-    "page_set": "../tough_layout_cases.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "tough_layout_cases_001.wpr": [

tools/perf/page_sets/data/typical_25.json

 {
-    "page_set": "../typical_25.json", 
     "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
     "archives": {
         "typical_25_000.wpr": [

tools/perf/page_sets/tough_memory_multi_tab.json

 {
   "description": "Mobile sites for exercising multi-tab memory issues",
-  "archive_data_file": "../data/key_mobile_sites.json",
-  "credentials_path": "../data/credentials.json",
+  "archive_data_file": "data/key_mobile_sites.json",
+  "credentials_path": "data/credentials.json",
   "user_agent_type": "mobile",
   "pages": [
     { "url": "https://www.google.com/#hl=en&q=barack+obama" },

tools/telemetry/telemetry/page/page_set.py

@@ -30,7 +30,7 @@ class PageSet(object):
     # Create a PageSetArchiveInfo object.
     if self.archive_data_file:
       self.wpr_archive_info = page_set_archive_info.PageSetArchiveInfo.FromFile(
-          os.path.join(self._base_dir, self.archive_data_file), file_path)
+          os.path.join(self._base_dir, self.archive_data_file))
     else:
       self.wpr_archive_info = None
 

tools/telemetry/telemetry/page/page_set_archive_info.py

@@ -13,16 +13,13 @@ from telemetry.page import cloud_storage
 
 
 class PageSetArchiveInfo(object):
-  def __init__(self, archive_data_file_path, page_set_file_path, data):
-    self._archive_data_file_path = archive_data_file_path
-    self._archive_data_file_dir = os.path.dirname(archive_data_file_path)
+  def __init__(self, file_path, data):
+    self._file_path = file_path
+    self._base_dir = os.path.dirname(file_path)
 
     # Ensure directory exists.
-    if not os.path.exists(self._archive_data_file_dir):
-      os.makedirs(self._archive_data_file_dir)
-
-    # Back pointer to the page set file.
-    self._page_set_file_path = page_set_file_path
+    if not os.path.exists(self._base_dir):
+      os.makedirs(self._base_dir)
 
     # Download all .wpr files.
     for archive_path in data['archives']:
@@ -61,12 +58,12 @@ class PageSetArchiveInfo(object):
     self.temp_target_wpr_file_path = None
 
   @classmethod
-  def FromFile(cls, file_path, page_set_file_path):
+  def FromFile(cls, file_path):
     if os.path.exists(file_path):
       with open(file_path, 'r') as f:
         data = json.load(f)
-        return cls(file_path, page_set_file_path, data)
-    return cls(file_path, page_set_file_path, {'archives': {}})
+        return cls(file_path, data)
+    return cls(file_path, {'archives': {}})
 
   def WprFilePathForPage(self, page):
     if self.temp_target_wpr_file_path:
@@ -119,21 +116,18 @@ class PageSetArchiveInfo(object):
     metadata['description'] = (
         'Describes the Web Page Replay archives for a page set. Don\'t edit by '
         'hand! Use record_wpr for updating.')
-    # Pointer from the metadata to the page set .json file.
-    metadata['page_set'] = os.path.relpath(self._page_set_file_path,
-                                           self._archive_data_file_dir)
     metadata['archives'] = self._wpr_file_to_urls.copy()
     # Don't write data for abandoned archives.
     abandoned_wpr_files = self._AbandonedWprFiles()
     for wpr_file in abandoned_wpr_files:
       del metadata['archives'][wpr_file]
 
-    with open(self._archive_data_file_path, 'w') as f:
+    with open(self._file_path, 'w') as f:
       json.dump(metadata, f, indent=4)
       f.flush()
 
   def _WprFileNameToPath(self, wpr_file):
-    return os.path.abspath(os.path.join(self._archive_data_file_dir, wpr_file))
+    return os.path.abspath(os.path.join(self._base_dir, wpr_file))
 
   def _NextWprFileName(self):
     """Creates a new file name for a wpr archive file."""
@@ -152,7 +146,7 @@ class PageSetArchiveInfo(object):
     if not base:
       # If we're creating a completely new info file, use the base name of the
       # page set file.
-      base = os.path.splitext(os.path.basename(self._page_set_file_path))[0]
+      base = os.path.splitext(os.path.basename(self._file_path))[0]
     new_filename = '%s_%03d.wpr' % (base, highest_number + 1)
     return new_filename, self._WprFileNameToPath(new_filename)
 

tools/telemetry/telemetry/page/page_set_archive_info_unittest.py

@@ -48,8 +48,7 @@ class TestPageSetArchiveInfo(unittest.TestCase):
 
     # Create the PageSetArchiveInfo object to be tested.
     self.archive_info = page_set_archive_info.PageSetArchiveInfo.FromFile(
-        self.page_set_archive_info_file,
-        os.path.join(tempfile.gettempdir(), 'pageset.json'))
+        self.page_set_archive_info_file)
 
   def tearDown(self):
     shutil.rmtree(self.tmp_dir)
@@ -113,7 +112,7 @@ class TestPageSetArchiveInfo(unittest.TestCase):
     # Write only the page set without the corresponding metadata file.
     page_set_contents = ("""
     {
-        archive_data_file": "new-metadata.json",
+        archive_data_file": "new_archive_info.json",
         "pages": [
             {
                 "url": "%s",
@@ -121,16 +120,16 @@ class TestPageSetArchiveInfo(unittest.TestCase):
         ]
     }""" % url1)
 
-    page_set_file = os.path.join(self.tmp_dir, 'new.json')
+    page_set_file = os.path.join(self.tmp_dir, 'new_page_set.json')
     with open(page_set_file, 'w') as f:
       f.write(page_set_contents)
 
     self.page_set_archive_info_file = os.path.join(self.tmp_dir,
-                                                   'new-metadata.json')
+                                                   'new_archive_info.json')
 
     # Create the PageSetArchiveInfo object to be tested.
     self.archive_info = page_set_archive_info.PageSetArchiveInfo.FromFile(
-        self.page_set_archive_info_file, page_set_file)
+        self.page_set_archive_info_file)
 
     # Add a recording for all the pages.
     new_temp_recording = os.path.join(self.tmp_dir, 'recording.wpr')
@@ -145,7 +144,7 @@ class TestPageSetArchiveInfo(unittest.TestCase):
     self.archive_info.AddRecordedPages([page1.url])
 
     # Expected name for the recording (decided by PageSetArchiveInfo).
-    new_recording = os.path.join(self.tmp_dir, 'new_000.wpr')
+    new_recording = os.path.join(self.tmp_dir, 'new_archive_info_000.wpr')
 
     self.assertTrue(os.path.exists(new_recording))
     self.assertFalse(os.path.exists(new_temp_recording))
@@ -154,7 +153,6 @@ class TestPageSetArchiveInfo(unittest.TestCase):
     # Check that the archive info was written correctly.
     self.assertTrue(os.path.exists(self.page_set_archive_info_file))
     read_archive_info = page_set_archive_info.PageSetArchiveInfo.FromFile(
-        self.page_set_archive_info_file,
-        os.path.join(tempfile.gettempdir(), 'pageset.json'))
+        self.page_set_archive_info_file)
     self.assertEquals(new_recording,
                       read_archive_info.WprFilePathForPage(page1))