[telemetry] Allow public page set data and WPR archives.

BUG=332148
TEST=mv ~/.boto ~/.boto.bak; rm tools/perf/page_sets/data/key_silk_cases_000.wpr; tools/perf/run_benchmark silk.key_silk_cases -v; mv ~/.boto.bak ~/.boto

Review URL: https://codereview.chromium.org/126093006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@243447 0039d316-1c4b-4281-b951-d872f2087c98

tools/perf/page_sets/PRESUBMIT.py

@@ -23,7 +23,7 @@ def LoadSupport(input_api):
   return globals()['cloud_storage']
 
 
-def _SyncFilesToCloud(input_api, output_api):
+def _GetFilesNotInCloud(input_api):
   """Searches for .sha1 files and uploads them to Cloud Storage.
 
   It validates all the hashes and skips upload if not necessary.
@@ -33,22 +33,38 @@ def _SyncFilesToCloud(input_api, output_api):
 
   # Look in both buckets, in case the user uploaded the file manually. But this
   # script focuses on WPR archives, so it only uploads to the internal bucket.
-  hashes_in_cloud_storage = cloud_storage.List(cloud_storage.INTERNAL_BUCKET)
-  hashes_in_cloud_storage += cloud_storage.List(cloud_storage.PUBLIC_BUCKET)
+  hashes_in_cloud_storage = cloud_storage.List(cloud_storage.PUBLIC_BUCKET)
+  try:
+    hashes_in_cloud_storage += cloud_storage.List(cloud_storage.INTERNAL_BUCKET)
+  except (cloud_storage.PermissionError, cloud_storage.CredentialsError):
+    pass
 
-  results = []
+  files = []
   for affected_file in input_api.AffectedFiles(include_deletes=False):
     hash_path = affected_file.AbsoluteLocalPath()
-    file_path, extension = os.path.splitext(hash_path)
+    _, extension = os.path.splitext(hash_path)
     if extension != '.sha1':
       continue
 
     with open(hash_path, 'rb') as f:
       file_hash = f.read(1024).rstrip()
-    if file_hash in hashes_in_cloud_storage:
-      results.append(output_api.PresubmitNotifyResult(
-          'File already in Cloud Storage, skipping upload: %s' % hash_path))
-      continue
+    if file_hash not in hashes_in_cloud_storage:
+      files.append((hash_path, file_hash))
+
+  return files
+
+
+def _SyncFilesToCloud(input_api, output_api):
+  """Searches for .sha1 files and uploads them to Cloud Storage.
+
+  It validates all the hashes and skips upload if not necessary.
+  """
+
+  cloud_storage = LoadSupport(input_api)
+
+  results = []
+  for hash_path, file_hash in _GetFilesNotInCloud(input_api):
+    file_path, _ = os.path.splitext(hash_path)
 
     if not re.match('^([A-Za-z0-9]{40})$', file_hash):
       results.append(output_api.PresubmitError(
@@ -64,7 +80,19 @@ def _SyncFilesToCloud(input_api, output_api):
       continue
 
     try:
-      cloud_storage.Insert(cloud_storage.INTERNAL_BUCKET, file_hash, file_path)
+      bucket_input = raw_input('Uploading to Cloud Storage: %s\nIs this file '
+                               '[p]ublic or Google-[i]nternal?').lower()
+      if 'public'.startswith(bucket_input):
+        bucket = cloud_storage.PUBLIC_BUCKET
+      elif ('internal'.startswith(bucket_input) or
+            'google-internal'.startswith(bucket_input)):
+        bucket = cloud_storage.INTERNAL_BUCKET
+      else:
+        results.append(output_api.PresubmitError(
+            'Response was neither "public" nor "internal": %s' % bucket_input))
+        return results
+
+      cloud_storage.Insert(bucket, file_hash, file_path)
       results.append(output_api.PresubmitNotifyResult(
           'Uploaded file to Cloud Storage: %s' % hash_path))
     except cloud_storage.CloudStorageError, e:
@@ -74,9 +102,22 @@ def _SyncFilesToCloud(input_api, output_api):
   return results
 
 
+def _VerifyFilesInCloud(input_api, output_api):
+  """Searches for .sha1 files and uploads them to Cloud Storage.
+
+  It validates all the hashes and skips upload if not necessary.
+  """
+  results = []
+  for hash_path, _ in _GetFilesNotInCloud(input_api):
+    results.append(output_api.PresubmitError(
+        'Attemping to commit hash file, but corresponding '
+        'data file is not in Cloud Storage: %s' % hash_path))
+  return results
+
+
 def CheckChangeOnUpload(input_api, output_api):
   return _SyncFilesToCloud(input_api, output_api)
 
 
 def CheckChangeOnCommit(input_api, output_api):
-  return _SyncFilesToCloud(input_api, output_api)
+  return _VerifyFilesInCloud(input_api, output_api)

tools/perf/page_sets/data/test.json

-{
-    "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
-    "archives": {
-        "test_000.wpr": [
-            "http://techcrunch.com"
-        ]
-    }
-}

tools/perf/page_sets/data/test_000.wpr.sha1

-e0a14cfe60d16028e1d246723b33ffaf319db164
\ No newline at end of file

tools/telemetry/telemetry/core/backends/webdriver/webdriver_desktop_browser_finder.py

@@ -73,7 +73,7 @@ class PossibleDesktopIE(PossibleWebDriverBrowser):
     def DriverCreator():
       ie_driver_exe = os.path.join(util.GetTelemetryDir(), 'bin',
                                    'IEDriverServer_%s.exe' % self._architecture)
-      cloud_storage.GetIfChanged(cloud_storage.PUBLIC_BUCKET, ie_driver_exe)
+      cloud_storage.GetIfChanged(ie_driver_exe, cloud_storage.PUBLIC_BUCKET)
       return webdriver.Ie(executable_path=ie_driver_exe)
     return webdriver_ie_backend.WebDriverIEBackend(
         platform_backend, DriverCreator, self.finder_options.browser_options)

tools/telemetry/telemetry/core/platform/linux_platform_backend.py

@@ -67,9 +67,9 @@ class LinuxPlatformBackend(
 
     try:
       changed = cloud_storage.GetIfChanged(
-          cloud_storage.INTERNAL_BUCKET, ipfw_bin)
+          ipfw_bin, cloud_storage.INTERNAL_BUCKET)
       changed |= cloud_storage.GetIfChanged(
-          cloud_storage.INTERNAL_BUCKET, ipfw_mod)
+          ipfw_mod, cloud_storage.INTERNAL_BUCKET)
     except cloud_storage.CloudStorageError, e:
       logging.error(e)
       logging.error('You may proceed by manually installing dummynet. See: '
@@ -90,7 +90,7 @@ class LinuxPlatformBackend(
     os.environ['PATH'] += os.pathsep + telemetry_bin_dir
 
     try:
-      cloud_storage.GetIfChanged(cloud_storage.INTERNAL_BUCKET, avconv_bin)
+      cloud_storage.GetIfChanged(avconv_bin, cloud_storage.INTERNAL_BUCKET)
     except cloud_storage.CloudStorageError, e:
       logging.error(e)
       logging.error('You may proceed by manually installing avconv via:\n'

tools/telemetry/telemetry/core/platform/profiler/android_prebuilt_profiler_helper.py

@@ -28,8 +28,8 @@ def GetHostPath(profiler_binary):
 
 
 def GetIfChanged(profiler_binary):
-  cloud_storage.GetIfChanged(cloud_storage.PUBLIC_BUCKET,
-                             GetHostPath(profiler_binary))
+  cloud_storage.GetIfChanged(GetHostPath(profiler_binary),
+                             cloud_storage.PUBLIC_BUCKET)
 
 
 def InstallOnDevice(adb, profiler_binary):

tools/telemetry/telemetry/page/cloud_storage.py

@@ -106,8 +106,9 @@ def _RunCommand(args):
 
 
 def List(bucket):
-  stdout = _RunCommand(['ls', 'gs://%s' % bucket])
-  return [url.split('/')[-1] for url in stdout.splitlines()]
+  query = 'gs://%s/' % bucket
+  stdout = _RunCommand(['ls', query])
+  return [url[len(query):] for url in stdout.splitlines()]
 
 def Exists(bucket, remote_path):
   try:
@@ -140,7 +141,7 @@ def Insert(bucket, remote_path, local_path, publicly_readable=False):
   _RunCommand(command_and_args)
 
 
-def GetIfChanged(bucket, file_path):
+def GetIfChanged(file_path, bucket=None):
   """Gets the file at file_path if it has a hash file that doesn't match.
 
   If the file is not in Cloud Storage, log a warning instead of raising an
@@ -158,13 +159,22 @@ def GetIfChanged(bucket, file_path):
   if os.path.exists(file_path) and GetHash(file_path) == expected_hash:
     return False
 
-  try:
-    Get(bucket, expected_hash, file_path)
-  except NotFoundError:
-    logging.warning('Unable to update file %s from Cloud Storage.' % file_path)
-    return False
-
-  return True
+  if bucket:
+    buckets = [bucket]
+  else:
+    buckets = [PUBLIC_BUCKET, INTERNAL_BUCKET]
+
+  found = False
+  for bucket in buckets:
+    try:
+      Get(bucket, expected_hash, file_path)
+      found = True
+    except NotFoundError:
+      continue
+
+  if not found:
+    logging.warning('Unable to find file in Cloud Storage: %s', file_path)
+  return found
 
 
 def GetHash(file_path):

tools/telemetry/telemetry/page/page_set.py

@@ -57,7 +57,6 @@ class PageSet(object):
     # Attempt to download the credentials file.
     if self.credentials_path:
       cloud_storage.GetIfChanged(
-          cloud_storage.INTERNAL_BUCKET,
           os.path.join(self._base_dir, self.credentials_path))
 
     # Scan every serving directory for .sha1 files
@@ -77,7 +76,7 @@ class PageSet(object):
               os.path.join(dirpath, filename))
           if extension != '.sha1':
             continue
-          cloud_storage.GetIfChanged(cloud_storage.PUBLIC_BUCKET, path)
+          cloud_storage.GetIfChanged(path)
 
   @classmethod
   def FromFile(cls, file_path):

tools/telemetry/telemetry/page/page_set_archive_info.py

@@ -25,7 +25,7 @@ class PageSetArchiveInfo(object):
     for archive_path in data['archives']:
       archive_path = self._WprFileNameToPath(archive_path)
       try:
-        cloud_storage.GetIfChanged(cloud_storage.INTERNAL_BUCKET, archive_path)
+        cloud_storage.GetIfChanged(archive_path)
       except (cloud_storage.CredentialsError,
               cloud_storage.PermissionError) as e:
         if os.path.exists(archive_path):