[Telemetry] Fix a possible source of flakiness on Android.

With this, we survive killing the adb process which produces the same stack as:
http://build.chromium.org/p/chromium.perf/builders/Android%20GN/builds/764/steps/jsgamebench/logs/stdio

BUG=163661
TEST='killall -9 adb' while running a benchmark

Review URL: https://codereview.chromium.org/11348370

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@171088 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/browser_backend.py

@@ -14,6 +14,12 @@ from telemetry import util
 from telemetry import wpr_modes
 from telemetry import wpr_server
 
+
+class BrowserConnectionGoneException(
+    browser_gone_exception.BrowserGoneException):
+  pass
+
+
 class BrowserBackend(object):
   """A base class for browser backends. Provides basic functionality
   once a remote-debugger port has been established."""
@@ -42,17 +48,7 @@ class BrowserBackend(object):
     def IsBrowserUp():
       try:
         self._ListTabs()
-      except socket.error:
-        if not self.IsBrowserRunning():
-          raise browser_gone_exception.BrowserGoneException()
-        return False
-      except httplib.BadStatusLine:
-        if not self.IsBrowserRunning():
-          raise browser_gone_exception.BrowserGoneException()
-        return False
-      except urllib2.URLError:
-        if not self.IsBrowserRunning():
-          raise browser_gone_exception.BrowserGoneException()
+      except BrowserConnectionGoneException:
         return False
       else:
         return True
@@ -66,17 +62,22 @@ class BrowserBackend(object):
     return 'http://localhost:%i/json' % self._port
 
   def _ListTabs(self, timeout=None):
-    req = urllib2.urlopen(self._debugger_url, timeout=timeout)
-    data = req.read()
-    all_contexts = json.loads(data)
-    tabs = [ctx for ctx in all_contexts
-            if not ctx['url'].startswith('chrome-extension://')]
-    # FIXME(dtu): The remote debugger protocol returns in order of most
-    # recently created tab first. In order to convert it to the UI tab
-    # order, we just reverse the list, which assumes we can't move tabs.
-    # We should guarantee that the remote debugger returns in the UI tab order.
-    tabs.reverse()
-    return tabs
+    try:
+      req = urllib2.urlopen(self._debugger_url, timeout=timeout)
+      data = req.read()
+      all_contexts = json.loads(data)
+      tabs = [ctx for ctx in all_contexts
+              if not ctx['url'].startswith('chrome-extension://')]
+      # FIXME(dtu): The remote debugger protocol returns in order of most
+      # recently created tab first. In order to convert it to the UI tab
+      # order, we just reverse the list, which assumes we can't move tabs.
+      # We should guarantee that the remote debugger returns in UI tab order.
+      tabs.reverse()
+      return tabs
+    except (socket.error, httplib.BadStatusLine, urllib2.URLError):
+      if not self.IsBrowserRunning():
+        raise browser_gone_exception.BrowserGoneException()
+      raise BrowserConnectionGoneException()
 
   def NewTab(self, timeout=None):
     req = urllib2.urlopen(self._debugger_url + '/new', timeout=timeout)

tools/telemetry/telemetry/page_runner.py

@@ -8,6 +8,7 @@ import traceback
 import urlparse
 import random
 
+from telemetry import browser_gone_exception
 from telemetry import page_set_url_builder
 from telemetry import page_test
 from telemetry import tab_crash_exception
@@ -111,23 +112,34 @@ http://goto/read-src-internal, or create a new archive using --record.
     state = _RunState()
     try:
       for page in pages:
-        if not state.browser:
-          self._SetupBrowser(state, test, possible_browser, credentials_path,
-                             archive_path)
-        if not state.tab:
-          state.tab = state.browser.ConnectToNthTab(0)
-        if options.trace_dir:
-          self._SetupTracingTab(state)
-
-        try:
-          self._RunPage(options, page, state.tab, test, results)
-        except tab_crash_exception.TabCrashException:
-          # If we don't support tab control, just restart the browser.
-          # TODO(dtu): Create a new tab: crbug.com/155077, crbug.com/159852
-          state.Close()
-
-        if options.trace_dir and state.trace_tab:
-          self._EndTracing(state, options, page)
+        tries = 3
+        while tries:
+          try:
+            if not state.browser:
+              self._SetupBrowser(state, test, possible_browser,
+                                 credentials_path, archive_path)
+            if not state.tab:
+              state.tab = state.browser.ConnectToNthTab(0)
+            if options.trace_dir:
+              self._SetupTracingTab(state)
+
+            try:
+              self._RunPage(options, page, state.tab, test, results)
+            except tab_crash_exception.TabCrashException:
+              # If we don't support tab control, just restart the browser.
+              # TODO(dtu): Create a new tab: crbug.com/155077, crbug.com/159852
+              state.Close()
+
+            if options.trace_dir and state.trace_tab:
+              self._EndTracing(state, options, page)
+            break
+          except browser_gone_exception.BrowserGoneException:
+            logging.warning('Lost connection to browser. Retrying.')
+            state.Close()
+            tries -= 1
+            if not tries:
+              logging.error('Lost connection to browser 3 times. Failing.')
+              raise
     finally:
       state.Close()
 
@@ -150,6 +162,8 @@ http://goto/read-src-internal, or create a new archive using --record.
       logging.warning('Tab crashed: %s', page.url)
       results.AddFailure(page, ex, traceback.format_exc())
       raise
+    except browser_gone_exception.BrowserGoneException:
+      raise browser_gone_exception.BrowserGoneException()
     except Exception, ex:
       logging.error('Unexpected failure while running %s: %s',
                     page.url, traceback.format_exc())
@@ -174,6 +188,8 @@ http://goto/read-src-internal, or create a new archive using --record.
       logging.warning('Tab crashed: %s', page.url)
       results.AddFailure(page, ex, traceback.format_exc())
       raise
+    except browser_gone_exception.BrowserGoneException:
+      raise browser_gone_exception.BrowserGoneException()
     except Exception, ex:
       logging.error('Unexpected failure while running %s: %s',
                     page.url, traceback.format_exc())
@@ -294,5 +310,8 @@ http://goto/read-src-internal, or create a new archive using --record.
   def _CleanUpPage(self, page, tab, page_state): # pylint: disable=R0201
     if page.credentials and page_state.did_login:
       tab.browser.credentials.LoginNoLongerNeeded(tab, page.credentials)
-    tab.runtime.Evaluate("""window.chrome && chrome.benchmarking &&
-                            chrome.benchmarking.closeConnections()""")
+    try:
+      tab.runtime.Evaluate("""window.chrome && chrome.benchmarking &&
+                              chrome.benchmarking.closeConnections()""")
+    except Exception:
+      pass