Adding Netflix performance tests,

- To measure dropped frames/second.
- To calculate extrapolation of the utilization to play the test video.

Note: Moving Netflix helper functions to a separate class, so they can be reused easily for other Netflix tests.
Review URL: http://codereview.chromium.org/8562006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@111063 0039d316-1c4b-4281-b951-d872f2087c98

chrome/test/functional/netflix.py

@@ -9,99 +9,131 @@ import pyauto_functional
 import pyauto
 
 
-class NetflixTest(pyauto.PyUITest):
-  """Test case for Netflix player"""
+class NetflixTestHelper():
+  """Helper functions for Netflix tests.
+     
+  For sample usage, look at class NetflixTest.
+  """
 
-  # Netflix player states
-  _is_playing = '4'
+  # Netflix player states.
+  is_playing = '4'
 
-  _title_homepage = 'http://movies.netflix.com/WiHome'
-  _signout_page = 'https://account.netflix.com/Logout'
+  title_homepage = 'http://movies.netflix.com/WiHome'
+  signout_page = 'https://account.netflix.com/Logout'
   # 30 Rock
-  _test_title = 'http://movies.netflix.com/WiPlayer?'+ \
+  test_title = 'http://movies.netflix.com/WiPlayer?'+ \
                'movieid=70136124&trkid=2361637&t=30+Rock'
+  _pyauto = None
 
-  def tearDown(self):
-    self._SignOut()
-    pyauto.PyUITest.tearDown(self) 
+  def __init__(self, pyauto):
+    self._pyauto = pyauto
 
   def _IsNetflixPluginEnabled(self):
-    """Determine Netflix plugin availability and its state"""
-    return [x for x in self.GetPluginsInfo().Plugins() \
-               if x['name'] == 'Netflix' and x['enabled']]
+    """Determine Netflix plugin availability and its state."""
+    return [x for x in self._pyauto.GetPluginsInfo().Plugins() \
+            if x['name'] == 'Netflix' and x['enabled']]
 
   def _LoginToNetflix(self):
-    """Login to Netflix"""
-    credentials = self.GetPrivateInfo()['test_netflix_acct']
-    board_name = self.ChromeOSBoard()
+    """Login to Netflix."""
+    credentials = self._pyauto.GetPrivateInfo()['test_netflix_acct']
+    board_name = self._pyauto.ChromeOSBoard()
     assert credentials.get(board_name), \
-           'No netflix credentials for %s' % board_name
-    self.NavigateToURL(credentials['login_url'])
+           'No netflix credentials for %s.' % board_name
+    self._pyauto.NavigateToURL(credentials['login_url'])
     login_js = """
         document.getElementById('email').value='%s';
         document.getElementById('password').value='%s';
         window.domAutomationController.send('ok');
     """ % (credentials[board_name], credentials['password'])
-    self.assertEqual(self.ExecuteJavascript(login_js), 'ok',
-        msg='Failed to set login credentials')
-    self.assertTrue(self.SubmitForm('login-form'),
-        msg='Login to Netflix failed')
+    self._pyauto.assertEqual(self._pyauto.ExecuteJavascript(login_js), 'ok',
+        msg='Failed to set login credentials.')
+    self._pyauto.assertTrue(self._pyauto.SubmitForm('login-form'),
+        msg='Login to Netflix failed.')
     
+  def _GetVideoDroppedFrames(self, tab_index=0, windex=0):
+    """Returns total Netflix video dropped frames."""
+    js = """
+        var frames = nrdp.video.droppedFrames; 
+        window.domAutomationController.send(frames + '');
+    """
+    return int(self._pyauto.ExecuteJavascript(js, tab_index=tab_index,
+                                              windex=windex))
+
+  def _GetVideoFrames(self, tab_index=0, windex=0):
+    """Returns Netflix video total frames."""
+    js = """
+        var frames = nrdp.video.totalFrames; 
+        window.domAutomationController.send(frames + '');
+    """
+    return int(self._pyauto.ExecuteJavascript(js, tab_index=tab_index,
+                                              windex=windex))
+
   def _HandleInfobars(self):
     """Manage infobars, come up during the test.
 
     We expect password and Netflix infobars. Processing only Netflix infobar,
     since to start a vidoe, pressing the OK button is a must. We can keep other
-    inforbars open."""
-    self.WaitForInfobarCount(2)
-    tab_info = self.GetBrowserInfo()['windows'][0]['tabs'][0]
+    infobars open."""
+    self._pyauto.WaitForInfobarCount(2)
+    tab_info = self._pyauto.GetBrowserInfo()['windows'][0]['tabs'][0]
     infobars = tab_info['infobars']
     index = 0
     netflix_infobar_status = False
     for infobar in infobars:
        if infobar['buttons'][0] == 'OK':
-         self.PerformActionOnInfobar('accept', infobar_index=index)
+         self._pyauto.PerformActionOnInfobar('accept', infobar_index=index)
          netflix_infobar_status = True
        index = index + 1
-    self.assertTrue(netflix_infobar_status, 
-                    msg='Netflix infobar did not show up')
+    self._pyauto.assertTrue(netflix_infobar_status,
+                            msg='Netflix infobar did not show up')
          
   def _CurrentPlaybackTime(self):
-    """Returns the current playback time in seconds"""
-    time = self.ExecuteJavascript("""
+    """Returns the current playback time in seconds."""
+    time = self._pyauto.ExecuteJavascript("""
         time = nrdp.video.currentTime;
         window.domAutomationController.send(time + '');
     """)
     return int(float(time))
 
   def _SignOut(self):
-    """Sing out from Netflix Login"""
-    self.NavigateToURL(self._signout_page)
+    """Sing out from Netflix Login."""
+    self._pyauto.NavigateToURL(self._pyauto.signout_page)
 
   def _LoginAndStartPlaying(self):
-    """Login and start playing the video"""
-    self.assertTrue(self._IsNetflixPluginEnabled(), 
-                    msg='Netflix plugin is disabled or not available') 
-    self._LoginToNetflix()
-    self.assertTrue(self.WaitUntil(
-        lambda:self.GetActiveTabURL().spec(),
-        expect_retval=self._title_homepage),
-        msg='Login to Netflix failed')
-    self.NavigateToURL(self._test_title)
-    self._HandleInfobars()
-    self.assertTrue(self.WaitUntil(
-        lambda: self.ExecuteJavascript("""
+    """Login and start playing the video."""
+    self._pyauto.assertTrue(self._pyauto._IsNetflixPluginEnabled(), 
+                            msg='Netflix plugin is disabled or not available.')
+    self._pyauto._LoginToNetflix()
+    self._pyauto.assertTrue(self._pyauto.WaitUntil(
+        lambda:self._pyauto.GetActiveTabURL().spec(),
+        expect_retval=self._pyauto.title_homepage),
+        msg='Login to Netflix failed.')
+    self._pyauto.NavigateToURL(self._pyauto.test_title)
+    self._pyauto._HandleInfobars()
+    self._pyauto.assertTrue(self._pyauto.WaitUntil(
+        lambda: self._pyauto.ExecuteJavascript("""
             player_status = nrdp.video.readyState;
             window.domAutomationController.send(player_status + '');
-        """), expect_retval=self._is_playing),
-        msg='Player did not start playing the title')
+        """), expect_retval=self._pyauto.is_playing),
+        msg='Player did not start playing the title.')
+
+class NetflixTest(pyauto.PyUITest, NetflixTestHelper):
+  """Test case for Netflix player."""
+
+  def __init__(self, methodName='runTest', **kwargs):
+    pyauto.PyUITest.__init__(self, methodName, **kwargs)
+    NetflixTestHelper.__init__(self, self)
+
+  def tearDown(self):
+    self._SignOut()
+    pyauto.PyUITest.tearDown(self) 
 
   def testPlayerLoadsAndPlays(self):
-    """Test that Netflix player loads and plays the title"""
+    """Test that Netflix player loads and plays the title."""
     self._LoginAndStartPlaying()
 
   def testPlaying(self):
-    """Test that title playing progresses"""
+    """Test that title playing progresses."""
     self._LoginAndStartPlaying()
     title_length =  self.ExecuteJavascript("""
         time = nrdp.video.duration;
@@ -112,13 +144,13 @@ class NetflixTest(pyauto.PyUITest):
     current_time = 0
     count = 0
     while current_time < title_length:
-      # We want to test playing only for ten seconds
+      # We want to test playing only for ten seconds.
       count = count + 1
       if count == 10:
         break
       current_time = self._CurrentPlaybackTime()
       self.assertTrue(prev_time <= current_time,
-          msg='Prev playing time %s is greater than current time %s'
+          msg='Prev playing time %s is greater than current time %s.'
           % (prev_time, current_time))
       prev_time = current_time
       # play video for some time 
@@ -128,7 +160,7 @@ class NetflixTest(pyauto.PyUITest):
     # still pass. So re-verifying and assuming that player did play something
     # during last 10 seconds.
     self.assertTrue(current_time > 0,
-        msg='Netflix player didnot start playing')
+        msg='Netflix player didnot start playing.')
 
 
 if __name__ == '__main__':

chrome/test/functional/perf.py

@@ -41,6 +41,7 @@ import urlparse
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 
+from netflix import NetflixTestHelper
 import perf_snapshot
 import pyauto_utils
 import test_utils
@@ -537,6 +538,58 @@ class LiveWebappLoadTest(BasePerfTest):
     self._RunNewTabTest('NewTabDocs', _RunSingleDocsTabOpen)
 
 
+class NetflixPerfTest(BasePerfTest, NetflixTestHelper):
+  """Test Netflix video performance."""
+
+  def __init__(self, methodName='runTest', **kwargs):
+    pyauto.PyUITest.__init__(self, methodName, **kwargs)
+    NetflixTestHelper.__init__(self, self)
+
+  def tearDown(self):
+    self._SignOut()
+    pyauto.PyUITest.tearDown(self) 
+
+  def testNetflixDroppedFrames(self):
+    """Measures the Netflix video dropped frames/second. Runs for 60 secs."""
+    self._LoginAndStartPlaying()
+    # Ignore first 10 seconds of video playing so we get smooth videoplayback.
+    time.sleep(10)
+    init_dropped_frames = self._GetVideoDroppedFrames()
+    dropped_frames = []
+    prev_dropped_frames = 0
+    for _ in xrange(60):
+      # Ignoring initial dropped frames of first 10 seconds.
+      total_dropped_frames = self._GetVideoDroppedFrames() - init_dropped_frames
+      dropped_frames.append(total_dropped_frames - prev_dropped_frames)
+      prev_dropped_frames = total_dropped_frames
+      # Play the video for some time.
+      time.sleep(1)
+    self._PrintSummaryResults('NetflixDroppedFrames', dropped_frames, 'frames')
+
+  def testNetflixCPU(self):
+    """Measures the Netflix video CPU usage. Runs for 60 seconds."""
+    self._LoginAndStartPlaying()
+    # Ignore first 10 seconds of video playing so we get smooth videoplayback.
+    time.sleep(10)
+    init_dropped_frames = self._GetVideoDroppedFrames()
+    init_video_frames = self._GetVideoFrames()
+    cpu_usage_start = self._GetCPUUsage()
+    total_shown_frames = 0
+    # Play the video for some time.
+    time.sleep(60)
+    total_video_frames = self._GetVideoFrames() - init_video_frames
+    total_dropped_frames = self._GetVideoDroppedFrames() - init_dropped_frames
+    cpu_usage_end = self._GetCPUUsage()
+    fraction_non_idle_time = \
+        self._GetFractionNonIdleCPUTime(cpu_usage_start, cpu_usage_end)
+    # Counting extrapolation for utilization to play the video.
+    extrapolation_value = fraction_non_idle_time * \
+        (total_video_frames + total_dropped_frames) / total_video_frames
+    logging.info('Netflix CPU extrapolation: %.2f' % extrapolation_value)
+    self._OutputPerfGraphValue('extrapolation_NetflixCPUExtrapolation',
+                               extrapolation_value)
+
+
 class YoutubePerfTest(BasePerfTest, YoutubeTestHelper):
   """Test Youtube video performance."""
 
@@ -578,7 +631,7 @@ class YoutubePerfTest(BasePerfTest, YoutubeTestHelper):
     time.sleep(10)
 
   def testYoutubeDroppedFrames(self):
-    """Measures the Youtube video dropped frames. Runs for 60 secs."""
+    """Measures the Youtube video dropped frames/second. Runs for 60 secs."""
     self.StartVideoForPerformance()
     init_dropped_frames = self.GetVideoDroppedFrames()
     total_dropped_frames = 0