Commit ca2594ea authored by pasko's avatar pasko Committed by Commit bot

Set tracing categories explicitly in Clovis and Sandwich

Also restrict the categories in Sandwich to reduce the size of traces by 2x.

BUG=582080

Review-Url: https://codereview.chromium.org/2047463002
Cr-Commit-Position: refs/heads/master@{#398262}
parent 5cb2b197
...@@ -7,6 +7,7 @@ import copy ...@@ -7,6 +7,7 @@ import copy
import unittest import unittest
from activity_lens import (ActivityLens, _EventsTree) from activity_lens import (ActivityLens, _EventsTree)
import clovis_constants
import test_utils import test_utils
import tracing import tracing
...@@ -14,13 +15,15 @@ import tracing ...@@ -14,13 +15,15 @@ import tracing
class ActivityLensTestCase(unittest.TestCase): class ActivityLensTestCase(unittest.TestCase):
@classmethod @classmethod
def _EventsFromRawEvents(cls, raw_events): def _EventsFromRawEvents(cls, raw_events):
tracing_track = tracing.TracingTrack(None) tracing_track = tracing.TracingTrack(None,
clovis_constants.DEFAULT_CATEGORIES)
tracing_track.Handle( tracing_track.Handle(
'Tracing.dataCollected', {'params': {'value': raw_events}}) 'Tracing.dataCollected', {'params': {'value': raw_events}})
return tracing_track.GetEvents() return tracing_track.GetEvents()
def setUp(self): def setUp(self):
self.tracing_track = tracing.TracingTrack(None) self.tracing_track = tracing.TracingTrack(None,
clovis_constants.DEFAULT_CATEGORIES)
def testGetRendererMainThread(self): def testGetRendererMainThread(self):
first_renderer_tid = 12345 first_renderer_tid = 12345
......
...@@ -25,6 +25,7 @@ import devil_chromium ...@@ -25,6 +25,7 @@ import devil_chromium
from pylib import constants from pylib import constants
import activity_lens import activity_lens
import clovis_constants
import content_classification_lens import content_classification_lens
import controller import controller
import device_setup import device_setup
...@@ -121,7 +122,8 @@ def _LogRequests(url, clear_cache_override=None): ...@@ -121,7 +122,8 @@ def _LogRequests(url, clear_cache_override=None):
if clear_cache: if clear_cache:
connection.ClearCache() connection.ClearCache()
trace = loading_trace.LoadingTrace.RecordUrlNavigation( trace = loading_trace.LoadingTrace.RecordUrlNavigation(
url, connection, chrome_ctl.ChromeMetadata()) url, connection, chrome_ctl.ChromeMetadata(),
categories=clovis_constants.DEFAULT_CATEGORIES)
except controller.ChromeControllerError as e: except controller.ChromeControllerError as e:
e.Dump(sys.stderr) e.Dump(sys.stderr)
raise raise
......
# Copyright (c) 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Provides common constants for Clovis."""
QUEUING_CATEGORY = 'disabled-by-default-loading.resource'
# Categories to enable or disable for all traces collected. Disabled categories
# are prefixed with '-'.
DEFAULT_CATEGORIES = [
QUEUING_CATEGORY,
'blink',
'blink.net',
'blink.user_timing',
'devtools.timeline',
'disabled-by-default-blink.debug.layout',
'toplevel',
'v8',
'-cc', # Contains a lot of events, none of which we use.
]
...@@ -76,7 +76,7 @@ class LoadingTrace(object): ...@@ -76,7 +76,7 @@ class LoadingTrace(object):
@classmethod @classmethod
def RecordUrlNavigation( def RecordUrlNavigation(
cls, url, connection, chrome_metadata, additional_categories=None, cls, url, connection, chrome_metadata, categories,
timeout_seconds=devtools_monitor.DEFAULT_TIMEOUT_SECONDS): timeout_seconds=devtools_monitor.DEFAULT_TIMEOUT_SECONDS):
"""Create a loading trace by using controller to fetch url. """Create a loading trace by using controller to fetch url.
...@@ -84,8 +84,7 @@ class LoadingTrace(object): ...@@ -84,8 +84,7 @@ class LoadingTrace(object):
url: (str) url to fetch. url: (str) url to fetch.
connection: An opened devtools connection. connection: An opened devtools connection.
chrome_metadata: Dictionary of chrome metadata. chrome_metadata: Dictionary of chrome metadata.
additional_categories: ([str] or None) TracingTrack additional categories categories: as in tracing.TracingTrack
to capture.
timeout_seconds: monitoring connection timeout in seconds. timeout_seconds: monitoring connection timeout in seconds.
Returns: Returns:
...@@ -93,9 +92,7 @@ class LoadingTrace(object): ...@@ -93,9 +92,7 @@ class LoadingTrace(object):
""" """
page = page_track.PageTrack(connection) page = page_track.PageTrack(connection)
request = request_track.RequestTrack(connection) request = request_track.RequestTrack(connection)
trace = tracing.TracingTrack( trace = tracing.TracingTrack(connection, categories)
connection,
additional_categories=additional_categories)
start_date_str = datetime.datetime.utcnow().isoformat() start_date_str = datetime.datetime.utcnow().isoformat()
seconds_since_epoch=time.time() seconds_since_epoch=time.time()
connection.MonitorUrl(url, timeout_seconds=timeout_seconds) connection.MonitorUrl(url, timeout_seconds=timeout_seconds)
......
...@@ -12,12 +12,12 @@ import collections ...@@ -12,12 +12,12 @@ import collections
import itertools import itertools
import logging import logging
import tracing import clovis_constants
class QueuingLens(object): class QueuingLens(object):
"""Attaches queuing related trace events to request objects.""" """Attaches queuing related trace events to request objects."""
QUEUING_CATEGORY = tracing.QUEUING_CATEGORY QUEUING_CATEGORY = clovis_constants.QUEUING_CATEGORY
ASYNC_NAME = 'ScheduledResourceRequest' ASYNC_NAME = 'ScheduledResourceRequest'
READY_NAME = 'ScheduledResourceRequest.Ready' READY_NAME = 'ScheduledResourceRequest.Ready'
SET_PRIORITY_NAME = 'ScheduledResourceRequest.SetPriority' SET_PRIORITY_NAME = 'ScheduledResourceRequest.SetPriority'
......
...@@ -53,8 +53,8 @@ _MINIMALIST_TRACE_EVENTS = [ ...@@ -53,8 +53,8 @@ _MINIMALIST_TRACE_EVENTS = [
def TracingTrack(events): def TracingTrack(events):
return tracing.TracingTrack.FromJsonDict({ return tracing.TracingTrack.FromJsonDict({
'events': events, 'events': events,
'categories': (tracing.INITIAL_CATEGORIES + 'categories': (sandwich_runner._TRACING_CATEGORIES +
(sandwich_runner.MEMORY_DUMP_CATEGORY,))}) [sandwich_runner.MEMORY_DUMP_CATEGORY])})
def LoadingTrace(events): def LoadingTrace(events):
......
...@@ -34,6 +34,18 @@ MEMORY_DUMP_CATEGORY = 'disabled-by-default-memory-infra' ...@@ -34,6 +34,18 @@ MEMORY_DUMP_CATEGORY = 'disabled-by-default-memory-infra'
# network condition. # network condition.
_DEVTOOLS_TIMEOUT = 60 _DEVTOOLS_TIMEOUT = 60
# Categories to enable or disable for all traces collected. Disabled categories
# are prefixed with '-'.
_TRACING_CATEGORIES = [
'blink',
'blink.net',
'blink.user_timing',
'devtools.timeline',
'java',
'toplevel',
'v8',
'-cc', # A lot of unnecessary events are enabled by default in "cc".
]
def _CleanArtefactsFromPastRuns(output_directories_path): def _CleanArtefactsFromPastRuns(output_directories_path):
"""Cleans artifacts generated from past run in the output directory. """Cleans artifacts generated from past run in the output directory.
...@@ -147,13 +159,23 @@ class SandwichRunner(object): ...@@ -147,13 +159,23 @@ class SandwichRunner(object):
os.makedirs(run_path) os.makedirs(run_path)
self._chrome_ctl.SetNetworkEmulation( self._chrome_ctl.SetNetworkEmulation(
self._GetEmulatorNetworkCondition('browser')) self._GetEmulatorNetworkCondition('browser'))
additional_categories = [] categories = _TRACING_CATEGORIES
if self.record_memory_dumps: if self.record_memory_dumps:
additional_categories = [MEMORY_DUMP_CATEGORY] categories += [MEMORY_DUMP_CATEGORY]
# TODO(gabadie): add a way to avoid recording a trace. # TODO(gabadie): add a way to avoid recording a trace.
with self._chrome_ctl.Open() as connection: with self._chrome_ctl.Open() as connection:
if clear_cache: if clear_cache:
connection.ClearCache() connection.ClearCache()
# Binds all parameters of RecordUrlNavigation() to avoid repetition.
def RecordTrace():
return loading_trace.LoadingTrace.RecordUrlNavigation(
url=self.url,
connection=connection,
chrome_metadata=self._chrome_ctl.ChromeMetadata(),
categories=categories,
timeout_seconds=_DEVTOOLS_TIMEOUT)
if run_path is not None and self.record_video: if run_path is not None and self.record_video:
device = self._chrome_ctl.GetDevice() device = self._chrome_ctl.GetDevice()
if device is None: if device is None:
...@@ -161,19 +183,9 @@ class SandwichRunner(object): ...@@ -161,19 +183,9 @@ class SandwichRunner(object):
video_recording_path = os.path.join(run_path, VIDEO_FILENAME) video_recording_path = os.path.join(run_path, VIDEO_FILENAME)
with device_setup.RemoteSpeedIndexRecorder(device, connection, with device_setup.RemoteSpeedIndexRecorder(device, connection,
video_recording_path): video_recording_path):
trace = loading_trace.LoadingTrace.RecordUrlNavigation( trace = RecordTrace()
url=self.url,
connection=connection,
chrome_metadata=self._chrome_ctl.ChromeMetadata(),
additional_categories=additional_categories,
timeout_seconds=_DEVTOOLS_TIMEOUT)
else: else:
trace = loading_trace.LoadingTrace.RecordUrlNavigation( trace = RecordTrace()
url=self.url,
connection=connection,
chrome_metadata=self._chrome_ctl.ChromeMetadata(),
additional_categories=additional_categories,
timeout_seconds=_DEVTOOLS_TIMEOUT)
if run_path is not None: if run_path is not None:
trace_path = os.path.join(run_path, TRACE_FILENAME) trace_path = os.path.join(run_path, TRACE_FILENAME)
trace.ToJsonFile(trace_path) trace.ToJsonFile(trace_path)
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
"""Common utilities used in unit tests, within this directory.""" """Common utilities used in unit tests, within this directory."""
import clovis_constants
import dependency_graph import dependency_graph
import devtools_monitor import devtools_monitor
import loading_trace import loading_trace
...@@ -140,7 +141,8 @@ def LoadingTraceFromEvents(requests, page_events=None, trace_events=None): ...@@ -140,7 +141,8 @@ def LoadingTraceFromEvents(requests, page_events=None, trace_events=None):
request = FakeRequestTrack(requests) request = FakeRequestTrack(requests)
page_event_track = FakePageTrack(page_events if page_events else []) page_event_track = FakePageTrack(page_events if page_events else [])
if trace_events is not None: if trace_events is not None:
tracing_track = tracing.TracingTrack(None) tracing_track = tracing.TracingTrack(None,
clovis_constants.DEFAULT_CATEGORIES)
tracing_track.Handle('Tracing.dataCollected', tracing_track.Handle('Tracing.dataCollected',
{'params': {'value': [e for e in trace_events]}}) {'params': {'value': [e for e in trace_events]}})
else: else:
......
...@@ -41,6 +41,7 @@ import urlparse ...@@ -41,6 +41,7 @@ import urlparse
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
import clovis_constants
import controller import controller
import loading_trace import loading_trace
import options import options
...@@ -234,7 +235,8 @@ def RunTest(webserver, test_page, expected): ...@@ -234,7 +235,8 @@ def RunTest(webserver, test_page, expected):
connection.ClearCache() connection.ClearCache()
observed_seq = InitiatorSequence( observed_seq = InitiatorSequence(
loading_trace.LoadingTrace.RecordUrlNavigation( loading_trace.LoadingTrace.RecordUrlNavigation(
url, connection, chrome_controller.ChromeMetadata())) url, connection, chrome_controller.ChromeMetadata(),
categories=clovis_constants.DEFAULT_CATEGORIES))
if observed_seq == expected: if observed_seq == expected:
sys.stdout.write(' ok\n') sys.stdout.write(' ok\n')
return True return True
......
...@@ -4,43 +4,27 @@ ...@@ -4,43 +4,27 @@
"""Monitor tracing events on chrome via chrome remote debugging.""" """Monitor tracing events on chrome via chrome remote debugging."""
import bisect
import itertools import itertools
import logging import logging
import operator import operator
import clovis_constants
import devtools_monitor import devtools_monitor
QUEUING_CATEGORY = 'disabled-by-default-loading.resource'
_ENABLED_CATEGORIES = (
('toplevel', 'blink', 'v8', 'java', 'devtools.timeline',
'blink.user_timing', 'blink.net', 'disabled-by-default-blink.debug.layout')
+ (QUEUING_CATEGORY,))
_DISABLED_CATEGORIES = ('cc',) # Contains a lot of events, none of which we use.
INITIAL_CATEGORIES = (
_ENABLED_CATEGORIES
+ tuple('-' + cat for cat in _DISABLED_CATEGORIES))
class TracingTrack(devtools_monitor.Track): class TracingTrack(devtools_monitor.Track):
"""Grabs and processes trace event messages. """Grabs and processes trace event messages.
See https://goo.gl/Qabkqk for details on the protocol. See https://goo.gl/Qabkqk for details on the protocol.
""" """
def __init__(self, connection, additional_categories=None, def __init__(self, connection, categories, fetch_stream=False):
disabled_categories=None,
fetch_stream=False):
"""Initialize this TracingTrack. """Initialize this TracingTrack.
Args: Args:
connection: a DevToolsConnection. connection: a DevToolsConnection.
additional_categories: ([str] or None) If set, a list of additional categories: ([str] or None) If set, a list of categories to enable or
categories to add. This cannot be used to re-enable disable in Chrome tracing. Categories prefixed with '-' are
a category which is disabled by default (see disabled.
_DISABLED_CATEGORIES), nor to disable a category.
disabled_categories: If set, a set of categories from _ENABLED_CATEGORIES
to disable.
fetch_stream: if true, use a websocket stream to fetch tracing data rather fetch_stream: if true, use a websocket stream to fetch tracing data rather
than dataCollected events. It appears based on very limited testing that than dataCollected events. It appears based on very limited testing that
a stream is slower than the default reporting as dataCollected events. a stream is slower than the default reporting as dataCollected events.
...@@ -49,28 +33,7 @@ class TracingTrack(devtools_monitor.Track): ...@@ -49,28 +33,7 @@ class TracingTrack(devtools_monitor.Track):
if connection: if connection:
connection.RegisterListener('Tracing.dataCollected', self) connection.RegisterListener('Tracing.dataCollected', self)
categories_to_enable = _ENABLED_CATEGORIES self._categories = set(categories)
categories_to_disable = _DISABLED_CATEGORIES
if disabled_categories:
assert not any(cat.startswith('-') for cat in disabled_categories), (
'Specify categories to disable without an initial -')
assert set(disabled_categories).issubset(set(_ENABLED_CATEGORIES)), (
'Can only disable categories that are enabled by default')
categories_to_enable = (
set(categories_to_enable).difference(set(disabled_categories)))
categories_to_disable += disabled_categories
if additional_categories:
assert not any(cat.startswith('-') for cat in additional_categories), (
'Use disabled_categories to disable a category')
assert not (set(additional_categories) & set(_DISABLED_CATEGORIES)), (
'Cannot enable a disabled category')
categories_to_enable += tuple(additional_categories)
self._categories = set(
itertools.chain(categories_to_enable,
tuple('-' + cat for cat in categories_to_disable)))
params = {} params = {}
params['categories'] = ','.join(self._categories) params['categories'] = ','.join(self._categories)
if fetch_stream: if fetch_stream:
...@@ -171,7 +134,7 @@ class TracingTrack(devtools_monitor.Track): ...@@ -171,7 +134,7 @@ class TracingTrack(devtools_monitor.Track):
events = filter( events = filter(
lambda e : set(e.category.split(',')).intersection(categories), lambda e : set(e.category.split(',')).intersection(categories),
events) events)
tracing_track = TracingTrack(None) tracing_track = TracingTrack(None, clovis_constants.DEFAULT_CATEGORIES)
tracing_track._events = events tracing_track._events = events
tracing_track._categories = self._categories tracing_track._categories = self._categories
if categories is not None: if categories is not None:
...@@ -188,7 +151,7 @@ class TracingTrack(devtools_monitor.Track): ...@@ -188,7 +151,7 @@ class TracingTrack(devtools_monitor.Track):
return None return None
assert 'events' in json_dict assert 'events' in json_dict
events = [Event(e) for e in json_dict['events']] events = [Event(e) for e in json_dict['events']]
tracing_track = TracingTrack(None) tracing_track = TracingTrack(None, clovis_constants.DEFAULT_CATEGORIES)
tracing_track._categories = set(json_dict.get('categories', [])) tracing_track._categories = set(json_dict.get('categories', []))
tracing_track._events = events tracing_track._events = events
tracing_track._base_msec = events[0].start_msec if events else 0 tracing_track._base_msec = events[0].start_msec if events else 0
......
...@@ -42,7 +42,7 @@ class TracingTrackTestCase(unittest.TestCase): ...@@ -42,7 +42,7 @@ class TracingTrackTestCase(unittest.TestCase):
def setUp(self): def setUp(self):
self.tree_threshold = _IntervalTree._TRESHOLD self.tree_threshold = _IntervalTree._TRESHOLD
_IntervalTree._TRESHOLD = 2 # Expose more edge cases in the tree. _IntervalTree._TRESHOLD = 2 # Expose more edge cases in the tree.
self.track = TracingTrack(None, additional_categories=('A', 'B', 'C', 'D')) self.track = TracingTrack(None, ['A', 'B', 'C', 'D'])
def tearDown(self): def tearDown(self):
_IntervalTree._TRESHOLD = self.tree_threshold _IntervalTree._TRESHOLD = self.tree_threshold
...@@ -360,26 +360,6 @@ class TracingTrackTestCase(unittest.TestCase): ...@@ -360,26 +360,6 @@ class TracingTrackTestCase(unittest.TestCase):
self.assertSetEqual( self.assertSetEqual(
set('A'), self.track.Filter(categories=set('A')).Categories()) set('A'), self.track.Filter(categories=set('A')).Categories())
def testAdditionalCategories(self):
track = TracingTrack(None, additional_categories=('best-category-ever',))
self.assertIn('best-category-ever', track.Categories())
# Cannot re-enable a category.
with self.assertRaises(AssertionError):
TracingTrack(None, additional_categories=('cc',))
# Cannot disable categories via |additional_categories|.
with self.assertRaises(AssertionError):
TracingTrack(None, additional_categories=('-best-category-ever',))
def testDisabledCategories(self):
track = TracingTrack(None, disabled_categories=('toplevel',))
self.assertNotIn('toplevel', track.Categories())
self.assertIn('-toplevel', track.Categories())
# Can only disable categories that are enabled by default.
with self.assertRaises(AssertionError):
TracingTrack(None, disabled_categories=('best-category-ever',))
with self.assertRaises(AssertionError):
TracingTrack(None, disabled_categories=('cc',))
def _HandleEvents(self, events): def _HandleEvents(self, events):
self.track.Handle('Tracing.dataCollected', {'params': {'value': [ self.track.Handle('Tracing.dataCollected', {'params': {'value': [
self.EventToMicroseconds(e) for e in events]}}) self.EventToMicroseconds(e) for e in events]}})
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment