Resident set size (RSS) of multiple processes without double-counts on Linux.

It fixes logging and in-process duplication counting in tools/linux/procfs.py
at the same time.

BUG=324194
TEST=None
NOTRY=True

Review URL: https://codereview.chromium.org/98423003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@238295 0039d316-1c4b-4281-b951-d872f2087c98

tools/linux/procfs.py

@@ -18,7 +18,13 @@ import struct
 import sys
 
 
-LOGGER = logging.getLogger('procfs')
+class _NullHandler(logging.Handler):
+  def emit(self, record):
+    pass
+
+
+_LOGGER = logging.getLogger('procfs')
+_LOGGER.addHandler(_NullHandler())
 
 
 class ProcStat(object):
@@ -547,6 +553,7 @@ class ProcPagemap(object):
     total_vsize = 0
     in_process_dup = 0
     vma_internals = collections.OrderedDict()
+    process_pageframe_set = set()
 
     pagemap_fd = os.open(
         os.path.join('/proc', str(pid), 'pagemap'), os.O_RDONLY)
@@ -555,22 +562,21 @@ class ProcPagemap(object):
       swapped = 0
       vsize = 0
       pageframes = collections.defaultdict(int)
-      pageframes_set = set()
       begin_offset = ProcPagemap._offset(vma.begin)
       chunk_size = ProcPagemap._offset(vma.end) - begin_offset
       os.lseek(pagemap_fd, begin_offset, os.SEEK_SET)
       buf = os.read(pagemap_fd, chunk_size)
       if len(buf) < chunk_size:
-        LOGGER.warn('Failed to read pagemap at 0x%x.' % vma.begin)
+        _LOGGER.warn('Failed to read pagemap at 0x%x in %d.' % (vma.begin, pid))
       pagemap_values = struct.unpack(
           '=%dQ' % (len(buf) / ProcPagemap._BYTES_PER_PAGEMAP_VALUE), buf)
       for pagemap_value in pagemap_values:
         vsize += ProcPagemap._BYTES_PER_OS_PAGE
         if pagemap_value & ProcPagemap._MASK_PRESENT:
-          if (pagemap_value & ProcPagemap._MASK_PFN) in pageframes_set:
+          if (pagemap_value & ProcPagemap._MASK_PFN) in process_pageframe_set:
             in_process_dup += ProcPagemap._BYTES_PER_OS_PAGE
           else:
-            pageframes_set.add(pagemap_value & ProcPagemap._MASK_PFN)
+            process_pageframe_set.add(pagemap_value & ProcPagemap._MASK_PFN)
           if (pagemap_value & ProcPagemap._MASK_PFN) not in pageframes:
             present += ProcPagemap._BYTES_PER_OS_PAGE
           pageframes[pagemap_value & ProcPagemap._MASK_PFN] += 1
@@ -680,13 +686,12 @@ class _ProcessMemory(object):
 
 def main(argv):
   """The main function for manual testing."""
-
-  LOGGER.setLevel(logging.DEBUG)
+  _LOGGER.setLevel(logging.WARNING)
   handler = logging.StreamHandler()
-  handler.setLevel(logging.INFO)
-  formatter = logging.Formatter('%(message)s')
-  handler.setFormatter(formatter)
-  LOGGER.addHandler(handler)
+  handler.setLevel(logging.WARNING)
+  handler.setFormatter(logging.Formatter(
+      '%(asctime)s:%(name)s:%(levelname)s:%(message)s'))
+  _LOGGER.addHandler(handler)
 
   pids = []
   for arg in argv[1:]:

tools/multi-process-rss.py

+#!/usr/bin/env python
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Counts a resident set size (RSS) of multiple processes without double-counts.
+# If they share the same page frame, the page frame is counted only once.
+#
+# Usage:
+# ./multi-process-rss.py <pid>|<pid>r [...]
+#
+# If <pid> has 'r' at the end, all descendants of the process are accounted.
+#
+# Example:
+# ./multi-process-rss.py 12345 23456r
+#
+# The command line above counts the RSS of 1) process 12345, 2) process 23456
+# and 3) all descendant processes of process 23456.
+
+
+import collections
+import logging
+import os
+import psutil
+import sys
+
+
+if sys.platform.startswith('linux'):
+  _TOOLS_PATH = os.path.dirname(os.path.abspath(__file__))
+  _TOOLS_LINUX_PATH = os.path.join(_TOOLS_PATH, 'linux')
+  sys.path.append(_TOOLS_LINUX_PATH)
+  import procfs  # pylint: disable=F0401
+
+
+class _NullHandler(logging.Handler):
+  def emit(self, record):
+    pass
+
+
+_LOGGER = logging.getLogger('multi-process-rss')
+_LOGGER.addHandler(_NullHandler())
+
+
+def _recursive_get_children(pid):
+  children = psutil.Process(pid).get_children()
+  descendant = []
+  for child in children:
+    descendant.append(child.pid)
+    descendant.extend(_recursive_get_children(child.pid))
+  return descendant
+
+
+def list_pids(argv):
+  pids = []
+  for arg in argv[1:]:
+    try:
+      if arg.endswith('r'):
+        recursive = True
+        pid = int(arg[:-1])
+      else:
+        recursive = False
+        pid = int(arg)
+    except ValueError:
+      raise SyntaxError("%s is not an integer." % arg)
+    else:
+      pids.append(pid)
+    if recursive:
+      children = _recursive_get_children(pid)
+      pids.extend(children)
+
+  pids = sorted(set(pids), key=pids.index)  # uniq: maybe slow, but simple.
+
+  return pids
+
+
+def count_pageframes(pids):
+  pageframes = collections.defaultdict(int)
+  pagemap_dct = {}
+  for pid in pids:
+    maps = procfs.ProcMaps.load(pid)
+    pagemap_dct[pid] = procfs.ProcPagemap.load(pid, maps)
+
+  for pid, pagemap in pagemap_dct.iteritems():
+    for vma in pagemap.vma_internals.itervalues():
+      for pageframe, number in vma.pageframes.iteritems():
+        pageframes[pageframe] += number
+
+  return pageframes
+
+
+def main(argv):
+  logging_handler = logging.StreamHandler()
+  logging_handler.setLevel(logging.WARNING)
+  logging_handler.setFormatter(logging.Formatter(
+      '%(asctime)s:%(name)s:%(levelname)s:%(message)s'))
+
+  _LOGGER.setLevel(logging.WARNING)
+  _LOGGER.addHandler(logging_handler)
+
+  if sys.platform.startswith('linux'):
+    logging.getLogger('procfs').setLevel(logging.WARNING)
+    logging.getLogger('procfs').addHandler(logging_handler)
+    pids = list_pids(argv)
+    pageframes = count_pageframes(pids)
+  else:
+    _LOGGER.error('%s is not supported.' % sys.platform)
+    return 1
+
+  # TODO(dmikurube): Classify this total RSS.
+  print len(pageframes) * 4096
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))