Commit c5fab709 authored by Lalit Maganti's avatar Lalit Maganti Committed by Commit Bot

memory-infra: include computation of shared memory footprint as part of infra service output

This CL achieves the long present goal to compute the shared memory footprint
of each process in Chrome. It does this by utilising earlier work to perform
the computation and adds it to the existing struct of metrics reported
by the memory-infra service.

For more context, see https://goo.gl/3kPb9S

Bug: 781782
Change-Id: I6b9603e54ddd76e84fcb4898eb38d6be5c0c233f
Reviewed-on: https://chromium-review.googlesource.com/757139
Commit-Queue: Lalit Maganti <lalitm@chromium.org>
Reviewed-by: default avatarErik Chen <erikchen@chromium.org>
Reviewed-by: default avatarSteven Holte <holte@chromium.org>
Reviewed-by: default avatarGreg Kerr <kerrnel@chromium.org>
Reviewed-by: default avatarPrimiano Tucci <primiano@chromium.org>
Cr-Commit-Position: refs/heads/master@{#515933}
parent 26387bf1
......@@ -77,7 +77,8 @@ class ProcessMemoryMetricsEmitterFake : public ProcessMemoryMetricsEmitter {
};
OSMemDumpPtr GetFakeOSMemDump(uint32_t resident_set_kb,
uint32_t private_footprint_kb) {
uint32_t private_footprint_kb,
uint32_t shared_footprint_kb) {
using memory_instrumentation::mojom::VmRegion;
std::vector<memory_instrumentation::mojom::VmRegionPtr> vm_regions;
......@@ -94,7 +95,8 @@ OSMemDumpPtr GetFakeOSMemDump(uint32_t resident_set_kb,
500, // byte_stats_swapped,
200)); // byte_stats_proportional_resident
return memory_instrumentation::mojom::OSMemDump::New(
resident_set_kb, private_footprint_kb, std::move(vm_regions));
resident_set_kb, private_footprint_kb, shared_footprint_kb,
std::move(vm_regions));
}
void PopulateBrowserMetrics(GlobalMemoryDumpPtr& global_dump,
......@@ -108,7 +110,7 @@ void PopulateBrowserMetrics(GlobalMemoryDumpPtr& global_dump,
#endif
OSMemDumpPtr os_dump =
GetFakeOSMemDump(metrics_mb["Resident"] * 1024,
metrics_mb["PrivateMemoryFootprint"] * 1024);
metrics_mb["PrivateMemoryFootprint"] * 1024, 0);
pmd->os_dump = std::move(os_dump);
global_dump->process_dumps.push_back(std::move(pmd));
}
......@@ -142,7 +144,7 @@ void PopulateRendererMetrics(GlobalMemoryDumpPtr& global_dump,
pmd->chrome_dump->v8_total_kb = metrics_mb["V8"] * 1024;
OSMemDumpPtr os_dump =
GetFakeOSMemDump(metrics_mb["Resident"] * 1024,
metrics_mb["PrivateMemoryFootprint"] * 1024);
metrics_mb["PrivateMemoryFootprint"] * 1024, 0);
pmd->os_dump = std::move(os_dump);
pmd->pid = pid;
global_dump->process_dumps.push_back(std::move(pmd));
......@@ -183,7 +185,7 @@ void PopulateGpuMetrics(GlobalMemoryDumpPtr& global_dump,
metrics_mb["CommandBuffer"] * 1024;
OSMemDumpPtr os_dump =
GetFakeOSMemDump(metrics_mb["Resident"] * 1024,
metrics_mb["PrivateMemoryFootprint"] * 1024);
metrics_mb["PrivateMemoryFootprint"] * 1024, 0);
pmd->os_dump = std::move(os_dump);
global_dump->process_dumps.push_back(std::move(pmd));
}
......
......@@ -27,24 +27,28 @@ namespace {
constexpr uint32_t kProcessMallocTriggerKb = 2 * 1024 * 1024; // 2 Gig
OSMemDumpPtr GetFakeOSMemDump(uint32_t resident_set_kb,
uint32_t private_footprint_kb) {
uint32_t private_footprint_kb,
uint32_t shared_footprint_kb) {
using memory_instrumentation::mojom::VmRegion;
std::vector<memory_instrumentation::mojom::VmRegionPtr> vm_regions;
return memory_instrumentation::mojom::OSMemDump::New(
resident_set_kb, private_footprint_kb, std::move(vm_regions));
resident_set_kb, private_footprint_kb, shared_footprint_kb,
std::move(vm_regions));
}
void PopulateMetrics(GlobalMemoryDumpPtr* global_dump,
base::ProcessId pid,
ProcessType process_type,
uint32_t resident_set_kb,
uint32_t private_memory_kb) {
uint32_t private_memory_kb,
uint32_t shared_footprint_kb) {
ProcessMemoryDumpPtr pmd(
memory_instrumentation::mojom::ProcessMemoryDump::New());
pmd->pid = pid;
pmd->process_type = process_type;
pmd->chrome_dump = memory_instrumentation::mojom::ChromeMemDump::New();
pmd->os_dump = GetFakeOSMemDump(resident_set_kb, private_memory_kb);
pmd->os_dump =
GetFakeOSMemDump(resident_set_kb, private_memory_kb, shared_footprint_kb);
(*global_dump)->process_dumps.push_back(std::move(pmd));
}
......@@ -52,16 +56,16 @@ GlobalMemoryDumpPtr GetLargeMemoryDump() {
GlobalMemoryDumpPtr dump(
memory_instrumentation::mojom::GlobalMemoryDump::New());
PopulateMetrics(&dump, 1, ProcessType::BROWSER, kProcessMallocTriggerKb,
kProcessMallocTriggerKb);
kProcessMallocTriggerKb, kProcessMallocTriggerKb);
PopulateMetrics(&dump, 2, ProcessType::RENDERER, kProcessMallocTriggerKb,
kProcessMallocTriggerKb);
PopulateMetrics(&dump, 3, ProcessType::RENDERER, 1, 1);
kProcessMallocTriggerKb, kProcessMallocTriggerKb);
PopulateMetrics(&dump, 3, ProcessType::RENDERER, 1, 1, 1);
PopulateMetrics(&dump, 4, ProcessType::GPU, kProcessMallocTriggerKb,
kProcessMallocTriggerKb);
kProcessMallocTriggerKb, kProcessMallocTriggerKb);
PopulateMetrics(&dump, 5, ProcessType::OTHER, kProcessMallocTriggerKb,
kProcessMallocTriggerKb);
kProcessMallocTriggerKb, kProcessMallocTriggerKb);
PopulateMetrics(&dump, 6, ProcessType::RENDERER, kProcessMallocTriggerKb,
kProcessMallocTriggerKb);
kProcessMallocTriggerKb, kProcessMallocTriggerKb);
return dump;
}
......@@ -110,28 +114,28 @@ TEST_F(BackgroundProfilingTriggersTest, OnReceivedMemoryDump_EmptyCases) {
GlobalMemoryDumpPtr dump_browser(
memory_instrumentation::mojom::GlobalMemoryDump::New());
PopulateMetrics(&dump_browser, 1, ProcessType::BROWSER, 1, 1);
PopulateMetrics(&dump_browser, 1, ProcessType::BROWSER, 1, 1, 1);
triggers_.OnReceivedMemoryDump(true, std::move(dump_browser));
EXPECT_TRUE(triggers_.pids().empty());
triggers_.Reset();
GlobalMemoryDumpPtr dump_gpu(
memory_instrumentation::mojom::GlobalMemoryDump::New());
PopulateMetrics(&dump_gpu, 1, ProcessType::GPU, 1, 1);
PopulateMetrics(&dump_gpu, 1, ProcessType::GPU, 1, 1, 1);
triggers_.OnReceivedMemoryDump(true, std::move(dump_gpu));
EXPECT_TRUE(triggers_.pids().empty());
triggers_.Reset();
GlobalMemoryDumpPtr dump_renderer(
memory_instrumentation::mojom::GlobalMemoryDump::New());
PopulateMetrics(&dump_renderer, 1, ProcessType::RENDERER, 1, 1);
PopulateMetrics(&dump_renderer, 1, ProcessType::RENDERER, 1, 1, 1);
triggers_.OnReceivedMemoryDump(true, std::move(dump_renderer));
EXPECT_TRUE(triggers_.pids().empty());
triggers_.Reset();
GlobalMemoryDumpPtr dump_other(
memory_instrumentation::mojom::GlobalMemoryDump::New());
PopulateMetrics(&dump_other, 1, ProcessType::OTHER, 1, 1);
PopulateMetrics(&dump_other, 1, ProcessType::OTHER, 1, 1, 1);
triggers_.OnReceivedMemoryDump(true, std::move(dump_other));
EXPECT_TRUE(triggers_.pids().empty());
triggers_.Reset();
......
......@@ -166,12 +166,13 @@ class MockClientProcess : public mojom::ClientProcess {
ON_CALL(*this, RequestChromeMemoryDump(_, _))
.WillByDefault(
Invoke([](const MemoryDumpRequestArgs& args,
const RequestChromeMemoryDumpCallback& callback) {
Invoke([pid](const MemoryDumpRequestArgs& args,
const RequestChromeMemoryDumpCallback& callback) {
MemoryDumpArgs dump_args{MemoryDumpLevelOfDetail::DETAILED};
auto pmd =
std::make_unique<ProcessMemoryDump>(nullptr, dump_args);
auto* mad = pmd->CreateAllocatorDump("malloc");
auto* mad = pmd->CreateAllocatorDump(
"malloc", base::trace_event::MemoryAllocatorDumpGuid(pid));
mad->AddScalar(MemoryAllocatorDump::kNameSize,
MemoryAllocatorDump::kUnitsBytes, 1024);
......@@ -454,7 +455,9 @@ TEST_F(CoordinatorImplTest, GlobalMemoryDumpStruct) {
auto* bytes = MemoryAllocatorDump::kUnitsBytes;
const uint32_t kB = 1024;
pmd->CreateAllocatorDump("malloc")->AddScalar(size, bytes, 1 * kB);
pmd->CreateAllocatorDump("malloc",
base::trace_event::MemoryAllocatorDumpGuid(1))
->AddScalar(size, bytes, 1 * kB);
pmd->CreateAllocatorDump("malloc/ignored")
->AddScalar(size, bytes, 99 * kB);
......@@ -489,7 +492,8 @@ TEST_F(CoordinatorImplTest, GlobalMemoryDumpStruct) {
callback) {
MemoryDumpArgs dump_args{MemoryDumpLevelOfDetail::DETAILED};
auto pmd = std::make_unique<ProcessMemoryDump>(nullptr, dump_args);
auto* mad = pmd->CreateAllocatorDump("malloc");
auto* mad = pmd->CreateAllocatorDump(
"malloc", base::trace_event::MemoryAllocatorDumpGuid(2));
mad->AddScalar(MemoryAllocatorDump::kNameSize,
MemoryAllocatorDump::kUnitsBytes, 1024 * 2);
callback.Run(true, args.dump_guid, std::move(pmd));
......
......@@ -32,35 +32,31 @@ Node::Entry::ScalarUnits EntryUnitsFromString(std::string units) {
}
}
void AddEntryToNode(Node* node, const MemoryAllocatorDump::Entry& entry) {
switch (entry.entry_type) {
case MemoryAllocatorDump::Entry::EntryType::kUint64:
node->AddEntry(entry.name, EntryUnitsFromString(entry.units),
entry.value_uint64);
break;
case MemoryAllocatorDump::Entry::EntryType::kString:
node->AddEntry(entry.name, entry.value_string);
break;
}
}
} // namespace
// static
std::unique_ptr<GlobalDumpGraph> GraphProcessor::CreateMemoryGraph(
const std::map<ProcessId, ProcessMemoryDump>& process_dumps) {
const GraphProcessor::MemoryDumpMap& process_dumps) {
auto global_graph = std::make_unique<GlobalDumpGraph>();
// First pass: collects allocator dumps into a graph and populate
// with entries.
for (const auto& pid_to_dump : process_dumps) {
// There can be null entries in the map; simply filter these out.
if (!pid_to_dump.second)
continue;
auto* graph = global_graph->CreateGraphForProcess(pid_to_dump.first);
CollectAllocatorDumps(pid_to_dump.second, global_graph.get(), graph);
CollectAllocatorDumps(*pid_to_dump.second, global_graph.get(), graph);
}
// Second pass: generate the graph of edges between the nodes.
for (const auto& pid_to_dump : process_dumps) {
AddEdges(pid_to_dump.second, global_graph.get());
// There can be null entries in the map; simply filter these out.
if (!pid_to_dump.second)
continue;
AddEdges(*pid_to_dump.second, global_graph.get());
}
return global_graph;
......@@ -125,6 +121,11 @@ GraphProcessor::ComputeSharedFootprintFromGraph(
Node* global_root =
global_graph.shared_memory_graph()->root()->GetChild("global");
// If there are no global dumps then just return an empty map with no data.
if (!global_root) {
return std::map<base::ProcessId, uint64_t>();
}
struct GlobalNodeOwners {
std::list<Edge*> edges;
int max_priority = 0;
......@@ -224,9 +225,15 @@ void GraphProcessor::CollectAllocatorDumps(
// Copy any entries not already present into the node.
for (auto& entry : dump.entries()) {
// Check that there are not multiple entries with the same name.
DCHECK(node->entries()->find(entry.name) == node->entries()->end());
AddEntryToNode(node, entry);
switch (entry.entry_type) {
case MemoryAllocatorDump::Entry::EntryType::kUint64:
node->AddEntry(entry.name, EntryUnitsFromString(entry.units),
entry.value_uint64);
break;
case MemoryAllocatorDump::Entry::EntryType::kString:
node->AddEntry(entry.name, entry.value_string);
break;
}
}
}
}
......@@ -265,8 +272,8 @@ void GraphProcessor::AddEdges(
// static
void GraphProcessor::MarkImplicitWeakParentsRecursively(Node* node) {
// Ensure that we aren't in a bad state where we have an implicit node
// which doesn't have any children.
DCHECK(node->is_explicit() || !node->children()->empty());
// which doesn't have any children (which is not the root node).
DCHECK(node->is_explicit() || !node->children()->empty() || !node->parent());
// Check that at this stage, any node which is weak is only so because
// it was explicitly created as such.
......
......@@ -15,8 +15,9 @@ namespace memory_instrumentation {
class GraphProcessor {
public:
// This map does not own the pointers inside.
using MemoryDumpMap =
std::map<base::ProcessId, base::trace_event::ProcessMemoryDump>;
std::map<base::ProcessId, const base::trace_event::ProcessMemoryDump*>;
static std::unique_ptr<GlobalDumpGraph> CreateMemoryGraph(
const MemoryDumpMap& process_dumps);
......
......@@ -59,7 +59,7 @@ class GraphProcessorTest : public testing::Test {
};
TEST_F(GraphProcessorTest, SmokeComputeMemoryGraph) {
std::map<ProcessId, ProcessMemoryDump> process_dumps;
std::map<ProcessId, const ProcessMemoryDump*> process_dumps;
MemoryDumpArgs dump_args = {MemoryDumpLevelOfDetail::DETAILED};
ProcessMemoryDump pmd(new HeapProfilerSerializationState, dump_args);
......@@ -74,7 +74,7 @@ TEST_F(GraphProcessorTest, SmokeComputeMemoryGraph) {
auto* weak =
pmd.CreateWeakSharedGlobalAllocatorDump(MemoryAllocatorDumpGuid(1));
process_dumps.emplace(1, std::move(pmd));
process_dumps.emplace(1, &pmd);
auto global_dump = GraphProcessor::CreateMemoryGraph(process_dumps);
......@@ -112,7 +112,7 @@ TEST_F(GraphProcessorTest, SmokeComputeMemoryGraph) {
}
TEST_F(GraphProcessorTest, SmokeComputeSharedFootprint) {
std::map<ProcessId, ProcessMemoryDump> process_dumps;
std::map<ProcessId, const ProcessMemoryDump*> process_dumps;
MemoryDumpArgs dump_args = {MemoryDumpLevelOfDetail::DETAILED};
ProcessMemoryDump pmd1(new HeapProfilerSerializationState, dump_args);
......@@ -163,8 +163,8 @@ TEST_F(GraphProcessorTest, SmokeComputeSharedFootprint) {
pmd1.CreateSharedMemoryOwnershipEdge(p1_d2->guid(), token, 1);
pmd2.CreateSharedMemoryOwnershipEdge(p2_d1->guid(), token, 2);
process_dumps.emplace(1, std::move(pmd1));
process_dumps.emplace(2, std::move(pmd2));
process_dumps.emplace(1, &pmd1);
process_dumps.emplace(2, &pmd2);
auto graph = GraphProcessor::CreateMemoryGraph(process_dumps);
auto pid_to_sizes = GraphProcessor::ComputeSharedFootprintFromGraph(*graph);
......
......@@ -11,6 +11,7 @@
#include "base/strings/pattern.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
#include "services/resource_coordinator/memory_instrumentation/graph_processor.h"
#if defined(OS_MACOSX) && !defined(OS_IOS)
#include "base/mac/mac_util.h"
......@@ -185,20 +186,18 @@ void QueuedRequestDispatcher::Finalize(QueuedRequest* request,
// details for the child processes to get around sandbox restrictions on
// opening /proc pseudo files.
struct RawDumpsForProcess {
mojom::ProcessType process_type = mojom::ProcessType::OTHER;
const base::trace_event::ProcessMemoryDump* raw_chrome_dump = nullptr;
mojom::RawOSMemDump* raw_os_dump = nullptr;
};
std::map<base::ProcessId, RawDumpsForProcess> pid_to_results;
// All the pointers in the maps will continue to be owned by |request|
// which outlives these containers.
std::map<base::ProcessId, mojom::ProcessType> pid_to_process_type;
std::map<base::ProcessId, const base::trace_event::ProcessMemoryDump*>
pid_to_pmd;
std::map<base::ProcessId, mojom::RawOSMemDump*> pid_to_os_dump;
for (auto& response : request->responses) {
const base::ProcessId& original_pid = response.second.process_id;
RawDumpsForProcess& raw_dumps = pid_to_results[original_pid];
raw_dumps.process_type = response.second.process_type;
pid_to_process_type[original_pid] = response.second.process_type;
// |chrome_dump| can be nullptr if this was a OS-counters only response.
raw_dumps.raw_chrome_dump = response.second.chrome_dump.get();
pid_to_pmd[original_pid] = response.second.chrome_dump.get();
// |response| accumulates the replies received by each client process.
// Depending on the OS each client process might return 1 chrome + 1 OS
......@@ -213,9 +212,8 @@ void QueuedRequestDispatcher::Finalize(QueuedRequest* request,
#if defined(OS_LINUX)
for (const auto& kv : extra_os_dumps) {
auto pid = kv.first == base::kNullProcessId ? original_pid : kv.first;
RawDumpsForProcess& extra_result = pid_to_results[pid];
DCHECK_EQ(extra_result.raw_os_dump, nullptr);
extra_result.raw_os_dump = kv.second.get();
DCHECK_EQ(pid_to_os_dump[pid], nullptr);
pid_to_os_dump[pid] = kv.second.get();
}
#else
// This can be empty if the client disconnects before providing both
......@@ -228,58 +226,67 @@ void QueuedRequestDispatcher::Finalize(QueuedRequest* request,
DCHECK_EQ(base::kNullProcessId, kv.first);
// Check we don't receive duplicate OS dumps for the same process.
DCHECK_EQ(raw_dumps.raw_os_dump, nullptr);
DCHECK_EQ(pid_to_os_dump[original_pid], nullptr);
raw_dumps.raw_os_dump = kv.second.get();
pid_to_os_dump[original_pid] = kv.second.get();
}
#endif
} // for (response : request->responses)
// Generate the global memory graph from the map of pids to dumps, removing
// weak nodes.
std::unique_ptr<GlobalDumpGraph> global_graph =
GraphProcessor::CreateMemoryGraph(pid_to_pmd);
GraphProcessor::RemoveWeakNodesFromGraph(global_graph.get());
// Compute the shared memory footprint for each process from the graph.
std::map<base::ProcessId, uint64_t> shared_footprints =
GraphProcessor::ComputeSharedFootprintFromGraph(*global_graph);
// Build up the global dump by iterating on the |valid| process dumps.
mojom::GlobalMemoryDumpPtr global_dump(mojom::GlobalMemoryDump::New());
global_dump->process_dumps.reserve(pid_to_results.size());
for (const auto& kv : pid_to_results) {
const base::ProcessId pid = kv.first;
const RawDumpsForProcess& raw_dumps = kv.second;
global_dump->process_dumps.reserve(request->responses.size());
for (const auto& response : request->responses) {
base::ProcessId pid = response.second.process_id;
// These pointers are owned by |request|.
mojom::RawOSMemDump* raw_os_dump = pid_to_os_dump[pid];
auto* raw_chrome_dump = pid_to_pmd[pid];
// If we have the OS dump we should have the platform private footprint.
DCHECK(!raw_dumps.raw_os_dump ||
raw_dumps.raw_os_dump->platform_private_footprint);
DCHECK(!raw_os_dump || raw_os_dump->platform_private_footprint);
// Ignore incomplete results (can happen if the client crashes/disconnects).
const bool valid =
raw_dumps.raw_os_dump &&
(!request->wants_chrome_dumps() || raw_dumps.raw_chrome_dump) &&
(!request->wants_mmaps() ||
(raw_dumps.raw_os_dump &&
!raw_dumps.raw_os_dump->memory_maps.empty()));
const bool valid = raw_os_dump &&
(!request->wants_chrome_dumps() || raw_chrome_dump) &&
(!request->wants_mmaps() ||
(raw_os_dump && !raw_os_dump->memory_maps.empty()));
if (!valid)
continue;
mojom::OSMemDumpPtr os_dump = CreatePublicOSDump(*raw_dumps.raw_os_dump);
mojom::OSMemDumpPtr os_dump = CreatePublicOSDump(*raw_os_dump);
os_dump->shared_footprint_kb = shared_footprints[pid];
if (request->add_to_trace) {
tracing_observer->AddOsDumpToTraceIfEnabled(
request->args, pid, os_dump.get(),
&raw_dumps.raw_os_dump->memory_maps);
request->args, pid, os_dump.get(), &raw_os_dump->memory_maps);
}
if (request->args.level_of_detail ==
MemoryDumpLevelOfDetail::VM_REGIONS_ONLY_FOR_HEAP_PROFILER) {
DCHECK(request->wants_mmaps());
os_dump->memory_maps_for_heap_profiler =
std::move(raw_dumps.raw_os_dump->memory_maps);
std::move(raw_os_dump->memory_maps);
}
// TODO(hjd): not sure we need an empty instance for the !SUMMARY_ONLY
// requests. Check and make the else branch a nullptr otherwise.
mojom::ChromeMemDumpPtr chrome_dump =
request->should_return_summaries()
? CreateDumpSummary(*raw_dumps.raw_chrome_dump)
: mojom::ChromeMemDump::New();
request->should_return_summaries() ? CreateDumpSummary(*raw_chrome_dump)
: mojom::ChromeMemDump::New();
mojom::ProcessMemoryDumpPtr pmd = mojom::ProcessMemoryDump::New();
pmd->pid = pid;
pmd->process_type = raw_dumps.process_type;
pmd->process_type = pid_to_process_type[pid];
pmd->os_dump = std::move(os_dump);
pmd->chrome_dump = std::move(chrome_dump);
global_dump->process_dumps.push_back(std::move(pmd));
......
......@@ -156,6 +156,10 @@ struct OSMemDump {
// memory in kilobytes. For more details, see https://goo.gl/3kPb9S.
uint32 private_footprint_kb = 0;
// This is roughly non-private, anonymous, non-discardable, resident memory
// in kilobytes. For more details, see https://goo.gl/3kPb9S.
uint32 shared_footprint_kb = 0;
// This field is present only when the dump is obtained through
// Coordinator.GetVmRegionsForHeapProfiler(), empty in the generic case of
// RequestGlobalMemoryDump().
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment