diff -r 000000000000 -r 6474c204b198 xpcom/base/SystemMemoryReporter.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xpcom/base/SystemMemoryReporter.cpp Wed Dec 31 06:09:35 2014 +0100 @@ -0,0 +1,750 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/SystemMemoryReporter.h" + +#include "mozilla/Attributes.h" +#include "mozilla/Preferences.h" +#include "mozilla/unused.h" + +#include "nsIMemoryReporter.h" +#include "nsPrintfCString.h" +#include "nsString.h" +#include "nsTHashtable.h" +#include "nsHashKeys.h" + +#include +#include +#include +#include +#include +#include +#include + +// This file implements a Linux-specific, system-wide memory reporter. It +// gathers all the useful memory measurements obtainable from the OS in a +// single place, giving a high-level view of memory consumption for the entire +// machine/device. +// +// Other memory reporters measure part of a single process's memory consumption. +// This reporter is different in that it measures memory consumption of many +// processes, and they end up in a single reports tree. This is a slight abuse +// of the memory reporting infrastructure, and therefore the results are given +// their own "process" called "System", which means they show up in about:memory +// in their own section, distinct from the per-process sections. + +namespace mozilla { +namespace SystemMemoryReporter { + +#if !defined(XP_LINUX) +#error "This won't work if we're not on Linux." +#endif + +static bool +EndsWithLiteral(const nsCString& aHaystack, const char* aNeedle) +{ + int32_t idx = aHaystack.RFind(aNeedle); + return idx != -1 && idx + strlen(aNeedle) == aHaystack.Length(); +} + +static void +GetDirname(const nsCString& aPath, nsACString& aOut) +{ + int32_t idx = aPath.RFind("/"); + if (idx == -1) { + aOut.Truncate(); + } else { + aOut.Assign(Substring(aPath, 0, idx)); + } +} + +static void +GetBasename(const nsCString& aPath, nsACString& aOut) +{ + nsCString out; + int32_t idx = aPath.RFind("/"); + if (idx == -1) { + out.Assign(aPath); + } else { + out.Assign(Substring(aPath, idx + 1)); + } + + // On Android, some entries in /dev/ashmem end with "(deleted)" (e.g. + // "/dev/ashmem/libxul.so(deleted)"). We don't care about this modifier, so + // cut it off when getting the entry's basename. + if (EndsWithLiteral(out, "(deleted)")) { + out.Assign(Substring(out, 0, out.RFind("(deleted)"))); + } + out.StripChars(" "); + + aOut.Assign(out); +} + +static bool +IsNumeric(const char* s) +{ + MOZ_ASSERT(*s); // shouldn't see empty strings + while (*s) { + if (!isdigit(*s)) { + return false; + } + s++; + } + return true; +} + +static bool +IsAnonymous(const nsACString& aName) +{ + // Recent kernels (e.g. 3.5) have multiple [stack:nnnn] entries, where |nnnn| + // is a thread ID. However, [stack:nnnn] entries count both stack memory + // *and* anonymous memory because the kernel only knows about the start of + // each thread stack, not its end. So we treat such entries as anonymous + // memory instead of stack. This is consistent with older kernels that don't + // even show [stack:nnnn] entries. + return aName.IsEmpty() || + StringBeginsWith(aName, NS_LITERAL_CSTRING("[stack:")); +} + +class SystemReporter MOZ_FINAL : public nsIMemoryReporter +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + +#define REPORT_WITH_CLEANUP(_path, _units, _amount, _desc, _cleanup) \ + do { \ + size_t amount = _amount; /* evaluate _amount only once */ \ + if (amount > 0) { \ + nsresult rv; \ + rv = aHandleReport->Callback(NS_LITERAL_CSTRING("System"), _path, \ + KIND_NONHEAP, _units, amount, _desc, \ + aData); \ + if (NS_WARN_IF(NS_FAILED(rv))) { \ + _cleanup; \ + return rv; \ + } \ + } \ + } while (0) + +#define REPORT(_path, _amount, _desc) \ + REPORT_WITH_CLEANUP(_path, UNITS_BYTES, _amount, _desc, (void)0) + + NS_IMETHOD CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData) + { + if (!Preferences::GetBool("memory.system_memory_reporter")) { + return NS_OK; + } + + // Read relevant fields from /proc/meminfo. + int64_t memTotal = 0, memFree = 0; + nsresult rv = ReadMemInfo(&memTotal, &memFree); + + // Collect per-process reports from /proc//smaps. + int64_t totalPss = 0; + rv = CollectProcessReports(aHandleReport, aData, &totalPss); + NS_ENSURE_SUCCESS(rv, rv); + + // Report the non-process numbers. + int64_t other = memTotal - memFree - totalPss; + REPORT(NS_LITERAL_CSTRING("mem/other"), other, NS_LITERAL_CSTRING( +"Memory which is neither owned by any user-space process nor free. Note that " +"this includes memory holding cached files from the disk which can be " +"reclaimed by the OS at any time.")); + + REPORT(NS_LITERAL_CSTRING("mem/free"), memFree, NS_LITERAL_CSTRING( +"Memory which is free and not being used for any purpose.")); + + // Report reserved memory not included in memTotal. + rv = CollectPmemReports(aHandleReport, aData); + NS_ENSURE_SUCCESS(rv, rv); + + // Report zram usage statistics. + rv = CollectZramReports(aHandleReport, aData); + NS_ENSURE_SUCCESS(rv, rv); + + return rv; + } + +private: + // Keep this in sync with SystemReporter::kindPathSuffixes! + enum ProcessSizeKind { + AnonymousOutsideBrk = 0, + AnonymousBrkHeap = 1, + SharedLibrariesRX = 2, + SharedLibrariesRW = 3, + SharedLibrariesR = 4, + SharedLibrariesOther = 5, + OtherFiles = 6, + MainThreadStack = 7, + Vdso = 8, + + ProcessSizeKindLimit = 9 // must be last + }; + + static const char* kindPathSuffixes[ProcessSizeKindLimit]; + + // These are the cross-cutting measurements across all processes. + struct ProcessSizes + { + ProcessSizes() { memset(this, 0, sizeof(*this)); } + + size_t mSizes[ProcessSizeKindLimit]; + }; + + nsresult ReadMemInfo(int64_t* aMemTotal, int64_t* aMemFree) + { + FILE* f = fopen("/proc/meminfo", "r"); + if (!f) { + return NS_ERROR_FAILURE; + } + + int n1 = fscanf(f, "MemTotal: %" SCNd64 " kB\n", aMemTotal); + int n2 = fscanf(f, "MemFree: %" SCNd64 " kB\n", aMemFree); + + fclose(f); + + if (n1 != 1 || n2 != 1) { + return NS_ERROR_FAILURE; + } + + // Convert from KB to B. + *aMemTotal *= 1024; + *aMemFree *= 1024; + + return NS_OK; + } + + nsresult CollectProcessReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, + int64_t* aTotalPss) + { + *aTotalPss = 0; + ProcessSizes processSizes; + + DIR* d = opendir("/proc"); + if (NS_WARN_IF(!d)) { + return NS_ERROR_FAILURE; + } + struct dirent* ent; + while ((ent = readdir(d))) { + struct stat statbuf; + const char* pidStr = ent->d_name; + // Don't check the return value of stat() -- it can return -1 for these + // directories even when it has succeeded, apparently. + stat(pidStr, &statbuf); + if (S_ISDIR(statbuf.st_mode) && IsNumeric(pidStr)) { + nsCString processName("process("); + + // Get the command name from cmdline. If that fails, the pid is still + // shown. + nsPrintfCString cmdlinePath("/proc/%s/cmdline", pidStr); + FILE* f = fopen(cmdlinePath.get(), "r"); + if (f) { + static const size_t len = 256; + char buf[len]; + if (fgets(buf, len, f)) { + processName.Append(buf); + // A hack: replace forward slashes with '\\' so they aren't treated + // as path separators. Consumers of this reporter (such as + // about:memory) have to undo this change. + processName.ReplaceChar('/', '\\'); + processName.Append(", "); + } + fclose(f); + } + processName.Append("pid="); + processName.Append(pidStr); + processName.Append(")"); + + // Read the PSS values from the smaps file. + nsPrintfCString smapsPath("/proc/%s/smaps", pidStr); + f = fopen(smapsPath.get(), "r"); + if (!f) { + // Processes can terminate between the readdir() call above and now, + // so just skip if we can't open the file. + continue; + } + while (true) { + nsresult rv = ParseMapping(f, processName, aHandleReport, aData, + &processSizes, aTotalPss); + if (NS_FAILED(rv)) + break; + } + fclose(f); + } + } + closedir(d); + + // Report the "processes/" tree. + + for (size_t i = 0; i < ProcessSizeKindLimit; i++) { + nsAutoCString path("processes/"); + path.Append(kindPathSuffixes[i]); + + nsAutoCString desc("This is the sum of all processes' '"); + desc.Append(kindPathSuffixes[i]); + desc.Append("' numbers."); + + REPORT(path, processSizes.mSizes[i], desc); + } + + return NS_OK; + } + + nsresult ParseMapping(FILE* aFile, + const nsACString& aProcessName, + nsIHandleReportCallback* aHandleReport, + nsISupports* aData, + ProcessSizes* aProcessSizes, + int64_t* aTotalPss) + { + // The first line of an entry in /proc//smaps looks just like an entry + // in /proc//maps: + // + // address perms offset dev inode pathname + // 02366000-025d8000 rw-p 00000000 00:00 0 [heap] + + const int argCount = 8; + + unsigned long long addrStart, addrEnd; + char perms[5]; + unsigned long long offset; + // The 2.6 and 3.0 kernels allocate 12 bits for the major device number and + // 20 bits for the minor device number. Future kernels might allocate more. + // 64 bits ought to be enough for anybody. + char devMajor[17]; + char devMinor[17]; + unsigned int inode; + char path[1025]; + + // A path might not be present on this line; set it to the empty string. + path[0] = '\0'; + + // This is a bit tricky. Whitespace in a scanf pattern matches *any* + // whitespace, including newlines. We want this pattern to match a line + // with or without a path, but we don't want to look to a new line for the + // path. Thus we have %u%1024[^\n] at the end of the pattern. This will + // capture into the path some leading whitespace, which we'll later trim + // off. + int n = fscanf(aFile, + "%llx-%llx %4s %llx " + "%16[0-9a-fA-F]:%16[0-9a-fA-F] %u%1024[^\n]", + &addrStart, &addrEnd, perms, &offset, devMajor, + devMinor, &inode, path); + + // Eat up any whitespace at the end of this line, including the newline. + unused << fscanf(aFile, " "); + + // We might or might not have a path, but the rest of the arguments should + // be there. + if (n != argCount && n != argCount - 1) { + return NS_ERROR_FAILURE; + } + + nsAutoCString name, description; + ProcessSizeKind kind; + GetReporterNameAndDescription(path, perms, name, description, &kind); + + while (true) { + size_t pss = 0; + nsresult rv = ParseMapBody(aFile, aProcessName, name, description, + aHandleReport, aData, &pss); + if (NS_FAILED(rv)) + break; + + // Increment the appropriate aProcessSizes values, and the total. + aProcessSizes->mSizes[kind] += pss; + *aTotalPss += pss; + } + + return NS_OK; + } + + void GetReporterNameAndDescription(const char* aPath, + const char* aPerms, + nsACString& aName, + nsACString& aDesc, + ProcessSizeKind* aProcessSizeKind) + { + aName.Truncate(); + aDesc.Truncate(); + + // If aPath points to a file, we have its absolute path, plus some + // whitespace. Truncate this to its basename, and put the absolute path in + // the description. + nsAutoCString absPath; + absPath.Append(aPath); + absPath.StripChars(" "); + + nsAutoCString basename; + GetBasename(absPath, basename); + + if (basename.EqualsLiteral("[heap]")) { + aName.Append("anonymous/brk-heap"); + aDesc.Append("Memory in anonymous mappings within the boundaries " + "defined by brk() / sbrk(). This is likely to be just " + "a portion of the application's heap; the remainder " + "lives in other anonymous mappings. This corresponds to " + "'[heap]' in /proc//smaps."); + *aProcessSizeKind = AnonymousBrkHeap; + + } else if (basename.EqualsLiteral("[stack]")) { + aName.Append("main-thread-stack"); + aDesc.Append("The stack size of the process's main thread. This " + "corresponds to '[stack]' in /proc//smaps."); + *aProcessSizeKind = MainThreadStack; + + } else if (basename.EqualsLiteral("[vdso]")) { + aName.Append("vdso"); + aDesc.Append("The virtual dynamically-linked shared object, also known " + "as the 'vsyscall page'. This is a memory region mapped by " + "the operating system for the purpose of allowing processes " + "to perform some privileged actions without the overhead of " + "a syscall."); + *aProcessSizeKind = Vdso; + + } else if (!IsAnonymous(basename)) { + nsAutoCString dirname; + GetDirname(absPath, dirname); + + // Hack: A file is a shared library if the basename contains ".so" and + // its dirname contains "/lib", or if the basename ends with ".so". + if (EndsWithLiteral(basename, ".so") || + (basename.Find(".so") != -1 && dirname.Find("/lib") != -1)) { + aName.Append("shared-libraries/"); + + if (strncmp(aPerms, "r-x", 3) == 0) { + *aProcessSizeKind = SharedLibrariesRX; + } else if (strncmp(aPerms, "rw-", 3) == 0) { + *aProcessSizeKind = SharedLibrariesRW; + } else if (strncmp(aPerms, "r--", 3) == 0) { + *aProcessSizeKind = SharedLibrariesR; + } else { + *aProcessSizeKind = SharedLibrariesOther; + } + + } else { + aName.Append("other-files/"); + if (EndsWithLiteral(basename, ".xpi")) { + aName.Append("extensions/"); + } else if (dirname.Find("/fontconfig") != -1) { + aName.Append("fontconfig/"); + } + *aProcessSizeKind = OtherFiles; + } + + aName.Append(basename); + aDesc.Append(absPath); + + } else { + aName.Append("anonymous/outside-brk"); + aDesc.Append("Memory in anonymous mappings outside the boundaries " + "defined by brk() / sbrk()."); + *aProcessSizeKind = AnonymousOutsideBrk; + } + + aName.Append("/["); + aName.Append(aPerms); + aName.Append("]"); + + // Append the permissions. This is useful for non-verbose mode in + // about:memory when the filename is long and goes of the right side of the + // window. + aDesc.Append(" ["); + aDesc.Append(aPerms); + aDesc.Append("]"); + } + + nsresult ParseMapBody( + FILE* aFile, + const nsACString& aProcessName, + const nsACString& aName, + const nsACString& aDescription, + nsIHandleReportCallback* aHandleReport, + nsISupports* aData, + size_t* aPss) + { + // Most of the lines in the body look like this: + // + // Size: 132 kB + // Rss: 20 kB + // Pss: 20 kB + // + // We're only interested in Pss. In newer kernels, the last line in the + // body has a different form: + // + // VmFlags: rd wr mr mw me dw ac + // + // The strings after "VmFlags: " vary. + + char desc[1025]; + int64_t sizeKB; + int n = fscanf(aFile, "%1024[a-zA-Z_]: %" SCNd64 " kB\n", desc, &sizeKB); + if (n == EOF || n == 0) { + return NS_ERROR_FAILURE; + } else if (n == 1 && strcmp(desc, "VmFlags") == 0) { + // This is the "VmFlags:" line. Chew up the rest of it. + fscanf(aFile, "%*1024[a-z ]\n"); + return NS_ERROR_FAILURE; + } + + // Only report "Pss" values. + if (strcmp(desc, "Pss") == 0) { + *aPss = sizeKB * 1024; + + // Don't report zero values. + if (*aPss == 0) { + return NS_OK; + } + + nsAutoCString path("mem/processes/"); + path.Append(aProcessName); + path.Append("/"); + path.Append(aName); + + REPORT(path, *aPss, aDescription); + } else { + *aPss = 0; + } + + return NS_OK; + } + + nsresult CollectPmemReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData) + { + // The pmem subsystem allocates physically contiguous memory for + // interfacing with hardware. In order to ensure availability, + // this memory is reserved during boot, and allocations are made + // within these regions at runtime. + // + // There are typically several of these pools allocated at boot. + // The /sys/kernel/pmem_regions directory contains a subdirectory + // for each one. Within each subdirectory, the files we care + // about are "size" (the total amount of physical memory) and + // "mapped_regions" (a list of the current allocations within that + // area). + DIR* d = opendir("/sys/kernel/pmem_regions"); + if (!d) { + if (NS_WARN_IF(errno != ENOENT)) { + return NS_ERROR_FAILURE; + } + // If ENOENT, system doesn't use pmem. + return NS_OK; + } + + struct dirent* ent; + while ((ent = readdir(d))) { + const char* name = ent->d_name; + uint64_t size; + int scanned; + + // Skip "." and ".." (and any other dotfiles). + if (name[0] == '.') { + continue; + } + + // Read the total size. The file gives the size in decimal and + // hex, in the form "13631488(0xd00000)"; we parse the former. + nsPrintfCString sizePath("/sys/kernel/pmem_regions/%s/size", name); + FILE* sizeFile = fopen(sizePath.get(), "r"); + if (NS_WARN_IF(!sizeFile)) { + continue; + } + scanned = fscanf(sizeFile, "%" SCNu64, &size); + if (NS_WARN_IF(scanned != 1)) { + continue; + } + fclose(sizeFile); + + // Read mapped regions; format described below. + uint64_t freeSize = size; + nsPrintfCString regionsPath("/sys/kernel/pmem_regions/%s/mapped_regions", + name); + FILE* regionsFile = fopen(regionsPath.get(), "r"); + if (regionsFile) { + static const size_t bufLen = 4096; + char buf[bufLen]; + while (fgets(buf, bufLen, regionsFile)) { + int pid; + + // Skip header line. + if (strncmp(buf, "pid #", 5) == 0) { + continue; + } + // Line format: "pid N:" + zero or more "(Start,Len) ". + // N is decimal; Start and Len are in hex. + scanned = sscanf(buf, "pid %d", &pid); + if (NS_WARN_IF(scanned != 1)) { + continue; + } + for (const char* nextParen = strchr(buf, '('); + nextParen != nullptr; + nextParen = strchr(nextParen + 1, '(')) { + uint64_t mapStart, mapLen; + + scanned = sscanf(nextParen + 1, "%" SCNx64 ",%" SCNx64, + &mapStart, &mapLen); + if (NS_WARN_IF(scanned != 2)) { + break; + } + + nsPrintfCString path("mem/pmem/used/%s/segment(pid=%d, " + "offset=0x%" PRIx64 ")", name, pid, mapStart); + nsPrintfCString desc("Physical memory reserved for the \"%s\" pool " + "and allocated to a buffer.", name); + REPORT_WITH_CLEANUP(path, UNITS_BYTES, mapLen, desc, + (fclose(regionsFile), closedir(d))); + freeSize -= mapLen; + } + } + fclose(regionsFile); + } + + nsPrintfCString path("mem/pmem/free/%s", name); + nsPrintfCString desc("Physical memory reserved for the \"%s\" pool and " + "unavailable to the rest of the system, but not " + "currently allocated.", name); + REPORT_WITH_CLEANUP(path, UNITS_BYTES, freeSize, desc, closedir(d)); + } + closedir(d); + return NS_OK; + } + + uint64_t + ReadSizeFromFile(const char* aFilename) + { + FILE* sizeFile = fopen(aFilename, "r"); + if (NS_WARN_IF(!sizeFile)) { + return 0; + } + + uint64_t size = 0; + fscanf(sizeFile, "%" SCNu64, &size); + fclose(sizeFile); + + return size; + } + + nsresult + CollectZramReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData) + { + // zram usage stats files can be found under: + // /sys/block/zram + // |--> disksize - Maximum amount of uncompressed data that can be + // stored on the disk (bytes) + // |--> orig_data_size - Uncompressed size of data in the disk (bytes) + // |--> compr_data_size - Compressed size of the data in the disk (bytes) + // |--> num_reads - Number of attempted reads to the disk (count) + // |--> num_writes - Number of attempted writes to the disk (count) + // + // Each file contains a single integer value in decimal form. + + DIR* d = opendir("/sys/block"); + if (!d) { + if (NS_WARN_IF(errno != ENOENT)) { + return NS_ERROR_FAILURE; + } + + return NS_OK; + } + + struct dirent* ent; + while ((ent = readdir(d))) { + const char* name = ent->d_name; + + // Skip non-zram entries. + if (strncmp("zram", name, 4) != 0) { + continue; + } + + // Report disk size statistics. + nsPrintfCString diskSizeFile("/sys/block/%s/disksize", name); + nsPrintfCString origSizeFile("/sys/block/%s/orig_data_size", name); + + uint64_t diskSize = ReadSizeFromFile(diskSizeFile.get()); + uint64_t origSize = ReadSizeFromFile(origSizeFile.get()); + uint64_t unusedSize = diskSize - origSize; + + nsPrintfCString diskUsedPath("zram-disksize/%s/used", name); + nsPrintfCString diskUsedDesc( + "The uncompressed size of data stored in \"%s.\" " + "This excludes zero-filled pages since " + "no memory is allocated for them.", name); + REPORT_WITH_CLEANUP(diskUsedPath, UNITS_BYTES, origSize, + diskUsedDesc, closedir(d)); + + nsPrintfCString diskUnusedPath("zram-disksize/%s/unused", name); + nsPrintfCString diskUnusedDesc( + "The amount of uncompressed data that can still be " + "be stored in \"%s\"", name); + REPORT_WITH_CLEANUP(diskUnusedPath, UNITS_BYTES, unusedSize, + diskUnusedDesc, closedir(d)); + + // Report disk accesses. + nsPrintfCString readsFile("/sys/block/%s/num_reads", name); + nsPrintfCString writesFile("/sys/block/%s/num_writes", name); + + uint64_t reads = ReadSizeFromFile(readsFile.get()); + uint64_t writes = ReadSizeFromFile(writesFile.get()); + + nsPrintfCString readsDesc( + "The number of reads (failed or successful) done on " + "\"%s\"", name); + nsPrintfCString readsPath("zram-accesses/%s/reads", name); + REPORT_WITH_CLEANUP(readsPath, UNITS_COUNT_CUMULATIVE, reads, + readsDesc, closedir(d)); + + nsPrintfCString writesDesc( + "The number of writes (failed or successful) done " + "on \"%s\"", name); + nsPrintfCString writesPath("zram-accesses/%s/writes", name); + REPORT_WITH_CLEANUP(writesPath, UNITS_COUNT_CUMULATIVE, writes, + writesDesc, closedir(d)); + + // Report compressed data size. + nsPrintfCString comprSizeFile("/sys/block/%s/compr_data_size", name); + uint64_t comprSize = ReadSizeFromFile(comprSizeFile.get()); + + nsPrintfCString comprSizeDesc( + "The compressed size of data stored in \"%s\"", + name); + nsPrintfCString comprSizePath("zram-compr-data-size/%s", name); + REPORT_WITH_CLEANUP(comprSizePath, UNITS_BYTES, comprSize, + comprSizeDesc, closedir(d)); + } + + closedir(d); + return NS_OK; + } + +#undef REPORT +}; + +NS_IMPL_ISUPPORTS(SystemReporter, nsIMemoryReporter) + +// Keep this in sync with SystemReporter::ProcessSizeKind! +const char* SystemReporter::kindPathSuffixes[] = { + "anonymous/outside-brk", + "anonymous/brk-heap", + "shared-libraries/read-executable", + "shared-libraries/read-write", + "shared-libraries/read-only", + "shared-libraries/other", + "other-files", + "main-thread-stack", + "vdso" +}; + +void Init() +{ + RegisterStrongMemoryReporter(new SystemReporter()); +} + +} // namespace SystemMemoryReporter +} // namespace mozilla