You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by al...@apache.org on 2018/04/18 04:21:54 UTC
[01/12] mesos git commit: Refactored `DiskArtifact` in memory
profiler.
Repository: mesos
Updated Branches:
refs/heads/master 705bf1dfa -> ca21ca820
Refactored `DiskArtifact` in memory profiler.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ca21ca82
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ca21ca82
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ca21ca82
Branch: refs/heads/master
Commit: ca21ca82071f2c53d5817424569977728260da65
Parents: 081ef16
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 05:26:01 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 228 +++++++++++------------
3rdparty/libprocess/src/memory_profiler.hpp | 36 ++--
2 files changed, 126 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ca21ca82/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index 64048d3..b2174ba 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -300,15 +300,12 @@ Try<Path> getTemporaryDirectoryPath() {
}
+// TODO(alexr): Consider making this asynchronous.
Try<Nothing> generateJeprofFile(
- const Try<string>& inputPath,
+ const string& inputPath,
const string& options,
const string& outputPath)
{
- if (inputPath.isError()) {
- return Error("Cannot read input file: " + inputPath.error());
- }
-
// As jeprof doesn't have an option to specify an output file, we actually
// need `os::shell()` here instead of `os::spawn()`.
// Note that the three parameters *MUST NOT* be controllable by the user
@@ -319,7 +316,7 @@ Try<Nothing> generateJeprofFile(
Try<string> result = os::shell(strings::format(
"jeprof %s /proc/self/exe %s > %s",
options,
- inputPath.get(),
+ inputPath,
outputPath).get());
if (result.isError()) {
@@ -543,7 +540,7 @@ void MemoryProfiler::initialize()
route("/download/graph",
authenticationRealm,
DOWNLOAD_GRAPH_HELP(),
- &MemoryProfiler::downloadGraph);
+ &MemoryProfiler::downloadGraphProfile);
route("/statistics",
authenticationRealm,
@@ -582,91 +579,74 @@ void MemoryProfiler::ProfilingRun::extend(
}
-MemoryProfiler::DiskArtifact::DiskArtifact(const string& _filename)
- : filename(_filename),
- timestamp(Error("Not yet generated"))
+MemoryProfiler::DiskArtifact::DiskArtifact(
+ const std::string& path,
+ time_t id)
+ : path(path),
+ id(id)
{}
-const Try<time_t>& MemoryProfiler::DiskArtifact::id() const
+const time_t MemoryProfiler::DiskArtifact::getId() const
{
- return timestamp;
+ return id;
}
-Try<string> MemoryProfiler::DiskArtifact::path() const
+string MemoryProfiler::DiskArtifact::getPath() const
{
- Try<Path> tmpdir = getTemporaryDirectoryPath();
- if (tmpdir.isError()) {
- return tmpdir.error();
- }
-
- return path::join(tmpdir.get(), filename);
+ return path;
}
http::Response MemoryProfiler::DiskArtifact::asHttp() const
{
- Try<string> _path = path();
- if (_path.isError()) {
- return http::BadRequest(
- "Could not compute file path: " + _path.error() + ".\n");
- }
-
// If we get here, we want to serve the file that *should* be on disk.
// Verify that it still exists before attempting to serve it.
//
// TODO(bevers): Store a checksum and verify that it matches.
- if (!os::stat::isfile(_path.get())) {
+ if (!os::stat::isfile(path)) {
return http::BadRequest("Requested file was deleted from local disk.\n");
}
process::http::OK response;
response.type = response.PATH;
- response.path = _path.get();
+ response.path = path;
response.headers["Content-Type"] = "application/octet-stream";
response.headers["Content-Disposition"] =
- strings::format("attachment; filename=%s", _path.get()).get();
+ strings::format("attachment; filename=%s", path).get();
return response;
}
-Try<Nothing> MemoryProfiler::DiskArtifact::generate(
- time_t requestedTimestamp,
+Try<MemoryProfiler::DiskArtifact> MemoryProfiler::DiskArtifact::create(
+ const string& filename,
+ time_t timestamp,
std::function<Try<Nothing>(const string&)> generator)
{
- // Nothing to do if the requestd file already exists.
- if (timestamp.isSome() && timestamp.get() == requestedTimestamp) {
- return Nothing();
+ Try<Path> tmpdir = getTemporaryDirectoryPath();
+ if (tmpdir.isError()) {
+ return Error("Could not determine target path: " + tmpdir.error());
}
- Try<string> path_ = path();
- if (path_.isError()) {
- return Error("Could not determine target path: " + path_.get());
- }
+ const string path = path::join(tmpdir.get(), filename);
- Try<Nothing> result = generator(path_.get());
+ Try<Nothing> result = generator(path);
if (result.isError()) {
// The old file might still be fine on disk, but there's no good way to
// verify this hence we assume that the error rendered it unusable.
- timestamp = Error(result.error());
- return result;
+ return Error("Failed to create artifact: " + result.error());
}
- timestamp = requestedTimestamp;
-
- return Nothing();
+ return MemoryProfiler::DiskArtifact(path, timestamp);
}
MemoryProfiler::MemoryProfiler(const Option<string>& _authenticationRealm)
: ProcessBase("memory-profiler"),
- authenticationRealm(_authenticationRealm),
- jemallocRawProfile(RAW_PROFILE_FILENAME),
- jeprofSymbolizedProfile(SYMBOLIZED_PROFILE_FILENAME),
- jeprofGraph(GRAPH_FILENAME)
+ authenticationRealm(_authenticationRealm)
{}
@@ -764,12 +744,11 @@ Future<http::Response> MemoryProfiler::stop(
" raw profile through libprocess is currently not supported.\n");
}
- // If stop is successful or a no-op, `jemallocRawProfile.id()` will be set.
+ // If stop is successful or a no-op, `jemallocRawProfile` will be `Some`.
stopAndGenerateRawProfile();
- Try<time_t> generated = jemallocRawProfile.id();
- if (generated.isError()) {
- return http::BadRequest(generated.error() + ".\n");
+ if (rawProfile.isError()) {
+ return http::BadRequest(rawProfile.error() + ".\n");
}
Try<bool> stillActive = jemalloc::profilingActive();
@@ -782,7 +761,7 @@ Future<http::Response> MemoryProfiler::stop(
" jeprof must be installed on the host machine and generation of"
" these files can take several minutes.";
- const string id = stringify(generated.get());
+ const string id = stringify(rawProfile->getId());
JSON::Object result;
result.values["id"] = id;
@@ -791,7 +770,7 @@ Future<http::Response> MemoryProfiler::stop(
result.values["url_raw_profile"] =
"/" + this->self().id + "/download/raw?id=" + id;
- result.values["url_graph"] =
+ result.values["url_graph_profile"] =
"/" + this->self().id + "/download/graph?id=" + id;
result.values["url_symbolized_profile"] =
@@ -813,25 +792,25 @@ Future<http::Response> MemoryProfiler::downloadRawProfile(
"Invalid parameter 'id': " + requestedId.error() + ".\n");
}
- if (jemallocRawProfile.id().isError()) {
+ if (currentRun.isSome() && !requestedId.isSome()) {
+ return http::BadRequest(
+ "A profiling run is currently in progress. To download results of the"
+ " previous run, please pass an 'id' explicitly.\n");
+ }
+
+ if (rawProfile.isError()) {
return http::BadRequest(
- "No heap profile exists: " + jemallocRawProfile.id().error() + ".\n");
+ "Cannot access raw profile: " + rawProfile.error() + ".\n");
}
// Only requests for the latest available version are allowed.
if (requestedId.isSome() &&
- (requestedId.get() != jemallocRawProfile.id().get())) {
+ (requestedId.get() != rawProfile->getId())) {
return http::BadRequest(
"Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
}
- if (currentRun.isSome() && !requestedId.isSome()) {
- return http::BadRequest(
- "A profiling run is currently in progress. To download results of the"
- " previous run, please pass an 'id' explicitly.\n");
- }
-
- return jemallocRawProfile.asHttp();
+ return rawProfile->asHttp();
}
@@ -847,46 +826,52 @@ Future<http::Response> MemoryProfiler::downloadSymbolizedProfile(
"Invalid parameter 'id': " + requestedId.error() + ".\n");
}
- if (jemallocRawProfile.id().isError()) {
+ if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
- "No source profile exists: " + jemallocRawProfile.id().error() + ".\n");
+ "A profiling run is currently in progress. To download results of the"
+ " previous run, please pass an 'id' explicitly.\n");
}
- // Only requests for the latest available version are allowed.
- if (requestedId.isSome() &&
- (requestedId.get() != jemallocRawProfile.id().get())) {
+ if (rawProfile.isError()) {
return http::BadRequest(
- "Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
+ "No source profile exists: " + rawProfile.error() + ".\n");
}
- if (currentRun.isSome() && !requestedId.isSome()) {
+ const string rawProfilePath = rawProfile->getPath();
+ const time_t rawProfileId = rawProfile->getId();
+
+ // Only requests for the latest available version are allowed.
+ if (requestedId.isSome() && (requestedId.get() != rawProfileId)) {
return http::BadRequest(
- "A profiling run is currently in progress. To download results of the"
- " previous run, please pass an 'id' explicitly.\n");
+ "Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
}
- // Generate the profile for the latest available version,
+ // Generate the profile for the latest available version
// or return the cached file on disk.
- Try<string> rawProfilePath = jemallocRawProfile.path();
- Try<Nothing> result = jeprofSymbolizedProfile.generate(
- jemallocRawProfile.id().get(),
- [rawProfilePath](const string& outputPath) -> Try<Nothing> {
- return generateJeprofFile(
- rawProfilePath,
- "--text",
- outputPath);
- });
-
- if (result.isError()) {
- return http::BadRequest(
- "Could not generate file: " + result.error() + ".\n");
+ if (symbolizedProfile.isError() ||
+ (symbolizedProfile->getId() != rawProfileId)) {
+ symbolizedProfile = DiskArtifact::create(
+ SYMBOLIZED_PROFILE_FILENAME,
+ rawProfileId,
+ [rawProfilePath](const string& outputPath) -> Try<Nothing> {
+ return generateJeprofFile(
+ rawProfilePath,
+ "--text",
+ outputPath);
+ });
}
- return jeprofSymbolizedProfile.asHttp();
+ if (symbolizedProfile.isSome()) {
+ return symbolizedProfile->asHttp();
+ } else {
+ const string message = "Cannot generate file: " + symbolizedProfile.error();
+ LOG(WARNING) << message;
+ return http::BadRequest(message + ".\n");
+ }
}
-Future<http::Response> MemoryProfiler::downloadGraph(
+Future<http::Response> MemoryProfiler::downloadGraphProfile(
const http::Request& request,
const Option<http::authentication::Principal>&)
{
@@ -898,42 +883,47 @@ Future<http::Response> MemoryProfiler::downloadGraph(
"Invalid parameter 'id': " + requestedId.error() + ".\n");
}
- if (jemallocRawProfile.id().isError()) {
+ if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
- "No source profile exists: " + jemallocRawProfile.id().error() + ".\n");
+ "A profiling run is currently in progress. To download results of the"
+ " previous run, please pass an 'id' explicitly.\n");
}
- // Only requests for the latest available version are allowed.
- if (requestedId.isSome() &&
- (requestedId.get() != jemallocRawProfile.id().get())) {
+ if (rawProfile.isError()) {
return http::BadRequest(
- "Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
+ "No source profile exists: " + rawProfile.error() + ".\n");
}
- if (currentRun.isSome() && !requestedId.isSome()) {
+ const string rawProfilePath = rawProfile->getPath();
+ const time_t rawProfileId = rawProfile->getId();
+
+ // Only requests for the latest available version are allowed.
+ if (requestedId.isSome() && (requestedId.get() != rawProfileId)) {
return http::BadRequest(
- "A profiling run is currently in progress. To download results of the"
- " previous run, please pass an 'id' explicitly.\n");
+ "Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
}
- // Generate the profile for the latest available version,
+ // Generate the profile for the latest available version
// or return the cached file on disk.
- Try<string> rawProfilePath = jemallocRawProfile.path();
- Try<Nothing> result = jeprofGraph.generate(
- jemallocRawProfile.id().get(),
- [rawProfilePath](const string& outputPath) -> Try<Nothing> {
- return generateJeprofFile(
- rawProfilePath,
- "--svg",
- outputPath);
- });
-
- if (result.isError()) {
- return http::BadRequest(
- "Could not generate file: " + result.error() + ".\n");
+ if (graphProfile.isError() || (graphProfile->getId() != rawProfileId)) {
+ graphProfile = DiskArtifact::create(
+ GRAPH_FILENAME,
+ rawProfileId,
+ [rawProfilePath](const string& outputPath) -> Try<Nothing> {
+ return generateJeprofFile(
+ rawProfilePath,
+ "--svg",
+ outputPath);
+ });
}
- return jeprofGraph.asHttp();
+ if (graphProfile.isSome()) {
+ return graphProfile->asHttp();
+ } else {
+ const string message = "Cannot generate file: " + graphProfile.error();
+ LOG(WARNING) << message;
+ return http::BadRequest(message + ".\n");
+ }
}
@@ -974,7 +964,7 @@ Future<http::Response> MemoryProfiler::state(
JSON::Object profilerState;
profilerState.values["jemalloc_detected"] = detected;
- profilerState.values["tmpdir"] = stringify(
+ profilerState.values["tmp_dir"] = stringify(
temporaryDirectory.getOrElse("Not yet generated"));
{
@@ -983,8 +973,8 @@ Future<http::Response> MemoryProfiler::state(
runInformation.values["id"] = currentRun->id;
runInformation.values["remaining_seconds"] =
currentRun->timer.timeout().remaining().secs();
- } else if (jemallocRawProfile.id().isSome()) {
- runInformation.values["id"] = jemallocRawProfile.id().get();
+ } else if (rawProfile.isSome()) {
+ runInformation.values["id"] = rawProfile->getId();
runInformation.values["remaining_seconds"] = 0;
} else {
runInformation.values["id"] = JSON::Null();
@@ -1104,7 +1094,9 @@ void MemoryProfiler::stopAndGenerateRawProfile()
return;
}
- Try<Nothing> generated = jemallocRawProfile.generate(
+ // We store the new artifact even in case of error to surface it to the user.
+ rawProfile = DiskArtifact::create(
+ RAW_PROFILE_FILENAME,
runId,
[](const string& outputPath) -> Try<Nothing> {
// Make sure we actually have permissions to write to the file and that
@@ -1126,8 +1118,8 @@ void MemoryProfiler::stopAndGenerateRawProfile()
return jemalloc::dump(outputPath);
});
- if (generated.isError()) {
- LOG(WARNING) << "Could not dump profile: " + generated.error();
+ if (rawProfile.isError()) {
+ LOG(WARNING) << "Cannot dump profile: " + rawProfile.error();
}
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/ca21ca82/3rdparty/libprocess/src/memory_profiler.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.hpp b/3rdparty/libprocess/src/memory_profiler.hpp
index adcb49b..617f8de 100644
--- a/3rdparty/libprocess/src/memory_profiler.hpp
+++ b/3rdparty/libprocess/src/memory_profiler.hpp
@@ -77,7 +77,7 @@ private:
const Option<http::authentication::Principal>&);
// Generates and returns a call graph in svg format.
- Future<http::Response> downloadGraph(
+ Future<http::Response> downloadGraphProfile(
const http::Request& request,
const Option<http::authentication::Principal>&);
@@ -120,36 +120,32 @@ private:
class DiskArtifact
{
public:
- DiskArtifact(const std::string& filename);
-
- const Try<time_t>& id() const;
-
- // This is generated lazily to avoid creating a temporary
- // directory unless we actually need it.
- Try<std::string> path() const;
-
- // If the file with the given timestamp does not exist on disk, calls the
- // supplied `generator` function that should write it to the specified
- // location.
- Try<Nothing> generate(
+ static Try<DiskArtifact> create(
+ const std::string& filename,
time_t timestamp,
std::function<Try<Nothing>(const std::string& outputPath)> generator);
+ const time_t getId() const;
+
+ std::string getPath() const;
+
// Generates an error response if the file doesn't exist, or a download
// if it does.
http::Response asHttp() const;
private:
- std::string filename;
+ DiskArtifact(const std::string& path, time_t id);
- // A timestamp of the last successful creation that serves as unique id,
- // or the reason why it couldn't be created.
- Try<time_t> timestamp;
+ std::string path;
+ time_t id;
};
- DiskArtifact jemallocRawProfile;
- DiskArtifact jeprofSymbolizedProfile;
- DiskArtifact jeprofGraph;
+ // This profile is obtained by telling jemalloc to dump its stats to a file.
+ Try<DiskArtifact> rawProfile = Error("Not yet generated");
+
+ // These profiles are obtained by running `jeprof` on the `raw` profile.
+ Try<DiskArtifact> symbolizedProfile = Error("Not yet generated");
+ Try<DiskArtifact> graphProfile = Error("Not yet generated");
};
} // namespace process {
[06/12] mesos git commit: Added new --memory_profiling flag to agent
and master binaries.
Posted by al...@apache.org.
Added new --memory_profiling flag to agent and master binaries.
This flag allows explicit disabling of the memory profiler
endpoint in the master and agent binaries.
Review: https://reviews.apache.org/r/63370/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/58e224ef
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/58e224ef
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/58e224ef
Branch: refs/heads/master
Commit: 58e224efe4173fe5061c968b41712053b7c22a3e
Parents: 81c9978
Author: Benno Evers <be...@mesosphere.com>
Authored: Tue Apr 17 18:31:47 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
src/master/flags.cpp | 11 +++++++++++
src/master/flags.hpp | 1 +
src/master/main.cpp | 2 ++
src/slave/flags.cpp | 11 +++++++++++
src/slave/flags.hpp | 1 +
src/slave/main.cpp | 2 ++
6 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/58e224ef/src/master/flags.cpp
----------------------------------------------------------------------
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index dbb35be..4ad1375 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -627,6 +627,17 @@ mesos::internal::master::Flags::Flags()
"However, this port (along with `advertise_ip`) may be used to\n"
"access this master.");
+ // TODO(bevers): Switch the default to `true` after gathering some
+ // real-world experience.
+ add(&Flags::memory_profiling,
+ "memory_profiling",
+ "This setting controls whether the memory profiling functionality of\n"
+ "libprocess should be exposed when jemalloc is detected.\n"
+ "NOTE: Even if set to true, memory profiling will not work unless\n"
+ "jemalloc is loaded into the address space of the binary, either by\n"
+ "linking against it at compile-time or using `LD_PRELOAD`.",
+ false);
+
add(&Flags::zk,
"zk",
"ZooKeeper URL (used for leader election amongst masters).\n"
http://git-wip-us.apache.org/repos/asf/mesos/blob/58e224ef/src/master/flags.hpp
----------------------------------------------------------------------
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index 505786e..4715f2b 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -108,6 +108,7 @@ public:
Option<std::string> advertise_ip;
Option<std::string> advertise_port;
Option<flags::SecurePathOrValue> zk;
+ bool memory_profiling;
// Optional IP discover script that will set the Master IP.
// If set, its output is expected to be a valid parseable IP string.
http://git-wip-us.apache.org/repos/asf/mesos/blob/58e224ef/src/master/main.cpp
----------------------------------------------------------------------
diff --git a/src/master/main.cpp b/src/master/main.cpp
index 3def0f2..3f5b354 100644
--- a/src/master/main.cpp
+++ b/src/master/main.cpp
@@ -242,6 +242,8 @@ int main(int argc, char** argv)
}
}
+ os::setenv("LIBPROCESS_MEMORY_PROFILING", stringify(flags.memory_profiling));
+
// Log build information.
LOG(INFO) << "Build: " << build::DATE << " by " << build::USER;
LOG(INFO) << "Version: " << MESOS_VERSION;
http://git-wip-us.apache.org/repos/asf/mesos/blob/58e224ef/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index 4ea2286..e330e5f 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -1394,6 +1394,17 @@ mesos::internal::slave::Flags::Flags()
"this IPv6 address is only used to advertise IPv6 addresses for\n"
"containers running on the host network.\n");
+ // TODO(bevers): Switch the default to `true` after gathering
+ // some real-world experience.
+ add(&Flags::memory_profiling,
+ "memory_profiling",
+ "This setting controls whether the memory profiling functionality of\n"
+ "libprocess should be exposed when jemalloc is detected.\n"
+ "NOTE: Even if set to true, memory profiling will not work unless\n"
+ "jemalloc is loaded into the address space of the binary, either by\n"
+ "linking against it at compile-time or using `LD_PRELOAD`.",
+ false);
+
add(&Flags::domain,
"domain",
"Domain that the agent belongs to. Mesos currently only supports\n"
http://git-wip-us.apache.org/repos/asf/mesos/blob/58e224ef/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index beae47f..4195d3b 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -192,6 +192,7 @@ public:
Option<std::string> advertise_ip;
Option<std::string> advertise_port;
Option<flags::SecurePathOrValue> master;
+ bool memory_profiling;
Duration zk_session_timeout;
http://git-wip-us.apache.org/repos/asf/mesos/blob/58e224ef/src/slave/main.cpp
----------------------------------------------------------------------
diff --git a/src/slave/main.cpp b/src/slave/main.cpp
index dcc9448..ca35668 100644
--- a/src/slave/main.cpp
+++ b/src/slave/main.cpp
@@ -364,6 +364,8 @@ int main(int argc, char** argv)
os::setenv("LIBPROCESS_ADVERTISE_PORT", flags.advertise_port.get());
}
+ os::setenv("LIBPROCESS_MEMORY_PROFILING", stringify(flags.memory_profiling));
+
// Log build information.
LOG(INFO) << "Build: " << build::DATE << " by " << build::USER;
LOG(INFO) << "Version: " << MESOS_VERSION;
[09/12] mesos git commit: Fixed help messages in MemoryProfiler.
Posted by al...@apache.org.
Fixed help messages in MemoryProfiler.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/47e88dd9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/47e88dd9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/47e88dd9
Branch: refs/heads/master
Commit: 47e88dd987b179d7db7d21b79b70d824560f5a40
Parents: f5bd653
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 00:02:20 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/47e88dd9/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index 7a52b96..d935211 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -444,6 +444,7 @@ const std::string MemoryProfiler::DOWNLOAD_RAW_HELP()
"> id=VALUE Optional parameter to request a specific",
"> version of the profile."),
AUTHENTICATION(true),
+ None,
REFERENCES("[manpage]: http://jemalloc.net/jemalloc.3.html"));
}
@@ -473,7 +474,7 @@ const std::string MemoryProfiler::DOWNLOAD_GRAPH_HELP()
"Generates and returns a graph visualization."),
DESCRIPTION(
"Generates a graphical representation of the raw profile in the SVG",
- "Using this endpoint requires that that jeprof and dot are installed"
+ "Using this endpoint requires that that jeprof and dot are installed",
"on the host machine.",
"*NOTE*: Generating the returned file might take several minutes.",
"",
[10/12] mesos git commit: Simplified MemoryProfiler by removing a
method overload.
Posted by al...@apache.org.
Simplified MemoryProfiler by removing a method overload.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f2cb9073
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f2cb9073
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f2cb9073
Branch: refs/heads/master
Commit: f2cb9073c61c1089db559014abe5b6d2d6d71714
Parents: a3560c1
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 01:50:42 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 39 +++++++++---------------
3rdparty/libprocess/src/memory_profiler.hpp | 4 +--
2 files changed, 16 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f2cb9073/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index 4e19f16..2b3af3d 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -574,7 +574,7 @@ MemoryProfiler::ProfilingRun::ProfilingRun(
timer(delay(
duration,
profiler,
- &MemoryProfiler::_stopAndGenerateRawProfile))
+ &MemoryProfiler::stopAndGenerateRawProfile))
{}
@@ -587,7 +587,7 @@ void MemoryProfiler::ProfilingRun::extend(
timer = delay(
remaining + duration,
profiler,
- &MemoryProfiler::_stopAndGenerateRawProfile);
+ &MemoryProfiler::stopAndGenerateRawProfile);
}
@@ -765,7 +765,10 @@ Future<http::Response> MemoryProfiler::stop(
" raw profile through libprocess is currently not supported.\n");
}
- Try<time_t> generated = stopAndGenerateRawProfile();
+ // If stop is successful or a no-op, `jemallocRawProfile.id()` will be set.
+ stopAndGenerateRawProfile();
+
+ Try<time_t> generated = jemallocRawProfile.id();
if (generated.isError()) {
return http::BadRequest(generated.error() + ".\n");
}
@@ -799,24 +802,15 @@ Future<http::Response> MemoryProfiler::stop(
}
-// A simple wrapper to discard the result, so we can
-// use this as the target for `process::delay()`.
-void MemoryProfiler::_stopAndGenerateRawProfile()
-{
- stopAndGenerateRawProfile();
-}
-
-
-Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
+void MemoryProfiler::stopAndGenerateRawProfile()
{
ASSERT(detectJemalloc());
VLOG(1) << "Attempting to stop current profiling run";
- // Return the id of the last successful run if there is no current
- // profiling run.
+ // If there is no current profiling run, there is nothing to do.
if (!currentRun.isSome()) {
- return jemallocRawProfile.id();
+ return;
}
Try<bool> stopped = jemalloc::stopProfiling();
@@ -828,7 +822,7 @@ Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
// the problem will be clearly visible in the logs.
currentRun->extend(this, Seconds(5));
- return Error(stopped.error());
+ return;
}
// Heap profiling should not be active any more.
@@ -847,9 +841,10 @@ Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
// process stopped profiling independently of us.
// If there was some valuable, un-dumped data it is still possible to get
// it by starting a new run.
- return Error(
- "Memory profiling unexpectedly inactive; not dumping profile."
- " Ensure nothing else is interfacing with jemalloc in this process");
+ LOG(WARNING)
+ << "Memory profiling unexpectedly inactive; not dumping profile. Ensure"
+ << " nothing else is interfacing with jemalloc in this process";
+ return;
}
Try<Nothing> generated = jemallocRawProfile.generate(
@@ -875,12 +870,8 @@ Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
});
if (generated.isError()) {
- const string errorMessage = "Could not dump profile: " + generated.error();
- LOG(WARNING) << errorMessage;
- return Error(errorMessage);
+ LOG(WARNING) << "Could not dump profile: " + generated.error();
}
-
- return runId;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/f2cb9073/3rdparty/libprocess/src/memory_profiler.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.hpp b/3rdparty/libprocess/src/memory_profiler.hpp
index 5a359df..5cbe507 100644
--- a/3rdparty/libprocess/src/memory_profiler.hpp
+++ b/3rdparty/libprocess/src/memory_profiler.hpp
@@ -96,9 +96,7 @@ private:
// Internal functions and data members.
// Deactivates data collection and attempts to dump the raw profile to disk.
- Try<time_t> stopAndGenerateRawProfile();
-
- void _stopAndGenerateRawProfile();
+ void stopAndGenerateRawProfile();
// The authentication realm that the profiler's HTTP endpoints will be
// installed into.
[05/12] mesos git commit: Added MemoryProfiler class to Libprocess.
Posted by al...@apache.org.
Added MemoryProfiler class to Libprocess.
This class exposes profiling functionality of jemalloc
memory allocator when it is detected to be the memory
allocator of the current process.
In particular, it enables developers to access live
statistics of current memory usage, and to create
heap profiles that show where most memory is allocated.
Review: https://reviews.apache.org/r/63368/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/81c9978c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/81c9978c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/81c9978c
Branch: refs/heads/master
Commit: 81c9978c0f759f7e39f12ae5343a35aa123b0ba8
Parents: cae2d20
Author: Benno Evers <be...@mesosphere.com>
Authored: Tue Apr 17 18:31:36 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/Makefile.am | 6 +
3rdparty/libprocess/include/process/process.hpp | 1 +
3rdparty/libprocess/src/CMakeLists.txt | 6 +
3rdparty/libprocess/src/memory_profiler.cpp | 1139 ++++++++++++++++++
3rdparty/libprocess/src/memory_profiler.hpp | 163 +++
3rdparty/libprocess/src/process.cpp | 18 +-
6 files changed, 1332 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/81c9978c/3rdparty/libprocess/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/Makefile.am b/3rdparty/libprocess/Makefile.am
index c895d3a..70edb2e 100644
--- a/3rdparty/libprocess/Makefile.am
+++ b/3rdparty/libprocess/Makefile.am
@@ -205,6 +205,8 @@ libprocess_la_SOURCES = \
src/io.cpp \
src/latch.cpp \
src/logging.cpp \
+ src/memory_profiler.cpp \
+ src/memory_profiler.hpp \
src/metrics/metrics.cpp \
src/mime.cpp \
src/pid.cpp \
@@ -276,6 +278,10 @@ if ENABLE_STATIC_LIBPROCESS
libprocess_la_LDFLAGS = -static -fpic
endif
+if OS_LINUX
+libprocess_la_CPPFLAGS += -DLIBPROCESS_ALLOW_JEMALLOC
+endif
+
libprocess_la_LIBADD = \
$(LIB_GLOG) \
$(LIB_GPERFTOOLS) \
http://git-wip-us.apache.org/repos/asf/mesos/blob/81c9978c/3rdparty/libprocess/include/process/process.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/process.hpp b/3rdparty/libprocess/include/process/process.hpp
index c36df99..f68ecca 100644
--- a/3rdparty/libprocess/include/process/process.hpp
+++ b/3rdparty/libprocess/include/process/process.hpp
@@ -44,6 +44,7 @@ namespace process {
class EventQueue;
class Gate;
class Logging;
+class MemoryProfiler;
class Sequence;
namespace firewall {
http://git-wip-us.apache.org/repos/asf/mesos/blob/81c9978c/3rdparty/libprocess/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/CMakeLists.txt b/3rdparty/libprocess/src/CMakeLists.txt
index 0ce7dac..1eca50a 100644
--- a/3rdparty/libprocess/src/CMakeLists.txt
+++ b/3rdparty/libprocess/src/CMakeLists.txt
@@ -44,6 +44,8 @@ set(PROCESS_SRC
io.cpp
latch.cpp
logging.cpp
+ memory_profiler.cpp
+ memory_profiler.hpp
metrics/metrics.cpp
mime.cpp
pid.cpp
@@ -114,5 +116,9 @@ target_compile_definitions(
$<$<BOOL:${ENABLE_LOCK_FREE_EVENT_QUEUE}>:LOCK_FREE_EVENT_QUEUE>
$<$<BOOL:${ENABLE_LAST_IN_FIRST_OUT_FIXED_SIZE_SEMAPHORE}>:LAST_IN_FIRST_OUT_FIXED_SIZE_SEMAPHORE>)
+if (LINUX)
+ target_compile_definitions(process PRIVATE LIBPROCESS_ALLOW_JEMALLOC)
+endif ()
+
target_include_directories(process PUBLIC ../include)
target_include_directories(process PRIVATE .)
http://git-wip-us.apache.org/repos/asf/mesos/blob/81c9978c/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
new file mode 100644
index 0000000..995beea
--- /dev/null
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -0,0 +1,1139 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+#include "memory_profiler.hpp"
+
+#include <process/delay.hpp>
+#include <process/future.hpp>
+#include <process/help.hpp>
+#include <process/http.hpp>
+
+#include <stout/assert.hpp>
+#include <stout/format.hpp>
+#include <stout/json.hpp>
+#include <stout/option.hpp>
+#include <stout/os.hpp>
+#include <stout/path.hpp>
+
+#include <glog/logging.h>
+
+#include <chrono>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+
+using process::Future;
+using process::HELP;
+using process::TLDR;
+using process::DESCRIPTION;
+using process::AUTHENTICATION;
+
+
+// The main workflow to generate and download a heap profile
+// goes through the sequence of endpoints
+//
+// `/start?duration=T` -> `/download/{raw,graph,text}`
+//
+// A started profiling run will be stopped automatically after the
+// given duration has passed, but can be ended prematurely by accessing
+//
+// `/stop`
+//
+// Any started run has an associated unique id, which is intended to make
+// it easier for scripts to reliably download only those profiles that
+// they themselves generated. Human operators will mostly ignore it and
+// use the provided default value.
+//
+// The generated files are typically stored under the directory
+// `/tmp/libprocess.XXXXXX/jemalloc.{txt,svg,dump}`, where XXXXXX
+// stands for a random combination of letters. This directory, as well
+// as the files contained therein, is created lazily the first time it
+// is accessed.
+//
+// To avoid running out of disk space, every time a new file is
+// generated, the previous one is overwritten. The members `rawId`,
+// `graphId` and `textId` track which version, if any, of the
+// corresponding artifact is currently available on disk.
+//
+// Since this class, being a part of libprocess, will end up in
+// `libmesos.so` and thus possibly in applications that use their own
+// memory allocator, we carefully avoid actually linking this class
+// against `libjemalloc.so`. Instead, we use weak symbols to detect the
+// presence of jemalloc at runtime, and use a macro to hide these symbols
+// when building on platforms that don't support weak symbols.
+
+
+#ifdef LIBPROCESS_ALLOW_JEMALLOC
+
+extern "C" __attribute__((weak)) void malloc_stats_print(
+ void (*writecb)(void*, const char*),
+ void* opaque,
+ const char* opts);
+
+extern "C" __attribute__((weak)) int mallctl(
+ const char* opt, void* oldp, size_t* oldsz, void* newp, size_t newsz);
+
+#endif // LIBPROCESS_ALLOW_JEMALLOC
+
+
+namespace {
+
+constexpr char LIBPROCESS_DEFAULT_TMPDIR[] = "/tmp";
+constexpr char RAW_PROFILE_FILENAME[] = "profile.dump";
+constexpr char SYMBOLIZED_PROFILE_FILENAME[] = "symbolized-profile.dump";
+constexpr char GRAPH_FILENAME[] = "profile.svg";
+constexpr Duration MINIMUM_COLLECTION_TIME = Seconds(1);
+constexpr Duration DEFAULT_COLLECTION_TIME = Minutes(5);
+constexpr Duration MAXIMUM_COLLECTION_TIME = Hours(24);
+
+
+constexpr char JEMALLOC_NOT_DETECTED_MESSAGE[] = R"_(
+The current binary doesn't seem to be linked against jemalloc,
+or the currently used jemalloc library was compiled without
+support for statistics collection.
+
+If the current binary was not compiled against jemalloc,
+consider adding the path to libjemalloc to the LD_PRELOAD
+environment variable, for example LD_PRELOAD=/usr/lib/libjemalloc.so
+
+If you're running a mesos binary, and want to have it linked
+against jemalloc by default, consider using the
+--enable-jemalloc-allocator configuration option.)_";
+
+
+constexpr char JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE[] = R"_(
+The current process seems to be using jemalloc, but
+profiling couldn't be enabled.
+
+If you're using a custom version of libjemalloc, make sure
+that MALLOC_CONF="prof:true" is part of the environment. (The
+`/state` endpoint can be used to double-check the current malloc
+configuration)
+
+If the environment looks correct, make sure jemalloc was built with
+the --enable-stats and --enable-prof options enabled.
+
+If you're running a mesos binary that was built with the
+--enable-memory-profiling option enabled and you're still seeing this
+message, please consider filing a bug report.
+)_";
+
+
+// Size in bytes of the dummy file that gets written when hitting '/start'.
+constexpr int DUMMY_FILE_SIZE = 64 * 1024; // 64 KiB
+
+
+// The `detectJemalloc()`function below was taken from the folly library
+// (called `usingJEMalloc()` there), originally distributed by Facebook, Inc
+// under the Apache License.
+//
+// It checks whether jemalloc is used as the current malloc implementation
+// by allocating one byte and checking whether the threads allocation counter
+// increased. This requires jemalloc to have been compiled with
+// the `--enable-stats option`.
+bool detectJemalloc() noexcept {
+#ifndef LIBPROCESS_ALLOW_JEMALLOC
+ return false;
+#else
+ static const bool result = [] () noexcept {
+ // Some platforms (*cough* OSX *cough*) require weak symbol checks to be
+ // in the form if (mallctl != nullptr). Not if (mallctl) or if (!mallctl)
+ // (!!). http://goo.gl/xpmctm
+ if (mallctl == nullptr || malloc_stats_print == nullptr) {
+ return false;
+ }
+
+ // "volatile" because gcc optimizes out the reads from *counter, because
+ // it "knows" malloc doesn't modify global state...
+ volatile uint64_t* counter;
+ size_t counterLen = sizeof(uint64_t*);
+
+ if (mallctl("thread.allocatedp", static_cast<void*>(&counter), &counterLen,
+ nullptr, 0) != 0) {
+ return false;
+ }
+
+ if (counterLen != sizeof(uint64_t*)) {
+ return false;
+ }
+
+ uint64_t origAllocated = *counter;
+
+ // Static because otherwise clever compilers will find out that
+ // the ptr is not used and does not escape the scope, so they will
+ // just optimize away the malloc.
+ static const void* ptr = malloc(1);
+ if (!ptr) {
+ // wtf, failing to allocate 1 byte
+ return false;
+ }
+
+ return (origAllocated != *counter);
+ }();
+
+ return result;
+#endif
+}
+
+
+template<typename T>
+Try<T> readJemallocSetting(const char* name)
+{
+#ifdef LIBPROCESS_ALLOW_JEMALLOC
+ if (!detectJemalloc()) {
+ return Error(JEMALLOC_NOT_DETECTED_MESSAGE);
+ }
+
+ T value;
+ size_t size = sizeof(value);
+ int error = ::mallctl(name, &value, &size, nullptr, 0);
+
+ if (error) {
+ return Error(strings::format(
+ "Couldn't read option %s: %s", name, ::strerror(error)).get());
+ }
+
+ return value;
+#else
+ UNREACHABLE();
+#endif
+}
+
+
+// Returns an error on failure or the previous value on success.
+template<typename T>
+Try<T> updateJemallocSetting(const char* name, const T& value)
+{
+#ifdef LIBPROCESS_ALLOW_JEMALLOC
+ if (!detectJemalloc()) {
+ return Error(JEMALLOC_NOT_DETECTED_MESSAGE);
+ }
+
+ T previous;
+ size_t size = sizeof(previous);
+ int error = ::mallctl(
+ name, &previous, &size, const_cast<T*>(&value), sizeof(value));
+
+ if (error) {
+ return Error(strings::format(
+ "Couldn't write value %s for option %s: %s",
+ stringify(value), name, ::strerror(error)).get());
+ }
+
+ return previous;
+#else
+ UNREACHABLE();
+#endif
+}
+
+
+// Sadly, we cannot just use `updateJemallocSetting()` and ignore the result,
+// because some settings, in particular `prof.dump`, don't have previous value
+// to return.
+template<typename T>
+Try<Nothing> writeJemallocSetting(const char* name, const T& value)
+{
+#ifdef LIBPROCESS_ALLOW_JEMALLOC
+ if (!detectJemalloc()) {
+ return Error(JEMALLOC_NOT_DETECTED_MESSAGE);
+ }
+
+ int error = mallctl(
+ name, nullptr, nullptr, const_cast<T*>(&value), sizeof(value));
+
+ if (error) {
+ return Error(strings::format(
+ "Couldn't write value %s for option %s: %s",
+ stringify(value), name, ::strerror(error)).get());
+ }
+
+ return Nothing();
+#else
+ UNREACHABLE();
+#endif
+}
+
+
+// All generated disk artifacts, i.e. profiles and graph files, are stored
+// in this directory. It is generated lazily on the first call to
+// `getTemporaryDirectoryPath()` and never changed afterwards.
+// Locking is not needed because all accesses go through the same process,
+// MemoryProfiler, and hence are are always serialized with respect to each
+// other.
+//
+// TODO(bevers): This should be made available libprocess-global eventually,
+// but right now this is the only class that has a use for it.
+Option<Path> temporaryDirectory;
+
+
+Try<Path> getTemporaryDirectoryPath() {
+ if (temporaryDirectory.isSome()) {
+ return temporaryDirectory.get();
+ }
+
+ // TODO(bevers): Add a libprocess-specific override for the system-wide
+ // `TMPDIR`, for example `LIBPROCESS_TMPDIR`.
+ std::string tmpdir = os::getenv("TMPDIR")
+ .getOrElse(LIBPROCESS_DEFAULT_TMPDIR);
+
+ std::string pathTemplate = path::join(tmpdir, "libprocess.XXXXXX");
+
+ // TODO(bevers): Add an atexit-handler that cleans up the directory.
+ Try<std::string> dir = os::mkdtemp(pathTemplate);
+ if (dir.isError()) {
+ return Error(dir.error());
+ }
+
+ temporaryDirectory = dir.get();
+
+ VLOG(1) << "Using path " << dir.get() << " to store temporary files.";
+
+ return temporaryDirectory.get();
+}
+
+
+Try<Nothing> generateJeprofFile(
+ const Try<std::string>& inputPath,
+ const std::string& options,
+ const std::string& outputPath)
+{
+ if (inputPath.isError()) {
+ return Error("Cannot read input file: " + inputPath.error());
+ }
+
+ // As jeprof doesn't have an option to specify an output file, we actually
+ // need `os::shell()` here instead of `os::spawn()`.
+ // Note that the three parameters *MUST NOT* be controllable by the user
+ // accessing the HTTP endpoints, otherwise arbitrary shell commands could be
+ // trivially injected.
+ // Apart from that, we dont need to be as careful here as with the actual heap
+ // profile dump, because a failure will not crash the whole process.
+ Try<std::string> result = os::shell(strings::format(
+ "jeprof %s /proc/self/exe %s > %s",
+ options,
+ inputPath.get(),
+ outputPath).get());
+
+ if (result.isError()) {
+ return Error(
+ "Error trying to run jeprof: " + result.error() + "."
+ " Please make sure that jeprof is installed and that"
+ " the input file contains data.");
+ }
+
+ return Nothing();
+}
+
+
+// TODO(bevers): Implement `http::Request::extractFromRequest<T>(string key)`
+// instead of having this here.
+Result<time_t> extractIdFromRequest(const process::http::Request& request)
+{
+ Option<std::string> idParameter = request.url.query.get("id");
+ if (idParameter.isNone()) {
+ return None();
+ }
+
+ // Since `strtoll()` can legitimately return any value, we have to detect
+ // errors by checking if `errno` was set during the call.
+ errno = 0;
+ char* endptr;
+ int base = 10;
+ long long parsed = std::strtoll(idParameter->c_str(), &endptr, base);
+ if (errno) {
+ return Error(strerror(errno));
+ }
+
+ if (endptr != idParameter->c_str() + idParameter->size()) {
+ return Error("Garbage after parsed id");
+ }
+
+ return parsed;
+}
+
+} // namespace {
+
+
+namespace process {
+
+// Interface to interact with the `jemalloc` library.
+namespace jemalloc {
+
+Try<bool> startProfiling()
+{
+ return updateJemallocSetting("prof.active", true);
+}
+
+
+Try<bool> stopProfiling()
+{
+ return updateJemallocSetting("prof.active", false);
+}
+
+
+Try<bool> profilingActive()
+{
+ return readJemallocSetting<bool>("prof.active");
+}
+
+
+Try<Nothing> dump(const std::string& path)
+{
+ // A profile is dumped every time the 'prof.dump' setting is written to.
+ return writeJemallocSetting("prof.dump", path.c_str());
+}
+
+} // namespace jemalloc {
+
+
+const std::string MemoryProfiler::START_HELP()
+{
+ return HELP(
+ TLDR(
+ "Starts collection of stack traces."),
+ DESCRIPTION(
+ "Activates memory profiling.",
+ "The profiling works by statistically sampling the backtraces of",
+ "calls to `malloc()`. This requires some additional memory to store",
+ "the collected data. The required additional space is expected to",
+ "grow logarithmically."
+ "",
+ "Query Parameters:",
+ "> duration=VALUE How long to collect data before",
+ "> stopping. (default: 5mins)"),
+ AUTHENTICATION(true));
+}
+
+
+const std::string MemoryProfiler::STOP_HELP()
+{
+ return HELP(
+ TLDR(
+ "Stops memory profiling and dumps collected data."),
+ DESCRIPTION(
+ "Instructs the memory profiler to stop collecting data"
+ "and dumps a file containing the collected data to disk,"
+ "clearing that data from memory. Does nothing if profiling",
+ "was not started."),
+ AUTHENTICATION(true));
+}
+
+
+const std::string MemoryProfiler::DOWNLOAD_RAW_HELP()
+{
+ return HELP(
+ TLDR(
+ "Returns a raw memory profile."),
+ DESCRIPTION(
+ "Returns a file that was generated when the `/stop` endpoint was",
+ "last accessed. See the jemalloc [manual page][manpage]",
+ "for information about the file format.",
+ "",
+ "Query Parameters:",
+ "> id=VALUE Optional parameter to request a specific",
+ "> version of the profile."),
+ AUTHENTICATION(true),
+ REFERENCES("[manpage]: http://jemalloc.net/jemalloc.3.html"));
+}
+
+
+const std::string MemoryProfiler::DOWNLOAD_TEXT_HELP()
+{
+ return HELP(
+ TLDR(
+ "Generates and returns a symbolized memory profile."),
+ DESCRIPTION(
+ "Generates a symbolized profile.",
+ "Requires that the running binary was built with symbols, and that",
+ "jeprof is installed on the host machine.",
+ "*NOTE*: Generating the returned file might take several minutes.",
+ "",
+ "Query Parameters:",
+ "> id=VALUE Optional parameter to request a specific",
+ "> version of the generated profile."),
+ AUTHENTICATION(true));
+}
+
+
+const std::string MemoryProfiler::DOWNLOAD_GRAPH_HELP()
+{
+ return HELP(
+ TLDR(
+ "Generates and returns a graph visualization."),
+ DESCRIPTION(
+ "Generates a graphical representation of the raw profile in the SVG",
+ "Using this endpoint requires that that jeprof and dot are installed"
+ "on the host machine.",
+ "*NOTE*: Generating the returned file might take several minutes.",
+ "",
+ "Query Parameters:",
+ "> id=VALUE Optional parameter to request a specific",
+ "> version of the generated graph."),
+ AUTHENTICATION(true));
+}
+
+
+const std::string MemoryProfiler::STATISTICS_HELP()
+{
+ return HELP(
+ TLDR(
+ "Shows memory allocation statistics."),
+ DESCRIPTION(
+ "Memory allocation statistics as returned by `malloc_stats_print()`.",
+ "These track e.g. the total number of bytes allocated by the current",
+ "process and the bin-size of these allocations.",
+ "These statistics are unrelated to the profiling mechanism controlled",
+ "by the `/start` and `/stop` endpoints, and are always accurate.",
+ "",
+ "Returns a JSON object."),
+ AUTHENTICATION(true));
+}
+
+
+const std::string MemoryProfiler::STATE_HELP()
+{
+ return HELP(
+ TLDR(
+ "Shows the configuration of the memory-profiler process."),
+ DESCRIPTION(
+ "Current memory profiler state. This shows, for example, whether",
+ "jemalloc was detected, whether profiling is currently active and",
+ "the directory used to store temporary files.",
+ "",
+ "Returns a JSON object."),
+ AUTHENTICATION(true));
+}
+
+
+void MemoryProfiler::initialize()
+{
+ route("/start",
+ authenticationRealm,
+ START_HELP(),
+ &MemoryProfiler::start);
+
+ route("/stop",
+ authenticationRealm,
+ STOP_HELP(),
+ &MemoryProfiler::stop);
+
+ route("/download/raw",
+ authenticationRealm,
+ DOWNLOAD_RAW_HELP(),
+ &MemoryProfiler::downloadRaw);
+
+ route("/download/text",
+ authenticationRealm,
+ DOWNLOAD_TEXT_HELP(),
+ &MemoryProfiler::downloadTextProfile);
+
+ route("/download/graph",
+ authenticationRealm,
+ DOWNLOAD_GRAPH_HELP(),
+ &MemoryProfiler::downloadGraph);
+
+ route("/state",
+ authenticationRealm,
+ STATE_HELP(),
+ &MemoryProfiler::state);
+
+ route("/statistics",
+ authenticationRealm,
+ STATISTICS_HELP(),
+ &MemoryProfiler::statistics);
+}
+
+
+MemoryProfiler::MemoryProfiler(const Option<std::string>& _authenticationRealm)
+ : ProcessBase("memory-profiler"),
+ authenticationRealm(_authenticationRealm),
+ jemallocRawProfile(RAW_PROFILE_FILENAME),
+ jeprofSymbolizedProfile(SYMBOLIZED_PROFILE_FILENAME),
+ jeprofGraph(GRAPH_FILENAME)
+{}
+
+
+MemoryProfiler::ProfilingRun::ProfilingRun(
+ MemoryProfiler* profiler,
+ time_t id,
+ const Duration& duration)
+ : id(id),
+ timer(delay(
+ duration,
+ profiler,
+ &MemoryProfiler::_stopAndGenerateRawProfile))
+{}
+
+
+void MemoryProfiler::ProfilingRun::extend(
+ MemoryProfiler* profiler,
+ const Duration& duration)
+{
+ Duration remaining = timer.timeout().remaining();
+ Clock::cancel(timer);
+ timer = delay(
+ remaining + duration,
+ profiler,
+ &MemoryProfiler::_stopAndGenerateRawProfile);
+}
+
+
+MemoryProfiler::DiskArtifact::DiskArtifact(const std::string& _filename)
+ : filename(_filename),
+ timestamp(Error("Not yet generated."))
+{}
+
+
+const Try<time_t>& MemoryProfiler::DiskArtifact::id() const
+{
+ return timestamp;
+}
+
+
+Try<std::string> MemoryProfiler::DiskArtifact::path() const
+{
+ Try<Path> tmpdir = getTemporaryDirectoryPath();
+ if (tmpdir.isError()) {
+ return tmpdir.error();
+ }
+
+ return path::join(tmpdir.get(), filename);
+}
+
+
+http::Response MemoryProfiler::DiskArtifact::asHttp() const
+{
+ Try<std::string> _path = path();
+ if (_path.isError()) {
+ return http::BadRequest("Could not compute file path: " + _path.error());
+ }
+
+ // If we get here, we want to serve the file that *should* be on disk.
+ // Verify that it still exists before attempting to serve it.
+ //
+ // TODO(bevers): Store a checksum and verify that it matches.
+ if (!os::stat::isfile(_path.get())) {
+ return http::BadRequest("Requested file was deleted from local disk.");
+ }
+
+ process::http::OK response;
+ response.type = response.PATH;
+ response.path = _path.get();
+ response.headers["Content-Type"] = "application/octet-stream";
+ response.headers["Content-Disposition"] =
+ strings::format("attachment; filename=%s", _path.get()).get();
+
+ return response;
+}
+
+
+Try<Nothing> MemoryProfiler::DiskArtifact::generate(
+ time_t requestedTimestamp,
+ std::function<Try<Nothing>(const std::string&)> generator)
+{
+ // Nothing to do if the requestd file already exists.
+ if (timestamp.isSome() && timestamp.get() == requestedTimestamp) {
+ return Nothing();
+ }
+
+ Try<std::string> path_ = path();
+ if (path_.isError()) {
+ return Error("Could not determine target path: " + path_.get());
+ }
+
+ Try<Nothing> result = generator(path_.get());
+
+ if (result.isError()) {
+ // The old file might still be fine on disk, but there's no good way to
+ // verify so we assume that the error rendered it unusable.
+ timestamp = Error(result.error());
+ return result;
+ }
+
+ timestamp = requestedTimestamp;
+
+ return Nothing();
+}
+
+
+
+// TODO(bevers): Add a query parameter to select json or html format.
+// TODO(bevers): Add a query parameter to configure the sampling interval.
+Future<http::Response> MemoryProfiler::start(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&)
+{
+ if (!detectJemalloc()) {
+ return http::BadRequest(JEMALLOC_NOT_DETECTED_MESSAGE);
+ }
+
+ Duration duration = DEFAULT_COLLECTION_TIME;
+
+ // TODO(bevers): Introduce `http::Request::extractQueryParameter<T>(string)`
+ // instead of doing it ad-hoc here.
+ Option<std::string> durationParameter = request.url.query.get("duration");
+ if (durationParameter.isSome()) {
+ Try<Duration> parsed = Duration::parse(durationParameter.get());
+ if (parsed.isError()) {
+ return http::BadRequest(
+ "Could not parse parameter 'duration': " + parsed.error());
+ }
+ duration = parsed.get();
+ }
+
+ if (duration < MINIMUM_COLLECTION_TIME ||
+ duration > MAXIMUM_COLLECTION_TIME) {
+ return http::BadRequest(
+ "Duration '" + stringify(duration) + "' must be between "
+ + stringify(MINIMUM_COLLECTION_TIME) + " and "
+ + stringify(MAXIMUM_COLLECTION_TIME) + ".");
+ }
+
+ Try<bool> wasActive = jemalloc::startProfiling();
+ if (wasActive.isError()) {
+ return http::BadRequest(JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE);
+ }
+
+ if (!wasActive.get()) {
+ time_t id = std::chrono::system_clock::to_time_t(
+ std::chrono::system_clock::now());
+ currentRun = ProfilingRun(this, id, duration);
+ }
+
+ JSON::Object response;
+
+ // This can happpen when jemalloc was configured e.g. via the `MALLOC_CONF`
+ // environment variable. We don't touch it in this case.
+ if (!currentRun.isSome()) {
+ return http::Conflict("Heap profiling was started externally.");
+ }
+
+ std::string message = wasActive.get() ?
+ "Heap profiling is already active." :
+ "Successfully started new heap profiling run.";
+ message +=
+ " After the remaining time has elapsed, download the generated profile"
+ " at `/memory-profiler/download/raw?id=" + stringify(currentRun->id) + "`."
+ " Visit `/memory-profiler/stop` to end the run prematurely.";
+ response.values["id"] = currentRun->id;
+ // Adding 0.5 to round to nearest integer value.
+ response.values["remaining_seconds"] = stringify(static_cast<int>(
+ currentRun->timer.timeout().remaining().secs() + 0.5));
+ response.values["message"] = message;
+
+ return http::OK(response);
+}
+
+
+// TODO(bevers): Add a way to dump an intermediate profile without
+// stopping the data collection.
+Future<http::Response> MemoryProfiler::stop(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&)
+{
+ if (!detectJemalloc()) {
+ return http::BadRequest(JEMALLOC_NOT_DETECTED_MESSAGE);
+ }
+
+ Try<bool> active = jemalloc::profilingActive();
+ if (active.isError()) {
+ return http::BadRequest(
+ "Error interfacing with jemalloc: " + active.error());
+ }
+
+ if (!currentRun.isSome() && active.get()) {
+ // TODO(bevers): Allow stopping even in this case.
+ return http::BadRequest(
+ "Profiling is active, but was not started by libprocess."
+ " Accessing the raw profile through libprocess is currently"
+ " not supported.");
+ }
+
+ Try<time_t> generated = stopAndGenerateRawProfile();
+ if (generated.isError()) {
+ return http::BadRequest(generated.error());
+ }
+
+ Try<bool> stillActive = jemalloc::profilingActive();
+ CHECK(stillActive.isError() || !stillActive.get());
+
+ std::string message =
+ "Successfully stopped memory profiling run."
+ " Use one of the provided URLs to download results."
+ " Note that in order to generate graphs or symbolized profiles,"
+ " jeprof must be installed on the host machine and generation of"
+ " these files can take several minutes.";
+
+ std::string id = stringify(generated.get());
+
+ JSON::Object result;
+ result.values["id"] = id;
+ result.values["message"] = message;
+
+ result.values["url_raw_profile"] =
+ "./memory-profiler/download/raw?id=" + stringify(id);
+
+ result.values["url_graph"] =
+ "./memory-profiler/download/graph?id=" + stringify(id);
+
+ result.values["url_symbolized_profile"] =
+ "./memory-profiler/download/text?id=" + stringify(id);
+
+ return http::OK(result);
+}
+
+
+// A simple wrapper to discard the result, necessary so we can
+// use this as the target for `process::delay()`.
+void MemoryProfiler::_stopAndGenerateRawProfile()
+{
+ stopAndGenerateRawProfile();
+}
+
+
+Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
+{
+ ASSERT(detectJemalloc());
+
+ VLOG(1) << "Attempting to stop current profiling run.";
+
+ // Return the id of the last successful run if there is no current
+ // profiling run.
+ if (!currentRun.isSome()) {
+ return jemallocRawProfile.id();
+ }
+
+ Try<bool> stopped = jemalloc::stopProfiling();
+
+ if (stopped.isError()) {
+ LOG(WARNING) << "Failed to stop memory profiling: " << stopped.error();
+
+ // Don't give up. Probably it will fail again in the future, but at least
+ // the problem will be clearly visible in the logs.
+ currentRun->extend(this, Seconds(5));
+
+ return Error(stopped.error());
+ }
+
+ // Heap profiling should not be active any more.
+ // We won't retry stopping and generating a profile after this point:
+ // We're not actively sampling any more, and if the user still cares
+ // about this profile they will get the data with the next run.
+ Try<bool> stillActive = jemalloc::profilingActive();
+ CHECK(stillActive.isError() || !stillActive.get());
+
+ time_t runId = currentRun->id;
+ Clock::cancel(currentRun->timer);
+ currentRun = None();
+
+ if (!stopped.get()) {
+ // This is a weird state to end up in, apparently something else in this
+ // process stopped profiling independently of us.
+ // If there was some valuable, un-dumped data it is still possible to get
+ // it by starting a new run.
+ return Error(
+ "Memory profiling unexpectedly inactive; not dumping profile."
+ " Ensure nothing else is interfacing with jemalloc in this process.");
+ }
+
+ Try<Nothing> generated = jemallocRawProfile.generate(
+ runId,
+ [this](const std::string& outputPath) -> Try<Nothing> {
+ // Make sure we actually have permissions to write to the file and that
+ // there is at least a little bit space left on the device.
+ const std::string data(DUMMY_FILE_SIZE, '\0');
+ Try<Nothing> written = os::write(outputPath, data);
+ if (written.isError()) {
+ return Error(written.error());
+ }
+
+ // Verify independently that the file was actually written.
+ Try<Bytes> size = os::stat::size(outputPath);
+ if (size.isError() || size.get() != DUMMY_FILE_SIZE) {
+ return Error(strings::format(
+ "Couldn't verify integrity of dump file %s", outputPath).get());
+ }
+
+ // Finally, do the real dump.
+ return jemalloc::dump(outputPath);
+ });
+
+ if (generated.isError()) {
+ std::string errorMessage = "Could not dump profile: " + generated.error();
+ LOG(WARNING) << errorMessage;
+ return Error(errorMessage);
+ }
+
+ return runId;
+}
+
+
+Future<http::Response> MemoryProfiler::downloadRaw(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&)
+{
+ Result<time_t> requestedId = extractIdFromRequest(request);
+
+ // Verify that `id` has the correct version if it was explicitly passed.
+ if (requestedId.isError()) {
+ return http::BadRequest("Invalid parameter 'id': " + requestedId.error());
+ }
+
+ if (jemallocRawProfile.id().isError()) {
+ return http::BadRequest(
+ "No heap profile exists: " + jemallocRawProfile.id().error());
+ }
+
+ if (requestedId.isSome() &&
+ (requestedId.get() != jemallocRawProfile.id().get())) {
+ return http::BadRequest(
+ "Cannot serve requested id #" + stringify(requestedId.get()));
+ }
+
+ if (currentRun.isSome() && !requestedId.isSome()) {
+ return http::BadRequest(
+ "A profiling run is currently in progress. To download results of a"
+ " previous run, please pass an `id` explicitly.");
+ }
+
+ return jemallocRawProfile.asHttp();
+}
+
+
+Future<http::Response> MemoryProfiler::downloadGraph(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&)
+{
+ Result<time_t> requestedId = extractIdFromRequest(request);
+
+ // Verify that `id` has the correct version if it was explicitly passed.
+ if (requestedId.isError()) {
+ return http::BadRequest("Invalid parameter 'id': " + requestedId.error());
+ }
+
+ if (jemallocRawProfile.id().isError()) {
+ return http::BadRequest(
+ "No source profile exists: " + jemallocRawProfile.id().error());
+ }
+
+ if (currentRun.isSome() && !requestedId.isSome()) {
+ return http::BadRequest(
+ "A profiling run is currently in progress. To download results of a"
+ " previous run, please pass an `id` explicitly.");
+ }
+
+ time_t rawId = jemallocRawProfile.id().get();
+
+ // Use the latest version as default.
+ if (requestedId.isNone()) {
+ requestedId = rawId;
+ }
+
+ // Generate the graph with the given id, or return the cached file on disk.
+ Try<Nothing> result = jeprofGraph.generate(
+ rawId,
+ [&](const std::string& outputPath) -> Try<Nothing> {
+ if (!(requestedId.get() == jemallocRawProfile.id().get())) {
+ return Error("Requested outdated version.");
+ }
+
+ return generateJeprofFile(
+ jemallocRawProfile.path(),
+ "--svg",
+ outputPath);
+ });
+
+ if (result.isError()) {
+ return http::BadRequest("Could not generate file: " + result.error());
+ }
+
+ return jeprofGraph.asHttp();
+}
+
+
+Future<http::Response> MemoryProfiler::downloadTextProfile(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&)
+{
+ Result<time_t> requestedId = extractIdFromRequest(request);
+
+ // Verify that `id` has the correct version if it was explicitly passed.
+ if (requestedId.isError()) {
+ return http::BadRequest("Invalid parameter 'id': " + requestedId.error());
+ }
+
+ if (jemallocRawProfile.id().isError()) {
+ return http::BadRequest(
+ "No source profile exists: " + jemallocRawProfile.id().error());
+ }
+
+ if (currentRun.isSome() && !requestedId.isSome()) {
+ return http::BadRequest(
+ "A profiling run is currently in progress. To download results of a"
+ " previous run, please pass an `id` explicitly.");
+ }
+
+ time_t rawId = jemallocRawProfile.id().get();
+
+ // Use the latest version as default.
+ if (requestedId.isNone()) {
+ requestedId = rawId;
+ }
+
+ // Generate the profile with the given timestamp, or return the cached file
+ // on disk.
+ Try<Nothing> result = jeprofSymbolizedProfile.generate(
+ requestedId.get(),
+ [&](const std::string& outputPath) -> Try<Nothing>
+ {
+ if (!(requestedId.get() == jemallocRawProfile.id().get())) {
+ return Error("Requested outdated version.");
+ }
+
+ return generateJeprofFile(
+ jemallocRawProfile.path(),
+ "--text",
+ outputPath);
+ });
+
+ if (result.isError()) {
+ return http::BadRequest("Could not generate file: " + result.error());
+ }
+
+ return jeprofSymbolizedProfile.asHttp();
+}
+
+
+// TODO(bevers): Allow passing custom options via query parameters.
+Future<http::Response> MemoryProfiler::statistics(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&)
+{
+ if (!detectJemalloc()) {
+ return http::BadRequest(JEMALLOC_NOT_DETECTED_MESSAGE);
+ }
+
+ const std::string options = "J"; // 'J' selects JSON output format.
+
+ std::string statistics;
+
+#ifdef LIBPROCESS_ALLOW_JEMALLOC
+ ::malloc_stats_print([](void* opaque, const char* msg) {
+ std::string* statistics = static_cast<std::string*>(opaque);
+ *statistics += msg;
+ }, &statistics, options.c_str());
+#endif
+
+ return http::OK(statistics, "application/json; charset=utf-8");
+}
+
+
+Future<http::Response> MemoryProfiler::state(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&)
+{
+ bool detected = detectJemalloc();
+
+ JSON::Object state;
+
+ {
+ // State unrelated to jemalloc.
+ JSON::Object profilerState;
+ profilerState.values["jemalloc_detected"] = detected;
+
+ profilerState.values["tmpdir"] = stringify(
+ temporaryDirectory.getOrElse("Not yet generated."));
+
+ {
+ JSON::Object runInformation;
+ if (currentRun.isSome()) {
+ runInformation.values["id"] = currentRun->id;
+ runInformation.values["remaining_seconds"] =
+ currentRun->timer.timeout().remaining().secs();
+ } else if (jemallocRawProfile.id().isSome()) {
+ runInformation.values["id"] = jemallocRawProfile.id().get();
+ runInformation.values["remaining_seconds"] = 0;
+ } else {
+ runInformation.values["id"] = JSON::Null();
+ }
+
+ profilerState.values["current_run"] = std::move(runInformation);
+ }
+
+ state.values["memory_profiler"] = std::move(profilerState);
+ }
+
+ if (!detected) {
+ return http::OK(state);
+ }
+
+ {
+ // Holds relevant parts of the current jemalloc state.
+ JSON::Object jemallocState;
+
+ {
+ // Holds malloc configuration from various sources.
+ JSON::Object mallocConf;
+
+ // User-specified malloc configuration that was added via
+ // the `MALLOC_CONF` environment variable.
+ mallocConf.values["environment"] =
+ os::getenv("MALLOC_CONF").getOrElse("");
+
+ // Compile-time malloc configuration that was added at build time via
+ // the `--with-malloc-conf` flag.
+ Try<const char*> builtinMallocConf = readJemallocSetting<const char*>(
+ "config.malloc_conf");
+
+ if (builtinMallocConf.isError()) {
+ mallocConf.values["build_options"] = builtinMallocConf.error();
+ } else {
+ mallocConf.values["build_options"] = builtinMallocConf.get();
+ }
+
+ // TODO(bevers): System-wide jemalloc settings can be specified by
+ // creating a symlink at /etc/malloc.conf whose pointed-to value is read
+ // as an option string.
+ // Application-specific jemalloc settings can be specified by creating
+ // an externally visible symbol called `malloc_conf`.
+ // We should also display both of these here.
+
+ jemallocState.values["malloc_conf"] = std::move(mallocConf);
+ }
+
+ // Whether jemalloc was compiled with support for heap profiling.
+ Try<bool> profilingSupported = readJemallocSetting<bool>("config.prof");
+
+ if (profilingSupported.isError()) {
+ jemallocState.values["profiling_enabled"] = profilingSupported.error();
+ } else {
+ jemallocState.values["profiling_enabled"] = profilingSupported.get();
+ }
+
+ // Whether profiling is currently active.
+ Try<bool> profilingActive = readJemallocSetting<bool>("prof.active");
+
+ if (profilingActive.isError()) {
+ jemallocState.values["profiling_active"] = profilingActive.error();
+ } else {
+ jemallocState.values["profiling_active"] = profilingActive.get();
+ }
+
+ state.values["jemalloc"] = std::move(jemallocState);
+ }
+
+ return http::OK(state);
+}
+
+
+} // namespace process {
http://git-wip-us.apache.org/repos/asf/mesos/blob/81c9978c/3rdparty/libprocess/src/memory_profiler.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.hpp b/3rdparty/libprocess/src/memory_profiler.hpp
new file mode 100644
index 0000000..5a359df
--- /dev/null
+++ b/3rdparty/libprocess/src/memory_profiler.hpp
@@ -0,0 +1,163 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+#ifndef __PROCESS_MEMORYPROFILER_HPP__
+#define __PROCESS_MEMORYPROFILER_HPP__
+
+#include <functional>
+#include <string>
+
+#include <process/future.hpp>
+#include <process/http.hpp>
+#include <process/process.hpp>
+
+#include <stout/nothing.hpp>
+#include <stout/option.hpp>
+#include <stout/path.hpp>
+#include <stout/try.hpp>
+
+
+namespace process {
+
+// This class provides support for taking advantage of the introspection
+// and memory profiling capabilities of the jemalloc memory allocator.
+//
+// For user-facing documentation on how to use the facilities provided
+// by this class, see `docs/memory-profiling.md` in the mesos repository.
+//
+// For more details about the implementation, see the comments
+// in `memory_profiler.cpp`.
+
+class MemoryProfiler : public Process<MemoryProfiler>
+{
+public:
+ MemoryProfiler(const Option<std::string>& authenticationRealm);
+ virtual ~MemoryProfiler() {}
+
+protected:
+ virtual void initialize();
+
+private:
+ static const std::string START_HELP();
+ static const std::string STOP_HELP();
+ static const std::string STATISTICS_HELP();
+ static const std::string STATE_HELP();
+ static const std::string DOWNLOAD_RAW_HELP();
+ static const std::string DOWNLOAD_TEXT_HELP();
+ static const std::string DOWNLOAD_GRAPH_HELP();
+
+ // HTTP endpoints.
+ // Refer to the `HELP()` messages for detailed documentation.
+
+ // Starts memory profiling.
+ Future<http::Response> start(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&);
+
+ // Stops memory profiling and dumps collected data.
+ Future<http::Response> stop(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&);
+
+ // Returns a raw heap profile.
+ Future<http::Response> downloadRaw(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&);
+
+ // Generates and returns a call graph in svg format.
+ Future<http::Response> downloadGraph(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&);
+
+ // Generates and returns a symbolized heap profile.
+ Future<http::Response> downloadTextProfile(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&);
+
+ // Shows memory allocation statistics.
+ Future<http::Response> statistics(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&);
+
+ // Shows the configuration of the memory-profiler process.
+ Future<http::Response> state(
+ const http::Request& request,
+ const Option<http::authentication::Principal>&);
+
+ // Internal functions and data members.
+
+ // Deactivates data collection and attempts to dump the raw profile to disk.
+ Try<time_t> stopAndGenerateRawProfile();
+
+ void _stopAndGenerateRawProfile();
+
+ // The authentication realm that the profiler's HTTP endpoints will be
+ // installed into.
+ Option<std::string> authenticationRealm;
+
+ // Stores information about the current profiling run.
+ // When the timer reaches zero, a dump of the collected
+ // data will be attempted.
+ class ProfilingRun
+ {
+ public:
+ ProfilingRun(MemoryProfiler*, time_t, const Duration&);
+ void extend(MemoryProfiler*, const Duration&);
+
+ time_t id;
+ Timer timer;
+ };
+
+ Option<ProfilingRun> currentRun;
+
+ // Represents a file on the filesystem that is generated as the result
+ // of running some action.
+ class DiskArtifact
+ {
+ public:
+ DiskArtifact(const std::string& filename);
+
+ const Try<time_t>& id() const;
+
+ // This is generated lazily to avoid creating a temporary
+ // directory unless we actually need it.
+ Try<std::string> path() const;
+
+ // If the file with the given timestamp does not exist on disk, calls the
+ // supplied `generator` function that should write it to the specified
+ // location.
+ Try<Nothing> generate(
+ time_t timestamp,
+ std::function<Try<Nothing>(const std::string& outputPath)> generator);
+
+ // Generates an error response if the file doesn't exist, or a download
+ // if it does.
+ http::Response asHttp() const;
+
+ private:
+ std::string filename;
+
+ // A timestamp of the last successful creation that serves as unique id,
+ // or the reason why it couldn't be created.
+ Try<time_t> timestamp;
+ };
+
+ DiskArtifact jemallocRawProfile;
+
+ DiskArtifact jeprofSymbolizedProfile;
+
+ DiskArtifact jeprofGraph;
+};
+
+} // namespace process {
+
+#endif // __PROCESS_MEMORYPROFILER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/81c9978c/3rdparty/libprocess/src/process.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/process.cpp b/3rdparty/libprocess/src/process.cpp
index 9eb3746..21931db 100644
--- a/3rdparty/libprocess/src/process.cpp
+++ b/3rdparty/libprocess/src/process.cpp
@@ -116,6 +116,7 @@
#include "event_queue.hpp"
#include "gate.hpp"
#include "http_proxy.hpp"
+#include "memory_profiler.hpp"
#include "process_reference.hpp"
#include "socket_manager.hpp"
#include "run_queue.hpp"
@@ -247,6 +248,14 @@ struct Flags : public virtual flags::FlagsBase
"libprocess is listening may not match the address from\n"
"which libprocess connects to other actors.\n",
false);
+
+ // TODO(bevers): Set the default to `true` after gathering some
+ // real-world experience with this.
+ add(&Flags::memory_profiling,
+ "memory_profiling",
+ "If set to false, disables the memory profiling functionality\n"
+ "of libprocess.",
+ false);
}
Option<net::IP> ip;
@@ -255,6 +264,7 @@ struct Flags : public virtual flags::FlagsBase
Option<int> port;
Option<int> advertise_port;
bool require_peer_address_ip_match;
+ bool memory_profiling;
};
} // namespace internal {
@@ -1188,7 +1198,7 @@ bool initialize(
// | |--All other processes
// |
// |--logging
- // |--profiler
+ // |--(memory-)profiler
// |--processesRoute
//
// authenticator_manager
@@ -1207,6 +1217,12 @@ bool initialize(
// Create the global profiler process.
spawn(new Profiler(readwriteAuthenticationRealm), true);
+ // Create the global memory profiler process unless memory profiling
+ // was disabled.
+ if (libprocess_flags->memory_profiling) {
+ spawn(new MemoryProfiler(readwriteAuthenticationRealm), true);
+ }
+
// Create the global system statistics process.
spawn(new System(), true);
[12/12] mesos git commit: Cleaned up method ordering in memory
profiler.
Posted by al...@apache.org.
Cleaned up method ordering in memory profiler.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/963a289c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/963a289c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/963a289c
Branch: refs/heads/master
Commit: 963a289c6b3bfcec420092105c1d39837a261bbc
Parents: f2cb907
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 01:35:40 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 204 +++++++++++------------
3rdparty/libprocess/src/memory_profiler.hpp | 28 ++--
2 files changed, 113 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/963a289c/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index 2b3af3d..1991bbe 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -545,25 +545,16 @@ void MemoryProfiler::initialize()
DOWNLOAD_GRAPH_HELP(),
&MemoryProfiler::downloadGraph);
- route("/state",
- authenticationRealm,
- STATE_HELP(),
- &MemoryProfiler::state);
-
route("/statistics",
authenticationRealm,
STATISTICS_HELP(),
&MemoryProfiler::statistics);
-}
-
-MemoryProfiler::MemoryProfiler(const Option<string>& _authenticationRealm)
- : ProcessBase("memory-profiler"),
- authenticationRealm(_authenticationRealm),
- jemallocRawProfile(RAW_PROFILE_FILENAME),
- jeprofSymbolizedProfile(SYMBOLIZED_PROFILE_FILENAME),
- jeprofGraph(GRAPH_FILENAME)
-{}
+ route("/state",
+ authenticationRealm,
+ STATE_HELP(),
+ &MemoryProfiler::state);
+}
MemoryProfiler::ProfilingRun::ProfilingRun(
@@ -670,6 +661,14 @@ Try<Nothing> MemoryProfiler::DiskArtifact::generate(
}
+MemoryProfiler::MemoryProfiler(const Option<string>& _authenticationRealm)
+ : ProcessBase("memory-profiler"),
+ authenticationRealm(_authenticationRealm),
+ jemallocRawProfile(RAW_PROFILE_FILENAME),
+ jeprofSymbolizedProfile(SYMBOLIZED_PROFILE_FILENAME),
+ jeprofGraph(GRAPH_FILENAME)
+{}
+
// TODO(bevers): Add a query parameter to select json or html format.
// TODO(bevers): Add a query parameter to configure the sampling interval.
@@ -802,79 +801,6 @@ Future<http::Response> MemoryProfiler::stop(
}
-void MemoryProfiler::stopAndGenerateRawProfile()
-{
- ASSERT(detectJemalloc());
-
- VLOG(1) << "Attempting to stop current profiling run";
-
- // If there is no current profiling run, there is nothing to do.
- if (!currentRun.isSome()) {
- return;
- }
-
- Try<bool> stopped = jemalloc::stopProfiling();
-
- if (stopped.isError()) {
- LOG(WARNING) << "Failed to stop memory profiling: " << stopped.error();
-
- // Don't give up. Probably it will fail again in the future, but at least
- // the problem will be clearly visible in the logs.
- currentRun->extend(this, Seconds(5));
-
- return;
- }
-
- // Heap profiling should not be active any more.
- // We won't retry stopping and generating a profile after this point:
- // We're not actively sampling any more, and if the user still cares
- // about this profile they will get the data with the next run.
- Try<bool> stillActive = jemalloc::profilingActive();
- CHECK(stillActive.isError() || !stillActive.get());
-
- time_t runId = currentRun->id;
- Clock::cancel(currentRun->timer);
- currentRun = None();
-
- if (!stopped.get()) {
- // This is a weird state to end up in, apparently something else in this
- // process stopped profiling independently of us.
- // If there was some valuable, un-dumped data it is still possible to get
- // it by starting a new run.
- LOG(WARNING)
- << "Memory profiling unexpectedly inactive; not dumping profile. Ensure"
- << " nothing else is interfacing with jemalloc in this process";
- return;
- }
-
- Try<Nothing> generated = jemallocRawProfile.generate(
- runId,
- [this](const string& outputPath) -> Try<Nothing> {
- // Make sure we actually have permissions to write to the file and that
- // there is at least a little bit space left on the device.
- const string data(DUMMY_FILE_SIZE, '\0');
- Try<Nothing> written = os::write(outputPath, data);
- if (written.isError()) {
- return Error(written.error());
- }
-
- // Verify independently that the file was actually written.
- Try<Bytes> size = os::stat::size(outputPath);
- if (size.isError() || size.get() != DUMMY_FILE_SIZE) {
- return Error(strings::format(
- "Couldn't verify integrity of dump file %s", outputPath).get());
- }
-
- // Finally, do the real dump.
- return jemalloc::dump(outputPath);
- });
-
- if (generated.isError()) {
- LOG(WARNING) << "Could not dump profile: " + generated.error();
- }
-}
-
-
Future<http::Response> MemoryProfiler::downloadRaw(
const http::Request& request,
const Option<http::authentication::Principal>&)
@@ -908,7 +834,7 @@ Future<http::Response> MemoryProfiler::downloadRaw(
}
-Future<http::Response> MemoryProfiler::downloadGraph(
+Future<http::Response> MemoryProfiler::downloadTextProfile(
const http::Request& request,
const Option<http::authentication::Principal>&)
{
@@ -938,17 +864,19 @@ Future<http::Response> MemoryProfiler::downloadGraph(
requestedId = rawId;
}
- // Generate the graph with the given id, or return the cached file on disk.
- Try<Nothing> result = jeprofGraph.generate(
- rawId,
- [&](const string& outputPath) -> Try<Nothing> {
+ // Generate the profile with the given timestamp, or return the cached file
+ // on disk.
+ Try<Nothing> result = jeprofSymbolizedProfile.generate(
+ requestedId.get(),
+ [&](const string& outputPath) -> Try<Nothing>
+ {
if (!(requestedId.get() == jemallocRawProfile.id().get())) {
return Error("Requested version cannot be served");
}
return generateJeprofFile(
jemallocRawProfile.path(),
- "--svg",
+ "--text",
outputPath);
});
@@ -957,11 +885,11 @@ Future<http::Response> MemoryProfiler::downloadGraph(
"Could not generate file: " + result.error() + ".\n");
}
- return jeprofGraph.asHttp();
+ return jeprofSymbolizedProfile.asHttp();
}
-Future<http::Response> MemoryProfiler::downloadTextProfile(
+Future<http::Response> MemoryProfiler::downloadGraph(
const http::Request& request,
const Option<http::authentication::Principal>&)
{
@@ -991,19 +919,17 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
requestedId = rawId;
}
- // Generate the profile with the given timestamp, or return the cached file
- // on disk.
- Try<Nothing> result = jeprofSymbolizedProfile.generate(
- requestedId.get(),
- [&](const string& outputPath) -> Try<Nothing>
- {
+ // Generate the graph with the given id, or return the cached file on disk.
+ Try<Nothing> result = jeprofGraph.generate(
+ rawId,
+ [&](const string& outputPath) -> Try<Nothing> {
if (!(requestedId.get() == jemallocRawProfile.id().get())) {
return Error("Requested version cannot be served");
}
return generateJeprofFile(
jemallocRawProfile.path(),
- "--text",
+ "--svg",
outputPath);
});
@@ -1012,7 +938,7 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
"Could not generate file: " + result.error() + ".\n");
}
- return jeprofSymbolizedProfile.asHttp();
+ return jeprofGraph.asHttp();
}
@@ -1138,4 +1064,76 @@ Future<http::Response> MemoryProfiler::state(
}
+void MemoryProfiler::stopAndGenerateRawProfile()
+{
+ ASSERT(detectJemalloc());
+
+ VLOG(1) << "Attempting to stop current profiling run";
+
+ // If there is no current profiling run, there is nothing to do.
+ if (!currentRun.isSome()) {
+ return;
+ }
+
+ Try<bool> stopped = jemalloc::stopProfiling();
+
+ if (stopped.isError()) {
+ LOG(WARNING) << "Failed to stop memory profiling: " << stopped.error();
+
+ // Don't give up. Probably it will fail again in the future, but at least
+ // the problem will be clearly visible in the logs.
+ currentRun->extend(this, Seconds(5));
+
+ return;
+ }
+
+ // Heap profiling should not be active any more.
+ // We won't retry stopping and generating a profile after this point:
+ // We're not actively sampling any more, and if the user still cares
+ // about this profile they will get the data with the next run.
+ Try<bool> stillActive = jemalloc::profilingActive();
+ CHECK(stillActive.isError() || !stillActive.get());
+
+ time_t runId = currentRun->id;
+ Clock::cancel(currentRun->timer);
+ currentRun = None();
+
+ if (!stopped.get()) {
+ // This is a weird state to end up in, apparently something else in this
+ // process stopped profiling independently of us.
+ // If there was some valuable, un-dumped data it is still possible to get
+ // it by starting a new run.
+ LOG(WARNING)
+ << "Memory profiling unexpectedly inactive; not dumping profile. Ensure"
+ << " nothing else is interfacing with jemalloc in this process";
+ return;
+ }
+
+ Try<Nothing> generated = jemallocRawProfile.generate(
+ runId,
+ [this](const string& outputPath) -> Try<Nothing> {
+ // Make sure we actually have permissions to write to the file and that
+ // there is at least a little bit space left on the device.
+ const string data(DUMMY_FILE_SIZE, '\0');
+ Try<Nothing> written = os::write(outputPath, data);
+ if (written.isError()) {
+ return Error(written.error());
+ }
+
+ // Verify independently that the file was actually written.
+ Try<Bytes> size = os::stat::size(outputPath);
+ if (size.isError() || size.get() != DUMMY_FILE_SIZE) {
+ return Error(strings::format(
+ "Couldn't verify integrity of dump file %s", outputPath).get());
+ }
+
+ // Finally, do the real dump.
+ return jemalloc::dump(outputPath);
+ });
+
+ if (generated.isError()) {
+ LOG(WARNING) << "Could not dump profile: " + generated.error();
+ }
+}
+
} // namespace process {
http://git-wip-us.apache.org/repos/asf/mesos/blob/963a289c/3rdparty/libprocess/src/memory_profiler.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.hpp b/3rdparty/libprocess/src/memory_profiler.hpp
index 5cbe507..dc98998 100644
--- a/3rdparty/libprocess/src/memory_profiler.hpp
+++ b/3rdparty/libprocess/src/memory_profiler.hpp
@@ -10,8 +10,8 @@
// See the License for the specific language governing permissions and
// limitations under the License
-#ifndef __PROCESS_MEMORYPROFILER_HPP__
-#define __PROCESS_MEMORYPROFILER_HPP__
+#ifndef __PROCESS_MEMORY_PROFILER_HPP__
+#define __PROCESS_MEMORY_PROFILER_HPP__
#include <functional>
#include <string>
@@ -25,18 +25,16 @@
#include <stout/path.hpp>
#include <stout/try.hpp>
-
namespace process {
-// This class provides support for taking advantage of the introspection
-// and memory profiling capabilities of the jemalloc memory allocator.
+// This class provides support for memory profiling and introspection
+// using the capabilities of the jemalloc memory allocator.
//
// For user-facing documentation on how to use the facilities provided
// by this class, see `docs/memory-profiling.md` in the mesos repository.
//
// For more details about the implementation, see the comments
// in `memory_profiler.cpp`.
-
class MemoryProfiler : public Process<MemoryProfiler>
{
public:
@@ -49,11 +47,11 @@ protected:
private:
static const std::string START_HELP();
static const std::string STOP_HELP();
- static const std::string STATISTICS_HELP();
- static const std::string STATE_HELP();
static const std::string DOWNLOAD_RAW_HELP();
static const std::string DOWNLOAD_TEXT_HELP();
static const std::string DOWNLOAD_GRAPH_HELP();
+ static const std::string STATISTICS_HELP();
+ static const std::string STATE_HELP();
// HTTP endpoints.
// Refer to the `HELP()` messages for detailed documentation.
@@ -73,13 +71,13 @@ private:
const http::Request& request,
const Option<http::authentication::Principal>&);
- // Generates and returns a call graph in svg format.
- Future<http::Response> downloadGraph(
+ // Generates and returns a symbolized heap profile.
+ Future<http::Response> downloadTextProfile(
const http::Request& request,
const Option<http::authentication::Principal>&);
- // Generates and returns a symbolized heap profile.
- Future<http::Response> downloadTextProfile(
+ // Generates and returns a call graph in svg format.
+ Future<http::Response> downloadGraph(
const http::Request& request,
const Option<http::authentication::Principal>&);
@@ -93,7 +91,7 @@ private:
const http::Request& request,
const Option<http::authentication::Principal>&);
- // Internal functions and data members.
+ // Internal functions, helper classes, and data members.
// Deactivates data collection and attempts to dump the raw profile to disk.
void stopAndGenerateRawProfile();
@@ -150,12 +148,10 @@ private:
};
DiskArtifact jemallocRawProfile;
-
DiskArtifact jeprofSymbolizedProfile;
-
DiskArtifact jeprofGraph;
};
} // namespace process {
-#endif // __PROCESS_MEMORYPROFILER_HPP__
+#endif // __PROCESS_MEMORY_PROFILER_HPP__
[02/12] mesos git commit: Simplified download pipeline in
MemoryProfiler.
Posted by al...@apache.org.
Simplified download pipeline in MemoryProfiler.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/915bf398
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/915bf398
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/915bf398
Branch: refs/heads/master
Commit: 915bf398a930e2b7f0ec571fa5a5712d7bb3ca82
Parents: 963a289
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 03:07:32 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 61 +++++++++++-------------
1 file changed, 28 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/915bf398/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index 1991bbe..db3045e 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -818,6 +818,7 @@ Future<http::Response> MemoryProfiler::downloadRaw(
"No heap profile exists: " + jemallocRawProfile.id().error() + ".\n");
}
+ // Only requests for the latest available version are allowed.
if (requestedId.isSome() &&
(requestedId.get() != jemallocRawProfile.id().get())) {
return http::BadRequest(
@@ -851,31 +852,27 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
"No source profile exists: " + jemallocRawProfile.id().error() + ".\n");
}
+ // Only requests for the latest available version are allowed.
+ if (requestedId.isSome() &&
+ (requestedId.get() != jemallocRawProfile.id().get())) {
+ return http::BadRequest(
+ "Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
+ }
+
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
"A profiling run is currently in progress. To download results of the"
" previous run, please pass an 'id' explicitly.\n");
}
- time_t rawId = jemallocRawProfile.id().get();
-
- // Use the latest version as default.
- if (requestedId.isNone()) {
- requestedId = rawId;
- }
-
- // Generate the profile with the given timestamp, or return the cached file
- // on disk.
+ // Generate the profile for the latest available version,
+ // or return the cached file on disk.
+ Try<string> rawProfilePath = jemallocRawProfile.path();
Try<Nothing> result = jeprofSymbolizedProfile.generate(
- requestedId.get(),
- [&](const string& outputPath) -> Try<Nothing>
- {
- if (!(requestedId.get() == jemallocRawProfile.id().get())) {
- return Error("Requested version cannot be served");
- }
-
+ jemallocRawProfile.id().get(),
+ [rawProfilePath](const string& outputPath) -> Try<Nothing> {
return generateJeprofFile(
- jemallocRawProfile.path(),
+ rawProfilePath,
"--text",
outputPath);
});
@@ -906,29 +903,27 @@ Future<http::Response> MemoryProfiler::downloadGraph(
"No source profile exists: " + jemallocRawProfile.id().error() + ".\n");
}
+ // Only requests for the latest available version are allowed.
+ if (requestedId.isSome() &&
+ (requestedId.get() != jemallocRawProfile.id().get())) {
+ return http::BadRequest(
+ "Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
+ }
+
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
"A profiling run is currently in progress. To download results of the"
" previous run, please pass an 'id' explicitly.\n");
}
- time_t rawId = jemallocRawProfile.id().get();
-
- // Use the latest version as default.
- if (requestedId.isNone()) {
- requestedId = rawId;
- }
-
- // Generate the graph with the given id, or return the cached file on disk.
+ // Generate the profile for the latest available version,
+ // or return the cached file on disk.
+ Try<string> rawProfilePath = jemallocRawProfile.path();
Try<Nothing> result = jeprofGraph.generate(
- rawId,
- [&](const string& outputPath) -> Try<Nothing> {
- if (!(requestedId.get() == jemallocRawProfile.id().get())) {
- return Error("Requested version cannot be served");
- }
-
+ jemallocRawProfile.id().get(),
+ [rawProfilePath](const string& outputPath) -> Try<Nothing> {
return generateJeprofFile(
- jemallocRawProfile.path(),
+ rawProfilePath,
"--svg",
outputPath);
});
@@ -1111,7 +1106,7 @@ void MemoryProfiler::stopAndGenerateRawProfile()
Try<Nothing> generated = jemallocRawProfile.generate(
runId,
- [this](const string& outputPath) -> Try<Nothing> {
+ [](const string& outputPath) -> Try<Nothing> {
// Make sure we actually have permissions to write to the file and that
// there is at least a little bit space left on the device.
const string data(DUMMY_FILE_SIZE, '\0');
[07/12] mesos git commit: Ensured HTTP responses end with period and
linefeed.
Posted by al...@apache.org.
Ensured HTTP responses end with period and linefeed.
For consistency, all HTTP responses `MemoryProfiler` can produce now
end with a period and a linefeed character.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f5bd6534
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f5bd6534
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f5bd6534
Branch: refs/heads/master
Commit: f5bd65340257dbc7e1784f14a3837808d08be729
Parents: 58e224e
Author: Alexander Rukletsov <al...@apache.org>
Authored: Tue Apr 17 23:05:49 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 49 ++++++++++++++----------
1 file changed, 28 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f5bd6534/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index 995beea..7a52b96 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -108,7 +108,8 @@ environment variable, for example LD_PRELOAD=/usr/lib/libjemalloc.so
If you're running a mesos binary, and want to have it linked
against jemalloc by default, consider using the
---enable-jemalloc-allocator configuration option.)_";
+--enable-jemalloc-allocator configuration option.
+)_";
constexpr char JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE[] = R"_(
@@ -615,7 +616,8 @@ http::Response MemoryProfiler::DiskArtifact::asHttp() const
{
Try<std::string> _path = path();
if (_path.isError()) {
- return http::BadRequest("Could not compute file path: " + _path.error());
+ return http::BadRequest(
+ "Could not compute file path: " + _path.error() + ".\n");
}
// If we get here, we want to serve the file that *should* be on disk.
@@ -623,7 +625,7 @@ http::Response MemoryProfiler::DiskArtifact::asHttp() const
//
// TODO(bevers): Store a checksum and verify that it matches.
if (!os::stat::isfile(_path.get())) {
- return http::BadRequest("Requested file was deleted from local disk.");
+ return http::BadRequest("Requested file was deleted from local disk.\n");
}
process::http::OK response;
@@ -686,7 +688,7 @@ Future<http::Response> MemoryProfiler::start(
Try<Duration> parsed = Duration::parse(durationParameter.get());
if (parsed.isError()) {
return http::BadRequest(
- "Could not parse parameter 'duration': " + parsed.error());
+ "Could not parse parameter 'duration': " + parsed.error() + ".\n");
}
duration = parsed.get();
}
@@ -696,7 +698,7 @@ Future<http::Response> MemoryProfiler::start(
return http::BadRequest(
"Duration '" + stringify(duration) + "' must be between "
+ stringify(MINIMUM_COLLECTION_TIME) + " and "
- + stringify(MAXIMUM_COLLECTION_TIME) + ".");
+ + stringify(MAXIMUM_COLLECTION_TIME) + ".\n");
}
Try<bool> wasActive = jemalloc::startProfiling();
@@ -715,7 +717,7 @@ Future<http::Response> MemoryProfiler::start(
// This can happpen when jemalloc was configured e.g. via the `MALLOC_CONF`
// environment variable. We don't touch it in this case.
if (!currentRun.isSome()) {
- return http::Conflict("Heap profiling was started externally.");
+ return http::Conflict("Heap profiling was started externally.\n");
}
std::string message = wasActive.get() ?
@@ -748,7 +750,7 @@ Future<http::Response> MemoryProfiler::stop(
Try<bool> active = jemalloc::profilingActive();
if (active.isError()) {
return http::BadRequest(
- "Error interfacing with jemalloc: " + active.error());
+ "Error interfacing with jemalloc: " + active.error() + ".\n");
}
if (!currentRun.isSome() && active.get()) {
@@ -756,12 +758,12 @@ Future<http::Response> MemoryProfiler::stop(
return http::BadRequest(
"Profiling is active, but was not started by libprocess."
" Accessing the raw profile through libprocess is currently"
- " not supported.");
+ " not supported.\n");
}
Try<time_t> generated = stopAndGenerateRawProfile();
if (generated.isError()) {
- return http::BadRequest(generated.error());
+ return http::BadRequest(generated.error() + ".\n");
}
Try<bool> stillActive = jemalloc::profilingActive();
@@ -886,24 +888,25 @@ Future<http::Response> MemoryProfiler::downloadRaw(
// Verify that `id` has the correct version if it was explicitly passed.
if (requestedId.isError()) {
- return http::BadRequest("Invalid parameter 'id': " + requestedId.error());
+ return http::BadRequest(
+ "Invalid parameter 'id': " + requestedId.error() + ".\n");
}
if (jemallocRawProfile.id().isError()) {
return http::BadRequest(
- "No heap profile exists: " + jemallocRawProfile.id().error());
+ "No heap profile exists: " + jemallocRawProfile.id().error() + ".\n");
}
if (requestedId.isSome() &&
(requestedId.get() != jemallocRawProfile.id().get())) {
return http::BadRequest(
- "Cannot serve requested id #" + stringify(requestedId.get()));
+ "Cannot serve requested id #" + stringify(requestedId.get()) + ".\n");
}
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
"A profiling run is currently in progress. To download results of a"
- " previous run, please pass an `id` explicitly.");
+ " previous run, please pass an 'id' explicitly.\n");
}
return jemallocRawProfile.asHttp();
@@ -918,18 +921,19 @@ Future<http::Response> MemoryProfiler::downloadGraph(
// Verify that `id` has the correct version if it was explicitly passed.
if (requestedId.isError()) {
- return http::BadRequest("Invalid parameter 'id': " + requestedId.error());
+ return http::BadRequest(
+ "Invalid parameter 'id': " + requestedId.error() + ".\n");
}
if (jemallocRawProfile.id().isError()) {
return http::BadRequest(
- "No source profile exists: " + jemallocRawProfile.id().error());
+ "No source profile exists: " + jemallocRawProfile.id().error() + ".\n");
}
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
"A profiling run is currently in progress. To download results of a"
- " previous run, please pass an `id` explicitly.");
+ " previous run, please pass an 'id' explicitly.\n");
}
time_t rawId = jemallocRawProfile.id().get();
@@ -954,7 +958,8 @@ Future<http::Response> MemoryProfiler::downloadGraph(
});
if (result.isError()) {
- return http::BadRequest("Could not generate file: " + result.error());
+ return http::BadRequest(
+ "Could not generate file: " + result.error() + ".\n");
}
return jeprofGraph.asHttp();
@@ -969,18 +974,19 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
// Verify that `id` has the correct version if it was explicitly passed.
if (requestedId.isError()) {
- return http::BadRequest("Invalid parameter 'id': " + requestedId.error());
+ return http::BadRequest(
+ "Invalid parameter 'id': " + requestedId.error() + ".\n");
}
if (jemallocRawProfile.id().isError()) {
return http::BadRequest(
- "No source profile exists: " + jemallocRawProfile.id().error());
+ "No source profile exists: " + jemallocRawProfile.id().error() + ".\n");
}
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
"A profiling run is currently in progress. To download results of a"
- " previous run, please pass an `id` explicitly.");
+ " previous run, please pass an `id` explicitly.\n");
}
time_t rawId = jemallocRawProfile.id().get();
@@ -1007,7 +1013,8 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
});
if (result.isError()) {
- return http::BadRequest("Could not generate file: " + result.error());
+ return http::BadRequest(
+ "Could not generate file: " + result.error() + ".\n");
}
return jeprofSymbolizedProfile.asHttp();
[03/12] mesos git commit: Improved style, comments,
and messages in MemoryProfiled.
Posted by al...@apache.org.
Improved style, comments, and messages in MemoryProfiled.
This patch addresses the following:
- Single quote instead of backtick in user facing messages;
- Use process id instead of hardcoded name;
- Typos and more precise wording in messages and comments;
- Formatting of help endpoints;
- Indentation;
- No period at the end of error messages.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/03221b0b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/03221b0b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/03221b0b
Branch: refs/heads/master
Commit: 03221b0b56fb5432df50e77491717c9f2c3056ed
Parents: 47e88dd
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 00:48:53 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/include/process/process.hpp | 1 -
3rdparty/libprocess/src/memory_profiler.cpp | 247 ++++++++++---------
2 files changed, 125 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/03221b0b/3rdparty/libprocess/include/process/process.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/process.hpp b/3rdparty/libprocess/include/process/process.hpp
index f68ecca..c36df99 100644
--- a/3rdparty/libprocess/include/process/process.hpp
+++ b/3rdparty/libprocess/include/process/process.hpp
@@ -44,7 +44,6 @@ namespace process {
class EventQueue;
class Gate;
class Logging;
-class MemoryProfiler;
class Sequence;
namespace firewall {
http://git-wip-us.apache.org/repos/asf/mesos/blob/03221b0b/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index d935211..1fb89a6 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -106,42 +106,39 @@ If the current binary was not compiled against jemalloc,
consider adding the path to libjemalloc to the LD_PRELOAD
environment variable, for example LD_PRELOAD=/usr/lib/libjemalloc.so
-If you're running a mesos binary, and want to have it linked
+If you're running a mesos binary and want to have it linked
against jemalloc by default, consider using the
---enable-jemalloc-allocator configuration option.
-)_";
+--enable-jemalloc-allocator configuration option)_";
constexpr char JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE[] = R"_(
-The current process seems to be using jemalloc, but
-profiling couldn't be enabled.
+The current process seems to be using jemalloc, but profiling
+couldn't be enabled.
-If you're using a custom version of libjemalloc, make sure
-that MALLOC_CONF="prof:true" is part of the environment. (The
-`/state` endpoint can be used to double-check the current malloc
-configuration)
+If you're using a custom version of libjemalloc, make sure that
+MALLOC_CONF="prof:true" is part of the environment. (The '/state'
+endpoint can be used to double-check the current malloc
+configuration).
-If the environment looks correct, make sure jemalloc was built with
-the --enable-stats and --enable-prof options enabled.
+If the environment looks correct, make sure jemalloc was built
+with the --enable-stats and --enable-prof options enabled.
If you're running a mesos binary that was built with the
---enable-memory-profiling option enabled and you're still seeing this
-message, please consider filing a bug report.
-)_";
+--enable-memory-profiling option enabled and you're still seeing
+this message, please consider filing a bug report)_";
-// Size in bytes of the dummy file that gets written when hitting '/start'.
+// Size in bytes of the dummy file that gets written when hitting `/start`.
constexpr int DUMMY_FILE_SIZE = 64 * 1024; // 64 KiB
-// The `detectJemalloc()`function below was taken from the folly library
+// The `detectJemalloc()` function below was taken from the folly library
// (called `usingJEMalloc()` there), originally distributed by Facebook, Inc
// under the Apache License.
//
-// It checks whether jemalloc is used as the current malloc implementation
-// by allocating one byte and checking whether the threads allocation counter
-// increased. This requires jemalloc to have been compiled with
-// the `--enable-stats option`.
+// It checks whether jemalloc is used as the current malloc implementation by
+// allocating one byte and checking if the threads allocation counter increases.
+// This requires jemalloc to have been compiled with `--enable-stats`.
bool detectJemalloc() noexcept {
#ifndef LIBPROCESS_ALLOW_JEMALLOC
return false;
@@ -267,10 +264,10 @@ Try<Nothing> writeJemallocSetting(const char* name, const T& value)
// All generated disk artifacts, i.e. profiles and graph files, are stored
// in this directory. It is generated lazily on the first call to
-// `getTemporaryDirectoryPath()` and never changed afterwards.
-// Locking is not needed because all accesses go through the same process,
-// MemoryProfiler, and hence are are always serialized with respect to each
-// other.
+// `getTemporaryDirectoryPath()` and is never changed afterwards.
+// Locking is not needed because all accesses go through the same instance of
+// the `MemoryProfiler` process and hence are always serialized with respect
+// to each other.
//
// TODO(bevers): This should be made available libprocess-global eventually,
// but right now this is the only class that has a use for it.
@@ -284,8 +281,8 @@ Try<Path> getTemporaryDirectoryPath() {
// TODO(bevers): Add a libprocess-specific override for the system-wide
// `TMPDIR`, for example `LIBPROCESS_TMPDIR`.
- std::string tmpdir = os::getenv("TMPDIR")
- .getOrElse(LIBPROCESS_DEFAULT_TMPDIR);
+ std::string tmpdir =
+ os::getenv("TMPDIR").getOrElse(LIBPROCESS_DEFAULT_TMPDIR);
std::string pathTemplate = path::join(tmpdir, "libprocess.XXXXXX");
@@ -297,7 +294,7 @@ Try<Path> getTemporaryDirectoryPath() {
temporaryDirectory = dir.get();
- VLOG(1) << "Using path " << dir.get() << " to store temporary files.";
+ VLOG(1) << "Using path " << dir.get() << " to store temporary files";
return temporaryDirectory.get();
}
@@ -317,8 +314,8 @@ Try<Nothing> generateJeprofFile(
// Note that the three parameters *MUST NOT* be controllable by the user
// accessing the HTTP endpoints, otherwise arbitrary shell commands could be
// trivially injected.
- // Apart from that, we dont need to be as careful here as with the actual heap
- // profile dump, because a failure will not crash the whole process.
+ // Apart from that, we dont need to be as careful here as with the actual
+ // heap profile dump, because a failure will not crash the whole process.
Try<std::string> result = os::shell(strings::format(
"jeprof %s /proc/self/exe %s > %s",
options,
@@ -327,9 +324,8 @@ Try<Nothing> generateJeprofFile(
if (result.isError()) {
return Error(
- "Error trying to run jeprof: " + result.error() + "."
- " Please make sure that jeprof is installed and that"
- " the input file contains data.");
+ "Error trying to run jeprof: " + result.error() + ". Please make sure"
+ " that jeprof is installed and that the input file contains data");
}
return Nothing();
@@ -405,13 +401,14 @@ const std::string MemoryProfiler::START_HELP()
DESCRIPTION(
"Activates memory profiling.",
"The profiling works by statistically sampling the backtraces of",
- "calls to `malloc()`. This requires some additional memory to store",
+ "calls to 'malloc()'. This requires some additional memory to store",
"the collected data. The required additional space is expected to",
- "grow logarithmically."
+ "grow logarithmically.",
"",
- "Query Parameters:",
- "> duration=VALUE How long to collect data before",
- "> stopping. (default: 5mins)"),
+ "Query parameters:",
+ "",
+ "> duration=VALUE How long to collect data before",
+ "> stopping. (default: 5mins)"),
AUTHENTICATION(true));
}
@@ -425,7 +422,7 @@ const std::string MemoryProfiler::STOP_HELP()
"Instructs the memory profiler to stop collecting data"
"and dumps a file containing the collected data to disk,"
"clearing that data from memory. Does nothing if profiling",
- "was not started."),
+ "has not been started before."),
AUTHENTICATION(true));
}
@@ -433,87 +430,91 @@ const std::string MemoryProfiler::STOP_HELP()
const std::string MemoryProfiler::DOWNLOAD_RAW_HELP()
{
return HELP(
- TLDR(
- "Returns a raw memory profile."),
- DESCRIPTION(
- "Returns a file that was generated when the `/stop` endpoint was",
- "last accessed. See the jemalloc [manual page][manpage]",
- "for information about the file format.",
- "",
- "Query Parameters:",
- "> id=VALUE Optional parameter to request a specific",
- "> version of the profile."),
- AUTHENTICATION(true),
- None,
- REFERENCES("[manpage]: http://jemalloc.net/jemalloc.3.html"));
+ TLDR(
+ "Returns a raw memory profile."),
+ DESCRIPTION(
+ "Returns a file that was generated when the '/stop' endpoint",
+ "was last accessed. See the jemalloc [manual page][manpage] for",
+ "information about the file format.",
+ "",
+ "Query parameters:",
+ "",
+ "> id=VALUE Optional parameter to request a specific",
+ "> version of the profile."),
+ AUTHENTICATION(true),
+ None(),
+ REFERENCES("[manpage]: http://jemalloc.net/jemalloc.3.html"));
}
const std::string MemoryProfiler::DOWNLOAD_TEXT_HELP()
{
return HELP(
- TLDR(
- "Generates and returns a symbolized memory profile."),
- DESCRIPTION(
- "Generates a symbolized profile.",
- "Requires that the running binary was built with symbols, and that",
- "jeprof is installed on the host machine.",
- "*NOTE*: Generating the returned file might take several minutes.",
- "",
- "Query Parameters:",
- "> id=VALUE Optional parameter to request a specific",
- "> version of the generated profile."),
- AUTHENTICATION(true));
+ TLDR(
+ "Generates and returns a symbolized memory profile."),
+ DESCRIPTION(
+ "Generates a symbolized profile.",
+ "Requires that the running binary was built with symbols and that",
+ "jeprof is installed on the host machine.",
+ "",
+ "**NOTE:** Generating the returned file might take several minutes.",
+ "",
+ "Query parameters:",
+ "> id=VALUE Optional parameter to request a specific",
+ "> version of the generated profile."),
+ AUTHENTICATION(true));
}
const std::string MemoryProfiler::DOWNLOAD_GRAPH_HELP()
{
return HELP(
- TLDR(
- "Generates and returns a graph visualization."),
- DESCRIPTION(
- "Generates a graphical representation of the raw profile in the SVG",
- "Using this endpoint requires that that jeprof and dot are installed",
- "on the host machine.",
- "*NOTE*: Generating the returned file might take several minutes.",
- "",
- "Query Parameters:",
- "> id=VALUE Optional parameter to request a specific",
- "> version of the generated graph."),
- AUTHENTICATION(true));
+ TLDR(
+ "Generates and returns a graph visualization."),
+ DESCRIPTION(
+ "Generates a graphical representation of the raw profile in SVG.",
+ "Using this endpoint requires that that jeprof and dot are installed",
+ "on the host machine.",
+ "",
+ "**NOTE:** Generating the returned file might take several minutes.",
+ "",
+ "Query parameters:",
+ "",
+ "> id=VALUE Optional parameter to request a specific",
+ "> version of the generated graph."),
+ AUTHENTICATION(true));
}
const std::string MemoryProfiler::STATISTICS_HELP()
{
return HELP(
- TLDR(
- "Shows memory allocation statistics."),
- DESCRIPTION(
- "Memory allocation statistics as returned by `malloc_stats_print()`.",
- "These track e.g. the total number of bytes allocated by the current",
- "process and the bin-size of these allocations.",
- "These statistics are unrelated to the profiling mechanism controlled",
- "by the `/start` and `/stop` endpoints, and are always accurate.",
- "",
- "Returns a JSON object."),
- AUTHENTICATION(true));
+ TLDR(
+ "Shows memory allocation statistics."),
+ DESCRIPTION(
+ "Memory allocation statistics as returned by 'malloc_stats_print()'.",
+ "These track e.g. the total number of bytes allocated by the current",
+ "process and the bin-size of these allocations.",
+ "These statistics are unrelated to the profiling mechanism managed",
+ "by the '/start' and '/stop' endpoints, and are always accurate.",
+ "",
+ "Returns a JSON object."),
+ AUTHENTICATION(true));
}
const std::string MemoryProfiler::STATE_HELP()
{
return HELP(
- TLDR(
- "Shows the configuration of the memory-profiler process."),
- DESCRIPTION(
- "Current memory profiler state. This shows, for example, whether",
- "jemalloc was detected, whether profiling is currently active and",
- "the directory used to store temporary files.",
- "",
- "Returns a JSON object."),
- AUTHENTICATION(true));
+ TLDR(
+ "Shows the configuration of the memory profiler process."),
+ DESCRIPTION(
+ "Current memory profiler state. This shows, for example, whether",
+ "jemalloc was detected, whether profiling is currently active and",
+ "the directory used to store temporary files.",
+ "",
+ "Returns a JSON object."),
+ AUTHENTICATION(true));
}
@@ -592,7 +593,7 @@ void MemoryProfiler::ProfilingRun::extend(
MemoryProfiler::DiskArtifact::DiskArtifact(const std::string& _filename)
: filename(_filename),
- timestamp(Error("Not yet generated."))
+ timestamp(Error("Not yet generated"))
{}
@@ -658,7 +659,7 @@ Try<Nothing> MemoryProfiler::DiskArtifact::generate(
if (result.isError()) {
// The old file might still be fine on disk, but there's no good way to
- // verify so we assume that the error rendered it unusable.
+ // verify this hence we assume that the error rendered it unusable.
timestamp = Error(result.error());
return result;
}
@@ -677,7 +678,7 @@ Future<http::Response> MemoryProfiler::start(
const Option<http::authentication::Principal>&)
{
if (!detectJemalloc()) {
- return http::BadRequest(JEMALLOC_NOT_DETECTED_MESSAGE);
+ return http::BadRequest(std::string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
}
Duration duration = DEFAULT_COLLECTION_TIME;
@@ -704,7 +705,8 @@ Future<http::Response> MemoryProfiler::start(
Try<bool> wasActive = jemalloc::startProfiling();
if (wasActive.isError()) {
- return http::BadRequest(JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE);
+ return http::BadRequest(
+ std::string(JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE) + ".\n");
}
if (!wasActive.get()) {
@@ -715,7 +717,7 @@ Future<http::Response> MemoryProfiler::start(
JSON::Object response;
- // This can happpen when jemalloc was configured e.g. via the `MALLOC_CONF`
+ // This can happen when jemalloc was configured e.g. via the `MALLOC_CONF`
// environment variable. We don't touch it in this case.
if (!currentRun.isSome()) {
return http::Conflict("Heap profiling was started externally.\n");
@@ -724,15 +726,17 @@ Future<http::Response> MemoryProfiler::start(
std::string message = wasActive.get() ?
"Heap profiling is already active." :
"Successfully started new heap profiling run.";
+
message +=
- " After the remaining time has elapsed, download the generated profile"
- " at `/memory-profiler/download/raw?id=" + stringify(currentRun->id) + "`."
- " Visit `/memory-profiler/stop` to end the run prematurely.";
- response.values["id"] = currentRun->id;
+ " After the remaining time elapses, download the generated profile at '/" +
+ this->self().id + "/download/raw?id=" + stringify(currentRun->id) + "'." +
+ " Visit '/" + this->self().id + "/stop' to stop collection earlier.";
+
// Adding 0.5 to round to nearest integer value.
response.values["remaining_seconds"] = stringify(static_cast<int>(
currentRun->timer.timeout().remaining().secs() + 0.5));
response.values["message"] = message;
+ response.values["id"] = currentRun->id;
return http::OK(response);
}
@@ -745,7 +749,7 @@ Future<http::Response> MemoryProfiler::stop(
const Option<http::authentication::Principal>&)
{
if (!detectJemalloc()) {
- return http::BadRequest(JEMALLOC_NOT_DETECTED_MESSAGE);
+ return http::BadRequest(std::string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
}
Try<bool> active = jemalloc::profilingActive();
@@ -757,9 +761,8 @@ Future<http::Response> MemoryProfiler::stop(
if (!currentRun.isSome() && active.get()) {
// TODO(bevers): Allow stopping even in this case.
return http::BadRequest(
- "Profiling is active, but was not started by libprocess."
- " Accessing the raw profile through libprocess is currently"
- " not supported.\n");
+ "Profiling is active, but was not started by libprocess. Accessing the"
+ " raw profile through libprocess is currently not supported.\n");
}
Try<time_t> generated = stopAndGenerateRawProfile();
@@ -784,19 +787,19 @@ Future<http::Response> MemoryProfiler::stop(
result.values["message"] = message;
result.values["url_raw_profile"] =
- "./memory-profiler/download/raw?id=" + stringify(id);
+ "/" + this->self().id + "/download/raw?id=" + stringify(id);
result.values["url_graph"] =
- "./memory-profiler/download/graph?id=" + stringify(id);
+ "/" + this->self().id + "/download/graph?id=" + stringify(id);
result.values["url_symbolized_profile"] =
- "./memory-profiler/download/text?id=" + stringify(id);
+ "/" + this->self().id + "/download/text?id=" + stringify(id);
return http::OK(result);
}
-// A simple wrapper to discard the result, necessary so we can
+// A simple wrapper to discard the result, so we can
// use this as the target for `process::delay()`.
void MemoryProfiler::_stopAndGenerateRawProfile()
{
@@ -808,7 +811,7 @@ Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
{
ASSERT(detectJemalloc());
- VLOG(1) << "Attempting to stop current profiling run.";
+ VLOG(1) << "Attempting to stop current profiling run";
// Return the id of the last successful run if there is no current
// profiling run.
@@ -846,7 +849,7 @@ Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
// it by starting a new run.
return Error(
"Memory profiling unexpectedly inactive; not dumping profile."
- " Ensure nothing else is interfacing with jemalloc in this process.");
+ " Ensure nothing else is interfacing with jemalloc in this process");
}
Try<Nothing> generated = jemallocRawProfile.generate(
@@ -906,7 +909,7 @@ Future<http::Response> MemoryProfiler::downloadRaw(
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
- "A profiling run is currently in progress. To download results of a"
+ "A profiling run is currently in progress. To download results of the"
" previous run, please pass an 'id' explicitly.\n");
}
@@ -933,7 +936,7 @@ Future<http::Response> MemoryProfiler::downloadGraph(
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
- "A profiling run is currently in progress. To download results of a"
+ "A profiling run is currently in progress. To download results of the"
" previous run, please pass an 'id' explicitly.\n");
}
@@ -949,7 +952,7 @@ Future<http::Response> MemoryProfiler::downloadGraph(
rawId,
[&](const std::string& outputPath) -> Try<Nothing> {
if (!(requestedId.get() == jemallocRawProfile.id().get())) {
- return Error("Requested outdated version.");
+ return Error("Requested version cannot be served");
}
return generateJeprofFile(
@@ -986,8 +989,8 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
if (currentRun.isSome() && !requestedId.isSome()) {
return http::BadRequest(
- "A profiling run is currently in progress. To download results of a"
- " previous run, please pass an `id` explicitly.\n");
+ "A profiling run is currently in progress. To download results of the"
+ " previous run, please pass an 'id' explicitly.\n");
}
time_t rawId = jemallocRawProfile.id().get();
@@ -1004,7 +1007,7 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
[&](const std::string& outputPath) -> Try<Nothing>
{
if (!(requestedId.get() == jemallocRawProfile.id().get())) {
- return Error("Requested outdated version.");
+ return Error("Requested version cannot be served");
}
return generateJeprofFile(
@@ -1028,7 +1031,7 @@ Future<http::Response> MemoryProfiler::statistics(
const Option<http::authentication::Principal>&)
{
if (!detectJemalloc()) {
- return http::BadRequest(JEMALLOC_NOT_DETECTED_MESSAGE);
+ return http::BadRequest(std::string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
}
const std::string options = "J"; // 'J' selects JSON output format.
@@ -1060,7 +1063,7 @@ Future<http::Response> MemoryProfiler::state(
profilerState.values["jemalloc_detected"] = detected;
profilerState.values["tmpdir"] = stringify(
- temporaryDirectory.getOrElse("Not yet generated."));
+ temporaryDirectory.getOrElse("Not yet generated"));
{
JSON::Object runInformation;
[11/12] mesos git commit: Reconciled method names with the actions
they perform.
Posted by al...@apache.org.
Reconciled method names with the actions they perform.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/081ef161
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/081ef161
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/081ef161
Branch: refs/heads/master
Commit: 081ef161a2bd72eb61e354fe9f033f915a5f89cc
Parents: 915bf39
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 03:10:30 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 8 ++++----
3rdparty/libprocess/src/memory_profiler.hpp | 4 ++--
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/081ef161/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index db3045e..64048d3 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -533,12 +533,12 @@ void MemoryProfiler::initialize()
route("/download/raw",
authenticationRealm,
DOWNLOAD_RAW_HELP(),
- &MemoryProfiler::downloadRaw);
+ &MemoryProfiler::downloadRawProfile);
route("/download/text",
authenticationRealm,
DOWNLOAD_TEXT_HELP(),
- &MemoryProfiler::downloadTextProfile);
+ &MemoryProfiler::downloadSymbolizedProfile);
route("/download/graph",
authenticationRealm,
@@ -801,7 +801,7 @@ Future<http::Response> MemoryProfiler::stop(
}
-Future<http::Response> MemoryProfiler::downloadRaw(
+Future<http::Response> MemoryProfiler::downloadRawProfile(
const http::Request& request,
const Option<http::authentication::Principal>&)
{
@@ -835,7 +835,7 @@ Future<http::Response> MemoryProfiler::downloadRaw(
}
-Future<http::Response> MemoryProfiler::downloadTextProfile(
+Future<http::Response> MemoryProfiler::downloadSymbolizedProfile(
const http::Request& request,
const Option<http::authentication::Principal>&)
{
http://git-wip-us.apache.org/repos/asf/mesos/blob/081ef161/3rdparty/libprocess/src/memory_profiler.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.hpp b/3rdparty/libprocess/src/memory_profiler.hpp
index dc98998..adcb49b 100644
--- a/3rdparty/libprocess/src/memory_profiler.hpp
+++ b/3rdparty/libprocess/src/memory_profiler.hpp
@@ -67,12 +67,12 @@ private:
const Option<http::authentication::Principal>&);
// Returns a raw heap profile.
- Future<http::Response> downloadRaw(
+ Future<http::Response> downloadRawProfile(
const http::Request& request,
const Option<http::authentication::Principal>&);
// Generates and returns a symbolized heap profile.
- Future<http::Response> downloadTextProfile(
+ Future<http::Response> downloadSymbolizedProfile(
const http::Request& request,
const Option<http::authentication::Principal>&);
[08/12] mesos git commit: Removed std:: prefix in
"memory_profiler.cpp".
Posted by al...@apache.org.
Removed std:: prefix in "memory_profiler.cpp".
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a3560c16
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a3560c16
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a3560c16
Branch: refs/heads/master
Commit: a3560c16610f79b4b402e0c0af2dadbb16232fdc
Parents: 03221b0
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 18 01:01:02 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/libprocess/src/memory_profiler.cpp | 94 ++++++++++++------------
1 file changed, 47 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a3560c16/3rdparty/libprocess/src/memory_profiler.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/memory_profiler.cpp b/3rdparty/libprocess/src/memory_profiler.cpp
index 1fb89a6..4e19f16 100644
--- a/3rdparty/libprocess/src/memory_profiler.cpp
+++ b/3rdparty/libprocess/src/memory_profiler.cpp
@@ -12,6 +12,11 @@
#include "memory_profiler.hpp"
+#include <chrono>
+#include <fstream>
+#include <sstream>
+#include <string>
+
#include <process/delay.hpp>
#include <process/future.hpp>
#include <process/help.hpp>
@@ -26,18 +31,13 @@
#include <glog/logging.h>
-#include <chrono>
-#include <fstream>
-#include <sstream>
-#include <string>
-
-
using process::Future;
using process::HELP;
using process::TLDR;
using process::DESCRIPTION;
using process::AUTHENTICATION;
+using std::string;
// The main workflow to generate and download a heap profile
// goes through the sequence of endpoints
@@ -281,13 +281,13 @@ Try<Path> getTemporaryDirectoryPath() {
// TODO(bevers): Add a libprocess-specific override for the system-wide
// `TMPDIR`, for example `LIBPROCESS_TMPDIR`.
- std::string tmpdir =
+ const string tmpdir =
os::getenv("TMPDIR").getOrElse(LIBPROCESS_DEFAULT_TMPDIR);
- std::string pathTemplate = path::join(tmpdir, "libprocess.XXXXXX");
+ const string pathTemplate = path::join(tmpdir, "libprocess.XXXXXX");
// TODO(bevers): Add an atexit-handler that cleans up the directory.
- Try<std::string> dir = os::mkdtemp(pathTemplate);
+ Try<string> dir = os::mkdtemp(pathTemplate);
if (dir.isError()) {
return Error(dir.error());
}
@@ -301,9 +301,9 @@ Try<Path> getTemporaryDirectoryPath() {
Try<Nothing> generateJeprofFile(
- const Try<std::string>& inputPath,
- const std::string& options,
- const std::string& outputPath)
+ const Try<string>& inputPath,
+ const string& options,
+ const string& outputPath)
{
if (inputPath.isError()) {
return Error("Cannot read input file: " + inputPath.error());
@@ -316,7 +316,7 @@ Try<Nothing> generateJeprofFile(
// trivially injected.
// Apart from that, we dont need to be as careful here as with the actual
// heap profile dump, because a failure will not crash the whole process.
- Try<std::string> result = os::shell(strings::format(
+ Try<string> result = os::shell(strings::format(
"jeprof %s /proc/self/exe %s > %s",
options,
inputPath.get(),
@@ -336,7 +336,7 @@ Try<Nothing> generateJeprofFile(
// instead of having this here.
Result<time_t> extractIdFromRequest(const process::http::Request& request)
{
- Option<std::string> idParameter = request.url.query.get("id");
+ Option<string> idParameter = request.url.query.get("id");
if (idParameter.isNone()) {
return None();
}
@@ -384,7 +384,7 @@ Try<bool> profilingActive()
}
-Try<Nothing> dump(const std::string& path)
+Try<Nothing> dump(const string& path)
{
// A profile is dumped every time the 'prof.dump' setting is written to.
return writeJemallocSetting("prof.dump", path.c_str());
@@ -393,7 +393,7 @@ Try<Nothing> dump(const std::string& path)
} // namespace jemalloc {
-const std::string MemoryProfiler::START_HELP()
+const string MemoryProfiler::START_HELP()
{
return HELP(
TLDR(
@@ -413,7 +413,7 @@ const std::string MemoryProfiler::START_HELP()
}
-const std::string MemoryProfiler::STOP_HELP()
+const string MemoryProfiler::STOP_HELP()
{
return HELP(
TLDR(
@@ -427,7 +427,7 @@ const std::string MemoryProfiler::STOP_HELP()
}
-const std::string MemoryProfiler::DOWNLOAD_RAW_HELP()
+const string MemoryProfiler::DOWNLOAD_RAW_HELP()
{
return HELP(
TLDR(
@@ -447,7 +447,7 @@ const std::string MemoryProfiler::DOWNLOAD_RAW_HELP()
}
-const std::string MemoryProfiler::DOWNLOAD_TEXT_HELP()
+const string MemoryProfiler::DOWNLOAD_TEXT_HELP()
{
return HELP(
TLDR(
@@ -466,7 +466,7 @@ const std::string MemoryProfiler::DOWNLOAD_TEXT_HELP()
}
-const std::string MemoryProfiler::DOWNLOAD_GRAPH_HELP()
+const string MemoryProfiler::DOWNLOAD_GRAPH_HELP()
{
return HELP(
TLDR(
@@ -486,7 +486,7 @@ const std::string MemoryProfiler::DOWNLOAD_GRAPH_HELP()
}
-const std::string MemoryProfiler::STATISTICS_HELP()
+const string MemoryProfiler::STATISTICS_HELP()
{
return HELP(
TLDR(
@@ -503,7 +503,7 @@ const std::string MemoryProfiler::STATISTICS_HELP()
}
-const std::string MemoryProfiler::STATE_HELP()
+const string MemoryProfiler::STATE_HELP()
{
return HELP(
TLDR(
@@ -557,7 +557,7 @@ void MemoryProfiler::initialize()
}
-MemoryProfiler::MemoryProfiler(const Option<std::string>& _authenticationRealm)
+MemoryProfiler::MemoryProfiler(const Option<string>& _authenticationRealm)
: ProcessBase("memory-profiler"),
authenticationRealm(_authenticationRealm),
jemallocRawProfile(RAW_PROFILE_FILENAME),
@@ -591,7 +591,7 @@ void MemoryProfiler::ProfilingRun::extend(
}
-MemoryProfiler::DiskArtifact::DiskArtifact(const std::string& _filename)
+MemoryProfiler::DiskArtifact::DiskArtifact(const string& _filename)
: filename(_filename),
timestamp(Error("Not yet generated"))
{}
@@ -603,7 +603,7 @@ const Try<time_t>& MemoryProfiler::DiskArtifact::id() const
}
-Try<std::string> MemoryProfiler::DiskArtifact::path() const
+Try<string> MemoryProfiler::DiskArtifact::path() const
{
Try<Path> tmpdir = getTemporaryDirectoryPath();
if (tmpdir.isError()) {
@@ -616,7 +616,7 @@ Try<std::string> MemoryProfiler::DiskArtifact::path() const
http::Response MemoryProfiler::DiskArtifact::asHttp() const
{
- Try<std::string> _path = path();
+ Try<string> _path = path();
if (_path.isError()) {
return http::BadRequest(
"Could not compute file path: " + _path.error() + ".\n");
@@ -643,14 +643,14 @@ http::Response MemoryProfiler::DiskArtifact::asHttp() const
Try<Nothing> MemoryProfiler::DiskArtifact::generate(
time_t requestedTimestamp,
- std::function<Try<Nothing>(const std::string&)> generator)
+ std::function<Try<Nothing>(const string&)> generator)
{
// Nothing to do if the requestd file already exists.
if (timestamp.isSome() && timestamp.get() == requestedTimestamp) {
return Nothing();
}
- Try<std::string> path_ = path();
+ Try<string> path_ = path();
if (path_.isError()) {
return Error("Could not determine target path: " + path_.get());
}
@@ -678,14 +678,14 @@ Future<http::Response> MemoryProfiler::start(
const Option<http::authentication::Principal>&)
{
if (!detectJemalloc()) {
- return http::BadRequest(std::string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
+ return http::BadRequest(string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
}
Duration duration = DEFAULT_COLLECTION_TIME;
// TODO(bevers): Introduce `http::Request::extractQueryParameter<T>(string)`
// instead of doing it ad-hoc here.
- Option<std::string> durationParameter = request.url.query.get("duration");
+ Option<string> durationParameter = request.url.query.get("duration");
if (durationParameter.isSome()) {
Try<Duration> parsed = Duration::parse(durationParameter.get());
if (parsed.isError()) {
@@ -706,7 +706,7 @@ Future<http::Response> MemoryProfiler::start(
Try<bool> wasActive = jemalloc::startProfiling();
if (wasActive.isError()) {
return http::BadRequest(
- std::string(JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE) + ".\n");
+ string(JEMALLOC_PROFILING_NOT_ENABLED_MESSAGE) + ".\n");
}
if (!wasActive.get()) {
@@ -723,7 +723,7 @@ Future<http::Response> MemoryProfiler::start(
return http::Conflict("Heap profiling was started externally.\n");
}
- std::string message = wasActive.get() ?
+ string message = wasActive.get() ?
"Heap profiling is already active." :
"Successfully started new heap profiling run.";
@@ -749,7 +749,7 @@ Future<http::Response> MemoryProfiler::stop(
const Option<http::authentication::Principal>&)
{
if (!detectJemalloc()) {
- return http::BadRequest(std::string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
+ return http::BadRequest(string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
}
Try<bool> active = jemalloc::profilingActive();
@@ -773,27 +773,27 @@ Future<http::Response> MemoryProfiler::stop(
Try<bool> stillActive = jemalloc::profilingActive();
CHECK(stillActive.isError() || !stillActive.get());
- std::string message =
+ const string message =
"Successfully stopped memory profiling run."
" Use one of the provided URLs to download results."
" Note that in order to generate graphs or symbolized profiles,"
" jeprof must be installed on the host machine and generation of"
" these files can take several minutes.";
- std::string id = stringify(generated.get());
+ const string id = stringify(generated.get());
JSON::Object result;
result.values["id"] = id;
result.values["message"] = message;
result.values["url_raw_profile"] =
- "/" + this->self().id + "/download/raw?id=" + stringify(id);
+ "/" + this->self().id + "/download/raw?id=" + id;
result.values["url_graph"] =
- "/" + this->self().id + "/download/graph?id=" + stringify(id);
+ "/" + this->self().id + "/download/graph?id=" + id;
result.values["url_symbolized_profile"] =
- "/" + this->self().id + "/download/text?id=" + stringify(id);
+ "/" + this->self().id + "/download/text?id=" + id;
return http::OK(result);
}
@@ -854,10 +854,10 @@ Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
Try<Nothing> generated = jemallocRawProfile.generate(
runId,
- [this](const std::string& outputPath) -> Try<Nothing> {
+ [this](const string& outputPath) -> Try<Nothing> {
// Make sure we actually have permissions to write to the file and that
// there is at least a little bit space left on the device.
- const std::string data(DUMMY_FILE_SIZE, '\0');
+ const string data(DUMMY_FILE_SIZE, '\0');
Try<Nothing> written = os::write(outputPath, data);
if (written.isError()) {
return Error(written.error());
@@ -875,7 +875,7 @@ Try<time_t> MemoryProfiler::stopAndGenerateRawProfile()
});
if (generated.isError()) {
- std::string errorMessage = "Could not dump profile: " + generated.error();
+ const string errorMessage = "Could not dump profile: " + generated.error();
LOG(WARNING) << errorMessage;
return Error(errorMessage);
}
@@ -950,7 +950,7 @@ Future<http::Response> MemoryProfiler::downloadGraph(
// Generate the graph with the given id, or return the cached file on disk.
Try<Nothing> result = jeprofGraph.generate(
rawId,
- [&](const std::string& outputPath) -> Try<Nothing> {
+ [&](const string& outputPath) -> Try<Nothing> {
if (!(requestedId.get() == jemallocRawProfile.id().get())) {
return Error("Requested version cannot be served");
}
@@ -1004,7 +1004,7 @@ Future<http::Response> MemoryProfiler::downloadTextProfile(
// on disk.
Try<Nothing> result = jeprofSymbolizedProfile.generate(
requestedId.get(),
- [&](const std::string& outputPath) -> Try<Nothing>
+ [&](const string& outputPath) -> Try<Nothing>
{
if (!(requestedId.get() == jemallocRawProfile.id().get())) {
return Error("Requested version cannot be served");
@@ -1031,16 +1031,16 @@ Future<http::Response> MemoryProfiler::statistics(
const Option<http::authentication::Principal>&)
{
if (!detectJemalloc()) {
- return http::BadRequest(std::string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
+ return http::BadRequest(string(JEMALLOC_NOT_DETECTED_MESSAGE) + ".\n");
}
- const std::string options = "J"; // 'J' selects JSON output format.
+ const string options = "J"; // 'J' selects JSON output format.
- std::string statistics;
+ string statistics;
#ifdef LIBPROCESS_ALLOW_JEMALLOC
::malloc_stats_print([](void* opaque, const char* msg) {
- std::string* statistics = static_cast<std::string*>(opaque);
+ string* statistics = static_cast<string*>(opaque);
*statistics += msg;
}, &statistics, options.c_str());
#endif
[04/12] mesos git commit: Added jemalloc release tarball and build
rules.
Posted by al...@apache.org.
Added jemalloc release tarball and build rules.
Review: https://reviews.apache.org/r/63366/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cae2d20c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cae2d20c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cae2d20c
Branch: refs/heads/master
Commit: cae2d20c28af742d1b92f5882009cfd0f62dc9d6
Parents: 705bf1d
Author: Benno Evers <be...@mesosphere.com>
Authored: Tue Apr 17 18:31:31 2018 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 18 06:20:29 2018 +0200
----------------------------------------------------------------------
3rdparty/CMakeLists.txt | 25 +++++++++++
3rdparty/Makefile.am | 20 +++++++++
3rdparty/cmake/Versions.cmake | 2 +
3rdparty/jemalloc-5.0.1.tar.gz | Bin 0 -> 608494 bytes
3rdparty/versions.am | 1 +
CHANGELOG | 4 ++
cmake/CompilationConfigure.cmake | 14 ++++++
configure.ac | 78 +++++++++++++++++++++++++++++++++-
src/Makefile.am | 17 ++++++++
src/master/CMakeLists.txt | 3 ++
src/slave/CMakeLists.txt | 4 ++
11 files changed, 167 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/3rdparty/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
index 488e906..4eba87f 100644
--- a/3rdparty/CMakeLists.txt
+++ b/3rdparty/CMakeLists.txt
@@ -30,6 +30,7 @@ set(ELFIO_URL ${FETCH_URL}/elfio-${ELFIO_VERSION}.tar.gz)
set(GLOG_URL ${FETCH_URL}/glog-${GLOG_VERSION}.tar.gz)
set(GOOGLETEST_URL ${FETCH_URL}/googletest-release-${GOOGLETEST_VERSION}.tar.gz)
set(HTTP_PARSER_URL ${FETCH_URL}/http-parser-${HTTP_PARSER_VERSION}.tar.gz)
+set(JEMALLOC_URL ${FETCH_URL}/jemalloc-${JEMALLOC_VERSION}.tar.gz)
set(PICOJSON_URL ${FETCH_URL}/picojson-${PICOJSON_VERSION}.tar.gz)
set(PROTOBUF_URL ${FETCH_URL}/protobuf-${PROTOBUF_VERSION}.tar.gz)
set(LEVELDB_URL ${FETCH_URL}/leveldb-${LEVELDB_VERSION}.tar.gz)
@@ -973,6 +974,30 @@ ExternalProject_Add(
URL_HASH ${PROTOBUF_HASH})
+# Jemalloc: General-purpose malloc implementation.
+# http://jemalloc.net
+##################################################
+EXTERNAL(jemalloc ${JEMALLOC_VERSION} ${CMAKE_CURRENT_BINARY_DIR})
+add_library(jemalloc STATIC IMPORTED GLOBAL)
+add_dependencies(jemalloc ${JEMALLOC_TARGET})
+
+set_target_properties(
+ jemalloc PROPERTIES
+ IMPORTED_LOCATION ${JEMALLOC_ROOT}-build/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX})
+
+set(
+ JEMALLOC_CONFIGURE_COMMAND
+ ${JEMALLOC_ROOT}/configure --enable-stats --enable-prof --with-malloc-conf=prof:true,prof_active:false)
+
+ExternalProject_Add(
+ ${JEMALLOC_TARGET}
+ PREFIX ${JEMALLOC_CMAKE_ROOT}
+ CONFIGURE_COMMAND ${JEMALLOC_CONFIGURE_COMMAND}
+ INSTALL_COMMAND ${CMAKE_NOOP}
+ URL ${JEMALLOC_URL}
+ URL_HASH ${JEMALLOC_HASH})
+
+
# Apache ZooKeeper: C Client Library to ZooKeeper.
# https://zookeeper.apache.org/
##################################################
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/3rdparty/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
index 10b29c9..8d9fa85 100644
--- a/3rdparty/Makefile.am
+++ b/3rdparty/Makefile.am
@@ -59,6 +59,7 @@ GPERFTOOLS = gperftools-$(GPERFTOOLS_VERSION)
GRPC = grpc-$(GRPC_VERSION)
GTEST = $(GOOGLETEST)/googletest
HTTP_PARSER = http-parser-$(HTTP_PARSER_VERSION)
+JEMALLOC = jemalloc-$(JEMALLOC_VERSION)
LEVELDB = leveldb-$(LEVELDB_VERSION)
LIBEV = libev-$(LIBEV_VERSION)
NVML = nvml-$(NVML_VERSION)
@@ -79,6 +80,7 @@ EXTRA_DIST = \
$(GPERFTOOLS).tar.gz \
$(GRPC).tar.gz \
$(HTTP_PARSER).tar.gz \
+ $(JEMALLOC).tar.gz \
$(LEVELDB).tar.gz \
$(LIBEV).tar.gz \
$(NVML).tar.gz \
@@ -117,6 +119,7 @@ CLEAN_EXTRACTED = \
$(GPERFTOOLS) \
$(GRPC) \
$(HTTP_PARSER) \
+ $(JEMALLOC) \
$(LEVELDB) \
$(LIBEV) \
$(NVML) \
@@ -249,6 +252,23 @@ $(HTTP_PARSER)-build-stamp: libry_http_parser.la
touch $@
endif
+
+if WITH_BUNDLED_JEMALLOC
+JEMALLOC_CONFIGURE_ARGS = \
+ --enable-stats --enable-prof --with-malloc-conf="prof:true,prof_active:false"
+
+LIB_JEMALLOC = $(JEMALLOC)/libjemalloc.la
+
+$(LIB_JEMALLOC): $(JEMALLOC)-build-stamp
+
+$(JEMALLOC)-build-stamp: $(JEMALLOC)-stamp
+ cd $(JEMALLOC) && ./configure $(JEMALLOC_CONFIGURE_ARGS) $(CONFIGURE_ARGS)
+ cd $(JEMALLOC) && $(MAKE) $(AM_MAKEFLAGS)
+ touch $@
+
+ALL_LOCAL += $(LIB_JEMALLOC)
+endif
+
if WITH_BUNDLED_LIBEV
$(LIBEV)/libev.la: $(LIBEV)-build-stamp
$(LIBEV)-build-stamp: $(LIBEV)-stamp
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/3rdparty/cmake/Versions.cmake
----------------------------------------------------------------------
diff --git a/3rdparty/cmake/Versions.cmake b/3rdparty/cmake/Versions.cmake
index 605cbde..33577cc 100644
--- a/3rdparty/cmake/Versions.cmake
+++ b/3rdparty/cmake/Versions.cmake
@@ -12,6 +12,8 @@ set(GOOGLETEST_VERSION "1.8.0")
set(GOOGLETEST_HASH "SHA256=58A6F4277CA2BC8565222B3BBD58A177609E9C488E8A72649359BA51450DB7D8")
set(HTTP_PARSER_VERSION "2.6.2")
set(HTTP_PARSER_HASH "SHA256=80FFFC3B64EF6968CECDD4B299A96986007DFF4BD12AE6C58CBCB506959B90AD")
+set(JEMALLOC_VERSION "5.0.1")
+set(JEMALLOC_HASH "SHA256=18BD8A2135EA8CC4C9EFBE0378D9F7BE9C40E0804F34D941BF028D61F845198D")
set(LEVELDB_VERSION "1.19")
set(LEVELDB_HASH "SHA256=7D7A14AE825E66AABEB156C1C3FAE9F9A76D640EF6B40EDE74CC73DA937E5202")
set(LIBAPR_VERSION "1.5.2")
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/3rdparty/jemalloc-5.0.1.tar.gz
----------------------------------------------------------------------
diff --git a/3rdparty/jemalloc-5.0.1.tar.gz b/3rdparty/jemalloc-5.0.1.tar.gz
new file mode 100644
index 0000000..204e704
Binary files /dev/null and b/3rdparty/jemalloc-5.0.1.tar.gz differ
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/3rdparty/versions.am
----------------------------------------------------------------------
diff --git a/3rdparty/versions.am b/3rdparty/versions.am
index 63d879b..ada998d 100644
--- a/3rdparty/versions.am
+++ b/3rdparty/versions.am
@@ -28,6 +28,7 @@ GOOGLETEST_VERSION = 1.8.0
GPERFTOOLS_VERSION = 2.5
GRPC_VERSION = 1.10.0
HTTP_PARSER_VERSION = 2.6.2
+JEMALLOC_VERSION = 5.0.1
LEVELDB_VERSION = 1.19
LIBEV_VERSION = 4.22
NVML_VERSION = 352.79
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index d3809b2..09fff1d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -5,6 +5,10 @@ This release contains the following new features:
* [MESOS-8534] - **Experimental** A nested container is now allowed
to join a separate CNI network than its parent container.
+ * [MESOS-7944] - **Experimental** On Linux, Mesos can now be
+ configured to use the jemalloc allocator by default via the
+ `--enable-jemalloc-allocator` configuration option.
+
Release Notes - Mesos - Version 1.5.1 (WIP)
-------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/cmake/CompilationConfigure.cmake
----------------------------------------------------------------------
diff --git a/cmake/CompilationConfigure.cmake b/cmake/CompilationConfigure.cmake
index 08d154e..173089a 100644
--- a/cmake/CompilationConfigure.cmake
+++ b/cmake/CompilationConfigure.cmake
@@ -353,6 +353,20 @@ endif()
######################
string(COMPARE EQUAL ${CMAKE_SYSTEM_NAME} "Linux" LINUX)
+if (LINUX)
+ # We currenty only support using the bundled jemalloc on linux.
+ # While building it and linking against is actually not a problem
+ # on other platforms, to make it actually *useful* we need some
+ # additional platform-specific code in the mesos binaries that re-routes
+ # all existing malloc/free calls through jemalloc.
+ # On linux, that is not necessary because the default malloc implementation
+ # explicitly supports replacement via symbol interposition.
+ option(
+ ENABLE_JEMALLOC_ALLOCATOR
+ "Use jemalloc as memory allocator for the master and agent binaries."
+ FALSE)
+endif ()
+
# FREEBSD CONFIGURATION.
######################
string(COMPARE EQUAL ${CMAKE_SYSTEM_NAME} "FreeBSD" FREEBSD)
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 4c4085c..6e91ecf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -214,6 +214,13 @@ AC_ARG_ENABLE([java],
[do not build Java bindings]),
[], [enable_java=yes])
+# TODO(bevers): Eventually make this enabled by default.
+AC_ARG_ENABLE([jemalloc-allocator],
+ AS_HELP_STRING([--enable-jemalloc-allocator],
+ [Use jemalloc as the default memory allocator for the
+ mesos-master and mesos-agent binaries.]),
+ [], [enable_jemalloc_allocator=no])
+
# TODO(benh): Eventually make this enabled by default.
AC_ARG_ENABLE([last_in_first_out_fixed_size_semaphore],
AS_HELP_STRING([--enable-last-in-first-out-fixed-size-semaphore],
@@ -375,6 +382,13 @@ AC_ARG_WITH([http-parser],
location prefixed by the given path]),
[without_bundled_http_parser=yes], [])
+AC_ARG_WITH([jemalloc],
+ AS_HELP_STRING([--with-jemalloc@<:@=DIR@:>@],
+ [excludes building and using the bundled jemalloc
+ package in lieu of an installed version at a
+ location prefixed by the given path.]),
+ [without_bundled_jemalloc=yes], [])
+
AC_ARG_WITH([leveldb],
AS_HELP_STRING([--with-leveldb@<:@=DIR@:>@],
[excludes building and using the bundled LevelDB
@@ -504,6 +518,10 @@ AS_IF([test "x$enable_lock_free_event_queue" = "xyes"],
AS_IF([test "x$enable_lock_free_run_queue" = "xyes"],
[AC_DEFINE([LOCK_FREE_RUN_QUEUE])])
+# Check if we should link the mesos binaries against jemalloc.
+AM_CONDITIONAL([ENABLE_JEMALLOC_ALLOCATOR],
+ [test x"$enable_jemalloc_allocator" = "xyes"])
+
# Check to see if we should harden or not.
AM_CONDITIONAL([ENABLE_HARDENING], [test x"$enable_hardening" = "xyes"])
@@ -697,7 +715,7 @@ AM_CONDITIONAL([ENABLE_STATIC_LIBPROCESS], [true])
AC_MSG_NOTICE([Setting up build environment for ${target_cpu} ${target_os}])
# Determine the current OS (TODO(benh): Does autotools do this for us?).
-case "${target_os}" in
+case "${host_os}" in
linux*)
OS_NAME=linux
LIBS="$LIBS -lrt"
@@ -1081,6 +1099,64 @@ AM_CONDITIONAL([WITH_BUNDLED_HTTP_PARSER],
[test "x$with_bundled_http_parser" = "xyes"])
+# NB: Since malloc() and free() are declared in stdlib.h, and the
+# jemalloc-specific api is detected at runtime, we dont need any
+# jemalloc headers or modify CPPFLAGS here.
+if test "x$enable_jemalloc_allocator" = "xyes"; then
+ if test "x$OS_NAME" != "xlinux"; then
+ # Building and linking against jemalloc is actually not a problem
+ # on OSX even now, but to make it actually *useful* we would have
+ # have to re-route all existing malloc/free calls through jemalloc.
+ # The recommended way of doing that on Mac seems to be call
+ # `malloc_default_zone()` and to overwrite the hooks in the
+ # returned `malloc_zone_t`.
+ AC_MSG_ERROR([
+--------------------------------------------------------------------
+You have requested the default memory allocator to be replaced by
+jemalloc, but your operating system was detected to be $OS_NAME.
+
+This feature is currently only supported on linux.
+--------------------------------------------------------------------
+ ])
+ fi
+
+ if test -n "`echo $with_jemalloc`"; then
+ WITH_JEMALLOC="${with_jemalloc}"
+ fi
+
+ # Check if the user asked us to use a preinstalled jemalloc, or if
+ # they asked us to ignore all bundled libraries while compiling and
+ # linking.
+ if test "x$without_bundled_jemalloc" = "xyes" || \
+ test "x$enable_bundled" != "xyes"; then
+
+ # Avoid AC_SEARCH_LIBS here because it would implicitly add jemalloc
+ # to LIBS, but we dont want to link it against libmesos, only the
+ # individual binaries
+ AC_CHECK_LIB([jemalloc], [malloc_stats_print], [found_jemalloc=yes])
+ if test "x$found_jemalloc" = "xyes"; then
+ with_bundled_jemalloc=no
+ else
+ AC_MSG_ERROR([cannot find jemalloc
+-------------------------------------------------------------------
+You have requested the use of a non-bundled jemalloc but no suitable
+library could be found.
+
+You may want specify the location of jemalloc by providing a prefix
+path via --with-jemalloc=DIR, or check that the path you provided is
+correct if you're already doing this.
+-------------------------------------------------------------------
+ ])
+ fi
+ else
+ with_bundled_jemalloc=yes
+ fi
+fi
+
+AC_SUBST(WITH_JEMALLOC)
+AM_CONDITIONAL([WITH_BUNDLED_JEMALLOC],
+ [test "x$with_bundled_jemalloc" = "xyes"])
+
# TODO(benh): Consider using AS_IF instead of just shell 'if'
# statements for better autoconf style (the AS_IF macros also make
# sure variable dependencies are handled appropriately).
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 07eb138..9d610bb 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -33,6 +33,7 @@ GMOCK = $(GOOGLETEST)/googlemock
GOOGLETEST = 3rdparty/googletest-release-$(GOOGLETEST_VERSION)
GRPC = 3rdparty/grpc-$(GRPC_VERSION)
GTEST = $(GOOGLETEST)/googletest
+JEMALLOC = 3rdparty/jemalloc-$(JEMALLOC_VERSION)
LEVELDB = 3rdparty/leveldb-$(LEVELDB_VERSION)
LIBPROCESS = 3rdparty/libprocess
NVML = 3rdparty/nvml-$(NVML_VERSION)
@@ -183,6 +184,17 @@ GRPC_CPP_PLUGIN = @GRPC_CXX_PLUGIN@
endif
endif
+if WITH_BUNDLED_JEMALLOC
+# TODO(bennoe): We could call `jemalloc-conf --libs` to get the required
+# libs without hardcoding `-ldl`, but then a version bump could introduce
+# additional dependencies without us being aware of it.
+LIB_JEMALLOC = ../$(JEMALLOC)/lib/libjemalloc_pic.a -ldl
+else
+LIB_JEMALLOC = $(if $(WITH_JEMALLOC),\
+ -L$(WITH_JEMALLOC)/lib -ljemalloc -ldl,\
+ -ljemalloc -ldl)
+endif
+
if WITH_BUNDLED_LEVELDB
MESOS_CPPFLAGS += -I../$(LEVELDB)/include
@@ -1681,6 +1693,11 @@ mesos_local_SOURCES = local/main.cpp
mesos_local_CPPFLAGS = $(MESOS_CPPFLAGS)
mesos_local_LDADD = libmesos.la $(LDADD)
+if ENABLE_JEMALLOC_ALLOCATOR
+mesos_agent_LDADD += $(LIB_JEMALLOC)
+mesos_master_LDADD += $(LIB_JEMALLOC)
+endif
+
pkglibexec_PROGRAMS += mesos-fetcher
mesos_fetcher_SOURCES = launcher/fetcher.cpp
mesos_fetcher_CPPFLAGS = $(MESOS_CPPFLAGS)
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/src/master/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/master/CMakeLists.txt b/src/master/CMakeLists.txt
index b8953bd..f4b15b5 100644
--- a/src/master/CMakeLists.txt
+++ b/src/master/CMakeLists.txt
@@ -18,3 +18,6 @@
########################
add_executable(mesos-master main.cpp)
target_link_libraries(mesos-master PRIVATE mesos)
+if (ENABLE_JEMALLOC_ALLOCATOR)
+ target_link_libraries(mesos-master PRIVATE jemalloc)
+endif ()
http://git-wip-us.apache.org/repos/asf/mesos/blob/cae2d20c/src/slave/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/slave/CMakeLists.txt b/src/slave/CMakeLists.txt
index 943e8f5..e5fe32a 100644
--- a/src/slave/CMakeLists.txt
+++ b/src/slave/CMakeLists.txt
@@ -24,6 +24,10 @@ add_subdirectory(resource_estimators)
add_executable(mesos-agent main.cpp)
target_link_libraries(mesos-agent PRIVATE mesos)
+if (ENABLE_JEMALLOC_ALLOCATOR)
+ target_link_libraries(mesos-agent PRIVATE jemalloc)
+endif ()
+
# Add binary dependencies
add_dependencies(
mesos-agent