You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2016/04/25 23:32:30 UTC
[37/50] mesos git commit: Added flags to set size of completed
task/framework history.
Added flags to set size of completed task/framework history.
The default size of the buffers used to hold the state of completed
tasks/frameworks is very large. However, many users don't care much
about this information when requesting a master's state. Moreover, if a
large number of frameworks request this state simultaneously, the
master can quickly become overwhelmed because the process of generating
this state both blocks the master and takes up a lot of cycles. By
allowing the user to configure the size of the buffers used to hold
this state, we let the user decide how much state is needed.
This commit is based on a pull request generated by Felix Bechstein at:
https://github.com/apache/mesos/pull/82
Review: https://reviews.apache.org/r/42053/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/702c1870
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/702c1870
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/702c1870
Branch: refs/heads/0.26.x
Commit: 702c187035a6cbd099294a06b5fee917c8ee8de1
Parents: 8137265
Author: Kevin Klues <kl...@gmail.com>
Authored: Thu Jan 14 22:55:54 2016 -0800
Committer: Michael Park <mp...@apache.org>
Committed: Fri Feb 26 20:59:07 2016 -0800
----------------------------------------------------------------------
docs/configuration.md | 17 +++++++++++++++++
src/master/constants.cpp | 4 ++--
src/master/constants.hpp | 11 +++++------
src/master/flags.cpp | 10 ++++++++++
src/master/flags.hpp | 2 ++
src/master/master.cpp | 9 +++++----
src/master/master.hpp | 9 ++++++---
7 files changed, 47 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/702c1870/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index 72847e5..f3bbf49 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -437,6 +437,23 @@ file:///path/to/file (where file contains one of the above)</code></pre>
</tr>
<tr>
<td>
+ --max_completed_frameworks=VALUE
+ </td>
+ <td>
+ Maximum number of completed frameworks to store in memory. (default: 50)
+ </td>
+ </tr>
+ <tr>
+ <td>
+ --max_completed_tasks_per_framework=VALUE
+ </td>
+ <td>
+ Maximum number of completed tasks per framework to store in memory.
+ (default: 1000)
+ </td>
+ </tr>
+ <tr>
+ <td>
--max_slave_ping_timeouts=VALUE
</td>
<td>
http://git-wip-us.apache.org/repos/asf/mesos/blob/702c1870/src/master/constants.cpp
----------------------------------------------------------------------
diff --git a/src/master/constants.cpp b/src/master/constants.cpp
index 2b66b27..08bfe8d 100644
--- a/src/master/constants.cpp
+++ b/src/master/constants.cpp
@@ -37,8 +37,8 @@ const size_t DEFAULT_MAX_SLAVE_PING_TIMEOUTS = 5;
const Duration MIN_SLAVE_REREGISTER_TIMEOUT = Minutes(10);
const double RECOVERY_SLAVE_REMOVAL_PERCENT_LIMIT = 1.0; // 100%.
const size_t MAX_REMOVED_SLAVES = 100000;
-const uint32_t MAX_COMPLETED_FRAMEWORKS = 50;
-const uint32_t MAX_COMPLETED_TASKS_PER_FRAMEWORK = 1000;
+const size_t DEFAULT_MAX_COMPLETED_FRAMEWORKS = 50;
+const size_t DEFAULT_MAX_COMPLETED_TASKS_PER_FRAMEWORK = 1000;
const Duration WHITELIST_WATCH_INTERVAL = Seconds(5);
const uint32_t TASK_LIMIT = 100;
const std::string MASTER_INFO_LABEL = "info";
http://git-wip-us.apache.org/repos/asf/mesos/blob/702c1870/src/master/constants.hpp
----------------------------------------------------------------------
diff --git a/src/master/constants.hpp b/src/master/constants.hpp
index 51d4774..857396e 100644
--- a/src/master/constants.hpp
+++ b/src/master/constants.hpp
@@ -89,13 +89,12 @@ extern const double RECOVERY_SLAVE_REMOVAL_PERCENT_LIMIT;
// Maximum number of removed slaves to store in the cache.
extern const size_t MAX_REMOVED_SLAVES;
-// Maximum number of completed frameworks to store in the cache.
-// TODO(thomasm): Make configurable.
-extern const uint32_t MAX_COMPLETED_FRAMEWORKS;
+// Default maximum number of completed frameworks to store in the cache.
+extern const size_t DEFAULT_MAX_COMPLETED_FRAMEWORKS;
-// Maximum number of completed tasks per framework to store in the
-// cache. TODO(thomasm): Make configurable.
-extern const uint32_t MAX_COMPLETED_TASKS_PER_FRAMEWORK;
+// Default maximum number of completed tasks per framework
+// to store in the cache.
+extern const size_t DEFAULT_MAX_COMPLETED_TASKS_PER_FRAMEWORK;
// Time interval to check for updated watchers list.
extern const Duration WHITELIST_WATCH_INTERVAL;
http://git-wip-us.apache.org/repos/asf/mesos/blob/702c1870/src/master/flags.cpp
----------------------------------------------------------------------
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index 806e2da..0c192df 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -435,4 +435,14 @@ mesos::internal::master::Flags::Flags()
"\n"
"Currently there's no support for multiple authorizers.",
DEFAULT_AUTHORIZER);
+
+ add(&Flags::max_completed_frameworks,
+ "max_completed_frameworks",
+ "Maximum number of completed frameworks to store in memory.",
+ DEFAULT_MAX_COMPLETED_FRAMEWORKS);
+
+ add(&Flags::max_completed_tasks_per_framework,
+ "max_completed_tasks_per_framework",
+ "Maximum number of completed tasks per framework to store in memory.",
+ DEFAULT_MAX_COMPLETED_TASKS_PER_FRAMEWORK);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/702c1870/src/master/flags.hpp
----------------------------------------------------------------------
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index 5fd5d50..ece6e34 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -80,6 +80,8 @@ public:
Duration slave_ping_timeout;
size_t max_slave_ping_timeouts;
std::string authorizers;
+ size_t max_completed_frameworks;
+ size_t max_completed_tasks_per_framework;
#ifdef WITH_NETWORK_ISOLATOR
Option<size_t> max_executors_per_slave;
http://git-wip-us.apache.org/repos/asf/mesos/blob/702c1870/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index d02ccef..e8e8375 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -287,6 +287,7 @@ Master::Master(
contender(_contender),
detector(_detector),
authorizer(_authorizer),
+ frameworks(flags),
authenticator(None()),
metrics(new Metrics(*this)),
electedTime(None())
@@ -1968,7 +1969,7 @@ void Master::_subscribe(
FrameworkInfo frameworkInfo_ = frameworkInfo;
frameworkInfo_.mutable_id()->CopyFrom(newFrameworkId());
- Framework* framework = new Framework(this, frameworkInfo_, http);
+ Framework* framework = new Framework(this, flags, frameworkInfo_, http);
addFramework(framework);
@@ -2055,7 +2056,7 @@ void Master::_subscribe(
// elected Mesos master to which either an existing scheduler or a
// failed-over one is connecting. Create a Framework object and add
// any tasks it has that have been reported by reconnecting slaves.
- Framework* framework = new Framework(this, frameworkInfo, http);
+ Framework* framework = new Framework(this, flags, frameworkInfo, http);
// Add active tasks and executors to the framework.
foreachvalue (Slave* slave, slaves.registered) {
@@ -2268,7 +2269,7 @@ void Master::_subscribe(
FrameworkInfo frameworkInfo_ = frameworkInfo;
frameworkInfo_.mutable_id()->CopyFrom(newFrameworkId());
- Framework* framework = new Framework(this, frameworkInfo_, from);
+ Framework* framework = new Framework(this, flags, frameworkInfo_, from);
addFramework(framework);
@@ -2376,7 +2377,7 @@ void Master::_subscribe(
// elected Mesos master to which either an existing scheduler or a
// failed-over one is connecting. Create a Framework object and add
// any tasks it has that have been reported by reconnecting slaves.
- Framework* framework = new Framework(this, frameworkInfo, from);
+ Framework* framework = new Framework(this, flags, frameworkInfo, from);
// Add active tasks and executors to the framework.
foreachvalue (Slave* slave, slaves.registered) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/702c1870/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index ead8520..807d8d4 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1143,7 +1143,8 @@ private:
struct Frameworks
{
- Frameworks() : completed(MAX_COMPLETED_FRAMEWORKS) {}
+ Frameworks(const Flags& masterFlags)
+ : completed(masterFlags.max_completed_frameworks) {}
hashmap<FrameworkID, Framework*> registered;
boost::circular_buffer<std::shared_ptr<Framework>> completed;
@@ -1459,6 +1460,7 @@ private:
struct Framework
{
Framework(Master* const _master,
+ const Flags& masterFlags,
const FrameworkInfo& _info,
const process::UPID& _pid,
const process::Time& time = process::Clock::now())
@@ -1469,9 +1471,10 @@ struct Framework
active(true),
registeredTime(time),
reregisteredTime(time),
- completedTasks(MAX_COMPLETED_TASKS_PER_FRAMEWORK) {}
+ completedTasks(masterFlags.max_completed_tasks_per_framework) {}
Framework(Master* const _master,
+ const Flags& masterFlags,
const FrameworkInfo& _info,
const HttpConnection& _http,
const process::Time& time = process::Clock::now())
@@ -1482,7 +1485,7 @@ struct Framework
active(true),
registeredTime(time),
reregisteredTime(time),
- completedTasks(MAX_COMPLETED_TASKS_PER_FRAMEWORK) {}
+ completedTasks(masterFlags.max_completed_tasks_per_framework) {}
~Framework()
{