You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jp...@apache.org on 2018/10/25 15:46:04 UTC
[mesos] branch master updated (798cdab -> 5a0a36f)
This is an automated email from the ASF dual-hosted git repository.
jpeach pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from 798cdab Added MESOS-9340 to the 1.6.2 CHANGELOG.
new 91f45ee Refactor allocator configuration into a struct.
new 90eddb0 Add a flag to toggle per-framework metrics.
new 5a0a36f Add documentation for per-framework metrics flag.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/configuration/master.md | 11 ++++
include/mesos/allocator/allocator.hpp | 26 +++++----
src/master/allocator/mesos/allocator.hpp | 39 +++-----------
src/master/allocator/mesos/hierarchical.cpp | 57 ++++++++++----------
src/master/allocator/mesos/hierarchical.hpp | 20 +++----
src/master/allocator/mesos/metrics.cpp | 28 ++++++++--
src/master/allocator/mesos/metrics.hpp | 10 +++-
src/master/flags.cpp | 8 +++
src/master/flags.hpp | 1 +
src/master/framework.cpp | 2 +-
src/master/master.cpp | 21 +++++---
src/master/master.hpp | 2 +-
src/master/metrics.cpp | 70 ++++++++++++++++---------
src/master/metrics.hpp | 9 +++-
src/tests/allocator.hpp | 19 +++----
src/tests/api_tests.cpp | 4 +-
src/tests/hierarchical_allocator_benchmarks.cpp | 14 ++---
src/tests/hierarchical_allocator_tests.cpp | 14 ++---
src/tests/master_allocator_tests.cpp | 36 ++++++-------
src/tests/master_quota_tests.cpp | 20 +++----
src/tests/reservation_tests.cpp | 6 +--
src/tests/resource_offers_tests.cpp | 2 +-
src/tests/slave_recovery_tests.cpp | 2 +-
23 files changed, 241 insertions(+), 180 deletions(-)
[mesos] 01/03: Refactor allocator configuration into a struct.
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpeach pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 91f45ee495aedb86a838f0b77efbc5c6150eb04f
Author: Jacob Janco <jj...@gmail.com>
AuthorDate: Thu Oct 25 07:58:31 2018 -0700
Refactor allocator configuration into a struct.
There has been a proliferation of configuration passed into
the allocator. This commit collects that configuration into
an `Options` struct. Test fixes have been added as well.
Review: https://reviews.apache.org/r/68953/
---
include/mesos/allocator/allocator.hpp | 25 ++++++++-----
src/master/allocator/mesos/allocator.hpp | 39 +++++---------------
src/master/allocator/mesos/hierarchical.cpp | 47 +++++++++++--------------
src/master/allocator/mesos/hierarchical.hpp | 15 +++-----
src/master/master.cpp | 20 +++++++----
src/tests/allocator.hpp | 19 +++++-----
src/tests/api_tests.cpp | 4 +--
src/tests/hierarchical_allocator_benchmarks.cpp | 14 ++++----
src/tests/hierarchical_allocator_tests.cpp | 14 ++++----
src/tests/master_allocator_tests.cpp | 36 +++++++++----------
src/tests/master_quota_tests.cpp | 20 +++++------
src/tests/reservation_tests.cpp | 6 ++--
src/tests/resource_offers_tests.cpp | 2 +-
src/tests/slave_recovery_tests.cpp | 2 +-
14 files changed, 121 insertions(+), 142 deletions(-)
diff --git a/include/mesos/allocator/allocator.hpp b/include/mesos/allocator/allocator.hpp
index 61b2b84..4c4273a 100644
--- a/include/mesos/allocator/allocator.hpp
+++ b/include/mesos/allocator/allocator.hpp
@@ -42,6 +42,20 @@ namespace mesos {
namespace allocator {
/**
+ * Pass in configuration to the allocator.
+ */
+struct Options
+{
+ Duration allocationInterval = Seconds(1);
+ Option<std::set<std::string>> fairnessExcludeResourceNames = None();
+ bool filterGpuResources = true;
+ Option<DomainInfo> domain = None();
+ Option<std::vector<Resources>> minAllocatableResources = None();
+ size_t maxCompletedFrameworks = 0;
+};
+
+
+/**
* Basic model of an allocator: resources are allocated to a framework
* in the form of offers. A framework can refuse some resources in
* offers and run tasks in others. Allocated resources can have offer
@@ -89,7 +103,7 @@ public:
* allocations from the frameworks.
*/
virtual void initialize(
- const Duration& allocationInterval,
+ const Options& options,
const lambda::function<
void(const FrameworkID&,
const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&
@@ -97,14 +111,7 @@ public:
const lambda::function<
void(const FrameworkID&,
const hashmap<SlaveID, UnavailableResources>&)>&
- inverseOfferCallback,
- const Option<std::set<std::string>>&
- fairnessExcludeResourceNames = None(),
- bool filterGpuResources = true,
- const Option<DomainInfo>& domain = None(),
- const Option<std::vector<Resources>>&
- minAllocatableResources = None(),
- const size_t maxCompletedFrameworks = 0) = 0;
+ inverseOfferCallback) = 0;
/**
* Informs the allocator of the recovered state from the master.
diff --git a/src/master/allocator/mesos/allocator.hpp b/src/master/allocator/mesos/allocator.hpp
index a4d7f2b..2d83f38 100644
--- a/src/master/allocator/mesos/allocator.hpp
+++ b/src/master/allocator/mesos/allocator.hpp
@@ -47,7 +47,7 @@ public:
~MesosAllocator() override;
void initialize(
- const Duration& allocationInterval,
+ const mesos::allocator::Options& options,
const lambda::function<
void(const FrameworkID&,
const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&
@@ -55,13 +55,7 @@ public:
const lambda::function<
void(const FrameworkID&,
const hashmap<SlaveID, UnavailableResources>&)>&
- inverseOfferCallback,
- const Option<std::set<std::string>>&
- fairnessExcludeResourceNames = None(),
- bool filterGpuResources = true,
- const Option<DomainInfo>& domain = None(),
- const Option<std::vector<Resources>>& minAllocatableResources = None(),
- const size_t maxCompletedFrameworks = 0) override;
+ inverseOfferCallback) override;
void recover(
const int expectedAgentCount,
@@ -200,7 +194,7 @@ public:
using process::ProcessBase::initialize;
virtual void initialize(
- const Duration& allocationInterval,
+ const mesos::allocator::Options& options,
const lambda::function<
void(const FrameworkID&,
const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&
@@ -208,14 +202,7 @@ public:
const lambda::function<
void(const FrameworkID&,
const hashmap<SlaveID, UnavailableResources>&)>&
- inverseOfferCallback,
- const Option<std::set<std::string>>&
- fairnessExcludeResourceNames = None(),
- bool filterGpuResources = true,
- const Option<DomainInfo>& domain = None(),
- const Option<std::vector<Resources>>&
- minAllocatableResources = None(),
- const size_t maxCompletedFrameworks = 0) = 0;
+ inverseOfferCallback) = 0;
virtual void recover(
const int expectedAgentCount,
@@ -363,7 +350,7 @@ MesosAllocator<AllocatorProcess>::~MesosAllocator()
template <typename AllocatorProcess>
inline void MesosAllocator<AllocatorProcess>::initialize(
- const Duration& allocationInterval,
+ const mesos::allocator::Options& options,
const lambda::function<
void(const FrameworkID&,
const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&
@@ -371,24 +358,14 @@ inline void MesosAllocator<AllocatorProcess>::initialize(
const lambda::function<
void(const FrameworkID&,
const hashmap<SlaveID, UnavailableResources>&)>&
- inverseOfferCallback,
- const Option<std::set<std::string>>& fairnessExcludeResourceNames,
- bool filterGpuResources,
- const Option<DomainInfo>& domain,
- const Option<std::vector<Resources>>& minAllocatableResources,
- const size_t maxCompletedFrameworks)
+ inverseOfferCallback)
{
process::dispatch(
process,
&MesosAllocatorProcess::initialize,
- allocationInterval,
+ options,
offerCallback,
- inverseOfferCallback,
- fairnessExcludeResourceNames,
- filterGpuResources,
- domain,
- minAllocatableResources,
- maxCompletedFrameworks);
+ inverseOfferCallback);
}
diff --git a/src/master/allocator/mesos/hierarchical.cpp b/src/master/allocator/mesos/hierarchical.cpp
index c89a63d..eae5a12 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -47,6 +47,7 @@ using std::string;
using std::vector;
using mesos::allocator::InverseOfferStatus;
+using mesos::allocator::Options;
using process::after;
using process::Continue;
@@ -144,7 +145,7 @@ Framework::Framework(
void HierarchicalAllocatorProcess::initialize(
- const Duration& _allocationInterval,
+ const Options& _options,
const lambda::function<
void(const FrameworkID&,
const hashmap<string, hashmap<SlaveID, Resources>>&)>&
@@ -152,42 +153,35 @@ void HierarchicalAllocatorProcess::initialize(
const lambda::function<
void(const FrameworkID&,
const hashmap<SlaveID, UnavailableResources>&)>&
- _inverseOfferCallback,
- const Option<set<string>>& _fairnessExcludeResourceNames,
- bool _filterGpuResources,
- const Option<DomainInfo>& _domain,
- const Option<std::vector<Resources>>& _minAllocatableResources,
- const size_t maxCompletedFrameworks)
+ _inverseOfferCallback)
{
- allocationInterval = _allocationInterval;
+ options = _options;
offerCallback = _offerCallback;
inverseOfferCallback = _inverseOfferCallback;
- fairnessExcludeResourceNames = _fairnessExcludeResourceNames;
- filterGpuResources = _filterGpuResources;
- domain = _domain;
- minAllocatableResources = _minAllocatableResources;
initialized = true;
paused = false;
completedFrameworkMetrics =
BoundedHashMap<FrameworkID, process::Owned<FrameworkMetrics>>(
- maxCompletedFrameworks);
+ options.maxCompletedFrameworks);
// Resources for quota'ed roles are allocated separately and prior to
// non-quota'ed roles, hence a dedicated sorter for quota'ed roles is
// necessary.
- roleSorter->initialize(fairnessExcludeResourceNames);
- quotaRoleSorter->initialize(fairnessExcludeResourceNames);
+ roleSorter->initialize(options.fairnessExcludeResourceNames);
+ quotaRoleSorter->initialize(options.fairnessExcludeResourceNames);
VLOG(1) << "Initialized hierarchical allocator process";
// Start a loop to run allocation periodically.
PID<HierarchicalAllocatorProcess> _self = self();
+ // Set a temporary variable for the lambda capture.
+ Duration allocationInterval = options.allocationInterval;
loop(
None(), // Use `None` so we iterate outside the allocator process.
- [_allocationInterval]() {
- return after(_allocationInterval);
+ [allocationInterval]() {
+ return after(allocationInterval);
},
[_self](const Nothing&) {
return dispatch(_self, &HierarchicalAllocatorProcess::allocate)
@@ -1307,7 +1301,7 @@ void HierarchicalAllocatorProcess::recoverResources(
//
// TODO(alexr): If we allocated upon resource recovery
// (MESOS-3078), we would not need to increase the timeout here.
- timeout = std::max(allocationInterval, timeout.get());
+ timeout = std::max(options.allocationInterval, timeout.get());
// We need to disambiguate the function call to pick the correct
// `expire()` overload.
@@ -2427,14 +2421,15 @@ bool HierarchicalAllocatorProcess::isFiltered(
bool HierarchicalAllocatorProcess::allocatable(const Resources& resources)
{
- if (minAllocatableResources.isNone() ||
- CHECK_NOTNONE(minAllocatableResources).empty()) {
+ if (options.minAllocatableResources.isNone() ||
+ CHECK_NOTNONE(options.minAllocatableResources).empty()) {
return true;
}
Resources quantity = resources.createStrippedScalarQuantity();
foreach (
- const Resources& minResources, CHECK_NOTNONE(minAllocatableResources)) {
+ const Resources& minResources,
+ CHECK_NOTNONE(options.minAllocatableResources)) {
if (quantity.contains(minResources)) {
return true;
}
@@ -2535,7 +2530,7 @@ void HierarchicalAllocatorProcess::trackFrameworkUnderRole(
CHECK(!frameworkSorters.contains(role));
frameworkSorters.insert({role, Owned<Sorter>(frameworkSorterFactory())});
- frameworkSorters.at(role)->initialize(fairnessExcludeResourceNames);
+ frameworkSorters.at(role)->initialize(options.fairnessExcludeResourceNames);
foreachvalue (const Slave& slave, slaves) {
frameworkSorters.at(role)->add(slave.info.id(), slave.getTotal());
@@ -2682,14 +2677,14 @@ bool HierarchicalAllocatorProcess::isRemoteSlave(const Slave& slave) const
// If the slave has a configured domain (and it has been allowed to
// register with the master), the master must also have a configured
// domain.
- CHECK(domain.isSome());
+ CHECK(options.domain.isSome());
// The master will not startup if configured with a domain but no
// fault domain.
- CHECK(domain->has_fault_domain());
+ CHECK(options.domain->has_fault_domain());
const DomainInfo::FaultDomain::RegionInfo& masterRegion =
- domain->fault_domain().region();
+ options.domain->fault_domain().region();
const DomainInfo::FaultDomain::RegionInfo& slaveRegion =
slave.info.domain().fault_domain().region();
@@ -2704,7 +2699,7 @@ bool HierarchicalAllocatorProcess::isCapableOfReceivingAgent(
// Only offer resources from slaves that have GPUs to
// frameworks that are capable of receiving GPUs.
// See MESOS-5634.
- if (filterGpuResources && !frameworkCapabilities.gpuResources &&
+ if (options.filterGpuResources && !frameworkCapabilities.gpuResources &&
slave.hasGpu()) {
return false;
}
diff --git a/src/master/allocator/mesos/hierarchical.hpp b/src/master/allocator/mesos/hierarchical.hpp
index 7b31c9d..405f485 100644
--- a/src/master/allocator/mesos/hierarchical.hpp
+++ b/src/master/allocator/mesos/hierarchical.hpp
@@ -284,7 +284,7 @@ public:
}
void initialize(
- const Duration& allocationInterval,
+ const mesos::allocator::Options& options,
const lambda::function<
void(const FrameworkID&,
const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&
@@ -292,14 +292,7 @@ public:
const lambda::function<
void(const FrameworkID&,
const hashmap<SlaveID, UnavailableResources>&)>&
- inverseOfferCallback,
- const Option<std::set<std::string>>&
- fairnessExcludeResourceNames = None(),
- bool filterGpuResources = true,
- const Option<DomainInfo>& domain = None(),
- const Option<std::vector<Resources>>&
- minAllocatableResources = None(),
- const size_t maxCompletedFrameworks = 0) override;
+ inverseOfferCallback) override;
void recover(
const int _expectedAgentCount,
@@ -481,11 +474,11 @@ protected:
bool initialized;
bool paused;
+ mesos::allocator::Options options;
+
// Recovery data.
Option<int> expectedAgentCount;
- Duration allocationInterval;
-
lambda::function<
void(const FrameworkID&,
const hashmap<std::string, hashmap<SlaveID, Resources>>&)>
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 0c95c43..98a5bb1 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -754,16 +754,22 @@ void Master::initialize()
}
}
+ // Initialize the allocator options.
+ mesos::allocator::Options options;
+
+ options.allocationInterval = flags.allocation_interval;
+ options.fairnessExcludeResourceNames =
+ flags.fair_sharing_excluded_resource_names;
+ options.filterGpuResources = flags.filter_gpu_resources;
+ options.domain = flags.domain;
+ options.minAllocatableResources = CHECK_NOTERROR(minAllocatableResources);
+ options.maxCompletedFrameworks = flags.max_completed_frameworks;
+
// Initialize the allocator.
allocator->initialize(
- flags.allocation_interval,
+ options,
defer(self(), &Master::offer, lambda::_1, lambda::_2),
- defer(self(), &Master::inverseOffer, lambda::_1, lambda::_2),
- flags.fair_sharing_excluded_resource_names,
- flags.filter_gpu_resources,
- flags.domain,
- CHECK_NOTERROR(minAllocatableResources),
- flags.max_completed_frameworks);
+ defer(self(), &Master::inverseOffer, lambda::_1, lambda::_2));
// Parse the whitelist. Passing Allocator::updateWhitelist()
// callback is safe because we shut down the whitelistWatcher in
diff --git a/src/tests/allocator.hpp b/src/tests/allocator.hpp
index 15a5396..9437f58 100644
--- a/src/tests/allocator.hpp
+++ b/src/tests/allocator.hpp
@@ -33,6 +33,8 @@ using ::testing::DoDefault;
using ::testing::Invoke;
using ::testing::Return;
+using mesos::allocator::Options;
+
namespace mesos {
namespace internal {
namespace tests {
@@ -45,7 +47,7 @@ namespace tests {
ACTION_P(InvokeInitialize, allocator)
{
- allocator->real->initialize(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+ allocator->real->initialize(arg0, arg1, arg2);
}
@@ -247,9 +249,9 @@ public:
// to get the best of both worlds: the ability to use 'DoDefault'
// and no warnings when expectations are not explicit.
- ON_CALL(*this, initialize(_, _, _, _, _, _, _, _))
+ ON_CALL(*this, initialize(_, _, _))
.WillByDefault(InvokeInitialize(this));
- EXPECT_CALL(*this, initialize(_, _, _, _, _, _, _, _))
+ EXPECT_CALL(*this, initialize(_, _, _))
.WillRepeatedly(DoDefault());
ON_CALL(*this, recover(_, _))
@@ -390,19 +392,14 @@ public:
~TestAllocator() override {}
- MOCK_METHOD8(initialize, void(
- const Duration&,
+ MOCK_METHOD3(initialize, void(
+ const Options& options,
const lambda::function<
void(const FrameworkID&,
const hashmap<std::string, hashmap<SlaveID, Resources>>&)>&,
const lambda::function<
void(const FrameworkID&,
- const hashmap<SlaveID, UnavailableResources>&)>&,
- const Option<std::set<std::string>>&,
- bool,
- const Option<DomainInfo>&,
- const Option<std::vector<Resources>>&,
- const size_t maxCompletedFrameworks));
+ const hashmap<SlaveID, UnavailableResources>&)>&));
MOCK_METHOD2(recover, void(
const int expectedAgentCount,
diff --git a/src/tests/api_tests.cpp b/src/tests/api_tests.cpp
index c681b9b..97fd0bb 100644
--- a/src/tests/api_tests.cpp
+++ b/src/tests/api_tests.cpp
@@ -1196,7 +1196,7 @@ TEST_P(MasterAPITest, ReserveResources)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
@@ -1287,7 +1287,7 @@ TEST_P(MasterAPITest, UnreserveResources)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
diff --git a/src/tests/hierarchical_allocator_benchmarks.cpp b/src/tests/hierarchical_allocator_benchmarks.cpp
index bf9167b..5af09f7 100644
--- a/src/tests/hierarchical_allocator_benchmarks.cpp
+++ b/src/tests/hierarchical_allocator_benchmarks.cpp
@@ -47,6 +47,7 @@ using mesos::internal::master::allocator::HierarchicalDRFAllocator;
using mesos::internal::slave::AGENT_CAPABILITIES;
using mesos::allocator::Allocator;
+using mesos::allocator::Options;
using process::Clock;
using process::Future;
@@ -197,14 +198,15 @@ protected:
allocator = CHECK_NOTERROR(Allocator::create(
config.allocator, config.roleSorter, config.frameworkSorter));
+
+ Options options;
+ options.allocationInterval = config.allocationInterval;
+ options.minAllocatableResources = config.minAllocatableResources;
+
allocator->initialize(
- config.allocationInterval,
+ options,
CHECK_NOTNONE(offerCallback),
- {},
- None(),
- true,
- None(),
- config.minAllocatableResources);
+ {});
Stopwatch watch;
watch.start();
diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp
index 27fbd9c..3034d46 100644
--- a/src/tests/hierarchical_allocator_tests.cpp
+++ b/src/tests/hierarchical_allocator_tests.cpp
@@ -182,14 +182,16 @@ protected:
minAllocatableResources.push_back(CHECK_NOTERROR(Resources::parse(
"mem:" + stringify((double)MIN_MEM.bytes() / Bytes::MEGABYTES))));
+ Options options;
+ options.allocationInterval = flags.allocation_interval;
+ options.fairnessExcludeResourceNames =
+ flags.fair_sharing_excluded_resource_names;
+ options.minAllocatableResources = minAllocatableResources;
+
allocator->initialize(
- flags.allocation_interval,
+ options,
offerCallback.get(),
- inverseOfferCallback.get(),
- flags.fair_sharing_excluded_resource_names,
- true,
- None(),
- minAllocatableResources);
+ inverseOfferCallback.get());
}
SlaveInfo createSlaveInfo(const Resources& resources)
diff --git a/src/tests/master_allocator_tests.cpp b/src/tests/master_allocator_tests.cpp
index 88288ae..9dbab18 100644
--- a/src/tests/master_allocator_tests.cpp
+++ b/src/tests/master_allocator_tests.cpp
@@ -164,7 +164,7 @@ TYPED_TEST(MasterAllocatorTest, SingleFramework)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(&allocator);
ASSERT_SOME(master);
@@ -214,7 +214,7 @@ TYPED_TEST(MasterAllocatorTest, ResourcesUnused)
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(&allocator);
ASSERT_SOME(master);
@@ -324,7 +324,7 @@ TYPED_TEST(MasterAllocatorTest, OutOfOrderDispatch)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(&allocator);
ASSERT_SOME(master);
@@ -455,7 +455,7 @@ TYPED_TEST(MasterAllocatorTest, SchedulerFailover)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(&allocator);
ASSERT_SOME(master);
@@ -588,7 +588,7 @@ TYPED_TEST(MasterAllocatorTest, FrameworkExited)
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
master::Flags masterFlags = this->CreateMasterFlags();
masterFlags.allocation_interval = Milliseconds(50);
@@ -749,7 +749,7 @@ TYPED_TEST(MasterAllocatorTest, SlaveLost)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(&allocator);
ASSERT_SOME(master);
@@ -873,7 +873,7 @@ TYPED_TEST(MasterAllocatorTest, SlaveAdded)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
master::Flags masterFlags = this->CreateMasterFlags();
masterFlags.allocation_interval = Milliseconds(50);
@@ -969,7 +969,7 @@ TYPED_TEST(MasterAllocatorTest, TaskFinished)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
master::Flags masterFlags = this->CreateMasterFlags();
masterFlags.allocation_interval = Milliseconds(50);
@@ -1075,7 +1075,7 @@ TYPED_TEST(MasterAllocatorTest, CpusOnlyOfferedAndTaskLaunched)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
master::Flags masterFlags = this->CreateMasterFlags();
masterFlags.allocation_interval = Milliseconds(50);
@@ -1158,7 +1158,7 @@ TYPED_TEST(MasterAllocatorTest, MemoryOnlyOfferedAndTaskLaunched)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
master::Flags masterFlags = this->CreateMasterFlags();
masterFlags.allocation_interval = Milliseconds(50);
@@ -1359,7 +1359,7 @@ TYPED_TEST(MasterAllocatorTest, Whitelist)
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Future<Nothing> updateWhitelist1;
EXPECT_CALL(allocator, updateWhitelist(Option<hashset<string>>(hosts)))
@@ -1398,7 +1398,7 @@ TYPED_TEST(MasterAllocatorTest, RoleTest)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
master::Flags masterFlags = this->CreateMasterFlags();
masterFlags.roles = Some("role2");
@@ -1493,7 +1493,7 @@ TYPED_TEST(MasterAllocatorTest, FrameworkReregistersFirst)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(
&allocator, masterFlags);
@@ -1551,7 +1551,7 @@ TYPED_TEST(MasterAllocatorTest, FrameworkReregistersFirst)
{
TestAllocator<TypeParam> allocator2;
- EXPECT_CALL(allocator2, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator2, initialize(_, _, _));
Future<Nothing> addFramework;
EXPECT_CALL(allocator2, addFramework(_, _, _, _, _))
@@ -1619,7 +1619,7 @@ TYPED_TEST(MasterAllocatorTest, SlaveReregistersFirst)
{
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(&allocator);
@@ -1678,7 +1678,7 @@ TYPED_TEST(MasterAllocatorTest, SlaveReregistersFirst)
{
TestAllocator<TypeParam> allocator2;
- EXPECT_CALL(allocator2, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator2, initialize(_, _, _));
Future<Nothing> addSlave;
EXPECT_CALL(allocator2, addSlave(_, _, _, _, _, _))
@@ -1745,7 +1745,7 @@ TYPED_TEST(MasterAllocatorTest, RebalancedForUpdatedWeights)
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
// Start Mesos master.
master::Flags masterFlags = this->CreateMasterFlags();
@@ -1939,7 +1939,7 @@ TYPED_TEST(MasterAllocatorTest, NestedRoles)
TestAllocator<TypeParam> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
master::Flags masterFlags = this->CreateMasterFlags();
Try<Owned<cluster::Master>> master =
diff --git a/src/tests/master_quota_tests.cpp b/src/tests/master_quota_tests.cpp
index f669396..354a9e9 100644
--- a/src/tests/master_quota_tests.cpp
+++ b/src/tests/master_quota_tests.cpp
@@ -473,7 +473,7 @@ TEST_F(MasterQuotaTest, SetExistingQuota)
TEST_F(MasterQuotaTest, RemoveSingleQuota)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
@@ -648,7 +648,7 @@ TEST_F(MasterQuotaTest, Status)
TEST_F(MasterQuotaTest, InsufficientResourcesSingleAgent)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
@@ -708,7 +708,7 @@ TEST_F(MasterQuotaTest, InsufficientResourcesSingleAgent)
TEST_F(MasterQuotaTest, InsufficientResourcesMultipleAgents)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
@@ -783,7 +783,7 @@ TEST_F(MasterQuotaTest, InsufficientResourcesMultipleAgents)
TEST_F(MasterQuotaTest, AvailableResourcesSingleAgent)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
@@ -833,7 +833,7 @@ TEST_F(MasterQuotaTest, AvailableResourcesSingleAgent)
TEST_F(MasterQuotaTest, AvailableResourcesMultipleAgents)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
@@ -902,7 +902,7 @@ TEST_F(MasterQuotaTest, AvailableResourcesMultipleAgents)
TEST_F(MasterQuotaTest, AvailableResourcesAfterRescinding)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
@@ -1137,7 +1137,7 @@ TEST_F(MasterQuotaTest, RecoverQuotaEmptyCluster)
}
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
// Restart the master; configured quota should be recovered from the registry.
master->reset();
@@ -1170,7 +1170,7 @@ TEST_F(MasterQuotaTest, RecoverQuotaEmptyCluster)
TEST_F(MasterQuotaTest, NoAuthenticationNoAuthorization)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
// Disable http_readwrite authentication and authorization.
// TODO(alexr): Setting master `--acls` flag to `ACLs()` or `None()` seems
@@ -1276,7 +1276,7 @@ TEST_F(MasterQuotaTest, UnauthenticatedQuotaRequest)
TEST_F(MasterQuotaTest, AuthorizeGetUpdateQuotaRequests)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
// Setup ACLs so that only the default principal can modify quotas
// for `ROLE1` and read status.
@@ -1824,7 +1824,7 @@ TEST_F(MasterQuotaTest, DISABLED_ChildRoleDeleteParentQuota)
TEST_F(MasterQuotaTest, DISABLED_ClusterCapacityWithNestedRoles)
{
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
diff --git a/src/tests/reservation_tests.cpp b/src/tests/reservation_tests.cpp
index d693122..3f33885 100644
--- a/src/tests/reservation_tests.cpp
+++ b/src/tests/reservation_tests.cpp
@@ -601,7 +601,7 @@ TEST_F(ReservationTest, DropReserveTooLarge)
masterFlags.allocation_interval = Milliseconds(5);
masterFlags.roles = frameworkInfo.roles(0);
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator, masterFlags);
ASSERT_SOME(master);
@@ -2118,7 +2118,7 @@ TEST_F(ReservationTest, DropReserveWithDifferentRole)
masterFlags.allocation_interval = Milliseconds(5);
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator, masterFlags);
ASSERT_SOME(master);
@@ -2220,7 +2220,7 @@ TEST_F(ReservationTest, PreventUnreservingAlienResources)
masterFlags.allocation_interval = Milliseconds(5);
TestAllocator<> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator, masterFlags);
ASSERT_SOME(master);
diff --git a/src/tests/resource_offers_tests.cpp b/src/tests/resource_offers_tests.cpp
index 24800c2..08bb109 100644
--- a/src/tests/resource_offers_tests.cpp
+++ b/src/tests/resource_offers_tests.cpp
@@ -284,7 +284,7 @@ TEST_F(ResourceOffersTest, Request)
{
TestAllocator<master::allocator::HierarchicalDRFAllocator> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = StartMaster(&allocator);
ASSERT_SOME(master);
diff --git a/src/tests/slave_recovery_tests.cpp b/src/tests/slave_recovery_tests.cpp
index 5842ccf..4bf0229 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -3762,7 +3762,7 @@ TYPED_TEST(SlaveRecoveryTest, ReconcileTasksMissingFromSlave)
{
TestAllocator<master::allocator::HierarchicalDRFAllocator> allocator;
- EXPECT_CALL(allocator, initialize(_, _, _, _, _, _, _, _));
+ EXPECT_CALL(allocator, initialize(_, _, _));
Try<Owned<cluster::Master>> master = this->StartMaster(&allocator);
ASSERT_SOME(master);
[mesos] 03/03: Add documentation for per-framework metrics flag.
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpeach pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 5a0a36fbd8ba16aa14915cd2273efb1eb96fba7f
Author: Jacob Janco <jj...@gmail.com>
AuthorDate: Thu Oct 25 08:00:44 2018 -0700
Add documentation for per-framework metrics flag.
Review: https://reviews.apache.org/r/68957/
---
docs/configuration/master.md | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/docs/configuration/master.md b/docs/configuration/master.md
index f290e37..5754767 100644
--- a/docs/configuration/master.md
+++ b/docs/configuration/master.md
@@ -465,6 +465,17 @@ If not set, offers do not timeout.
</td>
</tr>
+<tr id="publish_per_framework_metrics">
+ <td>
+ --[no-]publish_per_framework_metrics
+ </td>
+ <td>
+If <code>true</code>, an extensive set of metrics for each active framework will
+be published. These metrics are useful for understanding cluster behavior,
+but can be overwhelming for very large numbers of frameworks. (default: true)
+ </td>
+</tr>
+
<tr id="rate_limits">
<td>
--rate_limits=VALUE
[mesos] 02/03: Add a flag to toggle per-framework metrics.
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpeach pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 90eddb08c7744337e5c43b15154851a3942001ed
Author: Jacob Janco <jj...@gmail.com>
AuthorDate: Thu Oct 25 07:59:48 2018 -0700
Add a flag to toggle per-framework metrics.
In clusters with high numbers of frameworks, it can be necessary
to control publishing of per framework metrics. Metrics are still
collected for active frameworks, but will not be pulished if this
flag is set. This allows future extraction of these metrics, e.g.
an API call or logs while keeping the code change simple.
Review: https://reviews.apache.org/r/68956/
---
include/mesos/allocator/allocator.hpp | 1 +
src/master/allocator/mesos/hierarchical.cpp | 10 +++--
src/master/allocator/mesos/hierarchical.hpp | 5 ++-
src/master/allocator/mesos/metrics.cpp | 28 ++++++++++--
src/master/allocator/mesos/metrics.hpp | 10 ++++-
src/master/flags.cpp | 8 ++++
src/master/flags.hpp | 1 +
src/master/framework.cpp | 2 +-
src/master/master.cpp | 1 +
src/master/master.hpp | 2 +-
src/master/metrics.cpp | 70 ++++++++++++++++++-----------
src/master/metrics.hpp | 9 +++-
12 files changed, 109 insertions(+), 38 deletions(-)
diff --git a/include/mesos/allocator/allocator.hpp b/include/mesos/allocator/allocator.hpp
index 4c4273a..2a6849b 100644
--- a/include/mesos/allocator/allocator.hpp
+++ b/include/mesos/allocator/allocator.hpp
@@ -52,6 +52,7 @@ struct Options
Option<DomainInfo> domain = None();
Option<std::vector<Resources>> minAllocatableResources = None();
size_t maxCompletedFrameworks = 0;
+ bool publishPerFrameworkMetrics = true;
};
diff --git a/src/master/allocator/mesos/hierarchical.cpp b/src/master/allocator/mesos/hierarchical.cpp
index eae5a12..cc8ab91 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -136,12 +136,13 @@ private:
Framework::Framework(
const FrameworkInfo& frameworkInfo,
const set<string>& _suppressedRoles,
- bool _active)
+ bool _active,
+ bool publishPerFrameworkMetrics)
: roles(protobuf::framework::getRoles(frameworkInfo)),
suppressedRoles(_suppressedRoles),
capabilities(frameworkInfo.capabilities()),
active(_active),
- metrics(new FrameworkMetrics(frameworkInfo)) {}
+ metrics(new FrameworkMetrics(frameworkInfo, publishPerFrameworkMetrics)) {}
void HierarchicalAllocatorProcess::initialize(
@@ -265,7 +266,10 @@ void HierarchicalAllocatorProcess::addFramework(
CHECK(!frameworks.contains(frameworkId));
frameworks.insert(
- {frameworkId, Framework(frameworkInfo, suppressedRoles, active)});
+ {frameworkId, Framework(frameworkInfo,
+ suppressedRoles,
+ active,
+ options.publishPerFrameworkMetrics)});
const Framework& framework = frameworks.at(frameworkId);
diff --git a/src/master/allocator/mesos/hierarchical.hpp b/src/master/allocator/mesos/hierarchical.hpp
index 405f485..a4425bc 100644
--- a/src/master/allocator/mesos/hierarchical.hpp
+++ b/src/master/allocator/mesos/hierarchical.hpp
@@ -81,7 +81,8 @@ struct Framework
Framework(
const FrameworkInfo& frameworkInfo,
const std::set<std::string>& suppressedRoles,
- bool active);
+ bool active,
+ bool publishPerFrameworkMetrics);
std::set<std::string> roles;
@@ -97,6 +98,8 @@ struct Framework
bool active;
+ bool publishPerFrameworkMetrics;
+
process::Owned<FrameworkMetrics> metrics;
};
diff --git a/src/master/allocator/mesos/metrics.cpp b/src/master/allocator/mesos/metrics.cpp
index 73e68eb..5533eb9 100644
--- a/src/master/allocator/mesos/metrics.cpp
+++ b/src/master/allocator/mesos/metrics.cpp
@@ -214,8 +214,11 @@ void Metrics::removeRole(const string& role)
}
-FrameworkMetrics::FrameworkMetrics(const FrameworkInfo& _frameworkInfo)
- : frameworkInfo(_frameworkInfo)
+FrameworkMetrics::FrameworkMetrics(
+ const FrameworkInfo& _frameworkInfo,
+ const bool _publishPerFrameworkMetrics)
+ : frameworkInfo(_frameworkInfo),
+ publishPerFrameworkMetrics(_publishPerFrameworkMetrics)
{
// TODO(greggomann): Calling `getRoles` below copies the roles from the
// framework info, which could become expensive if the number of roles grows
@@ -263,7 +266,7 @@ void FrameworkMetrics::addSubscribedRole(const string& role)
role + "/suppressed"));
CHECK(result.second);
- process::metrics::add(result.first->second);
+ addMetric(result.first->second);
}
@@ -272,10 +275,27 @@ void FrameworkMetrics::removeSubscribedRole(const string& role)
auto iter = suppressed.find(role);
CHECK(iter != suppressed.end());
- process::metrics::remove(iter->second);
+ removeMetric(iter->second);
suppressed.erase(iter);
}
+
+template <typename T>
+void FrameworkMetrics::addMetric(const T& metric) {
+ if (publishPerFrameworkMetrics) {
+ process::metrics::add(metric);
+ }
+}
+
+
+template <typename T>
+void FrameworkMetrics::removeMetric(const T& metric) {
+ if (publishPerFrameworkMetrics) {
+ process::metrics::remove(metric);
+ }
+}
+
+
} // namespace internal {
} // namespace allocator {
} // namespace master {
diff --git a/src/master/allocator/mesos/metrics.hpp b/src/master/allocator/mesos/metrics.hpp
index 34cc16b..a26278a 100644
--- a/src/master/allocator/mesos/metrics.hpp
+++ b/src/master/allocator/mesos/metrics.hpp
@@ -94,7 +94,9 @@ struct Metrics
struct FrameworkMetrics
{
- explicit FrameworkMetrics(const FrameworkInfo& _frameworkInfo);
+ FrameworkMetrics(
+ const FrameworkInfo& _frameworkInfo,
+ const bool _publishPerFrameworkMetrics);
~FrameworkMetrics();
@@ -106,8 +108,14 @@ struct FrameworkMetrics
void addSubscribedRole(const std::string& role);
void removeSubscribedRole(const std::string& role);
+ // Add or remove per-framework metrics.
+ template <typename T> void addMetric(const T& metric);
+ template <typename T> void removeMetric(const T& metric);
+
const FrameworkInfo frameworkInfo;
+ const bool publishPerFrameworkMetrics;
+
// Suppresion state metric (boolean 0 or 1) for each role.
hashmap<std::string, process::metrics::PushGauge> suppressed;
};
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index 6ad53ed..2677738 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -683,6 +683,14 @@ mesos::internal::master::Flags::Flags()
"If true, only agents with a configured domain can register.\n",
false);
+ add(&Flags::publish_per_framework_metrics,
+ "publish_per_framework_metrics",
+ "If true, an extensive set of metrics for each active framework will\n"
+ "be published. These metrics are useful for understanding cluster\n"
+ "behavior, but can be overwhelming for very large numbers of\n"
+ "frameworks.",
+ true);
+
add(&Flags::domain,
"domain",
"Domain that the master belongs to. Mesos currently only supports\n"
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index 4a26015..ed2d76a 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -99,6 +99,7 @@ public:
Duration registry_max_agent_age;
size_t registry_max_agent_count;
bool require_agent_domain;
+ bool publish_per_framework_metrics;
Option<DomainInfo> domain;
// The following flags are executable specific (e.g., since we only
diff --git a/src/master/framework.cpp b/src/master/framework.cpp
index 7cfe9f4..36eda9f 100644
--- a/src/master/framework.cpp
+++ b/src/master/framework.cpp
@@ -69,7 +69,7 @@ Framework::Framework(
reregisteredTime(time),
completedTasks(masterFlags.max_completed_tasks_per_framework),
unreachableTasks(masterFlags.max_unreachable_tasks_per_framework),
- metrics(_info)
+ metrics(_info, masterFlags.publish_per_framework_metrics)
{
CHECK(_info.has_id());
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 98a5bb1..704dfc0 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -764,6 +764,7 @@ void Master::initialize()
options.domain = flags.domain;
options.minAllocatableResources = CHECK_NOTERROR(minAllocatableResources);
options.maxCompletedFrameworks = flags.max_completed_frameworks;
+ options.publishPerFrameworkMetrics = flags.publish_per_framework_metrics;
// Initialize the allocator.
allocator->initialize(
diff --git a/src/master/master.hpp b/src/master/master.hpp
index ea7e924..e77babf 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -2574,7 +2574,7 @@ struct Framework
Option<process::Owned<
Heartbeater<mesos::scheduler::Event, v1::scheduler::Event>>> heartbeater;
- // This is used for per-framwork metrics.
+ // This is used for per-framework metrics.
FrameworkMetrics metrics;
private:
diff --git a/src/master/metrics.cpp b/src/master/metrics.cpp
index 56a7eef..f69ed52 100644
--- a/src/master/metrics.cpp
+++ b/src/master/metrics.cpp
@@ -552,8 +552,11 @@ void Metrics::incrementTasksStates(
}
-FrameworkMetrics::FrameworkMetrics(const FrameworkInfo& _frameworkInfo)
+FrameworkMetrics::FrameworkMetrics(
+ const FrameworkInfo& _frameworkInfo,
+ bool _publishPerFrameworkMetrics)
: frameworkInfo(_frameworkInfo),
+ publishPerFrameworkMetrics(_publishPerFrameworkMetrics),
subscribed(
getFrameworkMetricPrefix(frameworkInfo) + "subscribed"),
calls(
@@ -571,15 +574,14 @@ FrameworkMetrics::FrameworkMetrics(const FrameworkInfo& _frameworkInfo)
operations(
getFrameworkMetricPrefix(frameworkInfo) + "operations")
{
- process::metrics::add(subscribed);
-
- process::metrics::add(offers_sent);
- process::metrics::add(offers_accepted);
- process::metrics::add(offers_declined);
- process::metrics::add(offers_rescinded);
+ addMetric(subscribed);
+ addMetric(offers_sent);
+ addMetric(offers_accepted);
+ addMetric(offers_declined);
+ addMetric(offers_rescinded);
// Add metrics for scheduler calls.
- process::metrics::add(calls);
+ addMetric(calls);
for (int index = 0;
index < scheduler::Call::Type_descriptor()->value_count();
index++) {
@@ -598,11 +600,11 @@ FrameworkMetrics::FrameworkMetrics(const FrameworkInfo& _frameworkInfo)
strings::lower(descriptor->name()));
call_types.put(type, counter);
- process::metrics::add(counter);
+ addMetric(counter);
}
// Add metrics for scheduler events.
- process::metrics::add(events);
+ addMetric(events);
for (int index = 0;
index < scheduler::Event::Type_descriptor()->value_count();
index++) {
@@ -621,7 +623,7 @@ FrameworkMetrics::FrameworkMetrics(const FrameworkInfo& _frameworkInfo)
strings::lower(descriptor->name()));
event_types.put(type, counter);
- process::metrics::add(counter);
+ addMetric(counter);
}
// Add metrics for both active and terminal task states.
@@ -637,19 +639,19 @@ FrameworkMetrics::FrameworkMetrics(const FrameworkInfo& _frameworkInfo)
strings::lower(descriptor->name()));
terminal_task_states.put(state, counter);
- process::metrics::add(counter);
+ addMetric(counter);
} else {
PushGauge gauge = PushGauge(
getFrameworkMetricPrefix(frameworkInfo) + "tasks/active/" +
strings::lower(TaskState_Name(state)));
active_task_states.put(state, gauge);
- process::metrics::add(gauge);
+ addMetric(gauge);
}
}
// Add metrics for offer operations.
- process::metrics::add(operations);
+ addMetric(operations);
for (int index = 0;
index < Offer::Operation::Type_descriptor()->value_count();
index++) {
@@ -668,41 +670,41 @@ FrameworkMetrics::FrameworkMetrics(const FrameworkInfo& _frameworkInfo)
"operations/" + strings::lower(descriptor->name()));
operation_types.put(type, counter);
- process::metrics::add(counter);
+ addMetric(counter);
}
}
FrameworkMetrics::~FrameworkMetrics()
{
- process::metrics::remove(subscribed);
+ removeMetric(subscribed);
- process::metrics::remove(calls);
+ removeMetric(calls);
foreachvalue (const Counter& counter, call_types) {
- process::metrics::remove(counter);
+ removeMetric(counter);
}
process::metrics::remove(events);
foreachvalue (const Counter& counter, event_types) {
- process::metrics::remove(counter);
+ removeMetric(counter);
}
- process::metrics::remove(offers_sent);
- process::metrics::remove(offers_accepted);
- process::metrics::remove(offers_declined);
- process::metrics::remove(offers_rescinded);
+ removeMetric(offers_sent);
+ removeMetric(offers_accepted);
+ removeMetric(offers_declined);
+ removeMetric(offers_rescinded);
foreachvalue (const Counter& counter, terminal_task_states) {
- process::metrics::remove(counter);
+ removeMetric(counter);
}
foreachvalue (const PushGauge& gauge, active_task_states) {
- process::metrics::remove(gauge);
+ removeMetric(gauge);
}
process::metrics::remove(operations);
foreachvalue (const Counter& counter, operation_types) {
- process::metrics::remove(counter);
+ removeMetric(counter);
}
}
@@ -753,6 +755,22 @@ string getFrameworkMetricPrefix(const FrameworkInfo& frameworkInfo)
}
+template <typename T>
+void FrameworkMetrics::addMetric(const T& metric) {
+ if (publishPerFrameworkMetrics) {
+ process::metrics::add(metric);
+ }
+}
+
+
+template <typename T>
+void FrameworkMetrics::removeMetric(const T& metric) {
+ if (publishPerFrameworkMetrics) {
+ process::metrics::remove(metric);
+ }
+}
+
+
void FrameworkMetrics::incrementEvent(const scheduler::Event& event)
{
++CHECK_NOTNONE(event_types.get(event.type()));
diff --git a/src/master/metrics.hpp b/src/master/metrics.hpp
index e1da18e..f6bb89d 100644
--- a/src/master/metrics.hpp
+++ b/src/master/metrics.hpp
@@ -220,7 +220,9 @@ struct Metrics
struct FrameworkMetrics
{
- explicit FrameworkMetrics(const FrameworkInfo& _frameworkInfo);
+ FrameworkMetrics(
+ const FrameworkInfo& _frameworkInfo,
+ bool publishPerFrameworkMetrics);
~FrameworkMetrics();
@@ -247,8 +249,13 @@ struct FrameworkMetrics
void incrementOperation(const Offer::Operation& operation);
+ template <typename T> void addMetric(const T& metric);
+ template <typename T> void removeMetric(const T& metric);
+
const FrameworkInfo frameworkInfo;
+ bool publishPerFrameworkMetrics;
+
process::metrics::PushGauge subscribed;
process::metrics::Counter calls;