You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by gr...@apache.org on 2018/08/01 20:24:31 UTC
[14/16] mesos git commit: Updated tests to check per-framework
metrics.
Updated tests to check per-framework metrics.
Review: https://reviews.apache.org/r/67878/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3c85608a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3c85608a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3c85608a
Branch: refs/heads/master
Commit: 3c85608ada614196e1c4708fdc0998a40d6eb917
Parents: 9f39fe3
Author: Greg Mann <gr...@mesosphere.io>
Authored: Wed Aug 1 07:59:06 2018 -0700
Committer: Greg Mann <gr...@gmail.com>
Committed: Wed Aug 1 13:07:55 2018 -0700
----------------------------------------------------------------------
src/tests/master_tests.cpp | 139 +++++++++++++++++++++++++++++++++----
src/tests/scheduler_tests.cpp | 29 +++++++-
2 files changed, 153 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3c85608a/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 44b0ac3..91ce81b 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -51,6 +51,7 @@
#include "master/flags.hpp"
#include "master/master.hpp"
+#include "master/metrics.hpp"
#include "master/registry_operations.hpp"
#include "master/allocator/mesos/allocator.hpp"
@@ -745,11 +746,15 @@ TEST_F(MasterTest, StatusUpdateAck)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
- EXPECT_CALL(sched, registered(&driver, _, _));
+ FrameworkID frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(SaveArg<1>(&frameworkId));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
@@ -791,6 +796,25 @@ TEST_F(MasterTest, StatusUpdateAck)
// Ensure the slave gets a status update ACK.
AWAIT_READY(acknowledgement);
+ JSON::Object metrics = Metrics();
+
+ frameworkInfo.mutable_id()->CopyFrom(frameworkId);
+
+ const string prefix = master::getFrameworkMetricPrefix(frameworkInfo);
+
+ EXPECT_EQ(2, metrics.values[prefix + "calls"]);
+ EXPECT_EQ(1, metrics.values[prefix + "calls/accept"]);
+ EXPECT_EQ(1, metrics.values[prefix + "calls/acknowledge"]);
+
+ EXPECT_EQ(1, metrics.values[prefix + "offers/accepted"]);
+
+ EXPECT_EQ(1, metrics.values[prefix + "operations"]);
+ EXPECT_EQ(1, metrics.values[prefix + "operations/launch"]);
+
+ EXPECT_EQ(1, metrics.values[prefix + "events/update"]);
+
+ EXPECT_EQ(1, metrics.values[prefix + "tasks/active/task_running"]);
+
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
@@ -4343,13 +4367,15 @@ TEST_F(MasterTest, OfferTimeout)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
- Future<Nothing> registered;
+ FrameworkID frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
- .WillOnce(FutureSatisfy(®istered));
+ .WillOnce(SaveArg<1>(&frameworkId));
Future<vector<Offer>> offers1;
Future<vector<Offer>> offers2;
@@ -4367,7 +4393,6 @@ TEST_F(MasterTest, OfferTimeout)
driver.start();
- AWAIT_READY(registered);
AWAIT_READY(offers1);
ASSERT_EQ(1u, offers1->size());
@@ -4393,6 +4418,16 @@ TEST_F(MasterTest, OfferTimeout)
EXPECT_EQ(offers1.get()[0].resources(), offers2.get()[0].resources());
+ JSON::Object metrics = Metrics();
+
+ frameworkInfo.mutable_id()->CopyFrom(frameworkId);
+
+ const string prefix = master::getFrameworkMetricPrefix(frameworkInfo);
+
+ EXPECT_EQ(0, metrics.values[prefix + "offers/accepted"]);
+ EXPECT_EQ(2, metrics.values[prefix + "offers/sent"]);
+ EXPECT_EQ(1, metrics.values[prefix + "offers/rescinded"]);
+
driver.stop();
driver.join();
}
@@ -6019,11 +6054,15 @@ TEST_F(MasterTest, MasterFailoverLongLivedExecutor)
StartSlave(&detector, &containerizer, flags);
ASSERT_SOME(slave);
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+
MockScheduler sched;
- TestingMesosSchedulerDriver driver(&sched, &detector);
+ TestingMesosSchedulerDriver driver(&sched, &detector, frameworkInfo);
+ Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
- .Times(2);
+ .WillOnce(FutureArg<1>(&frameworkId))
+ .WillOnce(Return());
EXPECT_CALL(sched, disconnected(&driver));
@@ -6061,6 +6100,15 @@ TEST_F(MasterTest, MasterFailoverLongLivedExecutor)
AWAIT_READY(status1);
EXPECT_EQ(TASK_RUNNING, status1->state());
+ AWAIT_ASSERT_READY(frameworkId);
+
+ frameworkInfo.mutable_id()->CopyFrom(frameworkId.get());
+
+ const string metricPrefix = master::getFrameworkMetricPrefix(frameworkInfo);
+
+ JSON::Object metrics1 = Metrics();
+ EXPECT_EQ(1, metrics1.values[metricPrefix + "tasks/active/task_running"]);
+
// Fail over master.
master->reset();
master = StartMaster();
@@ -6078,6 +6126,9 @@ TEST_F(MasterTest, MasterFailoverLongLivedExecutor)
AWAIT_READY(offers2);
ASSERT_FALSE(offers2->empty());
+ JSON::Object metrics2 = Metrics();
+ EXPECT_EQ(1, metrics2.values[metricPrefix + "tasks/active/task_running"]);
+
// The second task is a just a copy of the first task (using the
// same executor and resources). We have to set a new task id.
TaskInfo task2 = task1;
@@ -6094,6 +6145,9 @@ TEST_F(MasterTest, MasterFailoverLongLivedExecutor)
AWAIT_READY(status2);
EXPECT_EQ(TASK_RUNNING, status2->state());
+ JSON::Object metrics3 = Metrics();
+ EXPECT_EQ(2, metrics3.values[metricPrefix + "tasks/active/task_running"]);
+
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
@@ -6483,6 +6537,8 @@ TEST_F(MasterTest, MaxCompletedFrameworksFlag)
const size_t totalFrameworks = 2;
const size_t maxFrameworksArray[] = {0, 1, 2};
+ const string ROLE = "foo";
+
foreach (const size_t maxFrameworks, maxFrameworksArray) {
master::Flags masterFlags = CreateMasterFlags();
masterFlags.max_completed_frameworks = maxFrameworks;
@@ -6494,11 +6550,18 @@ TEST_F(MasterTest, MaxCompletedFrameworksFlag)
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
+ // This is used to check per-framework metrics.
+ vector<FrameworkInfo> frameworkInfos;
+
for (size_t i = 0; i < totalFrameworks; i++) {
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.clear_roles();
+ frameworkInfo.add_roles(ROLE);
+
MockScheduler sched;
MesosSchedulerDriver schedDriver(
&sched,
- DEFAULT_FRAMEWORK_INFO,
+ frameworkInfo,
master.get()->pid,
DEFAULT_CREDENTIAL);
@@ -6506,13 +6569,16 @@ TEST_F(MasterTest, MaxCompletedFrameworksFlag)
EXPECT_CALL(sched, resourceOffers(_, _))
.WillRepeatedly(Return());
- Future<Nothing> schedRegistered;
+ Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(_, _, _))
- .WillOnce(FutureSatisfy(&schedRegistered));
+ .WillOnce(FutureArg<1>(&frameworkId));
schedDriver.start();
- AWAIT_READY(schedRegistered);
+ AWAIT_READY(frameworkId);
+
+ frameworkInfo.mutable_id()->CopyFrom(frameworkId.get());
+ frameworkInfos.push_back(frameworkInfo);
schedDriver.stop();
schedDriver.join();
@@ -6535,6 +6601,23 @@ TEST_F(MasterTest, MaxCompletedFrameworksFlag)
state.values["completed_frameworks"].as<JSON::Array>();
EXPECT_EQ(maxFrameworks, completedFrameworks->values.size());
+
+ // The first `evictedFrameworks` schedulers in the list should have their
+ // per-framework metrics GC'd, both from the master and the allocator. We
+ // check the 'suppressed' metric because it resides in the allocator.
+ const size_t evictedFrameworks = totalFrameworks - maxFrameworks;
+ JSON::Object metrics = Metrics();
+ for (size_t i = 0; i < totalFrameworks; i++) {
+ const string prefix = master::getFrameworkMetricPrefix(frameworkInfos[i]);
+ if (i < evictedFrameworks) {
+ EXPECT_NONE(metrics.at<JSON::Number>(prefix + "calls"));
+ EXPECT_NONE(metrics.at<JSON::Number>(
+ prefix + "roles/" + ROLE + "/suppressed"));
+ } else {
+ EXPECT_EQ(0, metrics.values[prefix + "subscribed"]);
+ EXPECT_EQ(0, metrics.values[prefix + "roles/" + ROLE + "/suppressed"]);
+ }
+ }
}
}
@@ -7803,6 +7886,13 @@ TEST_F(MasterTest, MultiRoleSchedulerUnsubscribeFromRole)
AWAIT_READY(offers);
ASSERT_EQ(1u, offers->size());
+ frameworkInfo.mutable_id()->CopyFrom(frameworkId.get());
+
+ const string prefix = master::getFrameworkMetricPrefix(frameworkInfo);
+
+ JSON::Object metrics1 = Metrics();
+ EXPECT_EQ(0, metrics1.values[prefix + "roles/foo/suppressed"]);
+
Resources resources = Resources::parse("cpus:1;mem:512").get();
Offer offer = offers.get()[0];
@@ -7827,7 +7917,6 @@ TEST_F(MasterTest, MultiRoleSchedulerUnsubscribeFromRole)
// Remove the role from the framework.
- frameworkInfo.mutable_id()->CopyFrom(frameworkId.get());
frameworkInfo.clear_roles();
MockScheduler sched2;
@@ -7853,6 +7942,9 @@ TEST_F(MasterTest, MultiRoleSchedulerUnsubscribeFromRole)
AWAIT_READY(registered2);
+ JSON::Object metrics2 = Metrics();
+ EXPECT_NONE(metrics2.at<JSON::Number>(prefix + "roles/foo/suppressed"));
+
// Wait for the agent to get the updated framework info.
AWAIT_READY(updateFrameworkMessage);
@@ -9368,7 +9460,9 @@ TEST_P(MasterTestPrePostReservationRefinement,
// We use this to capture offers from 'resourceOffers'.
Future<vector<Offer>> offers;
- EXPECT_CALL(sched, registered(&driver, _, _));
+ FrameworkID frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(SaveArg<1>(&frameworkId));
// The expectation for the first offer.
EXPECT_CALL(sched, resourceOffers(&driver, _))
@@ -9486,6 +9580,23 @@ TEST_P(MasterTestPrePostReservationRefinement,
.contains(allocatedResources(
unreservedCpus + unreservedDisk, frameworkInfo.roles(0))));
+ JSON::Object metrics = Metrics();
+
+ frameworkInfo.mutable_id()->CopyFrom(frameworkId);
+
+ const string prefix = master::getFrameworkMetricPrefix(frameworkInfo);
+
+ EXPECT_EQ(4, metrics.values[prefix + "calls"]);
+ EXPECT_EQ(4, metrics.values[prefix + "calls/accept"]);
+
+ EXPECT_EQ(4, metrics.values[prefix + "offers/accepted"]);
+
+ EXPECT_EQ(4, metrics.values[prefix + "operations"]);
+ EXPECT_EQ(1, metrics.values[prefix + "operations/reserve"]);
+ EXPECT_EQ(1, metrics.values[prefix + "operations/create"]);
+ EXPECT_EQ(1, metrics.values[prefix + "operations/destroy"]);
+ EXPECT_EQ(1, metrics.values[prefix + "operations/unreserve"]);
+
driver.stop();
driver.join();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/3c85608a/src/tests/scheduler_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/scheduler_tests.cpp b/src/tests/scheduler_tests.cpp
index bcdd0ec..0ee5b77 100644
--- a/src/tests/scheduler_tests.cpp
+++ b/src/tests/scheduler_tests.cpp
@@ -1515,6 +1515,8 @@ TEST_P(SchedulerTest, Revive)
TEST_P(SchedulerTest, Suppress)
{
+ const string ROLE = "foo";
+
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -1548,12 +1550,17 @@ TEST_P(SchedulerTest, Suppress)
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers1));
+ v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
+
+ frameworkInfo.clear_roles();
+ frameworkInfo.add_roles(ROLE);
+
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
- subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
+ subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
mesos.send(call);
}
@@ -1562,6 +1569,8 @@ TEST_P(SchedulerTest, Suppress)
v1::FrameworkID frameworkId(subscribed->framework_id());
+ frameworkInfo.mutable_id()->CopyFrom(frameworkId);
+
AWAIT_READY(offers1);
ASSERT_FALSE(offers1->offers().empty());
@@ -1587,6 +1596,20 @@ TEST_P(SchedulerTest, Suppress)
mesos.send(call);
}
+ // Wait for the master to process the DECLINE call.
+ Clock::pause();
+ Clock::settle();
+ Clock::resume();
+
+ JSON::Object metrics1 = Metrics();
+
+ const string prefix =
+ master::getFrameworkMetricPrefix(devolve(frameworkInfo));
+
+ EXPECT_EQ(1, metrics1.values[prefix + "subscribed"]);
+ EXPECT_EQ(1, metrics1.values[prefix + "offers/declined"]);
+ EXPECT_EQ(0, metrics1.values[prefix + "roles/" + ROLE + "/suppressed"]);
+
Future<Nothing> suppressOffers =
FUTURE_DISPATCH(_, &MesosAllocatorProcess::suppressOffers);
@@ -1604,6 +1627,10 @@ TEST_P(SchedulerTest, Suppress)
Clock::pause();
Clock::settle();
+ JSON::Object metrics2 = Metrics();
+
+ EXPECT_EQ(1, metrics2.values[prefix + "roles/" + ROLE + "/suppressed"]);
+
// No offers should be sent within 100 mins because the framework
// suppressed offers.
Clock::advance(Minutes(100));