You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ya...@apache.org on 2016/09/07 19:43:23 UTC

[3/3] mesos git commit: Add unit tests for sharing of resources.

Add unit tests for sharing of resources.

Initial set of tests. More test coverage in progress before releasing
this feature.

Review: https://reviews.apache.org/r/45964/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f8a9008e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f8a9008e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f8a9008e

Branch: refs/heads/master
Commit: f8a9008e8973984baf16908b2fee1c526d71b1c4
Parents: ca064f6
Author: Anindya Sinha <an...@apple.com>
Authored: Wed Sep 7 08:35:26 2016 -0700
Committer: Jiang Yan Xu <xu...@apple.com>
Committed: Wed Sep 7 12:24:10 2016 -0700

----------------------------------------------------------------------
 src/tests/hierarchical_allocator_tests.cpp | 110 ++++++++++++++++++++++++
 src/tests/master_validation_tests.cpp      |  46 +++++++++-
 src/tests/mesos.hpp                        |   4 +-
 src/tests/persistent_volume_tests.cpp      | 101 ++++++++++++++++++++++
 src/tests/sorter_tests.cpp                 |  89 +++++++++++++++++++
 5 files changed, 343 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/f8a9008e/src/tests/hierarchical_allocator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp
index 24698ff..b217921 100644
--- a/src/tests/hierarchical_allocator_tests.cpp
+++ b/src/tests/hierarchical_allocator_tests.cpp
@@ -1273,6 +1273,116 @@ TEST_F(HierarchicalAllocatorTest, UpdateAllocation)
 }
 
 
+// This test verifies that `updateAllocation()` supports creating and
+// destroying shared persistent volumes.
+TEST_F(HierarchicalAllocatorTest, UpdateAllocationSharedPersistentVolume)
+{
+  Clock::pause();
+
+  initialize();
+
+  SlaveInfo slave = createSlaveInfo("cpus:100;mem:100;disk(role1):100");
+  allocator->addSlave(slave.id(), slave, None(), slave.resources(), {});
+
+  // Initially, all the resources are allocated.
+  FrameworkInfo framework = createFrameworkInfo("role1");
+  allocator->addFramework(
+      framework.id(), framework, hashmap<SlaveID, Resources>());
+
+  Future<Allocation> allocation = allocations.get();
+  AWAIT_READY(allocation);
+  EXPECT_EQ(framework.id(), allocation.get().frameworkId);
+  EXPECT_EQ(1u, allocation.get().resources.size());
+  EXPECT_TRUE(allocation.get().resources.contains(slave.id()));
+  EXPECT_EQ(slave.resources(), Resources::sum(allocation.get().resources));
+
+  // Construct an offer operation for the framework's allocation.
+  Resource volume = Resources::parse("disk", "5", "role1").get();
+  volume.mutable_disk()->mutable_persistence()->set_id("ID");
+  volume.mutable_disk()->mutable_volume()->set_container_path("data");
+  volume.mutable_shared();
+
+  Offer::Operation create;
+  create.set_type(Offer::Operation::CREATE);
+  create.mutable_create()->add_volumes()->CopyFrom(volume);
+
+  // Ensure the offer operation can be applied.
+  Try<Resources> update =
+    Resources::sum(allocation.get().resources).apply(create);
+
+  ASSERT_SOME(update);
+
+  // Update the allocation in the allocator.
+  allocator->updateAllocation(
+      framework.id(),
+      slave.id(),
+      Resources::sum(allocation.get().resources),
+      {create});
+
+  // Now recover the resources, and expect the next allocation to
+  // contain the updated resources.
+  allocator->recoverResources(
+      framework.id(),
+      slave.id(),
+      update.get(),
+      None());
+
+  Clock::advance(flags.allocation_interval);
+
+  allocation = allocations.get();
+  AWAIT_READY(allocation);
+  EXPECT_EQ(framework.id(), allocation.get().frameworkId);
+  EXPECT_EQ(1u, allocation.get().resources.size());
+  EXPECT_TRUE(allocation.get().resources.contains(slave.id()));
+
+  // The allocation should be the slave's resources with the offer
+  // operation applied.
+  update = Resources(slave.resources()).apply(create);
+  ASSERT_SOME(update);
+
+  EXPECT_NE(Resources(slave.resources()),
+            Resources::sum(allocation.get().resources));
+
+  EXPECT_EQ(update.get(), Resources::sum(allocation.get().resources));
+
+  // Construct an offer operation for the framework's allocation to
+  // destroy the shared volume.
+  Offer::Operation destroy;
+  destroy.set_type(Offer::Operation::DESTROY);
+  destroy.mutable_destroy()->add_volumes()->CopyFrom(volume);
+
+  // Update the allocation in the allocator.
+  allocator->updateAllocation(
+      framework.id(),
+      slave.id(),
+      Resources::sum(allocation.get().resources),
+      {destroy});
+
+  // The resources to recover should be equal to the agent's original
+  // resources now that the shared volume is created and then destroyed.
+  ASSERT_SOME_EQ(slave.resources(), update.get().apply(destroy));
+
+  // Now recover the amount of `slave.resources()` and expect the
+  // next allocation to equal `slave.resources()`.
+  allocator->recoverResources(
+      framework.id(),
+      slave.id(),
+      slave.resources(),
+      None());
+
+  Clock::advance(flags.allocation_interval);
+
+  allocation = allocations.get();
+  AWAIT_READY(allocation);
+  EXPECT_EQ(framework.id(), allocation.get().frameworkId);
+  EXPECT_EQ(1u, allocation.get().resources.size());
+  EXPECT_TRUE(allocation.get().resources.contains(slave.id()));
+
+  EXPECT_EQ(Resources(slave.resources()),
+            Resources::sum(allocation.get().resources));
+}
+
+
 // This test ensures that a call to 'updateAvailable' succeeds when the
 // allocator has sufficient available resources.
 TEST_F(HierarchicalAllocatorTest, UpdateAvailableSuccess)

http://git-wip-us.apache.org/repos/asf/mesos/blob/f8a9008e/src/tests/master_validation_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_validation_tests.cpp b/src/tests/master_validation_tests.cpp
index 8bb4c37..3a5cfa2 100644
--- a/src/tests/master_validation_tests.cpp
+++ b/src/tests/master_validation_tests.cpp
@@ -605,13 +605,51 @@ TEST_F(DestroyOperationValidationTest, PersistentVolumes)
   Offer::Operation::Destroy destroy;
   destroy.add_volumes()->CopyFrom(volume1);
 
-  EXPECT_NONE(operation::validate(destroy, volumes));
+  EXPECT_NONE(operation::validate(destroy, volumes, {}, {}));
 
   Resource cpus = Resources::parse("cpus", "2", "*").get();
 
   destroy.add_volumes()->CopyFrom(cpus);
 
-  EXPECT_SOME(operation::validate(destroy, volumes));
+  EXPECT_SOME(operation::validate(destroy, volumes, {}, {}));
+}
+
+
+// This test verifies that DESTROY for shared persistent volumes
+// is only valid when the volumes are no longer in use.
+TEST_F(DestroyOperationValidationTest, SharedPersistentVolumeInUse)
+{
+  Resource cpus = Resources::parse("cpus", "1", "*").get();
+  Resource mem = Resources::parse("mem", "5", "*").get();
+  Resource disk1 = createDiskResource(
+      "50", "role1", "1", "path1", None(), true); // Shared.
+  Resource disk2 = createDiskResource("100", "role1", "2", "path2");
+
+  Resources volumes;
+  volumes += disk1;
+  volumes += disk2;
+
+  hashmap<FrameworkID, Resources> usedResources;
+  FrameworkID frameworkId1;
+  FrameworkID frameworkId2;
+  frameworkId1.set_value("id1");
+  frameworkId2.set_value("id2");
+
+  // Add used resources for 1st framework.
+  usedResources[frameworkId1] = Resources(cpus) + mem + disk1 + disk2;
+
+  // Add used resources for 2nd framework.
+  usedResources[frameworkId2] = Resources(cpus) + mem + disk1;
+
+  Offer::Operation::Destroy destroy;
+  destroy.add_volumes()->CopyFrom(disk1);
+
+  EXPECT_SOME(operation::validate(destroy, volumes, usedResources, {}));
+
+  usedResources[frameworkId1] -= disk1;
+  usedResources[frameworkId2] -= disk1;
+
+  EXPECT_NONE(operation::validate(destroy, volumes, usedResources, {}));
 }
 
 
@@ -623,8 +661,8 @@ TEST_F(DestroyOperationValidationTest, UnknownPersistentVolume)
   Offer::Operation::Destroy destroy;
   destroy.add_volumes()->CopyFrom(volume);
 
-  EXPECT_NONE(operation::validate(destroy, volume));
-  EXPECT_SOME(operation::validate(destroy, Resources()));
+  EXPECT_NONE(operation::validate(destroy, volume, {}, {}));
+  EXPECT_SOME(operation::validate(destroy, Resources(), {}, {}));
 }
 
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/f8a9008e/src/tests/mesos.hpp
----------------------------------------------------------------------
diff --git a/src/tests/mesos.hpp b/src/tests/mesos.hpp
index 4cae54b..9bcdaf1 100644
--- a/src/tests/mesos.hpp
+++ b/src/tests/mesos.hpp
@@ -731,7 +731,7 @@ inline hashmap<std::string, double> convertToHashmap(
 }
 
 
-// Helpers for creating reserve operations.
+// Helpers for creating offer operations.
 inline Offer::Operation RESERVE(const Resources& resources)
 {
   Offer::Operation operation;
@@ -741,7 +741,6 @@ inline Offer::Operation RESERVE(const Resources& resources)
 }
 
 
-// Helpers for creating unreserve operations.
 inline Offer::Operation UNRESERVE(const Resources& resources)
 {
   Offer::Operation operation;
@@ -751,7 +750,6 @@ inline Offer::Operation UNRESERVE(const Resources& resources)
 }
 
 
-// Helpers for creating offer operations.
 inline Offer::Operation CREATE(const Resources& volumes)
 {
   Offer::Operation operation;

http://git-wip-us.apache.org/repos/asf/mesos/blob/f8a9008e/src/tests/persistent_volume_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/persistent_volume_tests.cpp b/src/tests/persistent_volume_tests.cpp
index a6f97c4..d07ff1f 100644
--- a/src/tests/persistent_volume_tests.cpp
+++ b/src/tests/persistent_volume_tests.cpp
@@ -890,6 +890,107 @@ TEST_P(PersistentVolumeTest, AccessPersistentVolume)
 }
 
 
+// This test verifies that multiple tasks can be launched on a shared
+// persistent volume and write to it simultaneously.
+TEST_P(PersistentVolumeTest, SharedPersistentVolumeMultipleTasks)
+{
+  Try<Owned<cluster::Master>> master = StartMaster();
+  ASSERT_SOME(master);
+
+  slave::Flags slaveFlags = CreateSlaveFlags();
+  slaveFlags.resources = "cpus:2;mem:1024;disk(role1):1024";
+
+  Owned<MasterDetector> detector = master.get()->createDetector();
+  Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), slaveFlags);
+  ASSERT_SOME(slave);
+
+  FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+  frameworkInfo.set_role("role1");
+
+  MockScheduler sched;
+  MesosSchedulerDriver driver(
+      &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
+
+  Future<FrameworkID> frameworkId;
+  EXPECT_CALL(sched, registered(&driver, _, _))
+    .WillOnce(FutureArg<1>(&frameworkId));
+
+  Future<vector<Offer>> offers;
+  EXPECT_CALL(sched, resourceOffers(&driver, _))
+    .WillOnce(FutureArg<1>(&offers))
+    .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+  driver.start();
+
+  AWAIT_READY(frameworkId);
+
+  AWAIT_READY(offers);
+  EXPECT_FALSE(offers.get().empty());
+
+  Offer offer = offers.get()[0];
+
+  Resources volume = createPersistentVolume(
+      Megabytes(64),
+      "role1",
+      "id1",
+      "path1",
+      None(),
+      None(),
+      frameworkInfo.principal(),
+      true); // Shared.
+
+  // Create 2 tasks which write distinct files in the shared volume.
+  TaskInfo task1 = createTask(
+      offer.slave_id(),
+      Resources::parse("cpus:1;mem:128;disk(role1):32").get() + volume,
+      "echo task1 > path1/file1");
+
+  TaskInfo task2 = createTask(
+      offer.slave_id(),
+      Resources::parse("cpus:1;mem:256;disk(role1):64").get() + volume,
+      "echo task2 > path1/file2");
+
+  // We should receive a TASK_RUNNING followed by a TASK_FINISHED for
+  // each of the 2 tasks. We do not check for the actual task state
+  // since it's not the primary objective of the test. We instead
+  // verify that the paths are created by the tasks after we receive
+  // enough status updates.
+  Future<TaskStatus> status1;
+  Future<TaskStatus> status2;
+  Future<TaskStatus> status3;
+  Future<TaskStatus> status4;
+
+  EXPECT_CALL(sched, statusUpdate(&driver, _))
+    .WillOnce(FutureArg<1>(&status1))
+    .WillOnce(FutureArg<1>(&status2))
+    .WillOnce(FutureArg<1>(&status3))
+    .WillOnce(FutureArg<1>(&status4));
+
+  driver.acceptOffers(
+      {offer.id()},
+      {CREATE(volume),
+       LAUNCH({task1, task2})});
+
+  // When all status updates are received, the tasks have finished
+  // writing to the paths.
+  AWAIT_READY(status1);
+  AWAIT_READY(status2);
+  AWAIT_READY(status3);
+  AWAIT_READY(status4);
+
+  const string& volumePath = slave::paths::getPersistentVolumePath(
+      slaveFlags.work_dir,
+      "role1",
+      "id1");
+
+  EXPECT_SOME_EQ("task1\n", os::read(path::join(volumePath, "file1")));
+  EXPECT_SOME_EQ("task2\n", os::read(path::join(volumePath, "file2")));
+
+  driver.stop();
+  driver.join();
+}
+
+
 // This test verifies that persistent volumes are recovered properly
 // after the slave restarts. The idea is to launch a command which
 // keeps testing if the persistent volume exists, and fails if it does

http://git-wip-us.apache.org/repos/asf/mesos/blob/f8a9008e/src/tests/sorter_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/sorter_tests.cpp b/src/tests/sorter_tests.cpp
index 821e30d..1f17c01 100644
--- a/src/tests/sorter_tests.cpp
+++ b/src/tests/sorter_tests.cpp
@@ -722,6 +722,95 @@ TEST_P(Sorter_BENCHMARK_Test, FullSort)
        << watch.elapsed() << endl;
 }
 
+
+// This test verifies that shared resources are properly accounted for in
+// the DRF sorter.
+TEST(SorterTest, SharedResources)
+{
+  DRFSorter sorter;
+
+  SlaveID slaveId;
+  slaveId.set_value("agentId");
+
+  Resource sharedDisk = createDiskResource(
+      "100", "role1", "id1", "path1", None(), true);
+
+  // Set totalResources to have disk of 1000, with disk 100 being shared.
+  Resources totalResources = Resources::parse(
+      "cpus:100;mem:100;disk(role1):900").get();
+  totalResources += sharedDisk;
+
+  sorter.add(slaveId, totalResources);
+
+  // Verify sort() works when shared resources are in the allocations.
+  sorter.add("a");
+  Resources aResources = Resources::parse("cpus:5;mem:5").get();
+  aResources += sharedDisk;
+  sorter.allocated("a", slaveId, aResources);
+
+  sorter.add("b");
+  Resources bResources = Resources::parse("cpus:6;mem:6").get();
+  sorter.allocated("b", slaveId, bResources);
+
+  // Shares: a = .1 (dominant: disk), b = .06 (dominant: cpus).
+  EXPECT_EQ(vector<string>({"b", "a"}), sorter.sort());
+
+  sorter.add("c");
+  Resources cResources = Resources::parse("cpus:1;mem:1").get();
+  cResources += sharedDisk;
+  sorter.allocated("c", slaveId, cResources);
+
+  // 'a' and 'c' share the same persistent volume which is the
+  // dominant resource for both of these clients.
+  // Shares: a = .1 (dominant: disk), b = .06 (dominant: cpus),
+  //         c = .1 (dominant: disk).
+  EXPECT_EQ(vector<string>({"b", "a", "c"}), sorter.sort());
+
+  sorter.remove("a");
+  Resources bUnallocated = Resources::parse("cpus:4;mem:4").get();
+  sorter.unallocated("b", slaveId, bUnallocated);
+
+  // Shares: b = .02 (dominant: cpus), c = .1 (dominant: disk).
+  EXPECT_EQ(vector<string>({"b", "c"}), sorter.sort());
+
+  sorter.add("d");
+  Resources dResources = Resources::parse("cpus:1;mem:5").get();
+  dResources += sharedDisk;
+  sorter.allocated("d", slaveId, dResources);
+
+  // Shares: b = .02 (dominant: cpus), c = .1 (dominant: disk),
+  //         d = .1 (dominant: disk).
+  EXPECT_EQ(vector<string>({"b", "c", "d"}), sorter.sort());
+
+  // Verify other basic allocator methods work when shared resources
+  // are in the allocations.
+  Resources removedResources = Resources::parse("cpus:50;mem:0").get();
+  sorter.remove(slaveId, removedResources);
+
+  // Total resources is now:
+  // cpus:50;mem:100;disk(role1):900;disk(role1)[id1]:100
+
+  // Shares: b = .04 (dominant: cpus), c = .1 (dominant: disk),
+  //         d = .1 (dominant: disk).
+  EXPECT_EQ(vector<string>({"b", "c", "d"}), sorter.sort());
+
+  Resources addedResources = Resources::parse("cpus:0;mem:100").get();
+  sorter.add(slaveId, addedResources);
+
+  // Total resources is now:
+  // cpus:50;mem:200;disk(role1):900;disk(role1)[id1]:100
+
+  // Shares: b = .04 (dominant: cpus), c = .1 (dominant: disk),
+  //         d = .1 (dominant: disk).
+  EXPECT_EQ(vector<string>({"b", "c", "d"}), sorter.sort());
+
+  EXPECT_TRUE(sorter.contains("b"));
+
+  EXPECT_FALSE(sorter.contains("a"));
+
+  EXPECT_EQ(3, sorter.count());
+}
+
 } // namespace tests {
 } // namespace internal {
 } // namespace mesos {