You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by mz...@apache.org on 2019/05/13 14:57:26 UTC
[mesos] branch 1.8.x updated (ea28002 -> 03ac09f)
This is an automated email from the ASF dual-hosted git repository.
mzhu pushed a change to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from ea28002 Added license for libseccomp to toplevel LICENSE file.
new 6c6f2ff Logged headroom related info in the allocator.
new 2299746 Avoided logging quota headroom info when there is no quota set.
new c557d9e Fixed a bug where random sorter fails to clear removed clients.
new 03ac09f Templatized two sorter tests for both random and DRF.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/master/allocator/mesos/hierarchical.cpp | 28 ++++++++++++++++++++++++++
src/master/allocator/sorter/random/sorter.cpp | 3 +++
src/tests/sorter_tests.cpp | 29 ++++++++++++++++++++-------
3 files changed, 53 insertions(+), 7 deletions(-)
[mesos] 02/04: Avoided logging quota headroom info when there is no
quota set.
Posted by mz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mzhu pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 2299746927c5acb13badc034cb02ada63c12bddc
Author: Meng Zhu <mz...@mesosphere.io>
AuthorDate: Mon May 13 15:58:43 2019 +0200
Avoided logging quota headroom info when there is no quota set.
Review: https://reviews.apache.org/r/70633
---
src/master/allocator/mesos/hierarchical.cpp | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/src/master/allocator/mesos/hierarchical.cpp b/src/master/allocator/mesos/hierarchical.cpp
index 282af46..88c0438 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -1795,9 +1795,11 @@ void HierarchicalAllocatorProcess::__allocate()
slave.getAvailable().revocable().scalars());
}
- LOG(INFO) << "Before allocation, required quota headroom is "
- << requiredHeadroom
- << " and available quota headroom is " << availableHeadroom;
+ if (!quotaGuarantees.empty()) {
+ LOG(INFO) << "Before allocation, required quota headroom is "
+ << requiredHeadroom
+ << " and available quota headroom is " << availableHeadroom;
+ }
// Due to the two stages in the allocation algorithm and the nature of
// shared resources being re-offerable even if already allocated, the
@@ -2120,11 +2122,13 @@ void HierarchicalAllocatorProcess::__allocate()
}
}
- LOG(INFO) << "After allocation, " << requiredHeadroom
- << " are required for quota headroom, "
- << heldBackForHeadroom << " were held back from "
- << heldBackAgentCount
- << " agents to ensure sufficient quota headroom";
+ if (!quotaGuarantees.empty()) {
+ LOG(INFO) << "After allocation, " << requiredHeadroom
+ << " are required for quota headroom, "
+ << heldBackForHeadroom << " were held back from "
+ << heldBackAgentCount
+ << " agents to ensure sufficient quota headroom";
+ }
if (offerable.empty()) {
VLOG(2) << "No allocations performed";
[mesos] 01/04: Logged headroom related info in the allocator.
Posted by mz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mzhu pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 6c6f2ffb6c2a59da3d89d5dbd51392b0efdba850
Author: Meng Zhu <mz...@mesosphere.io>
AuthorDate: Mon Apr 29 17:49:15 2019 -0700
Logged headroom related info in the allocator.
This patch logs `requiredHeadroom` and `availableHeadroom`
before each allocation cycle and logs `requiredHeadroom`,
resources and number of agents held back for quota `headroom`
as well as resources filtered at the end of each allocation
cycle.
While we also held resources back for quota headroom in the
first stage, we do not log it there. This is because in the
second stage, we try to allocate all resources (including the
ones held back in the first stage). Thus only resources held
back in the second stage are truly held back for the whole
allocation cycle.
Review: https://reviews.apache.org/r/70570
---
src/master/allocator/mesos/hierarchical.cpp | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/src/master/allocator/mesos/hierarchical.cpp b/src/master/allocator/mesos/hierarchical.cpp
index 64a076d..282af46 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -1795,6 +1795,10 @@ void HierarchicalAllocatorProcess::__allocate()
slave.getAvailable().revocable().scalars());
}
+ LOG(INFO) << "Before allocation, required quota headroom is "
+ << requiredHeadroom
+ << " and available quota headroom is " << availableHeadroom;
+
// Due to the two stages in the allocation algorithm and the nature of
// shared resources being re-offerable even if already allocated, the
// same shared resources can appear in two (and not more due to the
@@ -2005,6 +2009,18 @@ void HierarchicalAllocatorProcess::__allocate()
// are not part of the headroom (and therefore can't be used to satisfy
// quota guarantees).
+ // For logging purposes, we track the number of agents that had resources
+ // held back for quota headroom, as well as how many resources in total
+ // were held back.
+ //
+ // While we also held resources back for quota headroom in the first stage,
+ // we do not track it there. This is because in the second stage, we try to
+ // allocate all resources (including the ones held back in the first stage).
+ // Thus only resources held back in the second stage are truly held back for
+ // the whole allocation cycle.
+ ResourceQuantities heldBackForHeadroom;
+ size_t heldBackAgentCount = 0;
+
foreach (const SlaveID& slaveId, slaveIds) {
CHECK(slaves.contains(slaveId));
Slave& slave = slaves.at(slaveId);
@@ -2073,6 +2089,8 @@ void HierarchicalAllocatorProcess::__allocate()
if (!sufficientHeadroom) {
toAllocate -= headroomResources;
+ heldBackForHeadroom += headroomToAllocate;
+ ++heldBackAgentCount;
}
// If the framework filters these resources, ignore.
@@ -2102,6 +2120,12 @@ void HierarchicalAllocatorProcess::__allocate()
}
}
+ LOG(INFO) << "After allocation, " << requiredHeadroom
+ << " are required for quota headroom, "
+ << heldBackForHeadroom << " were held back from "
+ << heldBackAgentCount
+ << " agents to ensure sufficient quota headroom";
+
if (offerable.empty()) {
VLOG(2) << "No allocations performed";
} else {
[mesos] 03/04: Fixed a bug where random sorter fails to clear
removed clients.
Posted by mz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mzhu pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c557d9e0ad0581472413f7ab9b1ed850c29bfc6f
Author: Meng Zhu <mz...@mesosphere.io>
AuthorDate: Mon May 13 14:32:37 2019 +0200
Fixed a bug where random sorter fails to clear removed clients.
Review: https://reviews.apache.org/r/70630
---
src/master/allocator/sorter/random/sorter.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/master/allocator/sorter/random/sorter.cpp b/src/master/allocator/sorter/random/sorter.cpp
index 813f5b5..9899cfd 100644
--- a/src/master/allocator/sorter/random/sorter.cpp
+++ b/src/master/allocator/sorter/random/sorter.cpp
@@ -583,6 +583,9 @@ void RandomSorter::SortInfo::updateRelativeWeights()
activeInternalNodes.contains(node);
};
+ clients.clear();
+ weights.clear();
+
// Note, though we reserve here, the size of the vector will always
// grow (as we add more roles).
clients.reserve(sorter->clients.size());
[mesos] 04/04: Templatized two sorter tests for both random and DRF.
Posted by mz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mzhu pushed a commit to branch 1.8.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 03ac09fc1498a510a5fae60ca1af85475a674c7a
Author: Meng Zhu <mz...@mesosphere.io>
AuthorDate: Mon May 13 14:35:30 2019 +0200
Templatized two sorter tests for both random and DRF.
Review: https://reviews.apache.org/r/70631
---
src/tests/sorter_tests.cpp | 29 ++++++++++++++++++++++-------
1 file changed, 22 insertions(+), 7 deletions(-)
diff --git a/src/tests/sorter_tests.cpp b/src/tests/sorter_tests.cpp
index 9d52a80..1e4a789 100644
--- a/src/tests/sorter_tests.cpp
+++ b/src/tests/sorter_tests.cpp
@@ -880,9 +880,9 @@ TEST(DRFSorterTest, AddChildToInactiveLeaf)
// This test checks what happens when a sorter client is removed,
// which allows a leaf node to be collapsed into its parent node. This
// is basically the inverse situation to `AddChildToLeaf`.
-TEST(DRFSorterTest, RemoveLeafCollapseParent)
+TYPED_TEST(CommonSorterTest, RemoveLeafCollapseParent)
{
- DRFSorter sorter;
+ TypeParam sorter;
SlaveID slaveId;
slaveId.set_value("agentId");
@@ -904,20 +904,30 @@ TEST(DRFSorterTest, RemoveLeafCollapseParent)
sorter.allocated(
"a/c", slaveId, Resources::parse("cpus:5;mem:5").get());
- EXPECT_EQ(vector<string>({"b", "a/c", "a"}), sorter.sort());
+ // We sort the `sort()` output alphabetically here since we only
+ // want to verify the content of elements but not their order.
+ vector<string> clients = sorter.sort();
+ sort(clients.begin(), clients.end());
+
+ EXPECT_EQ(vector<string>({"a", "a/c", "b"}), clients);
sorter.remove("a/c");
- EXPECT_EQ(vector<string>({"b", "a"}), sorter.sort());
+ // We sort the `sort()` output alphabetically here since we only
+ // want to verify the content of elements but not their order.
+ clients = sorter.sort();
+ sort(clients.begin(), clients.end());
+
+ EXPECT_EQ(vector<string>({"a", "b"}), clients);
}
// This test checks what happens when a sorter client is removed and a
// leaf node can be collapsed into its parent node, we correctly
// propagate the `inactive` flag from leaf -> parent.
-TEST(DRFSorterTest, RemoveLeafCollapseParentInactive)
+TYPED_TEST(CommonSorterTest, RemoveLeafCollapseParentInactive)
{
- DRFSorter sorter;
+ TypeParam sorter;
SlaveID slaveId;
slaveId.set_value("agentId");
@@ -941,7 +951,12 @@ TEST(DRFSorterTest, RemoveLeafCollapseParentInactive)
sorter.deactivate("a");
- EXPECT_EQ(vector<string>({"b", "a/c"}), sorter.sort());
+ // We sort the `sort()` output alphabetically here since we only
+ // want to verify the content of elements but not their order.
+ vector<string> clients = sorter.sort();
+ sort(clients.begin(), clients.end());
+
+ EXPECT_EQ(vector<string>({"a/c", "b"}), clients);
sorter.remove("a/c");