You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by as...@apache.org on 2020/08/25 14:35:08 UTC
[mesos] branch master updated (d8647b0 -> 62931a0)
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from d8647b0 Refactored state recovery in `volume/csi` isolator.
new 7453735 Added protobuf messages for offer constraints on a string equality.
new 5e5d683 Implemented offer constraints on a string attribute equality.
new 62931a0 Added basic tests for offer constraints on a string attribute equality.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
include/mesos/scheduler/scheduler.proto | 56 ++++++++-
include/mesos/v1/scheduler/scheduler.proto | 56 ++++++++-
.../allocator/mesos/offer_constraints_filter.cpp | 49 +++++++-
.../master/offer_constraints_filter_tests.cpp | 131 +++++++++++++++++++++
4 files changed, 284 insertions(+), 8 deletions(-)
[mesos] 01/03: Added protobuf messages for offer constraints on a
string equality.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 74537354ec1b2af3e0e0584bb5f277660dff3614
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Thu Jul 16 15:48:03 2020 +0200
Added protobuf messages for offer constraints on a string equality.
This patch adds protobuf messages for setting offer constraints
on equality/non-equality of agent's (pseudo)attribute to a specified
string.
Both added contsraint predicates will evaluate to `true` when the
attribute is not TEXT. This way, schedulers will still perform all
filtration based on non-TEXT attributes on their own, which helps to
avoid subtle integration bugs caused by discrepancies in Scalar/Ranges
comparison between Mesos and the scheduler.
Given that schedulers seem to rarely put constraints on Scalar/Ranges
attributes in the real world, this should not prevent them from
obtaining performance benefits by setting offer constraints.
Review: https://reviews.apache.org/r/72745
---
include/mesos/scheduler/scheduler.proto | 56 ++++++++++++++++++++++++++++--
include/mesos/v1/scheduler/scheduler.proto | 56 ++++++++++++++++++++++++++++--
2 files changed, 108 insertions(+), 4 deletions(-)
diff --git a/include/mesos/scheduler/scheduler.proto b/include/mesos/scheduler/scheduler.proto
index 9e89c82..06d62f5 100644
--- a/include/mesos/scheduler/scheduler.proto
+++ b/include/mesos/scheduler/scheduler.proto
@@ -276,12 +276,64 @@ message AttributeConstraint {
// Yields `true` if the (pseudo)attribute does not exist.
message NotExists {}
- // TODO(asekretenko): add predicates for testing equality (MESOS-10172)
- // and regular expression matching (MESOS-10173).
+ // Predicates for (pseudo)attribute value equality.
+ //
+ // Given that existing schedulers tend to test SCALAR and RANGES attributes
+ // of offered agents for equality in ways that are not entirely trivial,
+ // we currently only support TEXT (string for pseudoattributes)
+ // equality.
+ //
+ // Both equality predicates will match (yield `true`) for attributes of
+ // SCALAR and RANGES types. This way, schedulers will still receive offers
+ // for SCALAR and RANGES attributes if a string equality constraint was
+ // specified to Mesos, and the scheduler's own logic will determine how to
+ // filter attributes of these types.
+ //
+ // For example: if a scheduler sets a constraint
+ //
+ // { "selector": {"attribute_name": "foo"},
+ // "predicate": {"text_equals": {"value":"2.0"}} }
+ //
+ // then agents with a SCALAR attribute
+ // {"name": "foo", "scalar": {"value": 1.0}}
+ // will match (since it's SCALAR, not TEXT!) and the scheduler side
+ // filtering will need to filter it according to its desired semantics.
+ //
+ // That said, it is strongly recommended to only use TEXT attributes
+ // on agents. For users with existing attributes, this can be done
+ // by adding a variant of the existing attribute that uses TEXT instead:
+ //
+ // --attributes=foo2:v1.0
+ // which gets parsed into
+ // {"name": "foo2", "text": {"value": "v1.0"}}, etc.
+
+ // Yields `true` if the (pseudo)attribute exists and is a string/TEXT
+ // equal to the specified `value`.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reason explained above.
+ message TextEquals {
+ required string value = 1;
+ }
+
+ // Yields `true` if the (pseudo)attribute does not exist or is a string/TEXT
+ // NOT equal to the specified `value`.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reason explained above.
+ message TextNotEquals {
+ required string value = 1;
+ }
+
+ // TODO(asekretenko): add predicates for regular expression matching
+ // (MESOS-10173).
oneof predicate {
Exists exists = 1;
NotExists not_exists = 2;
+
+ TextEquals text_equals = 3;
+ TextNotEquals text_not_equals = 4;
}
}
diff --git a/include/mesos/v1/scheduler/scheduler.proto b/include/mesos/v1/scheduler/scheduler.proto
index cd5a980..08fb10a 100644
--- a/include/mesos/v1/scheduler/scheduler.proto
+++ b/include/mesos/v1/scheduler/scheduler.proto
@@ -274,12 +274,64 @@ message AttributeConstraint {
// Yields `true` if the (pseudo)attribute does not exist.
message NotExists {}
- // TODO(asekretenko): add predicates for testing equality (MESOS-10172)
- // and regular expression matching (MESOS-10173).
+ // Predicates for (pseudo)attribute value equality.
+ //
+ // Given that existing schedulers tend to test SCALAR and RANGES attributes
+ // of offered agents for equality in ways that are not entirely trivial,
+ // we currently only support TEXT (string for pseudoattributes)
+ // equality.
+ //
+ // Both equality predicates will match (yield `true`) for attributes of
+ // SCALAR and RANGES types. This way, schedulers will still receive offers
+ // for SCALAR and RANGES attributes if a string equality constraint was
+ // specified to Mesos, and the scheduler's own logic will determine how to
+ // filter attributes of these types.
+ //
+ // For example: if a scheduler sets a constraint
+ //
+ // { "selector": {"attribute_name": "foo"},
+ // "predicate": {"text_equals": {"value":"2.0"}} }
+ //
+ // then agents with a SCALAR attribute
+ // {"name": "foo", "scalar": {"value": 1.0}}
+ // will match (since it's SCALAR, not TEXT!) and the scheduler side
+ // filtering will need to filter it according to its desired semantics.
+ //
+ // That said, it is strongly recommended to only use TEXT attributes
+ // on agents. For users with existing attributes, this can be done
+ // by adding a variant of the existing attribute that uses TEXT instead:
+ //
+ // --attributes=foo2:v1.0
+ // which gets parsed into
+ // {"name": "foo2", "text": {"value": "v1.0"}}, etc.
+
+ // Yields `true` if the (pseudo)attribute exists and is a string/TEXT
+ // equal to the specified `value`.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reason explained above.
+ message TextEquals {
+ required string value = 1;
+ }
+
+ // Yields `true` if the (pseudo)attribute does not exist or is a string/TEXT
+ // NOT equal to the specified `value`.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reason explained above.
+ message TextNotEquals {
+ required string value = 1;
+ }
+
+ // TODO(asekretenko): add predicates for regular expression matching
+ // (MESOS-10173).
oneof predicate {
Exists exists = 1;
NotExists not_exists = 2;
+
+ TextEquals text_equals = 3;
+ TextNotEquals text_not_equals = 4;
}
}
[mesos] 02/03: Implemented offer constraints on a string attribute
equality.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 5e5d6832b196a450cad55a16cfb202256bef5b37
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Mon Aug 3 18:35:06 2020 +0200
Implemented offer constraints on a string attribute equality.
Review: https://reviews.apache.org/r/72775
---
.../allocator/mesos/offer_constraints_filter.cpp | 49 ++++++++++++++++++++--
1 file changed, 45 insertions(+), 4 deletions(-)
diff --git a/src/master/allocator/mesos/offer_constraints_filter.cpp b/src/master/allocator/mesos/offer_constraints_filter.cpp
index ef8a948..d724fd0 100644
--- a/src/master/allocator/mesos/offer_constraints_filter.cpp
+++ b/src/master/allocator/mesos/offer_constraints_filter.cpp
@@ -87,6 +87,15 @@ public:
return Self(Exists{});
case AttributeConstraint::Predicate::kNotExists:
return Self(NotExists{});
+
+ case AttributeConstraint::Predicate::kTextEquals:
+ return Self(TextEquals{
+ std::move(*predicate.mutable_text_equals()->mutable_value())});
+
+ case AttributeConstraint::Predicate::kTextNotEquals:
+ return Self(TextNotEquals{
+ std::move(*predicate.mutable_text_not_equals()->mutable_value())});
+
case AttributeConstraint::Predicate::PREDICATE_NOT_SET:
return Error("Unknown predicate type");
}
@@ -113,9 +122,39 @@ private:
bool apply(const Attribute&) const { return false; }
};
- // TODO(asekretenko): Introduce offer constraints for attribute equality
- // (MESOS-10172) and regex match (MESOS-10173).
- using Predicate = Variant<Nothing, Exists, NotExists>;
+ struct TextEquals
+ {
+ string value;
+
+ bool apply(const Nothing&) const { return false; }
+ bool apply(const string& str) const { return str == value; }
+ bool apply(const Attribute& attr) const
+ {
+ return attr.type() != Value::TEXT || attr.text().value() == value;
+ }
+ };
+
+ struct TextNotEquals
+ {
+ string value;
+
+ bool apply(const Nothing&) const { return true; }
+ bool apply(const string& str) const { return str != value; }
+ bool apply(const Attribute& attr) const
+ {
+ return attr.type() != Value::TEXT || attr.text().value() != value;
+ }
+ };
+
+ // TODO(asekretenko): Introduce offer constraints for regex match
+ // (MESOS-10173).
+
+ using Predicate = Variant<
+ Nothing,
+ Exists,
+ NotExists,
+ TextEquals,
+ TextNotEquals>;
Predicate predicate;
@@ -130,7 +169,9 @@ private:
UNREACHABLE();
},
[&](const Exists& p) { return p.apply(attribute); },
- [&](const NotExists& p) { return p.apply(attribute); });
+ [&](const NotExists& p) { return p.apply(attribute); },
+ [&](const TextEquals& p) { return p.apply(attribute); },
+ [&](const TextNotEquals& p) { return p.apply(attribute); });
}
};
[mesos] 03/03: Added basic tests for offer constraints on a string
attribute equality.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 62931a04f2cd2a1951d04bd348ff23700cc73d9b
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Fri Aug 14 21:52:51 2020 +0200
Added basic tests for offer constraints on a string attribute equality.
Review: https://reviews.apache.org/r/72776
---
.../master/offer_constraints_filter_tests.cpp | 131 +++++++++++++++++++++
1 file changed, 131 insertions(+)
diff --git a/src/tests/master/offer_constraints_filter_tests.cpp b/src/tests/master/offer_constraints_filter_tests.cpp
index f88e201..84a1e91 100644
--- a/src/tests/master/offer_constraints_filter_tests.cpp
+++ b/src/tests/master/offer_constraints_filter_tests.cpp
@@ -128,6 +128,137 @@ TEST(OfferConstraintsFilter, NamedAttributeNotExists)
}
+// Tests a single TextEquals constraint on a named attribute.
+TEST(OfferConstraintsFilter, NamedAttributeTextEquals)
+{
+ Try<OfferConstraints> constraints = OfferConstraintsFromJSON(R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_equals": {"value": "baz"}}
+ }]
+ }]
+ }
+ }
+ })~");
+
+ ASSERT_SOME(constraints);
+
+ const Try<OfferConstraintsFilter> filter =
+ OfferConstraintsFilter::create(std::move(*constraints));
+
+ ASSERT_SOME(filter);
+
+ // Attribute exists, is a string and equals to the value in the constraint.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:baz")));
+
+ // Attribute is not a string.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:123")));
+
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:[1-17]")));
+
+ // Attribute has a wrong string value.
+ EXPECT_TRUE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:zab")));
+
+ // Attribute does not exist.
+ EXPECT_TRUE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("foo:baz")));
+}
+
+
+// Tests a single TextNotEquals constraint on a named attribute.
+TEST(OfferConstraintsFilter, NamedAttributeTextNotEquals)
+{
+ Try<OfferConstraints> constraints = OfferConstraintsFromJSON(R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_not_equals": {"value": "baz"}}
+ }]
+ }]
+ }
+ }
+ })~");
+
+ ASSERT_SOME(constraints);
+
+ const Try<OfferConstraintsFilter> filter =
+ OfferConstraintsFilter::create(std::move(*constraints));
+
+ ASSERT_SOME(filter);
+
+ // Attribute exists, is a string and equals to the value in the constraint.
+ EXPECT_TRUE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:baz")));
+
+ // Attribute is not a string.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:123")));
+
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:[1-17]")));
+
+ // Attribute has a different string value.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:zab")));
+
+ // Attribute does not exist.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("foo:baz")));
+}
+
+
+// Tests that when an attribute has several values, the constraint's predicate
+// is evaluated against the first one.
+TEST(OfferConstraintsFilter, TwoAttributesWithTheSameName)
+{
+ Try<OfferConstraints> constraints = OfferConstraintsFromJSON(R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_equals": {"value": "baz"}}
+ }]
+ }]
+ }
+ }
+ })~");
+
+ ASSERT_SOME(constraints);
+
+ const Try<OfferConstraintsFilter> filter =
+ OfferConstraintsFilter::create(std::move(*constraints));
+
+ ASSERT_SOME(filter);
+
+ // The selected attribute has a suitable value.
+ EXPECT_FALSE(filter->isAgentExcluded(
+ "roleA", slaveInfoWithAttributes("bar:baz;bar:foo")));
+
+ EXPECT_FALSE(filter->isAgentExcluded(
+ "roleA", slaveInfoWithAttributes("bar:123;bar:foo")));
+
+ // The selected attribute has an unsuitable value.
+ EXPECT_TRUE(filter->isAgentExcluded(
+ "roleA", slaveInfoWithAttributes("bar:foo;bar:baz")));
+
+ EXPECT_TRUE(filter->isAgentExcluded(
+ "roleA", slaveInfoWithAttributes("bar:foo;bar:123")));
+}
+
+
// Tests a single group of two constraints.
TEST(OfferConstraintsFilter, TwoConstraintsInGroup)
{