You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2019/11/08 13:25:16 UTC
[mesos] branch master updated (fc0069b -> b6bdc74)
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from fc0069b Fixed a compilation issue due to const capture in a lambda.
new d5b6ba7 Revamped attribute handling in stout.
new c005954 Added 'nodiscard' attribute to some member functions of Resources.
new 48d79c2 Improved error reporting in 'Resources::pushReservation()'.
new 78b071c Added 'source' field to operator reservation API.
new 398a13e Added 'source' field to scheduler reservation API.
new fb37dda Consolidated 'Master::authorizeReserveResources' overloads.
new 2313215 Rejected scheduler calls that include reservation updates.
new c44b4fb3 Rejected operator API calls that include reservation updates.
new 8e742eb Added function to compute a common reservation ancestor.
new b368d89 Updated 'Master::Http::_reserve' to pass along new 'source' field.
new 1aedc02 Updated validation of 'Reserve' call.
new f1a9d72 Updated offer operation resource validation for reservation updates.
new 1d225b4 Updated 'getResourceConversion()' for reservation updates.
new 09c830d Added authorization handling for reservations with `source`.
new b6bdc74 Added end-to-end test for operator API reservation updates.
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
3rdparty/stout/include/stout/abort.hpp | 6 +-
3rdparty/stout/include/stout/attributes.hpp | 26 +++-
3rdparty/stout/include/stout/exit.hpp | 2 +-
3rdparty/stout/include/stout/unimplemented.hpp | 2 +-
3rdparty/stout/include/stout/unreachable.hpp | 2 +-
include/mesos/master/master.proto | 1 +
include/mesos/mesos.proto | 1 +
include/mesos/resources.hpp | 19 ++-
include/mesos/v1/master/master.proto | 1 +
include/mesos/v1/mesos.proto | 1 +
src/common/resources.cpp | 39 +++++-
src/common/resources_utils.cpp | 23 +++-
src/master/http.cpp | 79 +++++++----
src/master/master.cpp | 148 +++++++++++++++------
src/master/master.hpp | 7 +-
src/master/validation.cpp | 57 ++++++++
src/tests/api_tests.cpp | 144 ++++++++++++++++++++
src/tests/master_authorization_tests.cpp | 177 +++++++++++++++++++++++++
src/tests/resources_tests.cpp | 66 +++++++++
19 files changed, 715 insertions(+), 86 deletions(-)
[mesos] 03/15: Improved error reporting in
'Resources::pushReservation()'.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 48d79c25f639d7cd79ca7a78c23ad0e680bcdcbb
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 13:57:16 2019 +0100
Improved error reporting in 'Resources::pushReservation()'.
Make sure to print the actual error message after an assertion
failure in `Resources::pushReservations()`.
Review: https://reviews.apache.org/r/71722/
---
src/common/resources.cpp | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/src/common/resources.cpp b/src/common/resources.cpp
index bfa9f3e..e3e1100 100644
--- a/src/common/resources.cpp
+++ b/src/common/resources.cpp
@@ -1722,7 +1722,10 @@ Resources Resources::pushReservation(
resourcesNoMutationWithoutExclusiveOwnership) {
Resource_ r_ = *resource_;
r_.resource.add_reservations()->CopyFrom(reservation);
- CHECK_NONE(Resources::validate(r_.resource));
+ Option<Error> validation = Resources::validate(r_.resource);
+ CHECK_NONE(validation)
+ << "Validation failed: " << *validation;
+
result.add(std::move(r_));
}
[mesos] 10/15: Updated 'Master::Http::_reserve' to pass along new
'source' field.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit b368d897d83df2f261e01fa7583798d80d098052
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:06:16 2019 +0100
Updated 'Master::Http::_reserve' to pass along new 'source' field.
Updated 'Master::Http::_reserve()' to correctly set the new `source`
field in the `Offer::Operation` created from operator API input.
Review: https://reviews.apache.org/r/71695/
---
src/master/http.cpp | 84 ++++++++++++++++++++++++++++++++-------------------
src/master/master.hpp | 1 +
2 files changed, 54 insertions(+), 31 deletions(-)
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 9efe4a3..1778664 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -1914,11 +1914,45 @@ Future<Response> Master::Http::reserve(
return BadRequest("Unable to decode query string: " + decode.error());
}
- const hashmap<string, string>& values = decode.get();
+ // Helper function to parse a list of resource from query parameters.
+ auto parseResourcesList = [](
+ const std::string& key,
+ const hashmap<string, string>& queryParameters)
+ -> Try<RepeatedPtrField<Resource>>
+ {
+ Option<string> value = queryParameters.get(key);
+ if (value.isNone()) {
+ return Error(
+ "Missing '" + key + "' query parameter in the request body");
+ }
- Option<string> value;
+ Try<JSON::Array> parse =
+ JSON::parse<JSON::Array>(value.get());
- value = values.get("slaveId");
+ if (parse.isError()) {
+ return Error(
+ "Error parsing '" + key +
+ "' query parameter in the request body: " + parse.error());
+ }
+
+ RepeatedPtrField<Resource> resources;
+ foreach (const JSON::Value& value, parse->values) {
+ Try<Resource> resource = ::protobuf::parse<Resource>(value);
+ if (resource.isError()) {
+ return Error(
+ "Error parsing '" + key +
+ "' query parameter in the request body: " + resource.error());
+ }
+
+ resources.Add()->CopyFrom(resource.get());
+ }
+
+ return resources;
+ };
+
+ const hashmap<string, string>& values = decode.get();
+
+ Option<string> value = values.get("slaveId");
if (value.isNone()) {
return BadRequest("Missing 'slaveId' query parameter in the request body");
}
@@ -1926,39 +1960,32 @@ Future<Response> Master::Http::reserve(
SlaveID slaveId;
slaveId.set_value(value.get());
- value = values.get("resources");
- if (value.isNone()) {
- return BadRequest(
- "Missing 'resources' query parameter in the request body");
- }
-
- Try<JSON::Array> parse =
- JSON::parse<JSON::Array>(value.get());
+ Try<RepeatedPtrField<Resource>> resources =
+ parseResourcesList("resources", values);
- if (parse.isError()) {
- return BadRequest(
- "Error in parsing 'resources' query parameter in the request body: " +
- parse.error());
+ if (resources.isError()) {
+ return BadRequest(resources.error());
}
- RepeatedPtrField<Resource> resources;
- foreach (const JSON::Value& value, parse->values) {
- Try<Resource> resource = ::protobuf::parse<Resource>(value);
- if (resource.isError()) {
- return BadRequest(
- "Error in parsing 'resources' query parameter in the request body: " +
- resource.error());
+ RepeatedPtrField<Resource> source;
+ if (values.contains("source")) {
+ Try<RepeatedPtrField<Resource>> parsedSource =
+ parseResourcesList("source", values);
+
+ if (parsedSource.isError()) {
+ return BadRequest(parsedSource.error());
}
- resources.Add()->CopyFrom(resource.get());
+ source = parsedSource.get();
}
- return _reserve(slaveId, resources, principal);
+ return _reserve(slaveId, source, resources.get(), principal);
}
Future<Response> Master::Http::_reserve(
const SlaveID& slaveId,
+ const RepeatedPtrField<Resource>& source,
const RepeatedPtrField<Resource>& resources,
const Option<Principal>& principal) const
{
@@ -1970,6 +1997,7 @@ Future<Response> Master::Http::_reserve(
// Create an operation.
Offer::Operation operation;
operation.set_type(Offer::Operation::RESERVE);
+ operation.mutable_reserve()->mutable_source()->CopyFrom(source);
operation.mutable_reserve()->mutable_resources()->CopyFrom(resources);
Option<Error> error = validateAndUpgradeResources(&operation);
@@ -2009,13 +2037,7 @@ Future<Response> Master::Http::reserveResources(
const RepeatedPtrField<Resource>& resources =
call.reserve_resources().resources();
- // Reject all `Reserve` operations whenever `source` is set
- // until we have a proper implementation in place.
- if (!source.empty()) {
- return BadRequest("Reservation updates not yet supported.");
- }
-
- return _reserve(slaveId, resources, principal);
+ return _reserve(slaveId, source, resources, principal);
}
diff --git a/src/master/master.hpp b/src/master/master.hpp
index d2392d7..8a14065 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1728,6 +1728,7 @@ private:
process::Future<process::http::Response> _reserve(
const SlaveID& slaveId,
+ const google::protobuf::RepeatedPtrField<Resource>& source,
const google::protobuf::RepeatedPtrField<Resource>& resources,
const Option<process::http::authentication::Principal>&
principal) const;
[mesos] 15/15: Added end-to-end test for operator API reservation
updates.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit b6bdc74c896303dc1775c68642023ee4513834b1
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:19:22 2019 +0100
Added end-to-end test for operator API reservation updates.
Added a new test to verify that reservations can be updated
using the operator API.
Review: https://reviews.apache.org/r/71725/
---
src/tests/api_tests.cpp | 144 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 144 insertions(+)
diff --git a/src/tests/api_tests.cpp b/src/tests/api_tests.cpp
index bd207ea..393f9a2 100644
--- a/src/tests/api_tests.cpp
+++ b/src/tests/api_tests.cpp
@@ -1439,6 +1439,150 @@ TEST_P(MasterAPITest, ReserveResources)
}
+// This test verifies that an operator can update existing reservation through
+// the `RESERVE_RESOURCES` call.
+TEST_P(MasterAPITest, ReservationUpdate)
+{
+ ContentType contentType = GetParam();
+
+ TestAllocator<> allocator;
+ EXPECT_CALL(allocator, initialize(_, _, _));
+
+ Try<Owned<cluster::Master>> master = StartMaster(&allocator);
+ ASSERT_SOME(master);
+
+ Future<SlaveID> slaveId;
+ EXPECT_CALL(allocator, addSlave(_, _, _, _, _, _))
+ .WillOnce(DoAll(InvokeAddSlave(&allocator),
+ FutureArg<0>(&slaveId)));
+
+ Future<Nothing> __recover = FUTURE_DISPATCH(_, &Slave::__recover);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
+ ASSERT_SOME(slave);
+
+ // Wait until the agent has finished recovery.
+ ASSERT_SOME(slave);
+ AWAIT_READY(__recover);
+
+ // It's actually impossible to construct `Resources` from the contents
+ // of `state["reserved_resources"]`, so instead we just compare against
+ // the expected JSON below.
+ JSON::Value resourcesJson = JSON::parse(R"_(
+ {
+ "cpus": 1.0,
+ "disk": 0.0,
+ "gpus": 0.0,
+ "mem": 10.0
+ }
+ )_").get();
+
+ Resources unreserved = Resources::parse("cpus:1;mem:10").get();
+ Resources dynamicallyReserved =
+ unreserved.pushReservation(createDynamicReservationInfo(
+ "role", DEFAULT_CREDENTIAL.principal()));
+
+ Resources dynamicallyReservedToFoo =
+ dynamicallyReserved.pushReservation(createDynamicReservationInfo(
+ "role/foo", DEFAULT_CREDENTIAL.principal()));
+
+ Resources dynamicallyReservedToBar =
+ dynamicallyReserved.pushReservation(createDynamicReservationInfo(
+ "role/bar", DEFAULT_CREDENTIAL.principal()));
+
+ // Helper function to attempt a reservation update from a given `source`
+ // to a given reservation using an operator API call.
+ auto attemptReservation = [&master, &slaveId, &contentType](
+ const Resources& source,
+ const Resources& resources,
+ const std::string& expectedResponseStatus) {
+ v1::master::Call v1Call;
+ v1Call.set_type(v1::master::Call::RESERVE_RESOURCES);
+ v1::master::Call::ReserveResources* reserveResources =
+ v1Call.mutable_reserve_resources();
+
+ reserveResources->mutable_agent_id()->CopyFrom(evolve(slaveId.get()));
+ reserveResources->mutable_source()->CopyFrom(evolve(source));
+ reserveResources->mutable_resources()->CopyFrom(evolve(resources));
+
+ Future<http::Response> response = http::post(
+ master.get()->pid,
+ "api/v1",
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL),
+ serialize(contentType, v1Call),
+ stringify(contentType));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(expectedResponseStatus, response);
+ };
+
+ // Helper function to verify that the resources on the agent are
+ // indeed reserved to the specified role.
+ auto verifyReservation = [&slave, &resourcesJson](
+ const std::string& intendedRole) {
+ Future<http::Response> response = http::get(
+ slave.get()->pid,
+ "state",
+ None(), // query
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
+
+ Try<JSON::Value> json = JSON::parse(response->body);
+ ASSERT_SOME(json);
+
+ Result<JSON::Object> reservations =
+ json->as<JSON::Object>().at<JSON::Object>("reserved_resources");
+
+ ASSERT_SOME(reservations);
+ EXPECT_EQ(reservations->values.size(), 1u);
+
+ foreachpair (
+ const std::string& role,
+ const JSON::Value& reservation,
+ reservations->values) {
+ EXPECT_EQ(role, intendedRole);
+ EXPECT_EQ(resourcesJson, reservation);
+ }
+ };
+
+ // Setup done, actual test can start now.
+ const std::string conflict = http::Conflict().status;
+ const std::string accepted = http::Accepted().status;
+
+ // Should fail, since there are no resources that are dynamically
+ // reserved to `role/bar`.
+ attemptReservation(
+ dynamicallyReservedToBar,
+ dynamicallyReservedToFoo,
+ conflict);
+
+ // Reserve unreserved resources to `role/foo`.
+ attemptReservation(
+ unreserved,
+ dynamicallyReservedToFoo,
+ accepted);
+
+ verifyReservation("role/foo");
+
+ // Should fail again, since there are still no resources that are
+ // dynamically reserved to `role/bar`.
+ attemptReservation(
+ dynamicallyReservedToBar,
+ dynamicallyReservedToFoo,
+ conflict);
+
+ // Update reservation from `role/foo` to `role/bar`.
+ attemptReservation(
+ dynamicallyReservedToFoo,
+ dynamicallyReservedToBar,
+ accepted);
+
+ verifyReservation("role/bar");
+}
+
+
// This test verifies that an operator can unreserve reserved resources through
// the `UNRESERVE_RESOURCES` call.
TEST_P(MasterAPITest, UnreserveResources)
[mesos] 01/15: Revamped attribute handling in stout.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit d5b6ba7f40cb10724d92554f44cae5bfd39ac961
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Thu Nov 7 14:55:13 2019 +0100
Revamped attribute handling in stout.
This makes several changes to the attribute compatibility layer
provided by stout:
* Add a new `STOUT_HAS_CPP_ATTRIBUTE` macro to test compiler
support for a given attribute.
* Renamed the non-namespaced `NORETURN` macro to `STOUT_NORETURN`.
* Preferred the use of the standardized `[[noreturn]]` syntax
if supported by the compiler.
* Fixed previous usages of `NORETURN` in the stout codebase.
* Added support for the `[[nodiscard]]` attribute.
Review: https://reviews.apache.org/r/71734/
---
3rdparty/stout/include/stout/abort.hpp | 6 ++++--
3rdparty/stout/include/stout/attributes.hpp | 26 ++++++++++++++++++++++----
3rdparty/stout/include/stout/exit.hpp | 2 +-
3rdparty/stout/include/stout/unimplemented.hpp | 2 +-
3rdparty/stout/include/stout/unreachable.hpp | 2 +-
5 files changed, 29 insertions(+), 9 deletions(-)
diff --git a/3rdparty/stout/include/stout/abort.hpp b/3rdparty/stout/include/stout/abort.hpp
index 43ed5ce..1917106 100644
--- a/3rdparty/stout/include/stout/abort.hpp
+++ b/3rdparty/stout/include/stout/abort.hpp
@@ -40,7 +40,8 @@
#define ABORT(...) _Abort(_ABORT_PREFIX, __VA_ARGS__)
-inline NORETURN void _Abort(const char* prefix, const char* message)
+STOUT_NORETURN
+inline void _Abort(const char* prefix, const char* message)
{
#ifndef __WINDOWS__
const size_t prefix_len = strlen(prefix);
@@ -87,7 +88,8 @@ inline NORETURN void _Abort(const char* prefix, const char* message)
}
-inline NORETURN void _Abort(const char* prefix, const std::string& message)
+STOUT_NORETURN
+inline void _Abort(const char* prefix, const std::string& message)
{
_Abort(prefix, message.c_str());
}
diff --git a/3rdparty/stout/include/stout/attributes.hpp b/3rdparty/stout/include/stout/attributes.hpp
index aa377db..a02ee79 100644
--- a/3rdparty/stout/include/stout/attributes.hpp
+++ b/3rdparty/stout/include/stout/attributes.hpp
@@ -13,12 +13,30 @@
#ifndef __STOUT_ATTRIBUTES_HPP__
#define __STOUT_ATTRIBUTES_HPP__
+#ifdef __has_cpp_attribute
+# define STOUT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+# define STOUT_HAS_CPP_ATTRIBUTE(x) 0
+#endif
-#ifdef __WINDOWS__
-#define NORETURN __declspec(noreturn)
+// We unconditionally require compiler support for `noreturn`,
+// both for legacy reasons and because the presence of this
+// attribute might affect code generation.
+#if STOUT_HAS_CPP_ATTRIBUTE(noreturn) > 0
+# define STOUT_NORETURN [[noreturn]]
+#elif defined(__WINDOWS__)
+# define STOUT_NORETURN __declspec(noreturn)
#else
-#define NORETURN __attribute__((noreturn))
-#endif // __WINDOWS__
+# define STOUT_NORETURN __attribute__((noreturn))
+#endif
+
+// Non-namespaced version for backwards compatibility.
+#define NORETURN STOUT_NORETURN
+#if STOUT_HAS_CPP_ATTRIBUTE(nodiscard) > 0
+# define STOUT_NODISCARD [[nodiscard]]
+#else
+# define STOUT_NODISCARD
+#endif
#endif // __STOUT_ATTRIBUTES_HPP__
diff --git a/3rdparty/stout/include/stout/exit.hpp b/3rdparty/stout/include/stout/exit.hpp
index 34585a0..25721d4 100644
--- a/3rdparty/stout/include/stout/exit.hpp
+++ b/3rdparty/stout/include/stout/exit.hpp
@@ -62,7 +62,7 @@ struct __Exit
stream() << "EXIT with status " << _status << ": ";
}
- NORETURN ~__Exit()
+ STOUT_NORETURN ~__Exit()
{
message.Flush();
exit(status);
diff --git a/3rdparty/stout/include/stout/unimplemented.hpp b/3rdparty/stout/include/stout/unimplemented.hpp
index ab6caa8..168a2f8 100644
--- a/3rdparty/stout/include/stout/unimplemented.hpp
+++ b/3rdparty/stout/include/stout/unimplemented.hpp
@@ -28,7 +28,7 @@
#define UNIMPLEMENTED Unimplemented(__func__, __FILE__, __LINE__)
-NORETURN inline void Unimplemented(
+STOUT_NORETURN inline void Unimplemented(
const char* function,
const char* file,
const int line)
diff --git a/3rdparty/stout/include/stout/unreachable.hpp b/3rdparty/stout/include/stout/unreachable.hpp
index d4b3bb0..e33bc01 100644
--- a/3rdparty/stout/include/stout/unreachable.hpp
+++ b/3rdparty/stout/include/stout/unreachable.hpp
@@ -22,7 +22,7 @@
#define UNREACHABLE() Unreachable(__FILE__, __LINE__)
-inline NORETURN void Unreachable(const char* file, int line)
+STOUT_NORETURN inline void Unreachable(const char* file, int line)
{
std::cerr << "Reached unreachable statement at " << file << ':'
<< line << std::endl;
[mesos] 13/15: Updated 'getResourceConversion()' for reservation
updates.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 1d225b4c0270f06b901f0fafd777a347aae921cd
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:19:11 2019 +0100
Updated 'getResourceConversion()' for reservation updates.
Updated the `getResourcesConversion()` function to correctly
handle the `source` field in `RESERVE` operations.
Review: https://reviews.apache.org/r/71719/
---
src/common/resources_utils.cpp | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/src/common/resources_utils.cpp b/src/common/resources_utils.cpp
index 4c958c8..17cc96d 100644
--- a/src/common/resources_utils.cpp
+++ b/src/common/resources_utils.cpp
@@ -119,11 +119,20 @@ Try<vector<TResourceConversion>> getResourceConversions(
return Error("Operation not supported");
case TOperation::RESERVE: {
- foreach (const TResource& reserved, operation.reserve().resources()) {
- // Note that we only allow "pushing" a single reservation at time.
- TResources consumed = TResources(reserved).popReservation();
- conversions.emplace_back(consumed, reserved);
+ TResources reserved(operation.reserve().resources());
+
+ // If the operation explicitly specifies a `source` we use that,
+ // otherwise we assume that the operation is "pushing" a single
+ // reservation. At this point, the resources in the operation
+ // should have been already sanity checked, so we don't have to
+ // repeat that here.
+ TResources consumed;
+ if (operation.reserve().source_size() > 0) {
+ consumed = TResources(operation.reserve().source());
+ } else {
+ consumed = reserved.popReservation();
}
+ conversions.emplace_back(consumed, reserved);
break;
}
[mesos] 04/15: Added 'source' field to operator reservation API.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 78b071c3e4f00f932ff5530b5fa97ea66b837ee9
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:01:13 2019 +0100
Added 'source' field to operator reservation API.
Added a new 'source' field to the operator API `ReserveResources`
protobuf to prepare for the pending reservation update changes.
Review: https://reviews.apache.org/r/71679/
---
include/mesos/master/master.proto | 1 +
include/mesos/v1/master/master.proto | 1 +
2 files changed, 2 insertions(+)
diff --git a/include/mesos/master/master.proto b/include/mesos/master/master.proto
index 315809c..8c22802 100644
--- a/include/mesos/master/master.proto
+++ b/include/mesos/master/master.proto
@@ -154,6 +154,7 @@ message Call {
// Reserve resources dynamically on a specific agent.
message ReserveResources {
required SlaveID slave_id = 1;
+ repeated Resource source = 3;
repeated Resource resources = 2;
}
diff --git a/include/mesos/v1/master/master.proto b/include/mesos/v1/master/master.proto
index 5c99112..40de358 100644
--- a/include/mesos/v1/master/master.proto
+++ b/include/mesos/v1/master/master.proto
@@ -155,6 +155,7 @@ message Call {
// Reserve resources dynamically on a specific agent.
message ReserveResources {
required AgentID agent_id = 1;
+ repeated Resource source = 3;
repeated Resource resources = 2;
}
[mesos] 08/15: Rejected operator API calls that include reservation
updates.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c44b4fb3580d3ca5349168552e068a2343b91f61
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:04:57 2019 +0100
Rejected operator API calls that include reservation updates.
Review: https://reviews.apache.org/r/71688/
---
src/master/http.cpp | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 60765c9..9efe4a3 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -2005,9 +2005,16 @@ Future<Response> Master::Http::reserveResources(
CHECK_EQ(mesos::master::Call::RESERVE_RESOURCES, call.type());
const SlaveID& slaveId = call.reserve_resources().slave_id();
+ const RepeatedPtrField<Resource>& source = call.reserve_resources().source();
const RepeatedPtrField<Resource>& resources =
call.reserve_resources().resources();
+ // Reject all `Reserve` operations whenever `source` is set
+ // until we have a proper implementation in place.
+ if (!source.empty()) {
+ return BadRequest("Reservation updates not yet supported.");
+ }
+
return _reserve(slaveId, resources, principal);
}
[mesos] 14/15: Added authorization handling for reservations with
`source`.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 09c830d87b88d4c2f386cb9ded5931528d6cf144
Author: Benjamin Bannier <bb...@apache.org>
AuthorDate: Fri Nov 8 14:19:16 2019 +0100
Added authorization handling for reservations with `source`.
This patch adds authorization handling for `RESERVE` operations
containing `source` fields. In order to stay backwards-compatible we add
a dedicated authorization branch for such operations which under the
hood translates each removed reservation to an `UNRESERVE` operation and
every added reservation as a `RESERVE` operation where we fall back to
existing authorization code for authorization.
Review: https://reviews.apache.org/r/71729/
---
src/master/master.cpp | 110 ++++++++++++++-----
src/tests/master_authorization_tests.cpp | 177 +++++++++++++++++++++++++++++++
2 files changed, 259 insertions(+), 28 deletions(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index e7609f3..14b90a5 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -3765,42 +3765,96 @@ Future<bool> Master::authorizeReserveResources(
request.mutable_subject()->CopyFrom(subject.get());
}
- // The operation will be authorized if the entity is allowed to make
- // reservations for all roles included in `reserve.resources`.
- // Add an element to `request.roles` for each unique role in the resources.
- hashset<string> roles;
vector<Future<bool>> authorizations;
- foreach (const Resource& resource, reserve.resources()) {
- // NOTE: We rely on the master to ensure that the resource is in the
- // post-reservation-refinement format. If there is a stack of reservations,
- // we perform authorization for the role of the most refined reservation,
- // since we only support "pushing" one reservation at a time. That is, all
- // of the previous reservations must have already been authorized.
- //
- // NOTE: If there is no reservation, we authorize the resource with the
- // default role '*' for backward compatibility.
- CHECK(!resource.has_role()) << resource;
- CHECK(!resource.has_reservation()) << resource;
- const string role = Resources::isReserved(resource)
- ? Resources::reservationRole(resource) : "*";
+ // We support `Reserve` operations with either `source` set or unset. If
+ // `source` is unset (an older API), it is possible to send calls which
+ // contain also resources whose reservations are unmodified; in that case all
+ // `Reserve` `resources` will be authorized. In the case where `source` is set
+ // we follow a narrower contract and e.g., only accept resources whose
+ // reservations are all modified, and require identical modifications for all
+ // passed `Resource`s.
+ //
+ // This e.g., means that we cannot upgrade calls without `source` to
+ // calls with `source` and use uniform handling. Instead we branch
+ // on whether `source` was set to select the authorization handling.
+ // Each fundamental reservation added in with with-`source` case can
+ // then be authorized using the historic, wider contract.
+ if (reserve.source().empty()) {
+ // The operation will be authorized if the entity is allowed to make
+ // reservations for all roles included in `reserve.resources`.
+ // Add an element to `request.roles` for each unique role in the resources.
+ hashset<string> roles;
+
+ foreach (const Resource& resource, reserve.resources()) {
+ // NOTE: We rely on the master to ensure that the resource is in the
+ // post-reservation-refinement format. If there is a stack of
+ // reservations, we perform authorization for the role of the most refined
+ // reservation, since we only support "pushing" one reservation at a time.
+ // That is, all of the previous reservations must have already been
+ // authorized.
+ //
+ // NOTE: If there is no reservation, we authorize the resource with the
+ // default role '*' for backward compatibility.
+ CHECK(!resource.has_role()) << resource;
+ CHECK(!resource.has_reservation()) << resource;
- if (!roles.contains(role)) {
- roles.insert(role);
+ const string role = Resources::isReserved(resource)
+ ? Resources::reservationRole(resource) : "*";
- request.mutable_object()->mutable_resource()->CopyFrom(resource);
+ if (!roles.contains(role)) {
+ roles.insert(role);
- // We also set the deprecated `object.value` field to support legacy
- // authorizers that have not been upgraded to look at `object.resource`.
- request.mutable_object()->set_value(role);
+ request.mutable_object()->mutable_resource()->CopyFrom(resource);
- authorizations.push_back(authorizer.get()->authorized(request));
+ // We also set the deprecated `object.value` field to support legacy
+ // authorizers that have not been upgraded to look at `object.resource`.
+ request.mutable_object()->set_value(role);
+
+ authorizations.push_back(authorizer.get()->authorized(request));
+ }
}
- }
- LOG(INFO) << "Authorizing principal '"
- << (principal.isSome() ? stringify(principal.get()) : "ANY")
- << "' to reserve resources '" << reserve.resources() << "'";
+ LOG(INFO) << "Authorizing principal '"
+ << (principal.isSome() ? stringify(principal.get()) : "ANY")
+ << "' to reserve resources '" << reserve.resources() << "'";
+ } else {
+ Resources source = reserve.source();
+ const Resources target = reserve.resources();
+ const Resources ancestor =
+ Resources::getReservationAncestor(source, target);
+
+ // Authorize `UNRESERVE` operations bringing `source` to `ancestor`.
+ while (source != ancestor) {
+ Offer::Operation::Unreserve unreserve;
+ unreserve.mutable_resources()->CopyFrom(source);
+
+ authorizations.push_back(
+ authorizeUnreserveResources(unreserve, principal));
+
+ source = source.popReservation();
+ }
+
+ // Authorize `RESERVE` operations bringing `ancestor` to `target`.
+ const RepeatedPtrField<Resource::ReservationInfo>& targetReservations =
+ reserve.resources(0).reservations();
+ const RepeatedPtrField<Resource::ReservationInfo>& ancestorReservations =
+ RepeatedPtrField<Resource>(ancestor).begin()->reservations();
+
+ // Skip reservations common among `source` and `resources`.
+ auto it = targetReservations.begin();
+ std::advance(it, ancestorReservations.size());
+
+ for (; it != targetReservations.end(); ++it) {
+ source = source.pushReservation(*it);
+
+ // We do not set `source` here to trigger the previous branch.
+ Offer::Operation::Reserve reserve;
+ reserve.mutable_resources()->CopyFrom(source);
+
+ authorizations.push_back(authorizeReserveResources(reserve, principal));
+ }
+ }
// NOTE: Empty authorizations are not valid and are checked by a validator.
// However under certain circumstances, this method can be called before
diff --git a/src/tests/master_authorization_tests.cpp b/src/tests/master_authorization_tests.cpp
index 06471aa..bc8155b 100644
--- a/src/tests/master_authorization_tests.cpp
+++ b/src/tests/master_authorization_tests.cpp
@@ -76,6 +76,8 @@ using mesos::internal::master::allocator::MesosAllocatorProcess;
using mesos::internal::slave::Slave;
+using mesos::v1::master::Call;
+
using mesos::master::detector::MasterDetector;
using mesos::master::detector::StandaloneMasterDetector;
@@ -86,6 +88,7 @@ using process::Owned;
using process::PID;
using process::Promise;
+using process::http::Accepted;
using process::http::Forbidden;
using process::http::OK;
using process::http::Response;
@@ -2764,6 +2767,180 @@ TEST_F(MasterAuthorizationTest, AuthorizedToRegisterNoStaticReservations)
AWAIT_READY(slaveRegisteredMessage);
}
+// This test verifies that when reserving resources the correct
+// `RESERVE_RESOURCES` or `UNRESERVE_RESOURCES` authorization
+// requests are performed.
+TEST_F(MasterAuthorizationTest, ReserveResources)
+{
+ Clock::pause();
+
+ MockAuthorizer authorizer;
+
+ Try<Owned<cluster::Master>> master = StartMaster(&authorizer);
+ ASSERT_SOME(master);
+
+ Future<SlaveRegisteredMessage> slaveRegisteredMessage =
+ FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ slave::Flags slaveFlags = CreateSlaveFlags();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), slaveFlags);
+ ASSERT_SOME(slave);
+
+ Clock::advance(slaveFlags.registration_backoff_factor);
+ AWAIT_READY(slaveRegisteredMessage);
+
+ const SlaveID& slaveId = slaveRegisteredMessage->slave_id();
+
+ // Reserve `cpus:0.1` for `foo`. This should trigger exactly one authorization
+ // request for `RESERVE_RESOURCES`. This is a base case.
+ {
+ Future<authorization::Request> request;
+ EXPECT_CALL(authorizer, authorized(_))
+ .WillOnce(DoAll(FutureArg<0>(&request), Return(true)));
+
+ Call call;
+ call.set_type(Call::RESERVE_RESOURCES);
+ Call::ReserveResources* reserveResources = call.mutable_reserve_resources();
+
+ reserveResources->mutable_agent_id()->set_value(slaveId.value());
+
+ v1::Resource resource = *v1::Resources::parse("cpus", "0.1", "*");
+ *reserveResources->add_source() = resource;
+
+ *resource.add_reservations() =
+ v1::createDynamicReservationInfo("foo", DEFAULT_CREDENTIAL.principal());
+ *reserveResources->add_resources() = resource;
+
+ Future<http::Response> response = http::post(
+ master.get()->pid,
+ "/api/v1",
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL),
+ stringify(JSON::protobuf(call)),
+ stringify(ContentType::JSON));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(Accepted().status, response);
+
+ AWAIT_READY(request);
+ ASSERT_EQ(authorization::Action::RESERVE_RESOURCES, request->action());
+ ASSERT_EQ(1, request->object().resource().reservations_size());
+ ASSERT_EQ("foo", request->object().resource().reservations(0).role());
+ }
+
+ // Reserve `cpus:0.1` for `foo` and refine to `foo/bar`. This should trigger
+ // two authorization requests, one for the reservation to `foo` and another
+ // one for the reservation refinement to `foo/bar`.
+ {
+ Future<authorization::Request> request1;
+ Future<authorization::Request> request2;
+ EXPECT_CALL(authorizer, authorized(_))
+ .WillOnce(DoAll(FutureArg<0>(&request1), Return(true)))
+ .WillOnce(DoAll(FutureArg<0>(&request2), Return(true)));
+
+ Call call;
+ call.set_type(Call::RESERVE_RESOURCES);
+ Call::ReserveResources* reserveResources = call.mutable_reserve_resources();
+
+ reserveResources->mutable_agent_id()->set_value(slaveId.value());
+
+ v1::Resource resource = *v1::Resources::parse("cpus", "0.1", "*");
+ *reserveResources->add_source() = resource;
+
+ *resource.add_reservations() =
+ v1::createDynamicReservationInfo("foo", DEFAULT_CREDENTIAL.principal());
+ *resource.add_reservations() = v1::createDynamicReservationInfo(
+ "foo/bar", DEFAULT_CREDENTIAL.principal());
+ *reserveResources->add_resources() = resource;
+
+ Future<http::Response> response = http::post(
+ master.get()->pid,
+ "/api/v1",
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL),
+ stringify(JSON::protobuf(call)),
+ stringify(ContentType::JSON));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(Accepted().status, response);
+
+ AWAIT_READY(request1);
+ ASSERT_EQ(authorization::Action::RESERVE_RESOURCES, request1->action());
+ ASSERT_EQ(1, request1->object().resource().reservations_size());
+ ASSERT_EQ("foo", request1->object().resource().reservations(0).role());
+
+ AWAIT_READY(request2);
+ ASSERT_EQ(authorization::Action::RESERVE_RESOURCES, request2->action());
+ ASSERT_EQ(2, request2->object().resource().reservations_size());
+ ASSERT_EQ("foo", request2->object().resource().reservations(0).role());
+ ASSERT_EQ("foo/bar", request2->object().resource().reservations(1).role());
+ }
+
+ // Re-reserve the `cpus:0.1` reserved above for `foo` and refined to `foo/bar`
+ // to `baz` and refined to `baz/bar`. This will trigger two authorization
+ // requests to unreserved the resource in the `foo` hierarchy, and two
+ // authorization request for the reservations in the `baz` hierarchy.
+ {
+ Future<authorization::Request> request1;
+ Future<authorization::Request> request2;
+ Future<authorization::Request> request3;
+ Future<authorization::Request> request4;
+ EXPECT_CALL(authorizer, authorized(_))
+ .WillOnce(DoAll(FutureArg<0>(&request1), Return(true)))
+ .WillOnce(DoAll(FutureArg<0>(&request2), Return(true)))
+ .WillOnce(DoAll(FutureArg<0>(&request3), Return(true)))
+ .WillOnce(DoAll(FutureArg<0>(&request4), Return(true)));
+
+ Call call;
+ call.set_type(Call::RESERVE_RESOURCES);
+ Call::ReserveResources* reserveResources = call.mutable_reserve_resources();
+
+ reserveResources->mutable_agent_id()->set_value(slaveId.value());
+
+ v1::Resource source = *v1::Resources::parse("cpus", "0.1", "*");
+ *source.add_reservations() =
+ v1::createDynamicReservationInfo("foo", DEFAULT_CREDENTIAL.principal());
+ *source.add_reservations() = v1::createDynamicReservationInfo(
+ "foo/bar", DEFAULT_CREDENTIAL.principal());
+
+ v1::Resource resource = *v1::Resources::parse("cpus", "0.1", "*");
+ *resource.add_reservations() =
+ v1::createDynamicReservationInfo("baz", DEFAULT_CREDENTIAL.principal());
+ *resource.add_reservations() = v1::createDynamicReservationInfo(
+ "baz/bar", DEFAULT_CREDENTIAL.principal());
+
+ *reserveResources->add_resources() = resource;
+ *reserveResources->add_source() = source;
+
+ Future<http::Response> response = http::post(
+ master.get()->pid,
+ "/api/v1",
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL),
+ stringify(JSON::protobuf(call)),
+ stringify(ContentType::JSON));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(Accepted().status, response);
+
+ AWAIT_READY(request1);
+ ASSERT_EQ(authorization::Action::UNRESERVE_RESOURCES, request1->action());
+ ASSERT_EQ(2, request1->object().resource().reservations_size());
+ ASSERT_EQ("foo", request1->object().resource().reservations(0).role());
+ ASSERT_EQ("foo/bar", request1->object().resource().reservations(1).role());
+
+ AWAIT_READY(request2);
+ ASSERT_EQ(authorization::Action::UNRESERVE_RESOURCES, request2->action());
+ ASSERT_EQ(1, request2->object().resource().reservations_size());
+ ASSERT_EQ("foo", request1->object().resource().reservations(0).role());
+
+ AWAIT_READY(request3);
+ ASSERT_EQ(authorization::Action::RESERVE_RESOURCES, request3->action());
+ ASSERT_EQ(1, request3->object().resource().reservations_size());
+ ASSERT_EQ("baz", request3->object().resource().reservations(0).role());
+
+ AWAIT_READY(request4);
+ ASSERT_EQ(authorization::Action::RESERVE_RESOURCES, request4->action());
+ ASSERT_EQ(2, request4->object().resource().reservations_size());
+ ASSERT_EQ("baz", request4->object().resource().reservations(0).role());
+ ASSERT_EQ("baz/bar", request4->object().resource().reservations(1).role());
+ }
+}
class MasterOperationAuthorizationTest
: public MesosTest,
[mesos] 12/15: Updated offer operation resource validation for
reservation updates.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit f1a9d7206c7f6fee8e435e8fbe8e69802b1e9388
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:08:40 2019 +0100
Updated offer operation resource validation for reservation updates.
Updated `validateAndUpgradeResources()` to also validate the `source`
field in a reservation.
Review: https://reviews.apache.org/r/71700/
---
src/common/resources_utils.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/common/resources_utils.cpp b/src/common/resources_utils.cpp
index 5e78997..4c958c8 100644
--- a/src/common/resources_utils.cpp
+++ b/src/common/resources_utils.cpp
@@ -618,6 +618,12 @@ Option<Error> validateAndUpgradeResources(Offer::Operation* operation)
return error;
}
+ error = Resources::validate(operation->reserve().source());
+
+ if (error.isSome()) {
+ return error;
+ }
+
break;
}
case Offer::Operation::UNRESERVE: {
[mesos] 11/15: Updated validation of 'Reserve' call.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 1aedc02e5cf8ab2a5aea7c9ed8327139ca9c0f0e
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:07:29 2019 +0100
Updated validation of 'Reserve' call.
Updated `master::validation::operation::validate(Reserve&)` to
account for the new `source` field in the reserve call, verifying
that `source` and `resources` have a common reservation ancestor.
Review: https://reviews.apache.org/r/71696/
---
src/master/validation.cpp | 57 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/src/master/validation.cpp b/src/master/validation.cpp
index a7ecefb..c5fbbdd 100644
--- a/src/master/validation.cpp
+++ b/src/master/validation.cpp
@@ -2214,6 +2214,63 @@ Option<Error> validate(
return Error("Invalid resources: " + error->message);
}
+ error = resource::validate(reserve.source());
+ if (error.isSome()) {
+ return Error("Invalid source: " + error->message);
+ }
+
+ if (reserve.source_size() > 0) {
+ Resources source = reserve.source();
+ Resources target = reserve.resources();
+
+ auto validateReservationResources =
+ [](const RepeatedPtrField<Resource>& resources) -> Option<Error> {
+ // Resources should not be empty.
+ if (resources.empty()) {
+ return Error("Resource cannot be empty");
+ }
+
+ // All passed resources should have identical reservations.
+ const RepeatedPtrField<Resource::ReservationInfo>& reservations =
+ resources.begin()->reservations();
+
+ if (!std::all_of(
+ resources.begin(),
+ resources.end(),
+ [&](const Resource& resource) {
+ return resource.reservations_size() == reservations.size() &&
+ std::equal(
+ resource.reservations().begin(),
+ resource.reservations().end(),
+ reservations.begin());
+ })) {
+ return Error(
+ "Mixed reservations are not supported" + stringify(resources));
+ }
+
+ return None();
+ };
+
+ error = validateReservationResources(source);
+ if (error.isSome()) {
+ return Error("Invalid source: " + error->message);
+ }
+
+ error = validateReservationResources(target);
+ if (error.isSome()) {
+ return Error("Invalid resources: " + error->message);
+ }
+
+ // Both operands should refer to identical resource quantities.
+ if (source.toUnreserved() != target.toUnreserved()) {
+ return Error(
+ "Source and target resources should refer to identical resource "
+ "quantities: " +
+ stringify(source.toUnreserved()) + " vs. " +
+ stringify(target.toUnreserved()));
+ }
+ }
+
error =
resource::internal::validateSingleResourceProvider(reserve.resources());
if (error.isSome()) {
[mesos] 02/15: Added 'nodiscard' attribute to some member functions
of Resources.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c00595446f00ee4d139ee45f829644a512795ffd
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Thu Nov 7 14:55:20 2019 +0100
Added 'nodiscard' attribute to some member functions of Resources.
Added the `[[nodiscard]]` attribute to `Resources::pushReservation()`
and `Resources::popReservation()`, in order to prevent mistakes by
authors incorrectly assuming that these functions would modify the
resources objects in place.
Review: https://reviews.apache.org/r/71735/
---
include/mesos/resources.hpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/include/mesos/resources.hpp b/include/mesos/resources.hpp
index b8aef28..dbfa9c9 100644
--- a/include/mesos/resources.hpp
+++ b/include/mesos/resources.hpp
@@ -523,11 +523,12 @@ public:
// Returns a `Resources` object with the new reservation added to the back.
// The new reservation must be a valid refinement of the current reservation.
- Resources pushReservation(const Resource::ReservationInfo& reservation) const;
+ STOUT_NODISCARD Resources pushReservation(
+ const Resource::ReservationInfo& reservation) const;
// Returns a `Resources` object with the last reservation removed.
// Every resource in `Resources` must have `resource.reservations_size() > 0`.
- Resources popReservation() const;
+ STOUT_NODISCARD Resources popReservation() const;
// Returns a `Resources` object with all of the reservations removed.
Resources toUnreserved() const;
[mesos] 06/15: Consolidated 'Master::authorizeReserveResources'
overloads.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit fb37ddaa847b12a139e4fcb1d107a923e1355cff
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:03:40 2019 +0100
Consolidated 'Master::authorizeReserveResources' overloads.
Removed one overload of `authorizeReserveResources()` that was
not being used anywhere.
Review: https://reviews.apache.org/r/71699/
---
src/master/master.cpp | 18 +++++-------------
src/master/master.hpp | 6 ------
2 files changed, 5 insertions(+), 19 deletions(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 2fdd6f7..bbbc076 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -3753,16 +3753,6 @@ Future<bool> Master::authorizeReserveResources(
const Offer::Operation::Reserve& reserve,
const Option<Principal>& principal)
{
- // Authorizing the reserve operation is equivalent to authorizing
- // the resources specified in the operation.
- return authorizeReserveResources(reserve.resources(), principal);
-}
-
-
-Future<bool> Master::authorizeReserveResources(
- const Resources& resources,
- const Option<Principal>& principal)
-{
if (authorizer.isNone()) {
return true; // Authorization is disabled.
}
@@ -3780,7 +3770,7 @@ Future<bool> Master::authorizeReserveResources(
// Add an element to `request.roles` for each unique role in the resources.
hashset<string> roles;
vector<Future<bool>> authorizations;
- foreach (const Resource& resource, resources) {
+ foreach (const Resource& resource, reserve.resources()) {
// NOTE: We rely on the master to ensure that the resource is in the
// post-reservation-refinement format. If there is a stack of reservations,
// we perform authorization for the role of the most refined reservation,
@@ -3810,7 +3800,7 @@ Future<bool> Master::authorizeReserveResources(
LOG(INFO) << "Authorizing principal '"
<< (principal.isSome() ? stringify(principal.get()) : "ANY")
- << "' to reserve resources '" << resources << "'";
+ << "' to reserve resources '" << reserve.resources() << "'";
// NOTE: Empty authorizations are not valid and are checked by a validator.
// However under certain circumstances, this method can be called before
@@ -4213,8 +4203,10 @@ Future<bool> Master::authorizeSlave(
// agent's principal and authorizing them helps prevent agents from
// advertising reserved resources of arbitrary roles.
if (!Resources(slaveInfo.resources()).reserved().empty()) {
+ Offer::Operation::Reserve reserve;
+ reserve.mutable_resources()->CopyFrom(slaveInfo.resources());
authorizations.push_back(
- authorizeReserveResources(slaveInfo.resources(), principal));
+ authorizeReserveResources(reserve, principal));
}
return authorization::collectAuthorizations(authorizations);
diff --git a/src/master/master.hpp b/src/master/master.hpp
index dc45028..d2392d7 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -779,12 +779,6 @@ protected:
const Offer::Operation::Reserve& reserve,
const Option<process::http::authentication::Principal>& principal);
- // Authorizes whether the provided `principal` is allowed to reserve
- // the specified `resources`.
- process::Future<bool> authorizeReserveResources(
- const Resources& resources,
- const Option<process::http::authentication::Principal>& principal);
-
/**
* Authorizes an `UNRESERVE` operation.
*
[mesos] 05/15: Added 'source' field to scheduler reservation API.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 398a13e6fe8956d463a571e93551331dcfbdabeb
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:02:31 2019 +0100
Added 'source' field to scheduler reservation API.
Added a new 'source' field to the `Reserve` message
in the scheduler reservation API.
Review: https://reviews.apache.org/r/71686/
---
include/mesos/mesos.proto | 1 +
include/mesos/v1/mesos.proto | 1 +
2 files changed, 2 insertions(+)
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 0cab154..7d77ddd 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -2053,6 +2053,7 @@ message Offer {
}
message Reserve {
+ repeated Resource source = 2;
repeated Resource resources = 1;
}
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index a2f6dbb..7abb6ba 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -2041,6 +2041,7 @@ message Offer {
}
message Reserve {
+ repeated Resource source = 2;
repeated Resource resources = 1;
}
[mesos] 07/15: Rejected scheduler calls that include reservation
updates.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 231321559485ea4b6e3024b86c74914065521e45
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:03:48 2019 +0100
Rejected scheduler calls that include reservation updates.
Until reservation updates are implemented, ensure
that calls making use of them are properly rejected.
Review: https://reviews.apache.org/r/71687/
---
src/master/master.cpp | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index bbbc076..e7609f3 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -4484,6 +4484,30 @@ void Master::accept(
foreach (Offer::Operation& operation, operations) {
Option<Error> error = validateAndUpgradeResources(&operation);
+
+ // Additional operation-specific validation.
+ if (error.isNone()) {
+ switch (operation.type()) {
+ case Offer::Operation::RESERVE:
+ if (!operation.reserve().source().empty()) {
+ error = Error("Reservation updates are not yet implemented"
+ " for the scheduler API.");
+ }
+ break;
+ case Offer::Operation::UNRESERVE:
+ case Offer::Operation::CREATE:
+ case Offer::Operation::DESTROY:
+ case Offer::Operation::GROW_VOLUME:
+ case Offer::Operation::SHRINK_VOLUME:
+ case Offer::Operation::CREATE_DISK:
+ case Offer::Operation::DESTROY_DISK:
+ case Offer::Operation::LAUNCH:
+ case Offer::Operation::LAUNCH_GROUP:
+ case Offer::Operation::UNKNOWN:
+ break;
+ }
+ }
+
if (error.isSome()) {
switch (operation.type()) {
case Offer::Operation::RESERVE:
[mesos] 09/15: Added function to compute a common reservation
ancestor.
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bennoe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 8e742ebb72918ce349025fc6bf1ef03f2b3c7e4d
Author: Benno Evers <be...@mesosphere.com>
AuthorDate: Fri Nov 8 14:05:06 2019 +0100
Added function to compute a common reservation ancestor.
Added a new function `Resources::getReservationAncestor()` to compute
a common reservation ancestor between two `Resources`.
Review: https://reviews.apache.org/r/71690/
---
include/mesos/resources.hpp | 14 +++++++++
src/common/resources.cpp | 34 ++++++++++++++++++++++
src/tests/resources_tests.cpp | 66 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 114 insertions(+)
diff --git a/include/mesos/resources.hpp b/include/mesos/resources.hpp
index dbfa9c9..da585db 100644
--- a/include/mesos/resources.hpp
+++ b/include/mesos/resources.hpp
@@ -376,6 +376,20 @@ public:
// E.g. MOUNT volume).
static bool shrink(Resource* resource, const Value::Scalar& target);
+ // Returns the most refined reserved resource that can be
+ // reached by modifications to `a`'s or `b`'s reservations.
+ //
+ // Both `a` and `b` must refer to identical resource quantities.
+ static Resource getReservationAncestor(const Resource& a, const Resource& b);
+
+ // Returns the most refined reserved resources that can be
+ // reached by modifications to `a`'s or `b`'s reservations.
+ //
+ // Both `a` and `b` must be non-empty, and must contain the
+ // same resources except for reservations.
+ static Resources getReservationAncestor(
+ const Resources& a, const Resources& b);
+
// Returns the summed up Resources given a hashmap<Key, Resources>.
//
// NOTE: While scalar resources such as "cpus" sum correctly,
diff --git a/src/common/resources.cpp b/src/common/resources.cpp
index e3e1100..88c5696 100644
--- a/src/common/resources.cpp
+++ b/src/common/resources.cpp
@@ -1318,6 +1318,40 @@ bool Resources::shrink(Resource* resource, const Value::Scalar& target)
}
+Resources Resources::getReservationAncestor(
+ const Resources& r1, const Resources& r2)
+{
+ CHECK(!r1.empty());
+ CHECK(!r2.empty());
+ CHECK(r1.toUnreserved() == r2.toUnreserved());
+
+ Resources result = r1.toUnreserved();
+ Resource ancestor = getReservationAncestor(*r1.begin(), *r2.begin());
+ foreach (
+ const Resource::ReservationInfo& reservation,
+ ancestor.reservations()) {
+ result = result.pushReservation(reservation);
+ }
+
+ return result;
+}
+
+
+Resource Resources::getReservationAncestor(
+ const Resource& r1, const Resource& r2)
+{
+ Resource ancestor = r1;
+ ancestor.clear_reservations();
+
+ const int size = std::min(r1.reservations_size(), r2.reservations_size());
+ for (int i = 0; i < size && r1.reservations(i) == r2.reservations(i); ++i) {
+ *ancestor.add_reservations() = r1.reservations(i);
+ }
+
+ return ancestor;
+}
+
+
/////////////////////////////////////////////////
// Public member functions.
/////////////////////////////////////////////////
diff --git a/src/tests/resources_tests.cpp b/src/tests/resources_tests.cpp
index b585465..bd04431 100644
--- a/src/tests/resources_tests.cpp
+++ b/src/tests/resources_tests.cpp
@@ -3595,6 +3595,72 @@ TEST(ResourcesTest, Evolve)
}
+TEST(ResourcesTest, ReservationAncestor)
+{
+ Resources baseResources = *Resources::parse("cpus:2;mem:10");
+
+ EXPECT_EQ(
+ baseResources,
+ Resources::getReservationAncestor(baseResources, baseResources));
+
+ Resources resources1 = baseResources.pushReservation(
+ createDynamicReservationInfo("foo"));
+
+ EXPECT_EQ(
+ resources1,
+ Resources::getReservationAncestor(resources1, resources1));
+
+ EXPECT_EQ(
+ baseResources,
+ Resources::getReservationAncestor(resources1, baseResources));
+
+ EXPECT_EQ(
+ baseResources,
+ Resources::getReservationAncestor(baseResources, resources1));
+
+ resources1 = resources1.pushReservation(
+ createDynamicReservationInfo("foo/bar"));
+
+ EXPECT_EQ(
+ resources1,
+ Resources::getReservationAncestor(resources1, resources1));
+
+ EXPECT_EQ(
+ baseResources,
+ Resources::getReservationAncestor(resources1, baseResources));
+
+ EXPECT_EQ(
+ baseResources,
+ Resources::getReservationAncestor(baseResources, resources1));
+
+ Resources resources2 = baseResources.pushReservation(
+ createDynamicReservationInfo("foo"));
+
+ EXPECT_EQ(
+ resources2,
+ Resources::getReservationAncestor(resources1, resources2));
+
+ EXPECT_EQ(
+ resources2,
+ Resources::getReservationAncestor(resources2, resources1));
+
+ EXPECT_NE(
+ resources1,
+ Resources::getReservationAncestor(baseResources, resources2));
+
+ EXPECT_EQ(
+ baseResources,
+ Resources::getReservationAncestor(baseResources, resources2));
+
+ Resources resources3 = resources2.pushReservation(
+ createDynamicReservationInfo("foo/baz"));
+
+ EXPECT_EQ(
+ resources2,
+ Resources::getReservationAncestor(resources1, resources3));
+}
+
+
TEST(SharedResourcesTest, Printing)
{
Resources volume = createPersistentVolume(