You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/09/06 19:50:34 UTC
[1/2] mesos git commit: Added storage-related offer operations.
Repository: mesos
Updated Branches:
refs/heads/master 2e084bd39 -> a4121c25d
Added storage-related offer operations.
Review: https://reviews.apache.org/r/58021/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a4121c25
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a4121c25
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a4121c25
Branch: refs/heads/master
Commit: a4121c25d31276717ae186c7ff4ef589ec1d6551
Parents: 48e39a4
Author: Jan Schlicht <ja...@mesosphere.io>
Authored: Wed Sep 6 11:18:47 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Sep 6 12:50:26 2017 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 25 ++++++++++
include/mesos/v1/mesos.proto | 25 ++++++++++
src/common/protobuf_utils.cpp | 91 +++++++++++++++++++++++++++++++++----
src/common/resources.cpp | 16 +++++++
src/common/resources_utils.cpp | 48 +++++++++++++++++++
src/master/master.cpp | 56 ++++++++++++++++++++++-
src/v1/resources.cpp | 16 +++++++
7 files changed, 267 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a4121c25/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index b3bf28b..3b9a6fd 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1793,6 +1793,10 @@ message Offer {
UNRESERVE = 3;
CREATE = 4;
DESTROY = 5;
+ CREATE_VOLUME = 7;
+ DESTROY_VOLUME = 8;
+ CREATE_BLOCK = 9;
+ DESTROY_BLOCK = 10;
}
// TODO(vinod): Deprecate this in favor of `LaunchGroup` below.
@@ -1829,6 +1833,23 @@ message Offer {
repeated Resource volumes = 1;
}
+ message CreateVolume {
+ required Resource source = 1;
+ required Resource.DiskInfo.Source.Type target_type = 2;
+ }
+
+ message DestroyVolume {
+ required Resource volume = 1;
+ }
+
+ message CreateBlock {
+ required Resource source = 1;
+ }
+
+ message DestroyBlock {
+ required Resource block = 1;
+ }
+
optional Type type = 1;
optional Launch launch = 2;
optional LaunchGroup launch_group = 7;
@@ -1836,6 +1857,10 @@ message Offer {
optional Unreserve unreserve = 4;
optional Create create = 5;
optional Destroy destroy = 6;
+ optional CreateVolume create_volume = 8;
+ optional DestroyVolume destroy_volume = 9;
+ optional CreateBlock create_block = 10;
+ optional DestroyBlock destroy_block = 11;
}
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/a4121c25/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 68b369e..85de9e3 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1776,6 +1776,10 @@ message Offer {
UNRESERVE = 3;
CREATE = 4;
DESTROY = 5;
+ CREATE_VOLUME = 7;
+ DESTROY_VOLUME = 8;
+ CREATE_BLOCK = 9;
+ DESTROY_BLOCK = 10;
}
// TODO(vinod): Deprecate this in favor of `LaunchGroup` below.
@@ -1812,6 +1816,23 @@ message Offer {
repeated Resource volumes = 1;
}
+ message CreateVolume {
+ required Resource source = 1;
+ required Resource.DiskInfo.Source.Type target_type = 2;
+ }
+
+ message DestroyVolume {
+ required Resource volume = 1;
+ }
+
+ message CreateBlock {
+ required Resource source = 1;
+ }
+
+ message DestroyBlock {
+ required Resource block = 1;
+ }
+
optional Type type = 1;
optional Launch launch = 2;
optional LaunchGroup launch_group = 7;
@@ -1819,6 +1840,10 @@ message Offer {
optional Unreserve unreserve = 4;
optional Create create = 5;
optional Destroy destroy = 6;
+ optional CreateVolume create_volume = 8;
+ optional DestroyVolume destroy_volume = 9;
+ optional CreateBlock create_block = 10;
+ optional DestroyBlock destroy_block = 11;
}
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/a4121c25/src/common/protobuf_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.cpp b/src/common/protobuf_utils.cpp
index 3ae68e9..04f44f6 100644
--- a/src/common/protobuf_utils.cpp
+++ b/src/common/protobuf_utils.cpp
@@ -439,15 +439,25 @@ void injectAllocationInfo(
Offer::Operation* operation,
const Resource::AllocationInfo& allocationInfo)
{
- auto inject = [](
- RepeatedPtrField<Resource>* resources,
- const Resource::AllocationInfo& allocationInfo) {
- foreach (Resource& resource, *resources) {
+ struct Injector
+ {
+ void operator()(
+ Resource& resource, const Resource::AllocationInfo& allocationInfo)
+ {
if (!resource.has_allocation_info()) {
resource.mutable_allocation_info()->CopyFrom(allocationInfo);
}
}
- };
+
+ void operator()(
+ RepeatedPtrField<Resource>* resources,
+ const Resource::AllocationInfo& allocationInfo)
+ {
+ foreach (Resource& resource, *resources) {
+ operator()(resource, allocationInfo);
+ }
+ }
+ } inject;
switch (operation->type()) {
case Offer::Operation::LAUNCH: {
@@ -521,6 +531,38 @@ void injectAllocationInfo(
break;
}
+ case Offer::Operation::CREATE_VOLUME: {
+ inject(
+ *operation->mutable_create_volume()->mutable_source(),
+ allocationInfo);
+
+ break;
+ }
+
+ case Offer::Operation::DESTROY_VOLUME: {
+ inject(
+ *operation->mutable_destroy_volume()->mutable_volume(),
+ allocationInfo);
+
+ break;
+ }
+
+ case Offer::Operation::CREATE_BLOCK: {
+ inject(
+ *operation->mutable_create_block()->mutable_source(),
+ allocationInfo);
+
+ break;
+ }
+
+ case Offer::Operation::DESTROY_BLOCK: {
+ inject(
+ *operation->mutable_destroy_block()->mutable_block(),
+ allocationInfo);
+
+ break;
+ }
+
case Offer::Operation::UNKNOWN:
break; // No-op.
}
@@ -529,13 +571,22 @@ void injectAllocationInfo(
void stripAllocationInfo(Offer::Operation* operation)
{
- auto strip = [](RepeatedPtrField<Resource>* resources) {
- foreach (Resource& resource, *resources) {
+ struct Stripper
+ {
+ void operator()(Resource& resource)
+ {
if (resource.has_allocation_info()) {
resource.clear_allocation_info();
}
}
- };
+
+ void operator()(RepeatedPtrField<Resource>* resources)
+ {
+ foreach (Resource& resource, *resources) {
+ operator()(resource);
+ }
+ }
+ } strip;
switch (operation->type()) {
case Offer::Operation::LAUNCH: {
@@ -595,6 +646,30 @@ void stripAllocationInfo(Offer::Operation* operation)
break;
}
+ case Offer::Operation::CREATE_VOLUME: {
+ strip(*operation->mutable_create_volume()->mutable_source());
+
+ break;
+ }
+
+ case Offer::Operation::DESTROY_VOLUME: {
+ strip(*operation->mutable_destroy_volume()->mutable_volume());
+
+ break;
+ }
+
+ case Offer::Operation::CREATE_BLOCK: {
+ strip(*operation->mutable_create_block()->mutable_source());
+
+ break;
+ }
+
+ case Offer::Operation::DESTROY_BLOCK: {
+ strip(*operation->mutable_destroy_block()->mutable_block());
+
+ break;
+ }
+
case Offer::Operation::UNKNOWN:
break; // No-op.
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/a4121c25/src/common/resources.cpp
----------------------------------------------------------------------
diff --git a/src/common/resources.cpp b/src/common/resources.cpp
index 797fc75..14b600c 100644
--- a/src/common/resources.cpp
+++ b/src/common/resources.cpp
@@ -1692,6 +1692,22 @@ Try<Resources> Resources::apply(const Offer::Operation& operation) const
break;
}
+ case Offer::Operation::CREATE_VOLUME:
+ // TODO(nfnt): Implement this.
+ break;
+
+ case Offer::Operation::DESTROY_VOLUME:
+ // TODO(nfnt): Implement this.
+ break;
+
+ case Offer::Operation::CREATE_BLOCK:
+ // TODO(nfnt): Implement this.
+ break;
+
+ case Offer::Operation::DESTROY_BLOCK:
+ // TODO(nfnt): Implement this.
+ break;
+
case Offer::Operation::UNKNOWN:
return Error("Unknown offer operation");
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/a4121c25/src/common/resources_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/resources_utils.cpp b/src/common/resources_utils.cpp
index 821bd09..e34cd8a 100644
--- a/src/common/resources_utils.cpp
+++ b/src/common/resources_utils.cpp
@@ -397,6 +397,54 @@ Option<Error> validateAndNormalizeResources(Offer::Operation* operation)
return None();
}
+ case Offer::Operation::CREATE_VOLUME: {
+ // TODO(mpark): Once we perform a sanity check validation for
+ // offer operations as specified in MESOS-7760, this should no
+ // longer have to be handled in this function.
+ if (!operation->has_create_volume()) {
+ return Error(
+ "A CREATE_VOLUME offer operation must have"
+ " the Offer.Operation.create_volume field set.");
+ }
+
+ return None();
+ }
+ case Offer::Operation::DESTROY_VOLUME: {
+ // TODO(mpark): Once we perform a sanity check validation for
+ // offer operations as specified in MESOS-7760, this should no
+ // longer have to be handled in this function.
+ if (!operation->has_destroy_volume()) {
+ return Error(
+ "A DESTROY_VOLUME offer operation must have"
+ " the Offer.Operation.destroy_volume field set.");
+ }
+
+ return None();
+ }
+ case Offer::Operation::CREATE_BLOCK: {
+ // TODO(mpark): Once we perform a sanity check validation for
+ // offer operations as specified in MESOS-7760, this should no
+ // longer have to be handled in this function.
+ if (!operation->has_create_block()) {
+ return Error(
+ "A CREATE_BLOCK offer operation must have"
+ " the Offer.Operation.create_block field set.");
+ }
+
+ return None();
+ }
+ case Offer::Operation::DESTROY_BLOCK: {
+ // TODO(mpark): Once we perform a sanity check validation for
+ // offer operations as specified in MESOS-7760, this should no
+ // longer have to be handled in this function.
+ if (!operation->has_destroy_block()) {
+ return Error(
+ "A DESTROY_BLOCK offer operation must have"
+ " the Offer.Operation.destroy_block field set.");
+ }
+
+ return None();
+ }
case Offer::Operation::UNKNOWN: {
// TODO(mpark): Once we perform a sanity check validation for
// offer operations as specified in MESOS-7760, this should no
http://git-wip-us.apache.org/repos/asf/mesos/blob/a4121c25/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 4fa05fa..14d94cf 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -4037,7 +4037,11 @@ void Master::accept(
case Offer::Operation::RESERVE:
case Offer::Operation::UNRESERVE:
case Offer::Operation::CREATE:
- case Offer::Operation::DESTROY: {
+ case Offer::Operation::DESTROY:
+ case Offer::Operation::CREATE_VOLUME:
+ case Offer::Operation::DESTROY_VOLUME:
+ case Offer::Operation::CREATE_BLOCK:
+ case Offer::Operation::DESTROY_BLOCK: {
drop(framework, operation, error->message);
break;
}
@@ -4083,7 +4087,11 @@ void Master::accept(
case Offer::Operation::RESERVE:
case Offer::Operation::UNRESERVE:
case Offer::Operation::CREATE:
- case Offer::Operation::DESTROY: {
+ case Offer::Operation::DESTROY:
+ case Offer::Operation::CREATE_VOLUME:
+ case Offer::Operation::DESTROY_VOLUME:
+ case Offer::Operation::CREATE_BLOCK:
+ case Offer::Operation::DESTROY_BLOCK: {
// No-op.
break;
}
@@ -4255,6 +4263,26 @@ void Master::accept(
break;
}
+ case Offer::Operation::CREATE_VOLUME: {
+ // TODO(nfnt): Implement authorization for 'CREATE_VOLUME'.
+ break;
+ }
+
+ case Offer::Operation::DESTROY_VOLUME: {
+ // TODO(nfnt): Implement authorization for 'DESTROY_VOLUME'.
+ break;
+ }
+
+ case Offer::Operation::CREATE_BLOCK: {
+ // TODO(nfnt): Implement authorization for 'CREATE_BLOCK'.
+ break;
+ }
+
+ case Offer::Operation::DESTROY_BLOCK: {
+ // TODO(nfnt): Implement authorization for 'DESTROY_BLOCK'.
+ break;
+ }
+
case Offer::Operation::UNKNOWN: {
// TODO(vinod): Send an error event to the scheduler?
LOG(WARNING) << "Ignoring unknown offer operation";
@@ -5029,6 +5057,30 @@ void Master::_accept(
break;
}
+ case Offer::Operation::CREATE_VOLUME: {
+ // TODO(nfnt): Provide an implementation for 'CREATE_VOLUME'.
+ drop(framework, operation, "Unimplemented");
+ break;
+ }
+
+ case Offer::Operation::DESTROY_VOLUME: {
+ // TODO(nfnt): Provide an implementation for 'DESTROY_VOLUME'.
+ drop(framework, operation, "Unimplemented");
+ break;
+ }
+
+ case Offer::Operation::CREATE_BLOCK: {
+ // TODO(nfnt): Provide an implementation for 'CREATE_BLOCK'.
+ drop(framework, operation, "Unimplemented");
+ break;
+ }
+
+ case Offer::Operation::DESTROY_BLOCK: {
+ // TODO(nfnt): Provide an implementation for 'DESTROY_BLOCK'.
+ drop(framework, operation, "Unimplemented");
+ break;
+ }
+
case Offer::Operation::UNKNOWN: {
LOG(WARNING) << "Ignoring unknown offer operation";
break;
http://git-wip-us.apache.org/repos/asf/mesos/blob/a4121c25/src/v1/resources.cpp
----------------------------------------------------------------------
diff --git a/src/v1/resources.cpp b/src/v1/resources.cpp
index 96913b7..a5cc155 100644
--- a/src/v1/resources.cpp
+++ b/src/v1/resources.cpp
@@ -1721,6 +1721,22 @@ Try<Resources> Resources::apply(const Offer::Operation& operation) const
break;
}
+ case Offer::Operation::CREATE_VOLUME:
+ // TODO(nfnt): Implement this.
+ break;
+
+ case Offer::Operation::DESTROY_VOLUME:
+ // TODO(nfnt): Implement this.
+ break;
+
+ case Offer::Operation::CREATE_BLOCK:
+ // TODO(nfnt): Implement this.
+ break;
+
+ case Offer::Operation::DESTROY_BLOCK:
+ // TODO(nfnt): Implement this.
+ break;
+
case Offer::Operation::UNKNOWN:
return Error("Unknown offer operation");
}
[2/2] mesos git commit: Introduce BLOCK and RAW disk types.
Posted by ji...@apache.org.
Introduce BLOCK and RAW disk types.
BLOCK and RAW disk types are low-level disk resources which will need
to be transformed into e.g., volumes by dedicated, still to implement
offer operations.
Review: https://reviews.apache.org/r/58047/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/48e39a4c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/48e39a4c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/48e39a4c
Branch: refs/heads/master
Commit: 48e39a4c0da049534f9f90bc66d38b1368b1807a
Parents: 2e084bd
Author: Benjamin Bannier <be...@mesosphere.io>
Authored: Wed Sep 6 11:17:43 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Sep 6 12:50:26 2017 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 3 ++-
include/mesos/v1/mesos.proto | 3 ++-
src/common/resources.cpp | 9 +++++++++
src/slave/paths.cpp | 2 ++
src/slave/slave.cpp | 2 ++
src/v1/resources.cpp | 9 +++++++++
6 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/48e39a4c/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index eede082..b3bf28b 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1313,12 +1313,13 @@ message Resource {
optional Volume volume = 2;
// Describes where a disk originates from.
- // TODO(jmlvanre): Add support for BLOCK devices.
message Source {
enum Type {
UNKNOWN = 0;
PATH = 1;
MOUNT = 2;
+ BLOCK = 3;
+ RAW = 4;
}
// A folder that can be located on a separate disk device. This
http://git-wip-us.apache.org/repos/asf/mesos/blob/48e39a4c/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 8c6246e..68b369e 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1296,12 +1296,13 @@ message Resource {
optional Volume volume = 2;
// Describes where a disk originates from.
- // TODO(jmlvanre): Add support for BLOCK devices.
message Source {
enum Type {
UNKNOWN = 0;
PATH = 1;
MOUNT = 2;
+ BLOCK = 3;
+ RAW = 4;
}
// A folder that can be located on a separate disk device. This
http://git-wip-us.apache.org/repos/asf/mesos/blob/48e39a4c/src/common/resources.cpp
----------------------------------------------------------------------
diff --git a/src/common/resources.cpp b/src/common/resources.cpp
index 8d43889..797fc75 100644
--- a/src/common/resources.cpp
+++ b/src/common/resources.cpp
@@ -848,6 +848,11 @@ Option<Error> Resources::validate(const Resource& resource)
case Resource::DiskInfo::Source::MOUNT:
// `PATH` and `MOUNT` contain only `optional` members.
break;
+ case Resource::DiskInfo::Source::BLOCK:
+ case Resource::DiskInfo::Source::RAW:
+ // TODO(bbannier): Update with validation once the exact format of
+ // `BLOCK` and `RAW` messages have taken some form.
+ break;
case Resource::DiskInfo::Source::UNKNOWN:
return Error(
"Unsupported 'DiskInfo.Source.Type' in "
@@ -2141,6 +2146,10 @@ ostream& operator<<(ostream& stream, const Resource::DiskInfo::Source& source)
return stream << "PATH"
<< (source.path().has_root() ? ":" + source.path().root()
: "");
+ case Resource::DiskInfo::Source::BLOCK:
+ return stream << "BLOCK";
+ case Resource::DiskInfo::Source::RAW:
+ return stream << "RAW";
case Resource::DiskInfo::Source::UNKNOWN:
return stream << "UNKNOWN";
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/48e39a4c/src/slave/paths.cpp
----------------------------------------------------------------------
diff --git a/src/slave/paths.cpp b/src/slave/paths.cpp
index 1a6943f..08177bc 100644
--- a/src/slave/paths.cpp
+++ b/src/slave/paths.cpp
@@ -517,6 +517,8 @@ string getPersistentVolumePath(
CHECK(volume.disk().source().mount().has_root());
return volume.disk().source().mount().root();
}
+ case Resource::DiskInfo::Source::BLOCK:
+ case Resource::DiskInfo::Source::RAW:
case Resource::DiskInfo::Source::UNKNOWN:
LOG(FATAL) << "Unsupported DiskInfo.Source.type";
break;
http://git-wip-us.apache.org/repos/asf/mesos/blob/48e39a4c/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index df920ec..6d1516a 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -531,6 +531,8 @@ void Slave::initialize()
#endif // __linux__
break;
}
+ case Resource::DiskInfo::Source::BLOCK:
+ case Resource::DiskInfo::Source::RAW:
case Resource::DiskInfo::Source::UNKNOWN: {
EXIT(EXIT_FAILURE)
<< "Unsupported 'DiskInfo.Source.Type' in '" << resource << "'";
http://git-wip-us.apache.org/repos/asf/mesos/blob/48e39a4c/src/v1/resources.cpp
----------------------------------------------------------------------
diff --git a/src/v1/resources.cpp b/src/v1/resources.cpp
index 508f3f8..96913b7 100644
--- a/src/v1/resources.cpp
+++ b/src/v1/resources.cpp
@@ -879,6 +879,11 @@ Option<Error> Resources::validate(const Resource& resource)
case Resource::DiskInfo::Source::MOUNT:
// `PATH` and `MOUNT` contain only `optional` members.
break;
+ case Resource::DiskInfo::Source::BLOCK:
+ case Resource::DiskInfo::Source::RAW:
+ // TODO(bbannier): Update with validation once the exact format of
+ // `BLOCK` and `RAW` messages have taken some form.
+ break;
case Resource::DiskInfo::Source::UNKNOWN:
return Error(
"Unsupported 'DiskInfo.Source.Type' in "
@@ -2170,6 +2175,10 @@ ostream& operator<<(ostream& stream, const Resource::DiskInfo::Source& source)
return stream << "PATH"
<< (source.path().has_root() ? ":" + source.path().root()
: "");
+ case Resource::DiskInfo::Source::BLOCK:
+ return stream << "BLOCK";
+ case Resource::DiskInfo::Source::RAW:
+ return stream << "RAW";
case Resource::DiskInfo::Source::UNKNOWN:
return stream << "UNKNOWN";
}