You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/12/02 01:32:11 UTC
[4/5] mesos git commit: Added the `ResourceProviderState` protobuf
for resource providers.
Added the `ResourceProviderState` protobuf for resource providers.
The `ResourceProviderState` protobuf includes the list of pending offer
operations, the total resources, and the current resource version UUID.
Note that the pending operations do not includes completed operations
that have not been acknowledged yet.
Review: https://reviews.apache.org/r/64075/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/089e789b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/089e789b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/089e789b
Branch: refs/heads/master
Commit: 089e789b7820de6b3fde3a94c98fb1d89194a618
Parents: 3adab34
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
Authored: Fri Dec 1 15:12:11 2017 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri Dec 1 17:32:03 2017 -0800
----------------------------------------------------------------------
src/CMakeLists.txt | 1 +
src/Makefile.am | 6 ++-
src/resource_provider/registry.hpp | 2 +-
src/resource_provider/state.hpp | 24 ++++++++++
src/resource_provider/state.proto | 51 +++++++++++++++++++++
src/resource_provider/storage/provider.cpp | 60 ++++++++++++++++++++++++-
6 files changed, 141 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/089e789b/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 15cda10..318cb40 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -71,6 +71,7 @@ PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/isolators/docker/volum
PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/provisioner/docker/message)
PROTOC_GENERATE(INTERNAL TARGET master/registry)
PROTOC_GENERATE(INTERNAL TARGET resource_provider/registry)
+PROTOC_GENERATE(INTERNAL TARGET resource_provider/state)
# BUILD PROTOBUFS.
http://git-wip-us.apache.org/repos/asf/mesos/blob/089e789b/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 3444388..08d29ab 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -366,6 +366,8 @@ CXX_PROTOS += \
messages/messages.pb.h \
resource_provider/registry.pb.cc \
resource_provider/registry.pb.h \
+ resource_provider/state.pb.cc \
+ resource_provider/state.pb.h \
slave/containerizer/mesos/provisioner/docker/message.pb.cc \
slave/containerizer/mesos/provisioner/docker/message.pb.h \
slave/containerizer/mesos/isolators/docker/volume/state.pb.cc \
@@ -930,6 +932,8 @@ libmesos_no_3rdparty_la_SOURCES = \
master/registry.proto \
messages/flags.proto \
messages/messages.proto \
+ resource_provider/registry.proto \
+ resource_provider/state.proto \
slave/containerizer/mesos/provisioner/docker/message.proto \
slave/containerizer/mesos/isolators/docker/volume/state.proto \
slave/containerizer/mesos/isolators/network/cni/spec.proto
@@ -1153,7 +1157,7 @@ libmesos_no_3rdparty_la_SOURCES += \
resource_provider/message.hpp \
resource_provider/registrar.hpp \
resource_provider/registry.hpp \
- resource_provider/registry.proto \
+ resource_provider/state.hpp \
resource_provider/validation.hpp \
sched/constants.hpp \
sched/flags.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/089e789b/src/resource_provider/registry.hpp
----------------------------------------------------------------------
diff --git a/src/resource_provider/registry.hpp b/src/resource_provider/registry.hpp
index 048cd6b..4c6c4d4 100644
--- a/src/resource_provider/registry.hpp
+++ b/src/resource_provider/registry.hpp
@@ -19,6 +19,6 @@
#define __RESOURCE_PROVIDER_REGISTRY_HPP__
// ONLY USEFUL AFTER RUNNING PROTOC.
-#include <resource_provider/registry.pb.h>
+#include "resource_provider/registry.pb.h"
#endif // __RESOURCE_PROVIDER_REGISTRY_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/089e789b/src/resource_provider/state.hpp
----------------------------------------------------------------------
diff --git a/src/resource_provider/state.hpp b/src/resource_provider/state.hpp
new file mode 100644
index 0000000..ab1318b
--- /dev/null
+++ b/src/resource_provider/state.hpp
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#ifndef __RESOURCE_PROVIDER_STATE_HPP__
+#define __RESOURCE_PROVIDER_STATE_HPP__
+
+// ONLY USEFUL AFTER RUNNING PROTOC.
+#include "resource_provider/state.pb.h"
+
+#endif // __RESOURCE_PROVIDER_STATE_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/089e789b/src/resource_provider/state.proto
----------------------------------------------------------------------
diff --git a/src/resource_provider/state.proto b/src/resource_provider/state.proto
new file mode 100644
index 0000000..ae30d16
--- /dev/null
+++ b/src/resource_provider/state.proto
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+import "mesos/mesos.proto";
+
+import "mesos/resource_provider/resource_provider.proto";
+
+package mesos.resource_provider.state;
+
+option java_package = "org.apache.mesos.resource_provider.state";
+option java_outer_classname = "Protos";
+
+
+message ResourceProviderState {
+ // This includes only pending operations. Operations that have
+ // unacknowledged statuses should be recovered through the status
+ // update manager.
+ repeated Event.Operation operations = 1;
+
+ // The total resources provided by this resource provider.
+ repeated Resource resources = 2;
+
+ // Used to establish the relationship between the operation and
+ // the resources that the operation is operating on. Each resource
+ // provider will keep a resource version UUID, and change it when
+ // it believes that the resources from this resource provider are
+ // out of sync from the master's view. The master will keep track
+ // of the last known resource version UUID for each resource
+ // provider, and attach the resource version UUID in each
+ // operation it sends out. The resource provider should reject
+ // operations that have a different resource version UUID than
+ // that it maintains, because this means the operation is
+ // operating on resources that might have already been
+ // invalidated.
+ required bytes resource_version_uuid = 3;
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/089e789b/src/resource_provider/storage/provider.cpp
----------------------------------------------------------------------
diff --git a/src/resource_provider/storage/provider.cpp b/src/resource_provider/storage/provider.cpp
index a57ef15..d35b0d0 100644
--- a/src/resource_provider/storage/provider.cpp
+++ b/src/resource_provider/storage/provider.cpp
@@ -30,6 +30,7 @@
#include <process/process.hpp>
#include <process/timeout.hpp>
+#include <mesos/resources.hpp>
#include <mesos/type_utils.hpp>
#include <mesos/resource_provider/resource_provider.hpp>
@@ -56,6 +57,7 @@
#include "internal/evolve.hpp"
#include "resource_provider/detector.hpp"
+#include "resource_provider/state.hpp"
#include "slave/container_daemon.hpp"
#include "slave/paths.hpp"
@@ -91,6 +93,8 @@ using mesos::internal::slave::ContainerDaemon;
using mesos::resource_provider::Call;
using mesos::resource_provider::Event;
+using mesos::resource_provider::state::ResourceProviderState;
+
using mesos::v1::resource_provider::Driver;
namespace mesos {
@@ -272,6 +276,8 @@ private:
Future<Nothing> prepareControllerService();
Future<Nothing> prepareNodeService();
+ void checkpointResourceProviderState();
+
enum State
{
RECOVERING,
@@ -302,6 +308,10 @@ private:
Option<csi::GetPluginInfoResponse> nodeInfo;
Option<csi::ControllerCapabilities> controllerCapabilities;
Option<string> nodeId;
+
+ list<Event::Operation> pendingOperations;
+ Resources totalResources;
+ string resourceVersion;
};
@@ -428,8 +438,34 @@ Future<Nothing> StorageLocalResourceProviderProcess::recover()
": " + realpath.error());
}
- if (realpath.isSome()) {
+ if (realpath.isNone()) {
+ resourceVersion = UUID::random().toBytes();
+ } else {
info.mutable_id()->set_value(Path(realpath.get()).basename());
+
+ const string statePath = slave::paths::getResourceProviderStatePath(
+ metaDir, slaveId, info.type(), info.name(), info.id());
+
+ Result<ResourceProviderState> resourceProviderState =
+ ::protobuf::read<ResourceProviderState>(statePath);
+
+ if (resourceProviderState.isError()) {
+ return Failure(
+ "Failed to read resource provider state from '" + statePath +
+ "': " + resourceProviderState.error());
+ }
+
+ if (resourceProviderState.isNone()) {
+ resourceVersion = UUID::random().toBytes();
+ } else {
+ foreach (const Event::Operation& operation,
+ resourceProviderState->operations()) {
+ pendingOperations.push_back(operation);
+ }
+
+ totalResources = resourceProviderState->resources();
+ resourceVersion = resourceProviderState->resource_version_uuid();
+ }
}
state = DISCONNECTED;
@@ -589,6 +625,9 @@ void StorageLocalResourceProviderProcess::operation(
}
CHECK_EQ(READY, state);
+
+ pendingOperations.push_back(operation);
+ checkpointResourceProviderState();
}
@@ -970,6 +1009,25 @@ Future<Nothing> StorageLocalResourceProviderProcess::prepareNodeService()
}
+void StorageLocalResourceProviderProcess::checkpointResourceProviderState()
+{
+ ResourceProviderState state;
+
+ foreach (const Event::Operation& operation, pendingOperations) {
+ state.add_operations()->CopyFrom(operation);
+ }
+
+ state.mutable_resources()->CopyFrom(totalResources);
+ state.set_resource_version_uuid(resourceVersion);
+
+ const string statePath = slave::paths::getResourceProviderStatePath(
+ metaDir, slaveId, info.type(), info.name(), info.id());
+
+ CHECK_SOME(slave::state::checkpoint(statePath, state))
+ << "Failed to checkpoint resource provider state to '" << statePath << "'";
+}
+
+
Try<Owned<LocalResourceProvider>> StorageLocalResourceProvider::create(
const http::URL& url,
const string& workDir,