You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by gr...@apache.org on 2020/08/03 17:56:44 UTC
[mesos] 01/03: Enabled pre-provisioned volumes in the volume
manager.
This is an automated email from the ASF dual-hosted git repository.
grag pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 8d545180294ae705f7b8a2fe1578653107ede540
Author: Greg Mann <gr...@mesosphere.io>
AuthorDate: Mon Aug 3 10:11:50 2020 -0700
Enabled pre-provisioned volumes in the volume manager.
This patch makes it possible to publish CSI volumes on an
agent which were pre-provisioned out of band.
Review: https://reviews.apache.org/r/72681/
---
src/csi/state.proto | 5 ++
src/csi/v0_volume_manager.cpp | 95 +++++++++++++++++++++++++++--------
src/csi/v0_volume_manager.hpp | 5 +-
src/csi/v0_volume_manager_process.hpp | 8 ++-
src/csi/v1_volume_manager.cpp | 95 +++++++++++++++++++++++++++--------
src/csi/v1_volume_manager.hpp | 5 +-
src/csi/v1_volume_manager_process.hpp | 8 ++-
src/csi/volume_manager.hpp | 18 +++++--
8 files changed, 190 insertions(+), 49 deletions(-)
diff --git a/src/csi/state.proto b/src/csi/state.proto
index 28ad5ef..af0ef1c 100644
--- a/src/csi/state.proto
+++ b/src/csi/state.proto
@@ -70,4 +70,9 @@ message VolumeState {
// hence needs cleanup. If set, the resource provider MUST transition the
// volume to `PUBLISHED` state during recovery.
bool node_publish_required = 7;
+
+ // Indicates that the volume was not created by a CSI plugin, but rather was
+ // pre-provisioned by some other means and then attached to the node using a
+ // CSI plugin.
+ bool pre_provisioned = 8;
}
diff --git a/src/csi/v0_volume_manager.cpp b/src/csi/v0_volume_manager.cpp
index b383598..5368440 100644
--- a/src/csi/v0_volume_manager.cpp
+++ b/src/csi/v0_volume_manager.cpp
@@ -452,8 +452,29 @@ Future<Nothing> VolumeManagerProcess::detachVolume(const string& volumeId)
}
-Future<Nothing> VolumeManagerProcess::publishVolume(const string& volumeId)
+Future<Nothing> VolumeManagerProcess::publishVolume(
+ const string& volumeId,
+ const Option<VolumeState>& volumeState)
{
+ if (volumeState.isSome()) {
+ if (!volumeState->pre_provisioned()) {
+ return Failure(
+ "Cannot specify volume state when publishing a volume unless that"
+ " volume is pre-provisioned");
+ }
+
+ if (volumeState->state() != VolumeState::VOL_READY &&
+ volumeState->state() != VolumeState::NODE_READY) {
+ return Failure(
+ "Cannot specify volume state when publishing a volume unless that"
+ " volume is in either the VOL_READY or NODE_READY state");
+ }
+
+ // This must be an untracked volume. Track it now before we continue.
+ volumes.put(volumeId, VolumeState(volumeState.get()));
+ checkpointVolumeState(volumeId);
+ }
+
if (!volumes.contains(volumeId)) {
return Failure("Cannot publish unknown volume '" + volumeId + "'");
}
@@ -728,16 +749,7 @@ Future<bool> VolumeManagerProcess::_deleteVolume(const std::string& volumeId)
// the future returned by the sequence ready as well.
return __deleteVolume(volumeId)
.then(process::defer(self(), [this, volumeId](bool deleted) {
- volumes.erase(volumeId);
-
- const string volumePath =
- paths::getVolumePath(rootDir, info.type(), info.name(), volumeId);
-
- Try<Nothing> rmdir = os::rmdir(volumePath);
- CHECK_SOME(rmdir) << "Failed to remove checkpointed volume state at '"
- << volumePath << "': " << rmdir.error();
-
- garbageCollectMountPath(volumeId);
+ removeVolume(volumeId);
return deleted;
}));
@@ -1051,6 +1063,13 @@ Future<Nothing> VolumeManagerProcess::_unpublishVolume(const string& volumeId)
if (volumeState.state() == VolumeState::NODE_READY) {
CHECK(volumeState.boot_id().empty());
+
+ if (volumeState.pre_provisioned()) {
+ // Since this volume was pre-provisioned, it has reached the end of its
+ // lifecycle. Remove it now.
+ removeVolume(volumeId);
+ }
+
return Nothing();
}
@@ -1063,9 +1082,16 @@ Future<Nothing> VolumeManagerProcess::_unpublishVolume(const string& volumeId)
}
if (!nodeCapabilities->stageUnstageVolume) {
- // Since this is a no-op, no need to checkpoint here.
- volumeState.set_state(VolumeState::NODE_READY);
- volumeState.clear_boot_id();
+ if (volumeState.pre_provisioned()) {
+ // Since this volume was pre-provisioned, it has reached the end of its
+ // lifecycle. Remove it now.
+ removeVolume(volumeId);
+ } else {
+ // Since this is a no-op, no need to checkpoint here.
+ volumeState.set_state(VolumeState::NODE_READY);
+ volumeState.clear_boot_id();
+ }
+
return Nothing();
}
@@ -1091,13 +1117,20 @@ Future<Nothing> VolumeManagerProcess::_unpublishVolume(const string& volumeId)
request.set_staging_target_path(stagingPath);
return call(NODE_SERVICE, &Client::nodeUnstageVolume, std::move(request))
- .then(process::defer(self(), [this, volumeId] {
+ .then(process::defer(self(), [this, volumeId, volumeState] {
CHECK(volumes.contains(volumeId));
- VolumeState& volumeState = volumes.at(volumeId).state;
- volumeState.set_state(VolumeState::NODE_READY);
- volumeState.clear_boot_id();
- checkpointVolumeState(volumeId);
+ if (volumeState.pre_provisioned()) {
+ // Since this volume was pre-provisioned, it has reached the end of its
+ // lifecycle. Remove it now.
+ removeVolume(volumeId);
+ } else {
+ VolumeState& volumeState = volumes.at(volumeId).state;
+ volumeState.set_state(VolumeState::NODE_READY);
+ volumeState.clear_boot_id();
+
+ checkpointVolumeState(volumeId);
+ }
return Nothing();
}));
@@ -1188,6 +1221,21 @@ void VolumeManagerProcess::garbageCollectMountPath(const string& volumeId)
}
+void VolumeManagerProcess::removeVolume(const string& volumeId)
+{
+ volumes.erase(volumeId);
+
+ const string volumePath =
+ paths::getVolumePath(rootDir, info.type(), info.name(), volumeId);
+
+ Try<Nothing> rmdir = os::rmdir(volumePath);
+ CHECK_SOME(rmdir) << "Failed to remove checkpointed volume state at '"
+ << volumePath << "': " << rmdir.error();
+
+ garbageCollectMountPath(volumeId);
+}
+
+
VolumeManager::VolumeManager(
const string& rootDir,
const CSIPluginInfo& info,
@@ -1297,11 +1345,16 @@ Future<Nothing> VolumeManager::detachVolume(const string& volumeId)
}
-Future<Nothing> VolumeManager::publishVolume(const string& volumeId)
+Future<Nothing> VolumeManager::publishVolume(
+ const string& volumeId,
+ const Option<VolumeState>& volumeState)
{
return recovered
.then(process::defer(
- process.get(), &VolumeManagerProcess::publishVolume, volumeId));
+ process.get(),
+ &VolumeManagerProcess::publishVolume,
+ volumeId,
+ volumeState));
}
diff --git a/src/csi/v0_volume_manager.hpp b/src/csi/v0_volume_manager.hpp
index 6920eb8..93183c2 100644
--- a/src/csi/v0_volume_manager.hpp
+++ b/src/csi/v0_volume_manager.hpp
@@ -37,6 +37,7 @@
#include "csi/metrics.hpp"
#include "csi/service_manager.hpp"
+#include "csi/state.hpp"
#include "csi/volume_manager.hpp"
namespace mesos {
@@ -95,7 +96,9 @@ public:
process::Future<Nothing> detachVolume(const std::string& volumeId) override;
- process::Future<Nothing> publishVolume(const std::string& volumeId) override;
+ process::Future<Nothing> publishVolume(
+ const std::string& volumeId,
+ const Option<state::VolumeState>& volumeState = None()) override;
process::Future<Nothing> unpublishVolume(
const std::string& volumeId) override;
diff --git a/src/csi/v0_volume_manager_process.hpp b/src/csi/v0_volume_manager_process.hpp
index d5a5eb2..7548c43 100644
--- a/src/csi/v0_volume_manager_process.hpp
+++ b/src/csi/v0_volume_manager_process.hpp
@@ -89,7 +89,9 @@ public:
process::Future<Nothing> detachVolume(const std::string& volumeId);
- process::Future<Nothing> publishVolume(const std::string& volumeId);
+ process::Future<Nothing> publishVolume(
+ const std::string& volumeId,
+ const Option<state::VolumeState>& volumeState = None());
process::Future<Nothing> unpublishVolume(const std::string& volumeId);
@@ -167,6 +169,10 @@ private:
void garbageCollectMountPath(const std::string& volumeId);
+ // Removes the metadata associated with a particular volume both
+ // from memory and from disk.
+ void removeVolume(const std::string& volumeId);
+
const std::string rootDir;
const CSIPluginInfo info;
const hashset<Service> services;
diff --git a/src/csi/v1_volume_manager.cpp b/src/csi/v1_volume_manager.cpp
index a9b80d0..7eae638 100644
--- a/src/csi/v1_volume_manager.cpp
+++ b/src/csi/v1_volume_manager.cpp
@@ -473,8 +473,29 @@ Future<Nothing> VolumeManagerProcess::detachVolume(const string& volumeId)
}
-Future<Nothing> VolumeManagerProcess::publishVolume(const string& volumeId)
+Future<Nothing> VolumeManagerProcess::publishVolume(
+ const string& volumeId,
+ const Option<VolumeState>& volumeState)
{
+ if (volumeState.isSome()) {
+ if (!volumeState->pre_provisioned()) {
+ return Failure(
+ "Cannot specify volume state when publishing a volume unless that"
+ " volume is pre-provisioned");
+ }
+
+ if (volumeState->state() != VolumeState::VOL_READY &&
+ volumeState->state() != VolumeState::NODE_READY) {
+ return Failure(
+ "Cannot specify volume state when publishing a volume unless that"
+ " volume is in either the VOL_READY or NODE_READY state");
+ }
+
+ // This must be an untracked volume. Track it now before we continue.
+ volumes.put(volumeId, VolumeState(volumeState.get()));
+ checkpointVolumeState(volumeId);
+ }
+
if (!volumes.contains(volumeId)) {
return Failure("Cannot publish unknown volume '" + volumeId + "'");
}
@@ -750,16 +771,7 @@ Future<bool> VolumeManagerProcess::_deleteVolume(const std::string& volumeId)
// the future returned by the sequence ready as well.
return __deleteVolume(volumeId)
.then(process::defer(self(), [this, volumeId](bool deleted) {
- volumes.erase(volumeId);
-
- const string volumePath =
- paths::getVolumePath(rootDir, info.type(), info.name(), volumeId);
-
- Try<Nothing> rmdir = os::rmdir(volumePath);
- CHECK_SOME(rmdir) << "Failed to remove checkpointed volume state at '"
- << volumePath << "': " << rmdir.error();
-
- garbageCollectMountPath(volumeId);
+ removeVolume(volumeId);
return deleted;
}));
@@ -1082,6 +1094,13 @@ Future<Nothing> VolumeManagerProcess::_unpublishVolume(const string& volumeId)
if (volumeState.state() == VolumeState::NODE_READY) {
CHECK(volumeState.boot_id().empty());
+
+ if (volumeState.pre_provisioned()) {
+ // Since this volume was pre-provisioned, it has reached the end of its
+ // lifecycle. Remove it now.
+ removeVolume(volumeId);
+ }
+
return Nothing();
}
@@ -1094,9 +1113,16 @@ Future<Nothing> VolumeManagerProcess::_unpublishVolume(const string& volumeId)
}
if (!nodeCapabilities->stageUnstageVolume) {
- // Since this is a no-op, no need to checkpoint here.
- volumeState.set_state(VolumeState::NODE_READY);
- volumeState.clear_boot_id();
+ if (volumeState.pre_provisioned()) {
+ // Since this volume was pre-provisioned, it has reached the end of its
+ // lifecycle. Remove it now.
+ removeVolume(volumeId);
+ } else {
+ // Since this is a no-op, no need to checkpoint here.
+ volumeState.set_state(VolumeState::NODE_READY);
+ volumeState.clear_boot_id();
+ }
+
return Nothing();
}
@@ -1122,13 +1148,20 @@ Future<Nothing> VolumeManagerProcess::_unpublishVolume(const string& volumeId)
request.set_staging_target_path(stagingPath);
return call(NODE_SERVICE, &Client::nodeUnstageVolume, std::move(request))
- .then(process::defer(self(), [this, volumeId] {
+ .then(process::defer(self(), [this, volumeId, volumeState] {
CHECK(volumes.contains(volumeId));
- VolumeState& volumeState = volumes.at(volumeId).state;
- volumeState.set_state(VolumeState::NODE_READY);
- volumeState.clear_boot_id();
- checkpointVolumeState(volumeId);
+ if (volumeState.pre_provisioned()) {
+ // Since this volume was pre-provisioned, it has reached the end of its
+ // lifecycle. Remove it now.
+ removeVolume(volumeId);
+ } else {
+ VolumeState& volumeState = volumes.at(volumeId).state;
+ volumeState.set_state(VolumeState::NODE_READY);
+ volumeState.clear_boot_id();
+
+ checkpointVolumeState(volumeId);
+ }
return Nothing();
}));
@@ -1222,6 +1255,21 @@ void VolumeManagerProcess::garbageCollectMountPath(const string& volumeId)
}
+void VolumeManagerProcess::removeVolume(const string& volumeId)
+{
+ volumes.erase(volumeId);
+
+ const string volumePath =
+ paths::getVolumePath(rootDir, info.type(), info.name(), volumeId);
+
+ Try<Nothing> rmdir = os::rmdir(volumePath);
+ CHECK_SOME(rmdir) << "Failed to remove checkpointed volume state at '"
+ << volumePath << "': " << rmdir.error();
+
+ garbageCollectMountPath(volumeId);
+}
+
+
VolumeManager::VolumeManager(
const string& rootDir,
const CSIPluginInfo& info,
@@ -1331,11 +1379,16 @@ Future<Nothing> VolumeManager::detachVolume(const string& volumeId)
}
-Future<Nothing> VolumeManager::publishVolume(const string& volumeId)
+Future<Nothing> VolumeManager::publishVolume(
+ const string& volumeId,
+ const Option<VolumeState>& volumeState)
{
return recovered
.then(process::defer(
- process.get(), &VolumeManagerProcess::publishVolume, volumeId));
+ process.get(),
+ &VolumeManagerProcess::publishVolume,
+ volumeId,
+ volumeState));
}
diff --git a/src/csi/v1_volume_manager.hpp b/src/csi/v1_volume_manager.hpp
index 4db11b5..2f7897d 100644
--- a/src/csi/v1_volume_manager.hpp
+++ b/src/csi/v1_volume_manager.hpp
@@ -37,6 +37,7 @@
#include "csi/metrics.hpp"
#include "csi/service_manager.hpp"
+#include "csi/state.hpp"
#include "csi/volume_manager.hpp"
namespace mesos {
@@ -95,7 +96,9 @@ public:
process::Future<Nothing> detachVolume(const std::string& volumeId) override;
- process::Future<Nothing> publishVolume(const std::string& volumeId) override;
+ process::Future<Nothing> publishVolume(
+ const std::string& volumeId,
+ const Option<state::VolumeState>& volumeState = None()) override;
process::Future<Nothing> unpublishVolume(
const std::string& volumeId) override;
diff --git a/src/csi/v1_volume_manager_process.hpp b/src/csi/v1_volume_manager_process.hpp
index df52c3d..b8a1ef7 100644
--- a/src/csi/v1_volume_manager_process.hpp
+++ b/src/csi/v1_volume_manager_process.hpp
@@ -89,7 +89,9 @@ public:
process::Future<Nothing> detachVolume(const std::string& volumeId);
- process::Future<Nothing> publishVolume(const std::string& volumeId);
+ process::Future<Nothing> publishVolume(
+ const std::string& volumeId,
+ const Option<state::VolumeState>& volumeState = None());
process::Future<Nothing> unpublishVolume(const std::string& volumeId);
@@ -167,6 +169,10 @@ private:
void garbageCollectMountPath(const std::string& volumeId);
+ // Removes the metadata associated with a particular volume both
+ // from memory and from disk.
+ void removeVolume(const std::string& volumeId);
+
const std::string rootDir;
const CSIPluginInfo info;
const hashset<Service> services;
diff --git a/src/csi/volume_manager.hpp b/src/csi/volume_manager.hpp
index 199cf44..57e7c51 100644
--- a/src/csi/volume_manager.hpp
+++ b/src/csi/volume_manager.hpp
@@ -38,6 +38,7 @@
#include "csi/metrics.hpp"
#include "csi/service_manager.hpp"
+#include "csi/state.hpp"
namespace mesos {
namespace csi {
@@ -126,11 +127,22 @@ public:
virtual process::Future<Nothing> detachVolume(
const std::string& volumeId) = 0;
- // Transitions a tracked volume to `PUBLISHED` state from any state above.
+ // Transitions a volume to `PUBLISHED` state. This method may be called on
+ // tracked or untracked volumes:
+ // * If `volumeState` is NONE, then `volumeId` must correspond to a tracked
+ // volume, and this method will transition the volume to `PUBLISHED` from
+ // any state above.
+ // * If `volumeState` is SOME, then `volumeId` must correspond to an untracked
+ // volume, and thus the ID should be unknown to the volume manager. The
+ // volume will be tracked by the manager and will become useable on the
+ // agent if the publish attempt succeeds.
virtual process::Future<Nothing> publishVolume(
- const std::string& volumeId) = 0;
+ const std::string& volumeId,
+ const Option<state::VolumeState>& volumeState = None()) = 0;
- // Transitions a tracked volume to `NODE_READY` state from any state below.
+ // Transitions a tracked volume to `NODE_READY` state from any other state.
+ // If the volume was untracked when it was published (a pre-provisioned
+ // volume), then the volume's metadata is removed from the volume manager.
virtual process::Future<Nothing> unpublishVolume(
const std::string& volumeId) = 0;
};