You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ch...@apache.org on 2019/02/13 03:51:29 UTC

[mesos] branch 1.7.x updated (ce291a4 -> 33c3d49)

This is an automated email from the ASF dual-hosted git repository.

chhsiao pushed a change to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git.


    from ce291a4  Added MESOS-9554 to the 1.7.x CHANGELOG.
     new 1d2471d  Persisted intentionally dropped operations in SLRP.
     new 5fe817f  Made SLRP recover node-published volumes after reboot.
     new d6c9c03  Cleaned up persistent volumes on SLRP disks.
     new d8458ae  Disallowed `DESTROY_DISK` on persistent volumes.
     new 33c3d49  Added MESOS-9537 and MESOS-9544 to the 1.7.2 CHANGELOG.

The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 CHANGELOG                                          |   2 +
 src/csi/state.proto                                |  12 +-
 src/master/validation.cpp                          |   7 +
 src/resource_provider/storage/provider.cpp         | 403 +++++++++++++++------
 src/resource_provider/storage/provider_process.hpp |  14 +-
 src/tests/master_validation_tests.cpp              |  12 +
 6 files changed, 342 insertions(+), 108 deletions(-)


[mesos] 03/05: Cleaned up persistent volumes on SLRP disks.

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git

commit d6c9c0377c70d47ed5ed941fbaa54bfab0a1fd8c
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Thu Jan 31 14:36:11 2019 -0800

    Cleaned up persistent volumes on SLRP disks.
    
    This patch limits SLRP to only support persistent volumes on MOUNT
    disks, and makes it clean up data in persistent volumes when processing
    `DESTROY` operations.
    
    NOTE: Persistent volumes backed by CSI disks that are created before
    upgrading to a Mesos version that does not include this fix are subject
    to data leakage. To ensure data security, these persistent volume must
    be consumed by a task at least once after the upgrade before being
    destroyed.
    
    Review: https://reviews.apache.org/r/69893
---
 src/resource_provider/storage/provider.cpp         | 98 ++++++++++++++++++++--
 src/resource_provider/storage/provider_process.hpp |  9 ++
 2 files changed, 101 insertions(+), 6 deletions(-)

diff --git a/src/resource_provider/storage/provider.cpp b/src/resource_provider/storage/provider.cpp
index a0b42a2..7b2f8b9 100644
--- a/src/resource_provider/storage/provider.cpp
+++ b/src/resource_provider/storage/provider.cpp
@@ -2881,16 +2881,28 @@ Future<Nothing> StorageLocalResourceProviderProcess::_applyOperation(
 
   switch (operation.info().type()) {
     case Offer::Operation::RESERVE:
-    case Offer::Operation::UNRESERVE:
-    case Offer::Operation::CREATE:
-    case Offer::Operation::DESTROY: {
-      // Synchronously apply the speculative operations to ensure that
-      // its result is reflected in the total resources before any of
-      // its succeeding operations is applied.
+    case Offer::Operation::UNRESERVE: {
+      // Synchronously apply the speculative operations to ensure that its
+      // result is reflected in the total resources before any of its succeeding
+      // operations is applied.
       return updateOperationStatus(
           operationUuid,
           getResourceConversions(operation.info()));
     }
+    case Offer::Operation::CREATE: {
+      // Synchronously create the persistent volumes to ensure that its result
+      // is reflected in the total resources before any of its succeeding
+      // operations is applied.
+      return updateOperationStatus(
+          operationUuid, applyCreate(operation.info()));
+    }
+    case Offer::Operation::DESTROY: {
+      // Synchronously clean up and destroy the persistent volumes to ensure
+      // that its result is reflected in the total resources before any of its
+      // succeeding operations is applied.
+      return updateOperationStatus(
+          operationUuid, applyDestroy(operation.info()));
+    }
     case Offer::Operation::CREATE_DISK: {
       CHECK(operation.info().has_create_disk());
 
@@ -3151,6 +3163,7 @@ Future<vector<ResourceConversion>>
 StorageLocalResourceProviderProcess::applyDestroyDisk(
     const Resource& resource)
 {
+  CHECK(!Resources::isPersistentVolume(resource));
   CHECK(resource.disk().source().type() == Resource::DiskInfo::Source::MOUNT ||
         resource.disk().source().type() == Resource::DiskInfo::Source::BLOCK);
   CHECK(resource.disk().source().has_id());
@@ -3211,6 +3224,79 @@ StorageLocalResourceProviderProcess::applyDestroyDisk(
 }
 
 
+Try<vector<ResourceConversion>>
+StorageLocalResourceProviderProcess::applyCreate(
+    const Offer::Operation& operation) const
+{
+  CHECK(operation.has_create());
+
+  foreach (const Resource& resource, operation.create().volumes()) {
+    CHECK(Resources::isPersistentVolume(resource));
+
+    // TODO(chhsiao): Support persistent BLOCK volumes.
+    if (resource.disk().source().type() != Resource::DiskInfo::Source::MOUNT) {
+      return Error(
+          "Cannot create persistent volume '" +
+          stringify(resource.disk().persistence().id()) + "' on a " +
+          stringify(resource.disk().source().type()) + " disk");
+    }
+  }
+
+  return getResourceConversions(operation);
+}
+
+
+Try<vector<ResourceConversion>>
+StorageLocalResourceProviderProcess::applyDestroy(
+    const Offer::Operation& operation) const
+{
+  CHECK(operation.has_destroy());
+
+  foreach (const Resource& resource, operation.destroy().volumes()) {
+    // TODO(chhsiao): Support cleaning up persistent BLOCK volumes, presumably
+    // with `dd` or any other utility to zero out the block device.
+    CHECK(Resources::isPersistentVolume(resource));
+    CHECK(resource.disk().source().type() == Resource::DiskInfo::Source::MOUNT);
+    CHECK(resource.disk().source().has_id());
+
+    const string& volumeId = resource.disk().source().id();
+    CHECK(volumes.contains(volumeId));
+
+    const VolumeState& volumeState = volumes.at(volumeId).state;
+
+    // NOTE: Data can only be written to the persistent volume when when it is
+    // in `PUBLISHED` state (i.e., mounted). Once a volume has been transitioned
+    // to `PUBLISHED`, we will set the `node_publish_required` field and always
+    // recover it back to `PUBLISHED` after a failover, until a `DESTROY_DISK`
+    // is applied, which only comes after `DESTROY`. So we only need to clean up
+    // the volume if it has the field set.
+    if (!volumeState.node_publish_required()) {
+      continue;
+    }
+
+    CHECK_EQ(VolumeState::PUBLISHED, volumeState.state());
+
+    const string targetPath = csi::paths::getMountTargetPath(
+        csi::paths::getMountRootDir(
+            slave::paths::getCsiRootDir(workDir),
+            info.storage().plugin().type(),
+            info.storage().plugin().name()),
+        volumeId);
+
+    // Only the data in the target path, but not itself, should be removed.
+    Try<Nothing> rmdir = os::rmdir(targetPath, true, false);
+    if (rmdir.isError()) {
+      return Error(
+          "Failed to remove persistent volume '" +
+          stringify(resource.disk().persistence().id()) + "' at '" +
+          targetPath + "': " + rmdir.error());
+    }
+  }
+
+  return getResourceConversions(operation);
+}
+
+
 Try<Nothing> StorageLocalResourceProviderProcess::updateOperationStatus(
     const id::UUID& operationUuid,
     const Try<vector<ResourceConversion>>& conversions)
diff --git a/src/resource_provider/storage/provider_process.hpp b/src/resource_provider/storage/provider_process.hpp
index adc4651..2ba24ae 100644
--- a/src/resource_provider/storage/provider_process.hpp
+++ b/src/resource_provider/storage/provider_process.hpp
@@ -308,9 +308,18 @@ private:
       const id::UUID& operationUuid,
       const Resource::DiskInfo::Source::Type& targetType,
       const Option<std::string>& targetProfile);
+
   process::Future<std::vector<ResourceConversion>> applyDestroyDisk(
       const Resource& resource);
 
+  // Synchronously creates persistent volumes.
+  Try<std::vector<ResourceConversion>> applyCreate(
+      const Offer::Operation& operation) const;
+
+  // Synchronously cleans up and destroys persistent volumes.
+  Try<std::vector<ResourceConversion>> applyDestroy(
+      const Offer::Operation& operation) const;
+
   // Synchronously updates `totalResources` and the operation status and
   // then asks the status update manager to send status updates.
   Try<Nothing> updateOperationStatus(


[mesos] 04/05: Disallowed `DESTROY_DISK` on persistent volumes.

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git

commit d8458ae078862e706f985b2749d987f5c5cb3823
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Mon Feb 4 23:26:20 2019 -0800

    Disallowed `DESTROY_DISK` on persistent volumes.
    
    `DESTROY_DISK` would bypass persistent volume cleanup and directly ask
    the CSI plugin to delete the backed volume. Since the CSI spec does not
    require the plugin to do data cleanup, to avoid data leakage, we require
    that if there is persistent volume on the CSI volume, it should be
    destroyed first.
    
    Review: https://reviews.apache.org/r/69894
---
 src/master/validation.cpp             |  7 +++++++
 src/tests/master_validation_tests.cpp | 12 ++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/src/master/validation.cpp b/src/master/validation.cpp
index 249e6b2..4144bc1 100644
--- a/src/master/validation.cpp
+++ b/src/master/validation.cpp
@@ -2548,6 +2548,13 @@ Option<Error> validate(const Offer::Operation::DestroyDisk& destroyDisk)
     return Error("'source' is neither a MOUNT or BLOCK disk resource");
   }
 
+  if (Resources::isPersistentVolume(source)) {
+    return Error(
+        "A disk resource containing a persistent volume " + stringify(source) +
+        " cannot be destroyed directly. Please destroy the persistent volume"
+        " first then destroy the disk resource");
+  }
+
   return None();
 }
 
diff --git a/src/tests/master_validation_tests.cpp b/src/tests/master_validation_tests.cpp
index 726d677..137841c 100644
--- a/src/tests/master_validation_tests.cpp
+++ b/src/tests/master_validation_tests.cpp
@@ -1864,9 +1864,13 @@ TEST(OperationValidationTest, DestroyDisk)
   Resource disk4 = createDiskResource(
       "40", "*", None(), None(), createDiskSourceMount());
 
+  Resource disk5 = createPersistentVolume(
+      Megabytes(50), "role", "id", "path", None(), createDiskSourceMount());
+
   disk1.mutable_provider_id()->set_value("provider1");
   disk2.mutable_provider_id()->set_value("provider2");
   disk3.mutable_provider_id()->set_value("provider3");
+  disk5.mutable_provider_id()->set_value("provider5");
 
   Offer::Operation::DestroyDisk destroyDisk;
   destroyDisk.mutable_source()->CopyFrom(disk1);
@@ -1894,6 +1898,14 @@ TEST(OperationValidationTest, DestroyDisk)
   EXPECT_TRUE(strings::contains(
       error->message,
       "'source' is not managed by a resource provider"));
+
+  destroyDisk.mutable_source()->CopyFrom(disk5);
+
+  error = operation::validate(destroyDisk);
+  ASSERT_SOME(error);
+  EXPECT_TRUE(strings::contains(
+      error->message,
+      "Please destroy the persistent volume first"));
 }
 
 


[mesos] 02/05: Made SLRP recover node-published volumes after reboot.

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git

commit 5fe817fab4e8b85a517a0190014b1eb2c17c9f76
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Fri Feb 1 15:29:33 2019 -0800

    Made SLRP recover node-published volumes after reboot.
    
    If a CSI volume has been node-published before a reboot, SLRP will now
    try to bring it back to node-published again. This is important to
    perform synchronous persistent volume cleanup for `DESTROY`.
    
    To achieve this, in addition to keeping track of the boot ID when a CSI
    volume is node-staged in `VolumeState.vol_ready_boot_id` (formerly
    `VolumeState.boot_id`), SLRP now also keeps track of the boot ID when
    the volume is node-published. This helps SLRP to better determine if a
    volume has been published before reboot.
    
    Review: https://reviews.apache.org/r/69892
---
 src/csi/state.proto                        |  12 +-
 src/resource_provider/storage/provider.cpp | 248 +++++++++++++++++++----------
 2 files changed, 176 insertions(+), 84 deletions(-)

diff --git a/src/csi/state.proto b/src/csi/state.proto
index 264a565..b5ccf16 100644
--- a/src/csi/state.proto
+++ b/src/csi/state.proto
@@ -60,9 +60,13 @@ message VolumeState {
   map<string, string> publish_info = 4;
 
   // This field is used to check if the node has been rebooted since the volume
-  // was transitioned to `VOL_READY` state. If yes, `NodeStageVolume` needs to
-  // be called to make the volume publishable again. It MUST be set to the boot
-  // ID of the node if the volume is in `VOL_READY` state, and SHOULD remain
-  // unset otherwise. This is an OPTIONAL field.
+  // is made publishable on the node. It MUST be set to the boot ID of the node
+  // when the volume is transitioned to `VOL_READY`, and SHOULD be cleared when
+  // the volume is transitioned to `NODE_READY`. This is an OPTIONAL field.
   string boot_id = 5;
+
+  // This field is used to check if the volume has been used by a container and
+  // hence needs cleanup. If set, the resource provider MUST transition the
+  // volume to `PUBLISHED` state during recovery.
+  bool node_publish_required = 7;
 }
diff --git a/src/resource_provider/storage/provider.cpp b/src/resource_provider/storage/provider.cpp
index c6e054e..a0b42a2 100644
--- a/src/resource_provider/storage/provider.cpp
+++ b/src/resource_provider/storage/provider.cpp
@@ -787,101 +787,177 @@ Future<Nothing> StorageLocalResourceProviderProcess::recoverVolumes()
 
     if (volumeState.isSome()) {
       volumes.put(volumeId, std::move(volumeState.get()));
-      VolumeData& volume = volumes.at(volumeId);
 
-      Future<Nothing> recovered = Nothing();
+      // To avoid any race with, e.g., `deleteVolume` calls, we sequentialize
+      // this lambda with any other operation on the same volume below, so the
+      // volume is guaranteed to exist in the deferred execution.
+      std::function<Future<Nothing>()> recoverVolume = defer(self(), [=]()
+          -> Future<Nothing> {
+        VolumeData& volume = volumes.at(volumeId);
+        Future<Nothing> recovered = Nothing();
 
-      if (VolumeState::State_IsValid(volume.state.state())) {
-        switch (volume.state.state()) {
-          case VolumeState::CREATED:
-          case VolumeState::NODE_READY: {
-            break;
-          }
-          case VolumeState::VOL_READY:
-          case VolumeState::PUBLISHED: {
-            if (volume.state.boot_id() != bootId) {
-              // The node has been restarted since the volume is made
-              // publishable, so it is reset to `NODE_READY` state.
-              volume.state.set_state(VolumeState::NODE_READY);
-              volume.state.clear_boot_id();
-              checkpointVolumeState(volumeId);
+        // First, bring the volume back to a "good" state.
+        if (VolumeState::State_IsValid(volume.state.state())) {
+          switch (volume.state.state()) {
+            case VolumeState::CREATED:
+            case VolumeState::NODE_READY: {
+              break;
             }
+            case VolumeState::VOL_READY:
+            case VolumeState::PUBLISHED: {
+              if (volume.state.boot_id() != bootId) {
+                // The node has been restarted since the volume is made
+                // publishable, so it is reset to `NODE_READY` state.
+                volume.state.set_state(VolumeState::NODE_READY);
+                volume.state.clear_boot_id();
+                checkpointVolumeState(volumeId);
+              }
 
-            break;
-          }
-          case VolumeState::CONTROLLER_PUBLISH: {
-            recovered = volume.sequence->add(std::function<Future<Nothing>()>(
-                defer(self(), &Self::controllerPublish, volumeId)));
+              break;
+            }
+            case VolumeState::CONTROLLER_PUBLISH: {
+              recovered = controllerPublish(volumeId);
 
-            break;
-          }
-          case VolumeState::CONTROLLER_UNPUBLISH: {
-            recovered = volume.sequence->add(std::function<Future<Nothing>()>(
-                defer(self(), &Self::controllerUnpublish, volumeId)));
+              break;
+            }
+            case VolumeState::CONTROLLER_UNPUBLISH: {
+              recovered = controllerUnpublish(volumeId);
 
-            break;
-          }
-          case VolumeState::NODE_STAGE: {
-            recovered = volume.sequence->add(std::function<Future<Nothing>()>(
-                defer(self(), &Self::nodeStage, volumeId)));
+              break;
+            }
+            case VolumeState::NODE_STAGE: {
+              recovered = nodeStage(volumeId);
 
-            break;
-          }
-          case VolumeState::NODE_UNSTAGE: {
-            recovered = volume.sequence->add(std::function<Future<Nothing>()>(
-                defer(self(), &Self::nodeUnstage, volumeId)));
+              break;
+            }
+            case VolumeState::NODE_UNSTAGE: {
+              if (volume.state.boot_id() != bootId) {
+                // The node has been restarted since the volume is made
+                // publishable, so it is reset to `NODE_READY` state.
+                volume.state.set_state(VolumeState::NODE_READY);
+                volume.state.clear_boot_id();
+                checkpointVolumeState(volumeId);
+              } else {
+                recovered = nodeUnstage(volumeId);
+              }
 
-            break;
-          }
-          case VolumeState::NODE_PUBLISH: {
-            if (volume.state.boot_id() != bootId) {
-              // The node has been restarted since `NodePublishVolume` was
-              // called, so it is reset to `NODE_READY` state.
-              volume.state.set_state(VolumeState::NODE_READY);
-              volume.state.clear_boot_id();
-              checkpointVolumeState(volumeId);
-            } else {
-              recovered = volume.sequence->add(std::function<Future<Nothing>()>(
-                  defer(self(), &Self::nodePublish, volumeId)));
+              break;
+            }
+            case VolumeState::NODE_PUBLISH: {
+              if (volume.state.boot_id() != bootId) {
+                // The node has been restarted since the volume is made
+                // publishable, so it is reset to `NODE_READY` state.
+                volume.state.set_state(VolumeState::NODE_READY);
+                volume.state.clear_boot_id();
+                checkpointVolumeState(volumeId);
+              } else {
+                recovered = nodePublish(volumeId);
+              }
+
+              break;
             }
+            case VolumeState::NODE_UNPUBLISH: {
+              if (volume.state.boot_id() != bootId) {
+                // The node has been restarted since the volume is made
+                // publishable, so it is reset to `NODE_READY` state.
+                volume.state.set_state(VolumeState::NODE_READY);
+                volume.state.clear_boot_id();
+                checkpointVolumeState(volumeId);
+              } else {
+                recovered = nodeUnpublish(volumeId);
+              }
 
-            break;
-          }
-          case VolumeState::NODE_UNPUBLISH: {
-            if (volume.state.boot_id() != bootId) {
-              // The node has been restarted since `NodeUnpublishVolume` was
-              // called, so it is reset to `NODE_READY` state.
-              volume.state.set_state(VolumeState::NODE_READY);
-              volume.state.clear_boot_id();
-              checkpointVolumeState(volumeId);
-            } else {
-              recovered = volume.sequence->add(std::function<Future<Nothing>()>(
-                  defer(self(), &Self::nodeUnpublish, volumeId)));
+              break;
+            }
+            case VolumeState::UNKNOWN: {
+              return Failure(
+                  "Volume '" + volumeId + "' is in " +
+                  stringify(volume.state.state()) + " state");
             }
 
-            break;
+            // NOTE: We avoid using a default clause for the following values in
+            // proto3's open enum to enable the compiler to detect missing enum
+            // cases for us. See: https://github.com/google/protobuf/issues/3917
+            case google::protobuf::kint32min:
+            case google::protobuf::kint32max: {
+              UNREACHABLE();
+            }
           }
-          case VolumeState::UNKNOWN: {
-            recovered = Failure(
-                "Volume '" + volumeId + "' is in " +
-                stringify(volume.state.state()) + " state");
+        } else {
+          return Failure("Volume '" + volumeId + "' is in UNDEFINED state");
+        }
 
-            break;
-          }
+        auto err = [](const string& volumeId, const string& message) {
+          LOG(ERROR)
+            << "Failed to recover volume '" << volumeId << "': " << message;
+        };
 
-          // NOTE: We avoid using a default clause for the following values in
-          // proto3's open enum to enable the compiler to detect missing enum
-          // cases for us. See: https://github.com/google/protobuf/issues/3917
-          case google::protobuf::kint32min:
-          case google::protobuf::kint32max: {
-            UNREACHABLE();
-          }
+        // Second, if the volume has been used by a container before recovery,
+        // we have to bring the volume back to `PUBLISHED` so data can be
+        // cleaned up synchronously upon `DESTROY`. Otherwise, we skip the error
+        // and continue recovery.
+        if (volume.state.node_publish_required()) {
+          recovered = recovered
+            .then(defer(self(), [this, volumeId]() -> Future<Nothing> {
+              const VolumeData& volume = volumes.at(volumeId);
+              Future<Nothing> published = Nothing();
+
+              CHECK(VolumeState::State_IsValid(volume.state.state()));
+
+              switch (volume.state.state()) {
+                case VolumeState::NODE_READY: {
+                  published = published
+                    .then(defer(self(), &Self::nodeStage, volumeId));
+
+                  // NOTE: We continue to the next case to recover the volume in
+                  // `VOL_READY` state once the above is done.
+                }
+                case VolumeState::VOL_READY: {
+                  published = published
+                    .then(defer(self(), &Self::nodePublish, volumeId));
+
+                  // NOTE: We continue to the next case to recover the volume in
+                  // `PUBLISHED` state once the above is done.
+                }
+                case VolumeState::PUBLISHED: {
+                  break;
+                }
+                case VolumeState::UNKNOWN:
+                case VolumeState::CREATED:
+                case VolumeState::CONTROLLER_PUBLISH:
+                case VolumeState::CONTROLLER_UNPUBLISH:
+                case VolumeState::NODE_STAGE:
+                case VolumeState::NODE_UNSTAGE:
+                case VolumeState::NODE_PUBLISH:
+                case VolumeState::NODE_UNPUBLISH: {
+                  UNREACHABLE();
+                }
+
+                // NOTE: We avoid using a default clause for the following
+                // values in proto3's open enum to enable the compiler to detect
+                // missing enum cases for us. See:
+                // https://github.com/google/protobuf/issues/3917
+                case google::protobuf::kint32min:
+                case google::protobuf::kint32max: {
+                  UNREACHABLE();
+                }
+              }
+
+              return published;
+            }))
+            .onFailed(std::bind(err, volumeId, lambda::_1))
+            .onDiscarded(std::bind(err, volumeId, "future discarded"));
+        } else {
+          recovered = recovered
+            .onFailed(std::bind(err, volumeId, lambda::_1))
+            .onDiscarded(std::bind(err, volumeId, "future discarded"))
+            .recover([](const Future<Nothing>& future) { return Nothing(); });
         }
-      } else {
-        recovered = Failure("Volume '" + volumeId + "' is in UNDEFINED state");
-      }
 
-      futures.push_back(recovered);
+        return recovered;
+      });
+
+      futures.push_back(volumes.at(volumeId).sequence->add(recoverVolume));
     }
   }
 
@@ -2422,6 +2498,13 @@ Future<Nothing> StorageLocalResourceProviderProcess::nodePublish(
       VolumeData& volume = volumes.at(volumeId);
 
       volume.state.set_state(VolumeState::PUBLISHED);
+
+      // NOTE: The `node_publish_required` field is always set up by the
+      // successful `nodePublish` call, as it indicates that a container is
+      // going to use the volume. However, it will not cleared by a
+      // `nodeUnpublish` call, but by a `deleteVolume` call instead.
+      volume.state.set_node_publish_required(true);
+
       checkpointVolumeState(volumeId);
 
       return Nothing();
@@ -2543,12 +2626,17 @@ Future<bool> StorageLocalResourceProviderProcess::deleteVolume(
     return controllerCapabilities.createDeleteVolume;
   }
 
-  const VolumeData& volume = volumes.at(volumeId);
+  VolumeData& volume = volumes.at(volumeId);
 
-  CHECK(VolumeState::State_IsValid(volume.state.state()));
+  // NOTE: The volume must have been cleaned up before the `deleteVolume` call
+  // is made, so it is no longer required to publish the volume.
+  volume.state.set_node_publish_required(false);
+  checkpointVolumeState(volumeId);
 
   Future<Nothing> deleted = Nothing();
 
+  CHECK(VolumeState::State_IsValid(volume.state.state()));
+
   switch (volume.state.state()) {
     case VolumeState::PUBLISHED:
     case VolumeState::NODE_PUBLISH:


[mesos] 05/05: Added MESOS-9537 and MESOS-9544 to the 1.7.2 CHANGELOG.

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git

commit 33c3d49db5a57a500fd2fee0ca81945d350ab80f
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Tue Feb 12 16:49:30 2019 -0800

    Added MESOS-9537 and MESOS-9544 to the 1.7.2 CHANGELOG.
---
 CHANGELOG | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 1d3c536..0d4c323 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -8,6 +8,8 @@ Release Notes - Mesos - Version 1.7.2 (WIP)
   * [MESOS-9531] - chown error handling is incorrect in createSandboxDirectory.
   * [MESOS-9532] - ResourceOffersTest.ResourceOfferWithMultipleSlaves is flaky.
   * [MESOS-9533] - CniIsolatorTest.ROOT_CleanupAfterReboot is flaky.
+  * [MESOS-9537] - SLRP sends inconsistent status updates for dropped operations.
+  * [MESOS-9544] - SLRP does not clean up destroyed persistent volumes.
   * [MESOS-9554] - Allocator might skip allocations because a single framework is incapable of receiving certain resources.
 
 Release Notes - Mesos - Version 1.7.1


[mesos] 01/05: Persisted intentionally dropped operations in SLRP.

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chhsiao pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git

commit 1d2471d7c1eba8f046aec635054662430f2653df
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Tue Jan 29 21:28:48 2019 -0800

    Persisted intentionally dropped operations in SLRP.
    
    If an operation is dropped intentionally (e.g., because of a resource
    version mismatch), the operation should be persisted so no conflicting
    status update would be generated for operation reconciliation.
    
    Review: https://reviews.apache.org/r/69858
---
 src/resource_provider/storage/provider.cpp         | 57 ++++++++++++++++------
 src/resource_provider/storage/provider_process.hpp |  5 +-
 2 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/src/resource_provider/storage/provider.cpp b/src/resource_provider/storage/provider.cpp
index c4068a5..c6e054e 100644
--- a/src/resource_provider/storage/provider.cpp
+++ b/src/resource_provider/storage/provider.cpp
@@ -1753,6 +1753,7 @@ void StorageLocalResourceProviderProcess::reconcileOperations(
       continue;
     }
 
+    // TODO(chhsiao): Consider sending `OPERATION_UNKNOWN` instead.
     dropOperation(
         uuid.get(),
         None(),
@@ -2879,32 +2880,58 @@ void StorageLocalResourceProviderProcess::dropOperation(
   LOG(WARNING)
     << "Dropping operation (uuid: " << operationUuid << "): " << message;
 
+  CHECK(!operations.contains(operationUuid));
+
   UpdateOperationStatusMessage update =
     protobuf::createUpdateOperationStatusMessage(
         protobuf::createUUID(operationUuid),
         protobuf::createOperationStatus(
             OPERATION_DROPPED,
-            operation.isSome() && operation->has_id()
-              ? operation->id() : Option<OperationID>::none(),
+            None(),
             message,
             None(),
-            id::UUID::random(),
+            None(),
             slaveId,
             info.id()),
         None(),
         frameworkId,
         slaveId);
 
-  auto die = [=](const string& message) {
-    LOG(ERROR)
-      << "Failed to update status of operation (uuid: " << operationUuid
-      << "): " << message;
-    fatal();
-  };
+  if (operation.isSome()) {
+    // This operation is dropped intentionally. We have to persist the operation
+    // in the resource provider state and retry the status update.
+    *update.mutable_status()->mutable_uuid() = protobuf::createUUID();
+    if (operation->has_id()) {
+      *update.mutable_status()->mutable_operation_id() = operation->id();
+    }
 
-  statusUpdateManager.update(std::move(update), false)
-    .onFailed(defer(self(), std::bind(die, lambda::_1)))
-    .onDiscarded(defer(self(), std::bind(die, "future discarded")));
+    operations[operationUuid] = protobuf::createOperation(
+        operation.get(),
+        update.status(),
+        frameworkId,
+        slaveId,
+        update.operation_uuid());
+
+    checkpointResourceProviderState();
+
+    auto die = [=](const string& message) {
+      LOG(ERROR)
+        << "Failed to update status of operation (uuid: " << operationUuid
+        << "): " << message;
+      fatal();
+    };
+
+    statusUpdateManager.update(std::move(update))
+      .onFailed(defer(self(), std::bind(die, lambda::_1)))
+      .onDiscarded(defer(self(), std::bind(die, "future discarded")));
+  } else {
+    // This operation is unknown to the resource provider because of a
+    // disconnection, and is being asked for reconciliation. In this case, we
+    // send a status update without a retry. If it is dropped because of another
+    // disconnection, another reconciliation will be triggered by the master
+    // after a reregistration.
+    sendOperationStatusUpdate(std::move(update));
+  }
 
   ++metrics.operations_dropped.at(
       operation.isSome() ? operation->type() : Offer::Operation::UNKNOWN);
@@ -3338,9 +3365,9 @@ void StorageLocalResourceProviderProcess::sendOperationStatusUpdate(
     update->mutable_framework_id()->CopyFrom(_update.framework_id());
   }
 
-  // The latest status should have been set by the status update manager.
-  CHECK(_update.has_latest_status());
-  update->mutable_latest_status()->CopyFrom(_update.latest_status());
+  if (_update.has_latest_status()) {
+    update->mutable_latest_status()->CopyFrom(_update.latest_status());
+  }
 
   auto err = [](const id::UUID& uuid, const string& message) {
     LOG(ERROR)
diff --git a/src/resource_provider/storage/provider_process.hpp b/src/resource_provider/storage/provider_process.hpp
index 36187fb..adc4651 100644
--- a/src/resource_provider/storage/provider_process.hpp
+++ b/src/resource_provider/storage/provider_process.hpp
@@ -295,7 +295,8 @@ private:
   // applied. Do nothing if the operation is already in a terminal state.
   process::Future<Nothing> _applyOperation(const id::UUID& operationUuid);
 
-  // Sends `OPERATION_DROPPED` without checkpointing the operation status.
+  // Sends `OPERATION_DROPPED` status update. The operation status will be
+  // checkpointed if `operation` is set.
   void dropOperation(
       const id::UUID& operationUuid,
       const Option<FrameworkID>& frameworkId,
@@ -323,8 +324,6 @@ private:
 
   void sendResourceProviderStateUpdate();
 
-  // NOTE: This is a callback for the status update manager and should
-  // not be called directly.
   void sendOperationStatusUpdate(
       const UpdateOperationStatusMessage& update);