You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bm...@apache.org on 2017/09/19 22:07:51 UTC
[3/4] mesos git commit: Added a minimal /files path for the executor
sandbox.
Added a minimal /files path for the executor sandbox.
This patch added support to attach a virtual path
/frameworks/FID/executors/EID/latest to the /files endpoint of the
agent, which can be used to navigate an executor's latest sandbox
without needing to know the agent's work directory.
Review: https://reviews.apache.org/r/62047/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/219ce65f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/219ce65f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/219ce65f
Branch: refs/heads/master
Commit: 219ce65f17669570fa2fe9f7e5bd346bf23d7544
Parents: f4cd0ee
Author: Zhitao Li <zh...@gmail.com>
Authored: Tue Sep 19 14:10:14 2017 -0700
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Tue Sep 19 14:14:37 2017 -0700
----------------------------------------------------------------------
src/slave/paths.cpp | 14 +++++++++
src/slave/paths.hpp | 7 +++++
src/slave/slave.cpp | 81 ++++++++++++++++++++++++++++++++++++++++++------
3 files changed, 92 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/219ce65f/src/slave/paths.cpp
----------------------------------------------------------------------
diff --git a/src/slave/paths.cpp b/src/slave/paths.cpp
index 08177bc..e8724bf 100644
--- a/src/slave/paths.cpp
+++ b/src/slave/paths.cpp
@@ -303,6 +303,20 @@ string getExecutorSentinelPath(
}
+string getExecutorVirtualPath(
+ const FrameworkID& frameworkId,
+ const ExecutorID& executorId)
+{
+ return path::join(
+ stringify(os::PATH_SEPARATOR) + FRAMEWORKS_DIR,
+ stringify(frameworkId),
+ EXECUTORS_DIR,
+ stringify(executorId),
+ EXECUTOR_RUNS_DIR,
+ LATEST_SYMLINK);
+}
+
+
string getExecutorLatestRunPath(
const string& rootDir,
const SlaveID& slaveId,
http://git-wip-us.apache.org/repos/asf/mesos/blob/219ce65f/src/slave/paths.hpp
----------------------------------------------------------------------
diff --git a/src/slave/paths.hpp b/src/slave/paths.hpp
index 51b481f..d021e6b 100644
--- a/src/slave/paths.hpp
+++ b/src/slave/paths.hpp
@@ -211,6 +211,13 @@ std::string getExecutorSentinelPath(
const ContainerID& containerId);
+// Returns the "virtual" path used to expose the executor's sandbox
+// via the /files endpoints: `/frameworks/FID/executors/EID/latest`.
+std::string getExecutorVirtualPath(
+ const FrameworkID& frameworkId,
+ const ExecutorID& executorId);
+
+
std::string getExecutorLatestRunPath(
const std::string& rootDir,
const SlaveID& slaveId,
http://git-wip-us.apache.org/repos/asf/mesos/blob/219ce65f/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index d66c72e..01020a6 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -5568,16 +5568,27 @@ void Slave::removeExecutor(Framework* framework, Executor* executor)
const string path = paths::getExecutorPath(
flags.work_dir, info.id(), framework->id(), executor->id);
- // Make sure we detach the "latest" symlink.
+ // Make sure we detach both real and virtual paths for "latest"
+ // symlink. We prefer users to use the virtual paths because
+ // they do not expose the `work_dir` and agent ID, but the real
+ // paths remains for compatibility reason.
const string latestPath = paths::getExecutorLatestRunPath(
flags.work_dir,
info.id(),
framework->id(),
executor->id);
+ const string virtualLatestPath = paths::getExecutorVirtualPath(
+ framework->id(),
+ executor->id);
+
os::utime(path); // Update the modification time.
garbageCollect(path)
- .then(defer(self(), &Self::detachFile, latestPath));
+ .then(defer(self(), [=]() {
+ detachFile(latestPath);
+ detachFile(virtualLatestPath);
+ return Nothing();
+ }));
}
if (executor->checkpoint) {
@@ -7287,15 +7298,32 @@ Executor* Framework::addExecutor(const ExecutorInfo& executorInfo)
executorId);
};
- // Attach the "latest" symlink. This allows for frameworks to
- // easily construct paths to the sandbox directory for a given task
- // without having to know the container id.
- string latestPath = paths::getExecutorLatestRunPath(
+ // We expose the executor's sandbox in the /files endpoints
+ // via the following paths:
+ //
+ // (1) /agent_workdir/frameworks/FID/executors/EID/runs/CID
+ // (2) /agent_workdir/frameworks/FID/executors/EID/runs/latest
+ // (3) /frameworks/FID/executors/EID/runs/latest
+ //
+ // Originally we just exposed the real path (1) and later
+ // exposed the 'latest' symlink (2) since it's not easy for
+ // users to know the run's container ID. We deprecated
+ // (1) and (2) by exposing a virtual path (3) since we do not
+ // want to expose the agent's work directory and it's not
+ // something users care about in this context.
+ //
+ // TODO(zhitao): Remove (1) and (2) per MESOS-7960 once we
+ // pass 2.0. They remain now for backwards compatibility.
+ const string latestPath = paths::getExecutorLatestRunPath(
slave->flags.work_dir,
slave->info.id(),
id(),
executorInfo.executor_id());
+ const string virtualLatestPath = paths::getExecutorVirtualPath(
+ id(),
+ executorInfo.executor_id());
+
slave->files->attach(executor->directory, latestPath, authorize)
.onAny(defer(
slave,
@@ -7304,6 +7332,14 @@ Executor* Framework::addExecutor(const ExecutorInfo& executorInfo)
executor->directory,
latestPath));
+ slave->files->attach(executor->directory, virtualLatestPath, authorize)
+ .onAny(defer(
+ slave,
+ &Slave::fileAttached,
+ lambda::_1,
+ executor->directory,
+ virtualLatestPath));
+
slave->files->attach(executor->directory, executor->directory, authorize)
.onAny(defer(
slave,
@@ -7470,15 +7506,32 @@ void Framework::recoverExecutor(
executorId);
};
- // Attach the "latest" symlink. This allows for frameworks to
- // easily construct paths to the sandbox directory for a given task
- // without having to know the container id.
- string latestPath = paths::getExecutorLatestRunPath(
+ // We expose the executor's sandbox in the /files endpoints
+ // via the following paths:
+ //
+ // (1) /agent_workdir/frameworks/FID/executors/EID/runs/CID
+ // (2) /agent_workdir/frameworks/FID/executors/EID/runs/latest
+ // (3) /frameworks/FID/executors/EID/runs/latest
+ //
+ // Originally we just exposed the real path (1) and later
+ // exposed the 'latest' symlink (2) since it's not easy for
+ // users to know the run's container ID. We deprecated
+ // (1) and (2) by exposing a virtual path (3) since we do not
+ // want to expose the agent's work directory and it's not
+ // something users care about in this context.
+ //
+ // TODO(zhitao): Remove (1) and (2) per MESOS-7960 once we
+ // pass 2.0. They remain now for backwards compatibility.
+ const string latestPath = paths::getExecutorLatestRunPath(
slave->flags.work_dir,
slave->info.id(),
id(),
state.id);
+ const string virtualLatestPath = paths::getExecutorVirtualPath(
+ id(),
+ state.id);
+
slave->files->attach(executor->directory, latestPath, authorize)
.onAny(defer(
slave,
@@ -7487,6 +7540,14 @@ void Framework::recoverExecutor(
executor->directory,
latestPath));
+ slave->files->attach(executor->directory, virtualLatestPath, authorize)
+ .onAny(defer(
+ slave,
+ &Slave::fileAttached,
+ lambda::_1,
+ executor->directory,
+ virtualLatestPath));
+
// Expose the executor's files.
slave->files->attach(executor->directory, executor->directory, authorize)
.onAny(defer(