You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 10:47:51 UTC
svn commit: r1132127 - in /incubator/mesos/trunk/src: examples/java/
launcher/ slave/
Author: benh
Date: Sun Jun 5 08:47:50 2011
New Revision: 1132127
URL: http://svn.apache.org/viewvc?rev=1132127&view=rev
Log:
Chmod the directory that the executor is run in so that the executor can
write files to that directory. Also introduces mesos-slave
--frameworks_home option which is appended to executors which are
specified with relative paths.
Modified:
incubator/mesos/trunk/src/examples/java/TestExecutor.java
incubator/mesos/trunk/src/examples/java/TestFramework.java
incubator/mesos/trunk/src/launcher/launcher.cpp
incubator/mesos/trunk/src/launcher/launcher.hpp
incubator/mesos/trunk/src/launcher/main.cpp
incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp
incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp
incubator/mesos/trunk/src/slave/slave.cpp
incubator/mesos/trunk/src/slave/slave.hpp
incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp
Modified: incubator/mesos/trunk/src/examples/java/TestExecutor.java
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/examples/java/TestExecutor.java?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/examples/java/TestExecutor.java (original)
+++ incubator/mesos/trunk/src/examples/java/TestExecutor.java Sun Jun 5 08:47:50 2011
@@ -1,4 +1,4 @@
-import java.io.File;
+import java.io.FileWriter;
import mesos.*;
public class TestExecutor extends Executor {
@@ -11,6 +11,8 @@ public class TestExecutor extends Execut
new Thread() { public void run() {
try {
System.out.println("Running task " + task.getTaskId());
+ FileWriter outputStream = new FileWriter("test_file.txt");
+ outputStream.write("test output text to write", 0, 25);
Thread.sleep(1000);
d.sendStatusUpdate(new TaskStatus(task.getTaskId(),
TaskState.TASK_FINISHED,
Modified: incubator/mesos/trunk/src/examples/java/TestFramework.java
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/examples/java/TestFramework.java?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/examples/java/TestFramework.java (original)
+++ incubator/mesos/trunk/src/examples/java/TestFramework.java Sun Jun 5 08:47:50 2011
@@ -30,7 +30,7 @@ public class TestFramework {
public ExecutorInfo getExecutorInfo(SchedulerDriver d) {
try {
return new ExecutorInfo(
- new File("./test_executor").getCanonicalPath(),
+ "java/test_executor",
new byte[0]);
} catch (Exception e) {
e.printStackTrace();
Modified: incubator/mesos/trunk/src/launcher/launcher.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/launcher/launcher.cpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/launcher/launcher.cpp (original)
+++ incubator/mesos/trunk/src/launcher/launcher.cpp Sun Jun 5 08:47:50 2011
@@ -37,13 +37,15 @@ ExecutorLauncher::ExecutorLauncher(Frame
const string& _user,
const string& _workDirectory,
const string& _slavePid,
+ const string& _mesosFrameworks,
const string& _mesosHome,
const string& _hadoopHome,
bool _redirectIO,
bool _shouldSwitchUser,
const map<string, string>& _params)
: frameworkId(_frameworkId), executorUri(_executorUri), user(_user),
- workDirectory(_workDirectory), slavePid(_slavePid), mesosHome(_mesosHome),
+ workDirectory(_workDirectory), slavePid(_slavePid),
+ mesosFrameworks(_mesosFrameworks), mesosHome(_mesosHome),
hadoopHome(_hadoopHome), redirectIO(_redirectIO),
shouldSwitchUser(_shouldSwitchUser), params(_params)
{}
@@ -93,13 +95,21 @@ void ExecutorLauncher::createWorkingDire
string dir = "";
if (workDirectory.find_first_of("/") == 0) // We got an absolute path, so
dir = "/"; // keep the leading slash
+
foreach (const string& token, tokens) {
dir += token;
if (mkdir(dir.c_str(), 0755) < 0 && errno != EEXIST)
- fatalerror("Failed to mkdir %s", dir.c_str());
+ fatalerror("Failed to mkdir %s.", dir.c_str());
dir += "/";
}
- // TODO: chown the final directory to the framework's user
+ if (shouldSwitchUser) {
+ struct passwd *passwd;
+ if ((passwd = getpwnam(user.c_str())) == NULL)
+ fatal("Failed to get username information for %s.", user.c_str());
+
+ if (chown(dir.c_str(), passwd->pw_uid, passwd->pw_gid) < 0)
+ fatalerror("Failed to chown framework's working directory.");
+ }
}
@@ -115,9 +125,7 @@ string ExecutorLauncher::fetchExecutor()
executor.find_first_of('\0') != string::npos) {
fatal("Illegal characters in executor path");
}
-
- // Grab the executor from HDFS if its path begins with hdfs://
- // TODO: Enforce some size limits on files we get from HDFS
+// Grab the executor from HDFS if its path begins with hdfs:// // TODO: Enforce some size limits on files we get from HDFS
if (executor.find("hdfs://") == 0) {
// Locate Hadoop's bin/hadoop script. If a Hadoop home was given to us by
// the slave (from the Mesos config file), use that. Otherwise check for
@@ -146,6 +154,25 @@ string ExecutorLauncher::fetchExecutor()
if (chmod(executor.c_str(), S_IRWXU | S_IRGRP | S_IXGRP |
S_IROTH | S_IXOTH) != 0)
fatalerror("chmod failed");
+ } else if (executor.find_first_of("/") != 0) {
+ // We got a non-Hadoop and non-absolute path.
+ // Try prepending MESOS_HOME to it.
+ if (mesosFrameworks != "") {
+ executor = mesosFrameworks + "/" + executor;
+ cout << "Prepended mesosFrameworks to executor path, making it: "
+ << executor << endl;
+ } else {
+ if (mesosHome != "") {
+ executor = mesosHome + "/frameworks/" + executor;
+ cout << "Prepended MESOS_HOME/frameworks/ to relative "
+ << "executor path, making it: " << executor << endl;
+ } else {
+ cout << "Neither MESOS_HOME nor FRAMEWORKS_HOME is set, so using "
+ << "relative executor path as is which will result in it "
+ << "relative to the directory Mesos created for the "
+ << "executor to run in." << endl;
+ }
+ }
}
// If the executor was a .tgz, untar it in the work directory. The .tgz
Modified: incubator/mesos/trunk/src/launcher/launcher.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/launcher/launcher.hpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/launcher/launcher.hpp (original)
+++ incubator/mesos/trunk/src/launcher/launcher.hpp Sun Jun 5 08:47:50 2011
@@ -35,6 +35,7 @@ protected:
string user;
string workDirectory; // Directory in which the framework should run
string slavePid;
+ string mesosFrameworks;
string mesosHome;
string hadoopHome;
bool redirectIO; // Whether to redirect stdout and stderr to files
@@ -44,9 +45,10 @@ protected:
public:
ExecutorLauncher(FrameworkID _frameworkId, const string& _executorUri,
const string& _user, const string& _workDirectory,
- const string& _slavePid, const string& _mesosHome,
- const string& _hadoopHome, bool _redirectIO,
- bool _shouldSwitchUser, const map<string, string>& _params);
+ const string& _slavePid, const string& _mesosFrameworks,
+ const string& _mesosHome, const string& _hadoopHome,
+ bool _redirectIO, bool _shouldSwitchUser,
+ const map<string, string>& _params);
virtual ~ExecutorLauncher();
Modified: incubator/mesos/trunk/src/launcher/main.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/launcher/main.cpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/launcher/main.cpp (original)
+++ incubator/mesos/trunk/src/launcher/main.cpp Sun Jun 5 08:47:50 2011
@@ -29,6 +29,7 @@ int main(int argc, char **argv)
getenvOrFail("MESOS_USER"),
getenvOrFail("MESOS_WORK_DIRECTORY"),
getenvOrFail("MESOS_SLAVE_PID"),
+ getenvOrFail("MESOS_FRAMEWORKS_HOME"),
getenvOrFail("MESOS_HOME"),
getenvOrFail("MESOS_HADOOP_HOME"),
lexical_cast<bool>(getenvOrFail("MESOS_REDIRECT_IO")),
Modified: incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp (original)
+++ incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp Sun Jun 5 08:47:50 2011
@@ -90,7 +90,7 @@ void LxcIsolationModule::startExecutor(F
<< fw->executorInfo.uri;
// Get location of Mesos install in order to find mesos-launcher.
- string mesosHome = slave->getConf().get("home", ".");
+ string mesosHome = slave->getParams().get("home", ".");
string mesosLauncher = mesosHome + "/mesos-launcher";
// Create a name for the container
@@ -122,10 +122,12 @@ void LxcIsolationModule::startExecutor(F
fw->user,
slave->getUniqueWorkDirectory(fw->id),
slave->self(),
- slave->getConf().get("home", ""),
- slave->getConf().get("hadoop_home", ""),
+ slave->getParams().get("frameworks_home",
+ ""),
+ slave->getParams().get("home", ""),
+ slave->getParams().get("hadoop_home", ""),
!slave->local,
- slave->getConf().get("switch_user", true),
+ slave->getParams().get("switch_user", true),
fw->executorInfo.params);
launcher->setupEnvironmentForLauncherMain();
Modified: incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp (original)
+++ incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp Sun Jun 5 08:47:50 2011
@@ -114,10 +114,11 @@ ExecutorLauncher* ProcessBasedIsolationM
fw->user,
slave->getUniqueWorkDirectory(fw->id),
slave->self(),
- slave->getConf().get("home", ""),
- slave->getConf().get("hadoop_home", ""),
+ slave->getParams().get("frameworks_home", ""),
+ slave->getParams().get("home", ""),
+ slave->getParams().get("hadoop_home", ""),
!slave->local,
- slave->getConf().get("switch_user", true),
+ slave->getParams().get("switch_user", true),
fw->executorInfo.params);
}
Modified: incubator/mesos/trunk/src/slave/slave.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/slave.cpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/slave.cpp (original)
+++ incubator/mesos/trunk/src/slave/slave.cpp Sun Jun 5 08:47:50 2011
@@ -79,11 +79,11 @@ Slave::Slave(Resources _resources, bool
isolationModule(_isolationModule) {}
-Slave::Slave(const Params& _conf, bool _local, IsolationModule *_module)
- : id(""), conf(_conf), local(_local), isolationModule(_module)
+Slave::Slave(const Params& _params, bool _local, IsolationModule *_module)
+ : id(""), params(_params), local(_local), isolationModule(_module)
{
- resources = Resources(conf.get<int32_t>("cpus", DEFAULT_CPUS),
- conf.get<int32_t>("mem", DEFAULT_MEM));
+ resources = Resources(params.get<int32_t>("cpus", DEFAULT_CPUS),
+ params.get<int32_t>("mem", DEFAULT_MEM));
}
@@ -106,6 +106,11 @@ void Slave::registerOptions(Configurator
"submitted them rather than the user running\n"
"the slave (requires setuid permission)\n",
true);
+ conf->addOption<string>("frameworks_home",
+ "Directory to prepend to relative executor paths.\n \
+ (default: MESOS_HOME/frameworks if MESOS_HOME\n \
+ exists, else defaults to directory which mesos\n \
+ creates when it launches executor.)");
}
@@ -535,10 +540,10 @@ void Slave::executorExited(FrameworkID f
string Slave::getUniqueWorkDirectory(FrameworkID fid)
{
string workDir;
- if (conf.contains("work_dir")) {
- workDir = conf["work_dir"];
- } else if (conf.contains("home")) {
- workDir = conf["home"] + "/work";
+ if (params.contains("work_dir")) {
+ workDir = params["work_dir"];
+ } else if (params.contains("home")) {
+ workDir = params["home"] + "/work";
} else {
workDir = "work";
}
@@ -565,7 +570,7 @@ string Slave::getUniqueWorkDirectory(Fra
}
-const Params& Slave::getConf()
+const Params& Slave::getParams()
{
- return conf;
+ return params;
}
Modified: incubator/mesos/trunk/src/slave/slave.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/slave.hpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/slave.hpp (original)
+++ incubator/mesos/trunk/src/slave/slave.hpp Sun Jun 5 08:47:50 2011
@@ -166,7 +166,7 @@ struct Executor
class Slave : public MesosProcess
{
public:
- Params conf;
+ Params params;
typedef unordered_map<FrameworkID, Framework*> FrameworkMap;
typedef unordered_map<FrameworkID, Executor*> ExecutorMap;
@@ -185,7 +185,7 @@ public:
public:
Slave(Resources resources, bool local, IsolationModule* isolationModule);
- Slave(const Params& conf, bool local, IsolationModule *isolationModule);
+ Slave(const Params& params, bool local, IsolationModule *isolationModule);
virtual ~Slave();
@@ -201,7 +201,7 @@ public:
string getUniqueWorkDirectory(FrameworkID fid);
- const Params& getConf();
+ const Params& getParams();
protected:
void operator () ();
Modified: incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp?rev=1132127&r1=1132126&r2=1132127&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp (original)
+++ incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp Sun Jun 5 08:47:50 2011
@@ -87,10 +87,11 @@ ExecutorLauncher* SolarisProjectIsolatio
fw->user,
slave->getWorkDirectory(fw->id),
slave->self(),
- slave->getConf().get("home", ""),
- slave->getConf().get("hadoop_home", ""),
+ slave->getParams().get("frameworks_home", ""),
+ slave->getParams().get("home", ""),
+ slave->getParams().get("hadoop_home", ""),
!slave->local,
- slave->getConf().get("switch_user", true),
+ slave->getParams().get("switch_user", true),
frameworkProject[fw->id]);
}
@@ -101,7 +102,7 @@ void SolarisProjectIsolationModule::Comm
LOG(INFO) << "Starting projd for project " << project;
// Get location of Mesos install in order to find projd.
- string mesosHome = slave->getConf().get("home", ".");
+ string mesosHome = slave->getParams().get("home", ".");
pid_t pid;
if ((pid = fork()) == -1)