You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 10:22:41 UTC
svn commit: r1132004 - in /incubator/mesos/trunk: frameworks/mpi/
frameworks/torque/ src/ src/launcher/ src/slave/
Author: benh
Date: Sun Jun 5 08:22:41 2011
New Revision: 1132004
URL: http://svn.apache.org/viewvc?rev=1132004&view=rev
Log:
Made MPI and Torque frameworks use new MESOS_HOME directory structure to
locate the Mesos Python libraries, and added a couple of misc fixes to
make this easier (launcher sets MESOS_HOME environment variable so that
executors can find it, and a Makefile fix to move the SWIG-generated .py
file for the Python library to lib/python).
Modified:
incubator/mesos/trunk/frameworks/mpi/nmpiexec
incubator/mesos/trunk/frameworks/mpi/startmpd.sh
incubator/mesos/trunk/frameworks/torque/start_pbs_mom.sh
incubator/mesos/trunk/frameworks/torque/torquesched.sh
incubator/mesos/trunk/src/Makefile.in
incubator/mesos/trunk/src/launcher/launcher.cpp
incubator/mesos/trunk/src/launcher/launcher.hpp
incubator/mesos/trunk/src/launcher/main.cpp
incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp
incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp
incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp
Modified: incubator/mesos/trunk/frameworks/mpi/nmpiexec
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/mpi/nmpiexec?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/mpi/nmpiexec (original)
+++ incubator/mesos/trunk/frameworks/mpi/nmpiexec Sun Jun 5 08:22:41 2011
@@ -1,6 +1,15 @@
#!/bin/bash
-PYTHON=python
-if [ "`uname`" == "SunOS" ]; then
- PYTHON=python2.6
+
+if [ "x$PYTHON" == "x" ]; then
+ PYTHON=python
+ if [ "`uname`" == "SunOS" ]; then
+ PYTHON=python2.6
+ fi
fi
-PYTHONPATH=`dirname $0`/../../src/swig/python $PYTHON ./nmpiexec.py $@
+
+if [ "x$MESOS_HOME" == "x" ]; then
+ MESOS_HOME="$(dirname $0)/../.."
+fi
+
+export PYTHONPATH=$MESOS_HOME/lib/python
+exec $PYTHON "$(dirname $0)/nmpiexec.py" $@
Modified: incubator/mesos/trunk/frameworks/mpi/startmpd.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/mpi/startmpd.sh?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/mpi/startmpd.sh (original)
+++ incubator/mesos/trunk/frameworks/mpi/startmpd.sh Sun Jun 5 08:22:41 2011
@@ -1,11 +1,15 @@
#!/bin/bash
-PYTHON=python
-
-if [ "`uname`" == "SunOS" ]; then
- PYTHON=python2.6
+if [ "x$PYTHON" == "x" ]; then
+ PYTHON=python
+ if [ "`uname`" == "SunOS" ]; then
+ PYTHON=python2.6
+ fi
fi
-export PYTHONPATH=`dirname $0`/../../src/swig/python:$PYTHONPATH
+if [ "x$MESOS_HOME" == "x" ]; then
+ MESOS_HOME="$(dirname $0)/../.."
+fi
-$PYTHON `dirname $0`/startmpd.py $@
+export PYTHONPATH=$MESOS_HOME/lib/python
+exec $PYTHON "$(dirname $0)/startmpd.py" $@
Modified: incubator/mesos/trunk/frameworks/torque/start_pbs_mom.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/torque/start_pbs_mom.sh?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/torque/start_pbs_mom.sh (original)
+++ incubator/mesos/trunk/frameworks/torque/start_pbs_mom.sh Sun Jun 5 08:22:41 2011
@@ -1,11 +1,15 @@
#!/bin/bash
-PYTHON=python
-
-if [ "`uname`" == "SunOS" ]; then
- PYTHON=python2.6
+if [ "x$PYTHON" == "x" ]; then
+ PYTHON=python
+ if [ "`uname`" == "SunOS" ]; then
+ PYTHON=python2.6
+ fi
fi
-export PYTHONPATH=`dirname $0`/../../src/swig/python:$PYTHONPATH
+if [ "x$MESOS_HOME" == "x" ]; then
+ MESOS_HOME="$(dirname $0)/../.."
+fi
-$PYTHON `dirname $0`/start_pbs_mom.py $@
+export PYTHONPATH=$MESOS_HOME/lib/python
+exec $PYTHON "$(dirname $0)/start_pbs_mob.py" $@
Modified: incubator/mesos/trunk/frameworks/torque/torquesched.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/torque/torquesched.sh?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/torque/torquesched.sh (original)
+++ incubator/mesos/trunk/frameworks/torque/torquesched.sh Sun Jun 5 08:22:41 2011
@@ -1,6 +1,15 @@
#!/bin/bash
-PYTHON=python
-if [ "`uname`" == "SunOS" ]; then
- PYTHON=python2.6
+
+if [ "x$PYTHON" == "x" ]; then
+ PYTHON=python
+ if [ "`uname`" == "SunOS" ]; then
+ PYTHON=python2.6
+ fi
fi
-PYTHONPATH=`dirname $0`/../../src/swig/python $PYTHON ./torquesched.py $@
+
+if [ "x$MESOS_HOME" == "x" ]; then
+ MESOS_HOME="$(dirname $0)/../.."
+fi
+
+export PYTHONPATH=$MESOS_HOME/lib/python
+exec $PYTHON "$(dirname $0)/torquesched.py" $@
Modified: incubator/mesos/trunk/src/Makefile.in
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/Makefile.in?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/src/Makefile.in (original)
+++ incubator/mesos/trunk/src/Makefile.in Sun Jun 5 08:22:41 2011
@@ -307,6 +307,7 @@ ifdef PYTHON_HEADERS
mkdir -p swig/python
$(SWIG) -c++ -python -threads -I@top_srcdir@/include -o swig/python/mesos_wrap.cpp -outdir swig/python @srcdir@/swig/mesos.i
$(CXX) $(CXXFLAGS) -I$(PYTHON_HEADERS) -shared -o $@ swig/python/mesos_wrap.cpp $(MESOS_SCHED_LIB) $(MESOS_EXEC_LIB) $(LDFLAGS) $(PYTHON_LDFLAGS) $(LIBS)
+ cp swig/python/mesos.py $(LIBDIR)/python/mesos.py
endif
$(WEBUI_FILES): | $(WEBUI_DIRECTORIES)
Modified: incubator/mesos/trunk/src/launcher/launcher.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/launcher/launcher.cpp?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/src/launcher/launcher.cpp (original)
+++ incubator/mesos/trunk/src/launcher/launcher.cpp Sun Jun 5 08:22:41 2011
@@ -37,11 +37,12 @@ ExecutorLauncher::ExecutorLauncher(Frame
const string& _user,
const string& _workDirectory,
const string& _slavePid,
+ const string& _mesosHome,
const string& _hadoopHome,
bool _redirectIO,
const map<string, string>& _params)
: frameworkId(_frameworkId), executorUri(_executorUri), user(_user),
- workDirectory(_workDirectory), slavePid(_slavePid),
+ workDirectory(_workDirectory), slavePid(_slavePid), mesosHome(_mesosHome),
hadoopHome(_hadoopHome), redirectIO(_redirectIO), params(_params)
{}
@@ -199,6 +200,11 @@ void ExecutorLauncher::setupEnvironment(
// Set LIBPROCESS_PORT so that we bind to a random free port.
setenv("LIBPROCESS_PORT", "0", true);
+
+ // Set MESOS_HOME so that Java and Python executors can find libraries
+ if (mesosHome != "") {
+ setenv("MESOS_HOME", mesosHome.c_str(), 1);
+ }
}
@@ -239,6 +245,7 @@ void ExecutorLauncher::setupEnvironmentF
setenv("MESOS_USER", user.c_str(), 1);
setenv("MESOS_WORK_DIRECTORY", workDirectory.c_str(), 1);
setenv("MESOS_SLAVE_PID", slavePid.c_str(), 1);
+ setenv("MESOS_HOME", mesosHome.c_str(), 1);
setenv("MESOS_HADOOP_HOME", hadoopHome.c_str(), 1);
setenv("MESOS_REDIRECT_IO", redirectIO ? "1" : "0", 1);
}
Modified: incubator/mesos/trunk/src/launcher/launcher.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/launcher/launcher.hpp?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/src/launcher/launcher.hpp (original)
+++ incubator/mesos/trunk/src/launcher/launcher.hpp Sun Jun 5 08:22:41 2011
@@ -35,6 +35,7 @@ protected:
string user;
string workDirectory; // Directory in which the framework should run
string slavePid;
+ string mesosHome;
string hadoopHome;
bool redirectIO; // Whether to redirect stdout and stderr to files
map<string, string> params; // Key-value params in framework's ExecutorInfo
@@ -42,8 +43,9 @@ protected:
public:
ExecutorLauncher(FrameworkID _frameworkId, const string& _executorUri,
const string& _user, const string& _workDirectory,
- const string& _slavePid, const string& _hadoopHome,
- bool _redirectIO, const map<string, string>& _params);
+ const string& _slavePid, const string& _mesosHome,
+ const string& _hadoopHome, bool _redirectIO,
+ const map<string, string>& _params);
virtual ~ExecutorLauncher();
Modified: incubator/mesos/trunk/src/launcher/main.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/launcher/main.cpp?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/src/launcher/main.cpp (original)
+++ incubator/mesos/trunk/src/launcher/main.cpp Sun Jun 5 08:22:41 2011
@@ -29,6 +29,7 @@ int main(int argc, char **argv)
getenvOrFail("MESOS_USER"),
getenvOrFail("MESOS_WORK_DIRECTORY"),
getenvOrFail("MESOS_SLAVE_PID"),
+ getenvOrFail("MESOS_HOME"),
getenvOrFail("MESOS_HADOOP_HOME"),
lexical_cast<bool>(getenvOrFail("MESOS_REDIRECT_IO")),
params).run();
Modified: incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp (original)
+++ incubator/mesos/trunk/src/slave/lxc_isolation_module.cpp Sun Jun 5 08:22:41 2011
@@ -122,6 +122,7 @@ void LxcIsolationModule::startExecutor(F
fw->user,
slave->getUniqueWorkDirectory(fw->id),
slave->self(),
+ slave->getConf().get("home", ""),
slave->getConf().get("hadoop_home", ""),
!slave->local,
fw->executorInfo.params);
Modified: incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp (original)
+++ incubator/mesos/trunk/src/slave/process_based_isolation_module.cpp Sun Jun 5 08:22:41 2011
@@ -114,6 +114,7 @@ ExecutorLauncher* ProcessBasedIsolationM
fw->user,
slave->getUniqueWorkDirectory(fw->id),
slave->self(),
+ slave->getConf().get("home", ""),
slave->getConf().get("hadoop_home", ""),
!slave->local,
fw->executorInfo.params);
Modified: incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp?rev=1132004&r1=1132003&r2=1132004&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp (original)
+++ incubator/mesos/trunk/src/slave/solaris_project_isolation_module.cpp Sun Jun 5 08:22:41 2011
@@ -87,6 +87,7 @@ ExecutorLauncher* SolarisProjectIsolatio
fw->user,
slave->getWorkDirectory(fw->id),
slave->self(),
+ slave->getConf().get("home", ""),
slave->getConf().get("hadoop_home", ""),
!slave->local,
frameworkProject[fw->id]);