You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2014/06/29 18:28:19 UTC
[1/2] git commit: Refactored the scheduler/executor driver to remove
calling 'stop'.
Repository: mesos
Updated Branches:
refs/heads/master 451c3b6a7 -> 294337466
Refactored the scheduler/executor driver to remove calling 'stop'.
Review: https://reviews.apache.org/r/23142
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0821ffec
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0821ffec
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0821ffec
Branch: refs/heads/master
Commit: 0821ffeca1ec958c1bb01b203d57d20de3933341
Parents: 451c3b6
Author: Benjamin Hindman <be...@gmail.com>
Authored: Fri Jun 27 12:07:30 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Jun 29 09:25:14 2014 -0700
----------------------------------------------------------------------
src/exec/exec.cpp | 6 ++++--
src/java/jni/org_apache_mesos_MesosExecutorDriver.cpp | 10 ++++++----
src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp | 10 ++++++----
src/python/native/mesos_executor_driver_impl.cpp | 2 --
src/python/native/mesos_scheduler_driver_impl.cpp | 2 --
5 files changed, 16 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0821ffec/src/exec/exec.cpp
----------------------------------------------------------------------
diff --git a/src/exec/exec.cpp b/src/exec/exec.cpp
index 346e39f..e09ac8d 100644
--- a/src/exec/exec.cpp
+++ b/src/exec/exec.cpp
@@ -598,8 +598,10 @@ MesosExecutorDriver::MesosExecutorDriver(Executor* _executor)
MesosExecutorDriver::~MesosExecutorDriver()
{
- // Just as in SchedulerProcess, we might wait here indefinitely if
- // MesosExecutorDriver::stop has not been invoked.
+ // Just like with the MesosSchedulerDriver it's possible to get a
+ // deadlock here. Otherwise we terminate the ExecutorProcess and
+ // wait for it before deleting.
+ terminate(process);
wait(process);
delete process;
http://git-wip-us.apache.org/repos/asf/mesos/blob/0821ffec/src/java/jni/org_apache_mesos_MesosExecutorDriver.cpp
----------------------------------------------------------------------
diff --git a/src/java/jni/org_apache_mesos_MesosExecutorDriver.cpp b/src/java/jni/org_apache_mesos_MesosExecutorDriver.cpp
index 32e9a28..8cbc798 100644
--- a/src/java/jni/org_apache_mesos_MesosExecutorDriver.cpp
+++ b/src/java/jni/org_apache_mesos_MesosExecutorDriver.cpp
@@ -390,10 +390,12 @@ JNIEXPORT void JNICALL Java_org_apache_mesos_MesosExecutorDriver_finalize
MesosExecutorDriver* driver =
(MesosExecutorDriver*) env->GetLongField(thiz, __driver);
- // Call stop just in case.
- driver->stop();
- driver->join();
-
+ // Note that we DO NOT want to call 'abort' or 'stop' as this may be
+ // misinterpreted by the executor. It is possible, however, that
+ // since we haven't called 'abort' or 'stop' there are still threads
+ // executing within the executor callbacks but the
+ // MesosExecutorDriver destructor will wait until this is not the
+ // case before returning.
delete driver;
jfieldID __executor = env->GetFieldID(clazz, "__executor", "J");
http://git-wip-us.apache.org/repos/asf/mesos/blob/0821ffec/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
----------------------------------------------------------------------
diff --git a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
index d0b7152..3498930 100644
--- a/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
+++ b/src/java/jni/org_apache_mesos_MesosSchedulerDriver.cpp
@@ -548,10 +548,12 @@ JNIEXPORT void JNICALL Java_org_apache_mesos_MesosSchedulerDriver_finalize
MesosSchedulerDriver* driver =
(MesosSchedulerDriver*) env->GetLongField(thiz, __driver);
- // Call stop just in case.
- driver->stop();
- driver->join();
-
+ // Note that we DO NOT want to call 'abort' or 'stop' as this may be
+ // misinterpreted by the scheduler. It is possible, however, that
+ // since we haven't called 'abort' or 'stop' there are still threads
+ // executing within the scheduler callbacks but the
+ // MesosSchedulerDriver destructor will wait until this is not the
+ // case before returning.
delete driver;
jfieldID __scheduler = env->GetFieldID(clazz, "__scheduler", "J");
http://git-wip-us.apache.org/repos/asf/mesos/blob/0821ffec/src/python/native/mesos_executor_driver_impl.cpp
----------------------------------------------------------------------
diff --git a/src/python/native/mesos_executor_driver_impl.cpp b/src/python/native/mesos_executor_driver_impl.cpp
index aa564ed..407d8d1 100644
--- a/src/python/native/mesos_executor_driver_impl.cpp
+++ b/src/python/native/mesos_executor_driver_impl.cpp
@@ -167,7 +167,6 @@ int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
}
if (self->driver != NULL) {
- self->driver->stop();
delete self->driver;
self->driver = NULL;
}
@@ -190,7 +189,6 @@ int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self)
{
if (self->driver != NULL) {
- self->driver->stop();
// We need to wrap the driver destructor in an "allow threads"
// macro since the MesosExecutorDriver destructor waits for the
// ExecutorProcess to terminate and there might be a thread that
http://git-wip-us.apache.org/repos/asf/mesos/blob/0821ffec/src/python/native/mesos_scheduler_driver_impl.cpp
----------------------------------------------------------------------
diff --git a/src/python/native/mesos_scheduler_driver_impl.cpp b/src/python/native/mesos_scheduler_driver_impl.cpp
index 1c82532..e014eed 100644
--- a/src/python/native/mesos_scheduler_driver_impl.cpp
+++ b/src/python/native/mesos_scheduler_driver_impl.cpp
@@ -214,7 +214,6 @@ int MesosSchedulerDriverImpl_init(MesosSchedulerDriverImpl* self,
if (self->driver != NULL) {
- self->driver->stop();
delete self->driver;
self->driver = NULL;
}
@@ -244,7 +243,6 @@ int MesosSchedulerDriverImpl_init(MesosSchedulerDriverImpl* self,
void MesosSchedulerDriverImpl_dealloc(MesosSchedulerDriverImpl* self)
{
if (self->driver != NULL) {
- self->driver->stop();
// We need to wrap the driver destructor in an "allow threads"
// macro since the MesosSchedulerDriver destructor waits for the
// SchedulerProcess to terminate and there might be a thread that
[2/2] git commit: Make sure we pass CXXFLAGS when building Python
eggs.
Posted by be...@apache.org.
Make sure we pass CXXFLAGS when building Python eggs.
Python's distuils doesn't look automagically at CXXFLAGS, so we need
to pass it along embedded in CFLAGS. This bug manifested on OS X using
clang because we build libmesos with C++11 but not the Python EGG, and
so the Python EGG can't find certain symbols in libmesos (because
C++11 compiled symbols can have different signatures).
In addition to making sure we pass CXXFLAGS, the CXXFLAGS that are
saved in PYTHON_CXXFLAGS (and the LDFLAGS for that matter) are not
finalized when we do our Python configure checks so some code was
moved around to make sure we do Java and Python at the bottom.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/29433746
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/29433746
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/29433746
Branch: refs/heads/master
Commit: 294337466ccdbbc933c17712d9ca6877def3f83e
Parents: 0821ffe
Author: Benjamin Hindman <be...@gmail.com>
Authored: Sun Jun 29 08:28:57 2014 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Sun Jun 29 09:26:31 2014 -0700
----------------------------------------------------------------------
configure.ac | 247 ++++++++++++++++++++++++++-------------------------
src/Makefile.am | 2 +-
2 files changed, 126 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/29433746/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 0a8555a..e747208 100644
--- a/configure.ac
+++ b/configure.ac
@@ -368,6 +368,29 @@ else
fi
+# Check if we should/can build with C++11.
+if test "x$with_cxx11" = "xyes"; then
+ AX_CXX_COMPILE_STDCXX_11([noext], [mandatory])
+
+ case "$host_os" in
+ darwin* )
+ # If we're using clang, we need to pass -stdlib=libc++ too.
+ if test "x$CLANG" = "xyes"; then
+ CXXFLAGS="$CXXFLAGS -stdlib=libc++"
+ fi
+
+ # GTEST on OSX needs its own tr1 tuple.
+ # TODO(dhamon): Update to gmock 1.7 and pass GTEST_LANG_CXX11 when in
+ # c++11 mode.
+ CXXFLAGS="$CXXFLAGS -DGTEST_USE_OWN_TR1_TUPLE=1"
+ ;;
+ esac
+
+ # Also pass the flags to 3rdparty libraries.
+ CONFIGURE_ARGS="$CONFIGURE_ARGS CXXFLAGS='$CXXFLAGS'"
+fi
+
+
# Check for pthreads (uses m4/acx_pthread.m4).
ACX_PTHREAD([], [AC_MSG_ERROR([failed to find pthreads])])
@@ -376,6 +399,101 @@ ACX_PTHREAD([], [AC_MSG_ERROR([failed to find pthreads])])
AC_CHECK_LIB(unwind, backtrace, LIBS="$LIBS -lunwind")
+AC_CHECK_LIB([z], [gzread], [],
+ [AC_MSG_ERROR([cannot find libz
+-------------------------------------------------------------------
+libz is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+
+AC_CHECK_LIB([curl], [curl_global_init], [],
+ [AC_MSG_ERROR([cannot find libcurl
+-------------------------------------------------------------------
+libcurl is required for mesos to build.
+-------------------------------------------------------------------
+])])
+
+
+# TODO(benh): Also check for md5 support so we can use the CRAM-MD5
+# mechanism. We can likely do a AC_CHECK_LIB looking for a particular
+# function only provided if md5 support is present.
+AC_CHECK_LIB([sasl2], [sasl_done], [],
+ [AC_MSG_ERROR([cannot find libsasl2
+-------------------------------------------------------------------
+We need libsasl2 for authentication!
+-------------------------------------------------------------------
+])])
+
+
+# Perform necessary configuration for network isolator.
+if test "x$with_network_isolator" = "xyes"; then
+ # Check for OS support.
+ AS_IF([test "$OS_NAME" = "linux"],
+ [],
+ [AC_MSG_ERROR([cannot build network isolator
+-------------------------------------------------------------------
+Network isolator is only supported on Linux!
+-------------------------------------------------------------------
+ ])])
+
+ # Check for libnl (both headers and libraries).
+ AC_CHECK_LIB([nl-3], [nl_has_capability], [],
+ [AC_MSG_ERROR([cannot find libnl-3
+-------------------------------------------------------------------
+We need libnl-3 for building network isolator!
+
+Please install libnl3 (version 3.2.24 or higher):
+http://www.infradead.org/~tgr/libnl/
+-------------------------------------------------------------------
+ ])])
+
+ AC_CHECK_HEADERS([netlink/netlink.h libnl3/netlink/netlink.h],
+ [break]
+ [AC_MSG_ERROR([cannot find libnl-3 headers
+-------------------------------------------------------------------
+We need libnl-3 headers for building network isolator!
+
+Please install libnl3 (version 3.2.24 or higher):
+http://www.infradead.org/~tgr/libnl/
+-------------------------------------------------------------------
+ ])])
+
+ # Check for libnl-route (both headers and libraries).
+ AC_CHECK_LIB([nl-route-3], [rtnl_link_veth_add], [],
+ [AC_MSG_ERROR([cannot find libnl-route-3
+-------------------------------------------------------------------
+We need libnl-route-3 for building network isolator!
+
+Please install libnl3 (version 3.2.24 or higher):
+http://www.infradead.org/~tgr/libnl/
+-------------------------------------------------------------------
+ ])])
+
+ AC_CHECK_HEADERS([netlink/route/link/veth.h libnl3/netlink/route/link/veth.h],
+ [break]
+ [AC_MSG_ERROR([cannot find libnl-route-3 headers
+-------------------------------------------------------------------
+We need libnl-route-3 headers for building network isolator!
+
+Please install libnl3 (version 3.2.24 or higher):
+http://www.infradead.org/~tgr/libnl/
+-------------------------------------------------------------------
+ ])])
+
+ # TODO(jieyu): Automatically detect the location where the libnl
+ # headers are installed.
+ LIBNL_CFLAGS=-I/usr/include/libnl3
+
+ AC_SUBST([LIBNL_CFLAGS])
+
+ AC_DEFINE([WITH_NETWORK_ISOLATOR])
+fi
+
+AM_CONDITIONAL([WITH_NETWORK_ISOLATOR],
+ [test "x$with_network_isolator" = "xyes"])
+
+
# TODO(benh): Consider using AS_IF instead of just shell 'if'
# statements for better autoconf style (the AS_IF macros also make
# sure variable dependencies are handled appropriately).
@@ -661,7 +779,9 @@ __EOF__
# Build that Python egg that links against that library. The build
# settings are propagated to distutils.
- CXX="$CXX" CC="$CC" CXXFLAGS="$CXXFLAGS" CFLAGS="$CFLAGS" \
+ # NOTE: We need to embed our CXXFLAGS in CFLAGS because distutils
+ # doesn't pull out CXXFLAGS automagically.
+ CXX="$CXX" CC="$CC" CFLAGS="$CFLAGS $CXXFLAGS" \
LDFLAGS="$LDFLAGS" $PYTHON setup.py build_ext --inplace \
--build-temp ./ 2>&1 >/dev/null
@@ -725,16 +845,16 @@ There are two possible workarounds for this issue:
# When clang is being used, make sure that the distutils python-
# config cflags extraction does not cause build errors (MESOS-1079).
# TODO(tillt): Remove this once Apple distributed an updated Python.
- PYTHON_CFLAGS="$CFLAGS"
PYTHON_CPPFLAGS="$CPPFLAGS"
+ PYTHON_CFLAGS="$CFLAGS $CXXFLAGS" # distutils requires we embed CXXFLAGS.
PYTHON_LDFLAGS="$LDFLAGS"
AS_IF([test "x$CLANG" = "xyes"],
- [PYTHON_CFLAGS="$PYTHON_CFLAGS -Qunused-arguments"
- PYTHON_CPPFLAGS="$PYTHON_CPPFLAGS -Qunused-arguments"])
+ [PYTHON_CPPFLAGS="$PYTHON_CPPFLAGS -Qunused-arguments"
+ PYTHON_CFLAGS="$PYTHON_CFLAGS -Qunused-arguments"])
- AC_SUBST([PYTHON_CFLAGS])
AC_SUBST([PYTHON_CPPFLAGS])
+ AC_SUBST([PYTHON_CFLAGS])
AC_SUBST([PYTHON_EGG_POSTFIX])
AC_SUBST([PYTHON_EGG_PUREPY_POSTFIX])
AC_SUBST([PYTHON]) # Used by the example shell scripts and src/Makefile.am.
@@ -752,123 +872,6 @@ fi
AM_CONDITIONAL([HAS_PYTHON], [test "x$has_python" = "xyes"])
-AC_CHECK_LIB([z], [gzread], [],
- [AC_MSG_ERROR([cannot find libz
--------------------------------------------------------------------
-libz is required for mesos to build.
--------------------------------------------------------------------
-])])
-
-AC_CHECK_LIB([curl], [curl_global_init], [],
- [AC_MSG_ERROR([cannot find libcurl
--------------------------------------------------------------------
-libcurl is required for mesos to build.
--------------------------------------------------------------------
-])])
-
-
-# Check if we should/can build with C++11.
-if test "x$with_cxx11" = "xyes"; then
- AX_CXX_COMPILE_STDCXX_11([noext], [mandatory])
-
- case "$host_os" in
- darwin* )
- # If we're using clang, we need to pass -stdlib=libc++ too.
- if test "x$CLANG" = "xyes"; then
- CXXFLAGS="$CXXFLAGS -stdlib=libc++"
- fi
-
- # GTEST on OSX needs its own tr1 tuple.
- # TODO(dhamon): Update to gmock 1.7 and pass GTEST_LANG_CXX11 when in
- # c++11 mode.
- CXXFLAGS="$CXXFLAGS -DGTEST_USE_OWN_TR1_TUPLE=1"
- ;;
- esac
-
- # Also pass the flags to 3rdparty libraries.
- CONFIGURE_ARGS="$CONFIGURE_ARGS CXXFLAGS='$CXXFLAGS'"
-fi
-
-
-# TODO(benh): Also check for md5 support so we can use the CRAM-MD5
-# mechanism. We can likely do a AC_CHECK_LIB looking for a particular
-# function only provided if md5 support is present.
-AC_CHECK_LIB([sasl2], [sasl_done], [],
- [AC_MSG_ERROR([cannot find libsasl2
--------------------------------------------------------------------
-We need libsasl2 for authentication!
--------------------------------------------------------------------
-])])
-
-
-# Perform necessary configuration for network isolator.
-if test "x$with_network_isolator" = "xyes"; then
- # Check for OS support.
- AS_IF([test "$OS_NAME" = "linux"],
- [],
- [AC_MSG_ERROR([cannot build network isolator
--------------------------------------------------------------------
-Network isolator is only supported on Linux!
--------------------------------------------------------------------
- ])])
-
- # Check for libnl (both headers and libraries).
- AC_CHECK_LIB([nl-3], [nl_has_capability], [],
- [AC_MSG_ERROR([cannot find libnl-3
--------------------------------------------------------------------
-We need libnl-3 for building network isolator!
-
-Please install libnl3 (version 3.2.24 or higher):
-http://www.infradead.org/~tgr/libnl/
--------------------------------------------------------------------
- ])])
-
- AC_CHECK_HEADERS([netlink/netlink.h libnl3/netlink/netlink.h],
- [break]
- [AC_MSG_ERROR([cannot find libnl-3 headers
--------------------------------------------------------------------
-We need libnl-3 headers for building network isolator!
-
-Please install libnl3 (version 3.2.24 or higher):
-http://www.infradead.org/~tgr/libnl/
--------------------------------------------------------------------
- ])])
-
- # Check for libnl-route (both headers and libraries).
- AC_CHECK_LIB([nl-route-3], [rtnl_link_veth_add], [],
- [AC_MSG_ERROR([cannot find libnl-route-3
--------------------------------------------------------------------
-We need libnl-route-3 for building network isolator!
-
-Please install libnl3 (version 3.2.24 or higher):
-http://www.infradead.org/~tgr/libnl/
--------------------------------------------------------------------
- ])])
-
- AC_CHECK_HEADERS([netlink/route/link/veth.h libnl3/netlink/route/link/veth.h],
- [break]
- [AC_MSG_ERROR([cannot find libnl-route-3 headers
--------------------------------------------------------------------
-We need libnl-route-3 headers for building network isolator!
-
-Please install libnl3 (version 3.2.24 or higher):
-http://www.infradead.org/~tgr/libnl/
--------------------------------------------------------------------
- ])])
-
- # TODO(jieyu): Automatically detect the location where the libnl
- # headers are installed.
- LIBNL_CFLAGS=-I/usr/include/libnl3
-
- AC_SUBST([LIBNL_CFLAGS])
-
- AC_DEFINE([WITH_NETWORK_ISOLATOR])
-fi
-
-
-AM_CONDITIONAL([WITH_NETWORK_ISOLATOR],
- [test "x$with_network_isolator" = "xyes"])
-
AM_CONDITIONAL([GIT_REPO], [test -d ${srcdir}"/.git"])
http://git-wip-us.apache.org/repos/asf/mesos/blob/29433746/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 9027927..bc1603a 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -924,7 +924,7 @@ $(MESOS_EGG): python/setup.py $(srcdir)/python/src/mesos.py \
cp -pf $(srcdir)/python/src/mesos.py python/src; \
fi
@LIBS="$(LIBS)" CC="$(CC)" CXX="$(CXX)" \
- CFLAGS="$(PYTHON_CFLAGS)" CPPFLAGS="$(PYTHON_CPPFLAGS)" \
+ CPPFLAGS="$(PYTHON_CPPFLAGS)" CFLAGS="$(PYTHON_CFLAGS)" \
LDFLAGS="$(PYTHON_LDFLAGS)" \
PYTHONPATH=$(DISTRIBUTE_EGG) $(PYTHON) python/setup.py bdist_egg