You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@mesos.apache.org by Benjamin Mahler <bm...@apache.org> on 2016/03/12 03:55:02 UTC
Re: [3/3] mesos git commit: New python lib with only the executor driver.
+vinod
This breaks the build for me on OS X, it appears this line is the culprit:
EXTRA_LINK_ARGS = ['-Wl,--as-needed']
This leads to the following:
clang++ -bundle -undefined dynamic_lookup -arch x86_64 -arch i386 -Wl,-F.
-L/usr/local/opt/subversion/lib -O2 -O2 -Wno-unused-local-typedef
-std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -DGTEST_LANG_CXX11
-Qunused-arguments -I/usr/local/opt/subversion/include/subversion-1
-I/usr/include/apr-1 -I/usr/include/apr-1.0 -Qunused-arguments
build/temp.macosx-10.11-intel-2.7/src/mesos/executor/mesos_executor_driver_impl.o
build/temp.macosx-10.11-intel-2.7/src/mesos/executor/module.o
build/temp.macosx-10.11-intel-2.7/src/mesos/executor/proxy_executor.o
/Users/bmahler/git/mesos/build/src/.libs/libmesos_no_3rdparty.a
/Users/bmahler/git/mesos/build/3rdparty/libprocess/.libs/libprocess.a
/Users/bmahler/git/mesos/build/3rdparty/leveldb-1.4/libleveldb.a
/Users/bmahler/git/mesos/build/3rdparty/zookeeper-3.4.5/src/c/.libs/libzookeeper_mt.a
/Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/glog-0.3.3/.libs/libglog.a
/Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/protobuf-2.5.0/src/.libs/libprotobuf.a
-o build/lib.macosx-10.11-intel-2.7/mesos/executor/_executor.so
-Wl,--as-needed -L/usr/local/opt/subversion/lib -levent_openssl -lcrypto
-lssl -levent_pthreads -levent -lsasl2 -lsvn_delta-1 -lsvn_subr-1 -lapr-1
-lcurl -lz
ld: unknown option: --as-needed
clang: error: linker command failed with exit code 1 (use -v to see
invocation)
On Fri, Mar 11, 2016 at 1:56 PM, <vi...@apache.org> wrote:
> New python lib with only the executor driver.
>
> This patch produces a new python egg, mesos.executor, which contains only
> the
> code needed to create a MesosExecutorDriver. By doing so, the linker can
> remove
> unused code in libmesos_no_3rdparty.a, and therefor not include any
> external
> dependencies in the resulting _mesos.so.
>
> Review: https://reviews.apache.org/r/41049/
>
>
> Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c81a52ec
> Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c81a52ec
> Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c81a52ec
>
> Branch: refs/heads/master
> Commit: c81a52ec22266e1f2beb61b224c0f0d9be82521f
> Parents: 482dc14
> Author: Steve Niemitz <sn...@twitter.com>
> Authored: Fri Mar 11 16:56:13 2016 -0500
> Committer: Vinod Kone <vi...@gmail.com>
> Committed: Fri Mar 11 16:56:13 2016 -0500
>
> ----------------------------------------------------------------------
> configure.ac | 7 +-
> src/Makefile.am | 33 +-
> src/python/executor/setup.py.in | 39 +
> src/python/executor/src/mesos/__init__.py | 10 +
> .../executor/src/mesos/executor/__init__.py | 17 +
> .../executor/mesos_executor_driver_impl.cpp | 347 ++++++++
> .../executor/mesos_executor_driver_impl.hpp | 103 +++
> .../executor/src/mesos/executor/module.cpp | 91 +++
> .../src/mesos/executor/proxy_executor.cpp | 273 +++++++
> .../src/mesos/executor/proxy_executor.hpp | 64 ++
> src/python/native/ext_modules.py.in | 151 ----
> src/python/native/setup.py.in | 9 +-
> src/python/native/src/mesos/native/__init__.py | 7 +-
> .../mesos/native/mesos_executor_driver_impl.cpp | 347 --------
> .../mesos/native/mesos_executor_driver_impl.hpp | 103 ---
> .../native/mesos_scheduler_driver_impl.cpp | 782 -------------------
> .../native/mesos_scheduler_driver_impl.hpp | 134 ----
> src/python/native/src/mesos/native/module.cpp | 100 ---
> src/python/native/src/mesos/native/module.hpp | 136 ----
> .../native/src/mesos/native/proxy_executor.cpp | 273 -------
> .../native/src/mesos/native/proxy_executor.hpp | 64 --
> .../native/src/mesos/native/proxy_scheduler.cpp | 384 ---------
> .../native/src/mesos/native/proxy_scheduler.hpp | 72 --
> src/python/native_common/common.hpp | 136 ++++
> src/python/native_common/ext_modules.py.in | 154 ++++
> src/python/scheduler/setup.py.in | 39 +
> src/python/scheduler/src/mesos/__init__.py | 10 +
> .../scheduler/src/mesos/scheduler/__init__.py | 17 +
> .../scheduler/mesos_scheduler_driver_impl.cpp | 782 +++++++++++++++++++
> .../scheduler/mesos_scheduler_driver_impl.hpp | 134 ++++
> .../scheduler/src/mesos/scheduler/module.cpp | 91 +++
> .../src/mesos/scheduler/proxy_scheduler.cpp | 384 +++++++++
> .../src/mesos/scheduler/proxy_scheduler.hpp | 72 ++
> 33 files changed, 2795 insertions(+), 2570 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/configure.ac
> ----------------------------------------------------------------------
> diff --git a/configure.ac b/configure.ac
> index a20382e..8e4f035 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -1315,8 +1315,13 @@ There are two possible workarounds for this issue:
> AC_CONFIG_FILES([src/python/setup.py])
> AC_CONFIG_FILES([src/python/cli/setup.py])
> AC_CONFIG_FILES([src/python/interface/setup.py])
> - AC_CONFIG_FILES([src/python/native/ext_modules.py])
> + AC_CONFIG_FILES([src/python/native_common/ext_modules.py])
> + AC_CONFIG_FILES([src/python/executor/setup.py])
> AC_CONFIG_FILES([src/python/native/setup.py])
> + AC_CONFIG_FILES([src/python/scheduler/setup.py])
> +
> +
> AC_CONFIG_LINKS([src/python/executor/ext_modules.py:src/python/native_common/ext_modules.py])
> +
> AC_CONFIG_LINKS([src/python/scheduler/ext_modules.py:src/python/native_common/ext_modules.py])
>
> # When clang is being used, make sure that the distutils python-
> # config cflags extraction does not cause build errors (MESOS-1079).
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/Makefile.am
> ----------------------------------------------------------------------
> diff --git a/src/Makefile.am b/src/Makefile.am
> index f59ae12..8abef3b 100644
> --- a/src/Makefile.am
> +++ b/src/Makefile.am
> @@ -1467,18 +1467,23 @@ PYTHON_SOURCE =
> \
> python/interface/src/mesos/interface/__init__.py \
> python/interface/src/mesos/v1/__init__.py \
> python/interface/src/mesos/v1/interface/__init__.py \
> + python/native_common/common.hpp \
> python/native/src/mesos/__init__.py \
> python/native/src/mesos/native/__init__.py \
> - python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> \
> - python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> \
> - python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp \
> - python/native/src/mesos/native/mesos_scheduler_driver_impl.hpp \
> - python/native/src/mesos/native/module.cpp \
> - python/native/src/mesos/native/module.hpp \
> - python/native/src/mesos/native/proxy_executor.cpp \
> - python/native/src/mesos/native/proxy_executor.hpp \
> - python/native/src/mesos/native/proxy_scheduler.cpp \
> - python/native/src/mesos/native/proxy_scheduler.hpp \
> + python/executor/src/mesos/__init__.py
> \
> + python/executor/src/mesos/executor/__init__.py \
> + python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp \
> + python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp \
> + python/executor/src/mesos/executor/module.cpp
> \
> + python/executor/src/mesos/executor/proxy_executor.cpp
> \
> + python/executor/src/mesos/executor/proxy_executor.hpp
> \
> + python/scheduler/src/mesos/__init__.py \
> + python/scheduler/src/mesos/scheduler/__init__.py \
> + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp \
> + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.hpp \
> + python/scheduler/src/mesos/scheduler/module.cpp \
> + python/scheduler/src/mesos/scheduler/proxy_scheduler.cpp \
> + python/scheduler/src/mesos/scheduler/proxy_scheduler.hpp \
> python/src/mesos/__init__.py
>
> EXTRA_DIST += $(PYTHON_SOURCE)
> @@ -1530,14 +1535,18 @@ $(PYTHON_SOURCE):
> MESOS_EGGS =
> \
> python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg
> \
>
> python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg
> \
> + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> \
>
> python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg
> \
> - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> \
> + python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
>
> MESOS_WHLS =
> \
> python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl
> \
>
> python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl
> \
> + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> \
>
> python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl
> \
> - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> \
> + python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
>
> # The python source is in directories of the form: python/interface. The
> make
> # target is of the form: python/dist/mesos.interface-0.20.0-py2.7.egg. To
> build
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/setup.py.in
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/setup.py.in b/src/python/executor/
> setup.py.in
> new file mode 100644
> index 0000000..76db97f
> --- /dev/null
> +++ b/src/python/executor/setup.py.in
> @@ -0,0 +1,39 @@
> +#!/usr/bin/env python
> +
> +# Licensed to the Apache Software Foundation (ASF) under one
> +# or more contributor license agreements. See the NOTICE file
> +# distributed with this work for additional information
> +# regarding copyright ownership. The ASF licenses this file
> +# to you under the Apache License, Version 2.0 (the
> +# "License"); you may not use this file except in compliance
> +# with the License. You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +
> +import ext_modules
> +
> +config = {
> + 'name': 'mesos.executor',
> + 'version': '@PACKAGE_VERSION@',
> + 'description': 'Mesos native executor driver implementation',
> + 'author': 'Apache Mesos',
> + 'author_email': 'dev@mesos.apache.org',
> + 'url': 'http://pypi.python.org/pypi/mesos.executor',
> + 'namespace_packages': [ 'mesos' ],
> + 'packages': [ 'mesos', 'mesos.executor' ],
> + 'package_dir': { '': 'src' },
> + 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ],
> + 'license': 'Apache 2.0',
> + 'keywords': 'mesos',
> + 'classifiers': [ ],
> + 'ext_modules': [ ext_modules.executor_module ]
> +}
> +
> +from setuptools import setup
> +setup(**config)
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/__init__.py
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/__init__.py
> b/src/python/executor/src/mesos/__init__.py
> new file mode 100644
> index 0000000..3fcba01
> --- /dev/null
> +++ b/src/python/executor/src/mesos/__init__.py
> @@ -0,0 +1,10 @@
> +# See
> http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
> +# Because python does not normally allow the contents of a package to be
> +# retrieved from more than one location, this code snippet ensures that
> the
> +# namespace package machinery is operating and that the current package is
> +# registered as a namespace package.
> +try:
> + __import__('pkg_resources').declare_namespace(__name__)
> +except ImportError:
> + from pkgutil import extend_path
> + __path__ = extend_path(__path__, __name__)
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/__init__.py
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/__init__.py
> b/src/python/executor/src/mesos/executor/__init__.py
> new file mode 100644
> index 0000000..6808019
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/__init__.py
> @@ -0,0 +1,17 @@
> +# Licensed to the Apache Software Foundation (ASF) under one
> +# or more contributor license agreements. See the NOTICE file
> +# distributed with this work for additional information
> +# regarding copyright ownership. The ASF licenses this file
> +# to you under the Apache License, Version 2.0 (the
> +# "License"); you may not use this file except in compliance
> +# with the License. You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +
> +from ._executor import MesosExecutorDriverImpl as MesosExecutorDriver
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> ----------------------------------------------------------------------
> diff --git
> a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> new file mode 100644
> index 0000000..4189411
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> @@ -0,0 +1,347 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <string>
> +
> +#include "common.hpp"
> +#include "mesos_executor_driver_impl.hpp"
> +#include "proxy_executor.hpp"
> +
> +using namespace mesos;
> +using namespace mesos::python;
> +
> +using std::cerr;
> +using std::endl;
> +using std::string;
> +using std::vector;
> +using std::map;
> +
> +
> +namespace mesos { namespace python {
> +
> +/**
> + * Python type object for MesosExecutorDriverImpl.
> + */
> +PyTypeObject MesosExecutorDriverImplType = {
> + PyObject_HEAD_INIT(NULL)
> + 0, /* ob_size */
> + "_mesos.MesosExecutorDriverImpl", /* tp_name */
> + sizeof(MesosExecutorDriverImpl), /* tp_basicsize */
> + 0, /* tp_itemsize */
> + (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */
> + 0, /* tp_print */
> + 0, /* tp_getattr */
> + 0, /* tp_setattr */
> + 0, /* tp_compare */
> + 0, /* tp_repr */
> + 0, /* tp_as_number */
> + 0, /* tp_as_sequence */
> + 0, /* tp_as_mapping */
> + 0, /* tp_hash */
> + 0, /* tp_call */
> + 0, /* tp_str */
> + 0, /* tp_getattro */
> + 0, /* tp_setattro */
> + 0, /* tp_as_buffer */
> + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> + "Private MesosExecutorDriver implementation", /* tp_doc */
> + (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */
> + (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */
> + 0, /* tp_richcompare */
> + 0, /* tp_weaklistoffset */
> + 0, /* tp_iter */
> + 0, /* tp_iternext */
> + MesosExecutorDriverImpl_methods, /* tp_methods */
> + 0, /* tp_members */
> + 0, /* tp_getset */
> + 0, /* tp_base */
> + 0, /* tp_dict */
> + 0, /* tp_descr_get */
> + 0, /* tp_descr_set */
> + 0, /* tp_dictoffset */
> + (initproc) MesosExecutorDriverImpl_init, /* tp_init */
> + 0, /* tp_alloc */
> + MesosExecutorDriverImpl_new, /* tp_new */
> +};
> +
> +
> +/**
> + * List of Python methods in MesosExecutorDriverImpl.
> + */
> +PyMethodDef MesosExecutorDriverImpl_methods[] = {
> + { "start",
> + (PyCFunction) MesosExecutorDriverImpl_start,
> + METH_NOARGS,
> + "Start the driver to connect to Mesos"
> + },
> + { "stop",
> + (PyCFunction) MesosExecutorDriverImpl_stop,
> + METH_NOARGS,
> + "Stop the driver, disconnecting from Mesos"
> + },
> + { "abort",
> + (PyCFunction) MesosExecutorDriverImpl_abort,
> + METH_NOARGS,
> + "Abort the driver, disallowing calls from and to the driver"
> + },
> + { "join",
> + (PyCFunction) MesosExecutorDriverImpl_join,
> + METH_NOARGS,
> + "Wait for a running driver to disconnect from Mesos"
> + },
> + { "run",
> + (PyCFunction) MesosExecutorDriverImpl_run,
> + METH_NOARGS,
> + "Start a driver and run it, returning when it disconnects from Mesos"
> + },
> + { "sendStatusUpdate",
> + (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate,
> + METH_VARARGS,
> + "Send a status update for a task"
> + },
> + { "sendFrameworkMessage",
> + (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage,
> + METH_VARARGS,
> + "Send a FrameworkMessage to a slave"
> + },
> + { NULL } /* Sentinel */
> +};
> +
> +
> +/**
> + * Create, but don't initialize, a new MesosExecutorDriverImpl
> + * (called by Python before init method).
> + */
> +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> + PyObject *args,
> + PyObject *kwds)
> +{
> + MesosExecutorDriverImpl *self;
> + self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0);
> + if (self != NULL) {
> + self->driver = NULL;
> + self->proxyExecutor = NULL;
> + self->pythonExecutor = NULL;
> + }
> + return (PyObject*) self;
> +}
> +
> +
> +/**
> + * Initialize a MesosExecutorDriverImpl with constructor arguments.
> + */
> +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> + PyObject *args,
> + PyObject *kwds)
> +{
> + PyObject *pythonExecutor = NULL;
> +
> + if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) {
> + return -1;
> + }
> +
> + if (pythonExecutor != NULL) {
> + PyObject* tmp = self->pythonExecutor;
> + Py_INCREF(pythonExecutor);
> + self->pythonExecutor = pythonExecutor;
> + Py_XDECREF(tmp);
> + }
> +
> + if (self->driver != NULL) {
> + delete self->driver;
> + self->driver = NULL;
> + }
> +
> + if (self->proxyExecutor != NULL) {
> + delete self->proxyExecutor;
> + self->proxyExecutor = NULL;
> + }
> +
> + self->proxyExecutor = new ProxyExecutor(self);
> + self->driver = new MesosExecutorDriver(self->proxyExecutor);
> +
> + return 0;
> +}
> +
> +
> +/**
> + * Free a MesosExecutorDriverImpl.
> + */
> +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver != NULL) {
> + // We need to wrap the driver destructor in an "allow threads"
> + // macro since the MesosExecutorDriver destructor waits for the
> + // ExecutorProcess to terminate and there might be a thread that
> + // is trying to acquire the GIL to call through the
> + // ProxyExecutor. It will only be after this thread executes that
> + // the ExecutorProcess might actually get a terminate.
> + Py_BEGIN_ALLOW_THREADS
> + delete self->driver;
> + Py_END_ALLOW_THREADS
> + self->driver = NULL;
> + }
> +
> + if (self->proxyExecutor != NULL) {
> + delete self->proxyExecutor;
> + self->proxyExecutor = NULL;
> + }
> +
> + MesosExecutorDriverImpl_clear(self);
> + self->ob_type->tp_free((PyObject*) self);
> +}
> +
> +
> +/**
> + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> + * See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> + visitproc visit,
> + void* arg)
> +{
> + Py_VISIT(self->pythonExecutor);
> + return 0;
> +}
> +
> +
> +/**
> + * Clear fields of a MesosExecutorDriverImpl that can participate in
> + * GC cycles. See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self)
> +{
> + Py_CLEAR(self->pythonExecutor);
> + return 0;
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> + return NULL;
> + }
> +
> + Status status = self->driver->start();
> + return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> + return NULL;
> + }
> +
> + Status status = self->driver->stop();
> + return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> + return NULL;
> + }
> +
> + Status status = self->driver->abort();
> + return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> + return NULL;
> + }
> +
> + Status status;
> + Py_BEGIN_ALLOW_THREADS
> + status = self->driver->join();
> + Py_END_ALLOW_THREADS
> + return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> + return NULL;
> + }
> +
> + Status status;
> + Py_BEGIN_ALLOW_THREADS
> + status = self->driver->run();
> + Py_END_ALLOW_THREADS
> + return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> + MesosExecutorDriverImpl* self,
> + PyObject* args)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> + return NULL;
> + }
> +
> + PyObject* statusObj = NULL;
> + TaskStatus taskStatus;
> + if (!PyArg_ParseTuple(args, "O", &statusObj)) {
> + return NULL;
> + }
> + if (!readPythonProtobuf(statusObj, &taskStatus)) {
> + PyErr_Format(PyExc_Exception,
> + "Could not deserialize Python TaskStatus");
> + return NULL;
> + }
> +
> + Status status = self->driver->sendStatusUpdate(taskStatus);
> + return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> + MesosExecutorDriverImpl* self,
> + PyObject* args)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> + return NULL;
> + }
> +
> + const char* data;
> + int length;
> + if (!PyArg_ParseTuple(args, "s#", &data, &length)) {
> + return NULL;
> + }
> +
> + Status status = self->driver->sendFrameworkMessage(string(data,
> length));
> + return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> +}
> +
> +} // namespace python {
> +} // namespace mesos {
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> ----------------------------------------------------------------------
> diff --git
> a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> new file mode 100644
> index 0000000..6e672f8
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> @@ -0,0 +1,103 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP
> +#define MESOS_EXECUTOR_DRIVER_IMPL_HPP
> +
> +#include <mesos/executor.hpp>
> +
> +
> +namespace mesos { namespace python {
> +
> +class ProxyExecutor;
> +
> +/**
> + * Python object structure for MesosExecutorDriverImpl objects.
> + */
> +struct MesosExecutorDriverImpl {
> + PyObject_HEAD
> + /* Type-specific fields go here. */
> + MesosExecutorDriver* driver;
> + ProxyExecutor* proxyExecutor;
> + PyObject* pythonExecutor;
> +};
> +
> +/**
> + * Python type object for MesosExecutorDriverImpl.
> + */
> +extern PyTypeObject MesosExecutorDriverImplType;
> +
> +/**
> + * List of Python methods in MesosExecutorDriverImpl.
> + */
> +extern PyMethodDef MesosExecutorDriverImpl_methods[];
> +
> +/**
> + * Create, but don't initialize, a new MesosExecutorDriverImpl
> + * (called by Python before init method).
> + */
> +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> + PyObject *args,
> + PyObject *kwds);
> +
> +/**
> + * Initialize a MesosExecutorDriverImpl with constructor arguments.
> + */
> +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> + PyObject *args,
> + PyObject *kwds);
> +
> +/**
> + * Free a MesosExecutorDriverImpl.
> + */
> +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self);
> +
> +/**
> + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> + * See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> + visitproc visit,
> + void* arg);
> +/**
> + * Clear fields of a MesosExecutorDriverImpl that can participate in
> + * GC cycles. See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self);
> +
> +// MesosExecutorDriverImpl methods.
> +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> + MesosExecutorDriverImpl* self,
> + PyObject* args);
> +
> +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> + MesosExecutorDriverImpl* self,
> + PyObject* args);
> +
> +} // namespace python {
> +} // namespace mesos {
> +
> +#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/module.cpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/module.cpp
> b/src/python/executor/src/mesos/executor/module.cpp
> new file mode 100644
> index 0000000..f8c6382
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/module.cpp
> @@ -0,0 +1,91 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +/**
> + * This file defines the _mesos.so binary module used by the Mesos Python
> API.
> + * This module contains private implementations of MesosSchedulerDriver
> and
> + * MesosExecutorDriver as Python types that get called from the public
> module
> + * called mesos (in <root>/src/python/src/mesos.py). This design was
> chosen
> + * so that most of the API (e.g. the Scheduler and Executor interfaces)
> can
> + * be written in Python, and only the parts that need to call into C++ are
> + * in C++. Note that the mesos module also contains public classes called
> + * MesosSchedulerDriver and MesosExecutorDriver. These call into the
> private
> + * _mesos.MesosSchedulerDriverImpl and _mesos.MesosExecutorDriverImpl.
> + */
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <iostream>
> +
> +#include <mesos/executor.hpp>
> +
> +#include "common.hpp"
> +#include "mesos_executor_driver_impl.hpp"
> +#include "proxy_executor.hpp"
> +
> +using namespace mesos;
> +using namespace mesos::python;
> +
> +using std::map;
> +using std::string;
> +using std::vector;
> +
> +
> +/**
> + * The Python module object for mesos_pb2 (which contains the protobuf
> + * classes generated for Python).
> + */
> +PyObject* mesos::python::mesos_pb2 = NULL;
> +
> +
> +namespace {
> +
> +/**
> + * Method list for our Python module.
> + */
> +PyMethodDef MODULE_METHODS[] = {
> + {NULL, NULL, 0, NULL} /* Sentinel */
> +};
> +
> +} // namespace {
> +
> +
> +/**
> + * Entry point called by Python to initialize our module.
> + */
> +PyMODINIT_FUNC init_executor()
> +{
> + // Ensure that the interpreter's threading support is enabled.
> + PyEval_InitThreads();
> +
> + // Import the mesos_pb2 module (on which we depend for protobuf classes)
> + mesos_pb2 = PyImport_ImportModule("mesos.interface.mesos_pb2");
> + if (mesos_pb2 == NULL)
> + return;
> +
> + // Initialize our Python types.
> + if (PyType_Ready(&MesosExecutorDriverImplType) < 0)
> + return;
> +
> + // Create the _mesos module and add our types to it.
> + PyObject* module = Py_InitModule("_executor", MODULE_METHODS);
> + Py_INCREF(&MesosExecutorDriverImplType);
> + PyModule_AddObject(module,
> + "MesosExecutorDriverImpl",
> + (PyObject*) &MesosExecutorDriverImplType);
> +}
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.cpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/proxy_executor.cpp
> b/src/python/executor/src/mesos/executor/proxy_executor.cpp
> new file mode 100644
> index 0000000..da1a49b
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/proxy_executor.cpp
> @@ -0,0 +1,273 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <iostream>
> +
> +#include "common.hpp"
> +#include "mesos_executor_driver_impl.hpp"
> +#include "proxy_executor.hpp"
> +
> +using namespace mesos;
> +
> +using std::cerr;
> +using std::endl;
> +using std::map;
> +using std::string;
> +using std::vector;
> +
> +namespace mesos {
> +namespace python {
> +
> +void ProxyExecutor::registered(ExecutorDriver* driver,
> + const ExecutorInfo& executorInfo,
> + const FrameworkInfo& frameworkInfo,
> + const SlaveInfo& slaveInfo)
> +{
> + InterpreterLock lock;
> +
> + PyObject* executorInfoObj = NULL;
> + PyObject* frameworkInfoObj = NULL;
> + PyObject* slaveInfoObj = NULL;
> + PyObject* res = NULL;
> +
> + executorInfoObj = createPythonProtobuf(executorInfo, "ExecutorInfo");
> + frameworkInfoObj = createPythonProtobuf(frameworkInfo, "FrameworkInfo");
> + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo");
> +
> + if (executorInfoObj == NULL ||
> + frameworkInfoObj == NULL ||
> + slaveInfoObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "registered",
> + (char*) "OOOO",
> + impl,
> + executorInfoObj,
> + frameworkInfoObj,
> + slaveInfoObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor registered" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(executorInfoObj);
> + Py_XDECREF(frameworkInfoObj);
> + Py_XDECREF(slaveInfoObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::reregistered(ExecutorDriver* driver,
> + const SlaveInfo& slaveInfo)
> +{
> + InterpreterLock lock;
> +
> + PyObject* slaveInfoObj = NULL;
> + PyObject* res = NULL;
> +
> + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo");
> +
> + if (slaveInfoObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "reregistered",
> + (char*) "OO",
> + impl,
> + slaveInfoObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor re-registered" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(slaveInfoObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::disconnected(ExecutorDriver* driver)
> +{
> + InterpreterLock lock;
> + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "disconnected",
> + (char*) "O",
> + impl);
> + if (res == NULL) {
> + cerr << "Failed to call executor's disconnected" << endl;
> + goto cleanup;
> + }
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::launchTask(ExecutorDriver* driver,
> + const TaskInfo& task)
> +{
> + InterpreterLock lock;
> +
> + PyObject* taskObj = NULL;
> + PyObject* res = NULL;
> +
> + taskObj = createPythonProtobuf(task, "TaskInfo");
> + if (taskObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "launchTask",
> + (char*) "OO",
> + impl,
> + taskObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor's launchTask" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(taskObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::killTask(ExecutorDriver* driver,
> + const TaskID& taskId)
> +{
> + InterpreterLock lock;
> +
> + PyObject* taskIdObj = NULL;
> + PyObject* res = NULL;
> +
> + taskIdObj = createPythonProtobuf(taskId, "TaskID");
> + if (taskIdObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "killTask",
> + (char*) "OO",
> + impl,
> + taskIdObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor's killTask" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(taskIdObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::frameworkMessage(ExecutorDriver* driver,
> + const string& data)
> +{
> + InterpreterLock lock;
> +
> + PyObject* res = NULL;
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "frameworkMessage",
> + (char*) "Os#",
> + impl,
> + data.data(),
> + data.length());
> + if (res == NULL) {
> + cerr << "Failed to call executor's frameworkMessage" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::shutdown(ExecutorDriver* driver)
> +{
> + InterpreterLock lock;
> + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "shutdown",
> + (char*) "O",
> + impl);
> + if (res == NULL) {
> + cerr << "Failed to call executor's shutdown" << endl;
> + goto cleanup;
> + }
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::error(ExecutorDriver* driver, const string& message)
> +{
> + InterpreterLock lock;
> + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "error",
> + (char*) "Os#",
> + impl,
> + message.data(),
> + message.length());
> + if (res == NULL) {
> + cerr << "Failed to call executor's error" << endl;
> + goto cleanup;
> + }
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + // No need for driver.stop(); it should stop itself.
> + }
> + Py_XDECREF(res);
> +}
> +
> +} // namespace python {
> +} // namespace mesos {
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.hpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/proxy_executor.hpp
> b/src/python/executor/src/mesos/executor/proxy_executor.hpp
> new file mode 100644
> index 0000000..23d64fd
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/proxy_executor.hpp
> @@ -0,0 +1,64 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +#ifndef PROXY_EXECUTOR_HPP
> +#define PROXY_EXECUTOR_HPP
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <string>
> +#include <vector>
> +
> +#include <mesos/executor.hpp>
> +
> +namespace mesos {
> +namespace python {
> +
> +struct MesosExecutorDriverImpl;
> +
> +/**
> + * Proxy Executor implementation that will call into Python.
> + */
> +class ProxyExecutor : public Executor
> +{
> +public:
> + explicit ProxyExecutor(MesosExecutorDriverImpl *_impl) : impl(_impl) {}
> +
> + virtual ~ProxyExecutor() {}
> +
> + virtual void registered(ExecutorDriver* driver,
> + const ExecutorInfo& executorInfo,
> + const FrameworkInfo& frameworkInfo,
> + const SlaveInfo& slaveInfo);
> + virtual void reregistered(ExecutorDriver* driver, const SlaveInfo&
> slaveInfo);
> + virtual void disconnected(ExecutorDriver* driver);
> + virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task);
> + virtual void killTask(ExecutorDriver* driver, const TaskID& taskId);
> + virtual void frameworkMessage(ExecutorDriver* driver,
> + const std::string& data);
> + virtual void shutdown(ExecutorDriver* driver);
> + virtual void error(ExecutorDriver* driver, const std::string& message);
> +
> +private:
> + MesosExecutorDriverImpl *impl;
> +};
> +
> +} // namespace python {
> +} // namespace mesos {
> +
> +#endif // PROXY_EXECUTOR_HPP
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/ext_modules.py.in
> ----------------------------------------------------------------------
> diff --git a/src/python/native/ext_modules.py.in b/src/python/native/
> ext_modules.py.in
> deleted file mode 100644
> index eb93864..0000000
> --- a/src/python/native/ext_modules.py.in
> +++ /dev/null
> @@ -1,151 +0,0 @@
> -# Licensed to the Apache Software Foundation (ASF) under one
> -# or more contributor license agreements. See the NOTICE file
> -# distributed with this work for additional information
> -# regarding copyright ownership. The ASF licenses this file
> -# to you under the Apache License, Version 2.0 (the
> -# "License"); you may not use this file except in compliance
> -# with the License. You may obtain a copy of the License at
> -#
> -# http://www.apache.org/licenses/LICENSE-2.0
> -#
> -# Unless required by applicable law or agreed to in writing, software
> -# distributed under the License is distributed on an "AS IS" BASIS,
> -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> -# See the License for the specific language governing permissions and
> -# limitations under the License.
> -
> -import errno
> -import glob
> -import os
> -import shutil
> -
> -from setuptools import Extension
> -
> -abs_top_srcdir = '@abs_top_srcdir@'
> -abs_top_builddir = '@abs_top_builddir@'
> -
> -src_python_native = os.path.join(
> - 'src', 'python', 'native', 'src', 'mesos', 'native')
> -
> -leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> -zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.5', 'src', 'c')
> -libprocess = os.path.join('3rdparty', 'libprocess')
> -
> -# Even though a statically compiled libprocess should include glog,
> -# libev, gperftools, and protobuf before installation this isn't the
> -# case, so while a libtool managed build will correctly pull in these
> -# libraries when building the final result, we need to explicitly
> -# include them here (or more precisely, down where we actually include
> -# libev.a and libprofiler.a).
> -glog = os.path.join(libprocess, '3rdparty', 'glog-0.3.3')
> -gperftools = os.path.join(libprocess, '3rdparty', 'gperftools-2.0')
> -protobuf = os.path.join(libprocess, '3rdparty', 'protobuf-2.5.0')
> -
> -# Build the list of source files. Note that each source must be
> -# relative to our current directory (where this script lives).
> -SOURCES = [
> - os.path.join('src', 'mesos', 'native', file)
> - for file in os.listdir(os.path.join(abs_top_srcdir,
> src_python_native))
> - if file.endswith('.cpp')
> -]
> -
> -INCLUDE_DIRS = [
> - os.path.join(abs_top_srcdir, 'include'),
> - os.path.join(abs_top_builddir, 'include'),
> - # Needed for the *.pb.h protobuf includes.
> - os.path.join(abs_top_builddir, 'include', 'mesos'),
> - os.path.join(abs_top_builddir, 'src'),
> - os.path.join(abs_top_builddir, src_python_native),
> - os.path.join(abs_top_builddir, protobuf, 'src'),
> -]
> -
> -LIBRARY_DIRS = []
> -
> -EXTRA_OBJECTS = [
> - os.path.join(abs_top_builddir, 'src', '.libs',
> 'libmesos_no_3rdparty.a'),
> - os.path.join(abs_top_builddir, libprocess, '.libs', 'libprocess.a')
> -]
> -
> -# For leveldb, we need to check for the presence of libleveldb.a, since
> -# it is possible to disable leveldb inside mesos.
> -libglog = os.path.join(abs_top_builddir, glog, '.libs', 'libglog.a')
> -libleveldb = os.path.join(abs_top_builddir, leveldb, 'libleveldb.a')
> -libzookeeper = os.path.join(
> - abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> -libprotobuf = os.path.join(
> - abs_top_builddir, protobuf, 'src', '.libs', 'libprotobuf.a')
> -
> -if os.path.exists(libleveldb):
> - EXTRA_OBJECTS.append(libleveldb)
> -else:
> - EXTRA_OBJECTS.append('-lleveldb')
> -
> -if os.path.exists(libzookeeper):
> - EXTRA_OBJECTS.append(libzookeeper)
> -else:
> - EXTRA_OBJECTS.append('-lzookeeper_mt')
> -
> -if os.path.exists(libglog):
> - EXTRA_OBJECTS.append(libglog)
> -else:
> - EXTRA_OBJECTS.append('-lglog')
> -
> -if os.path.exists(libprotobuf):
> - EXTRA_OBJECTS.append(libprotobuf)
> -else:
> - EXTRA_OBJECTS.append('-lprotobuf')
> -
> -
> -# libev is a special case because it needs to be enabled only when
> -# libevent *is not* enabled through the top level ./configure.
> -#
> -# TODO(hartem): this entire block MUST be removed once libev is deprecated
> -# in favor of libevent.
> -if '@ENABLE_LIBEVENT_TRUE@' == '#':
> - libev = os.path.join(libprocess, '3rdparty', 'libev-4.15')
> - libev = os.path.join(abs_top_builddir, libev, '.libs', 'libev.a')
> -
> - if os.path.exists(libev):
> - EXTRA_OBJECTS.append(libev)
> - else:
> - EXTRA_OBJECTS.append('-lev')
> -
> -
> -# For gperftools, we need to check for the presence of libprofiler.a,
> since
> -# it is possible to disable perftools inside libprocess.
> -libprofiler = os.path.join(
> - abs_top_builddir, gperftools, '.libs', 'libprofiler.a')
> -
> -if os.path.exists(libprofiler):
> - EXTRA_OBJECTS.append(libprofiler)
> -
> -EXTRA_LINK_ARGS = []
> -
> -# Add any flags from LDFLAGS.
> -if 'LDFLAGS' in os.environ:
> - for flag in os.environ['LDFLAGS'].split():
> - EXTRA_LINK_ARGS.append(flag)
> -
> -# Add any libraries from LIBS.
> -if 'LIBS' in os.environ:
> - for library in os.environ['LIBS'].split():
> - EXTRA_LINK_ARGS.append(library)
> -
> -DEPENDS = [
> - os.path.join(abs_top_srcdir, 'src', 'python', source)
> - for source in SOURCES
> -]
> -
> -# Note that we add EXTRA_OBJECTS to our dependency list to make sure
> -# that we rebuild this module when one of them changes (e.g.,
> -# libprocess).
> -mesos_module = \
> - Extension('mesos.native._mesos',
> - sources = SOURCES,
> - include_dirs = INCLUDE_DIRS,
> - library_dirs = LIBRARY_DIRS,
> - extra_objects = EXTRA_OBJECTS,
> - extra_link_args = EXTRA_LINK_ARGS,
> - depends = EXTRA_OBJECTS,
> - language = 'c++',
> - )
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/setup.py.in
> ----------------------------------------------------------------------
> diff --git a/src/python/native/setup.py.in b/src/python/native/setup.py.in
> index 49ed612..10a5002 100644
> --- a/src/python/native/setup.py.in
> +++ b/src/python/native/setup.py.in
> @@ -16,8 +16,6 @@
> # See the License for the specific language governing permissions and
> # limitations under the License.
>
> -import ext_modules
> -
> config = {
> 'name': 'mesos.native',
> 'version': '@PACKAGE_VERSION@',
> @@ -28,13 +26,12 @@ config = {
> 'namespace_packages': [ 'mesos' ],
> 'packages': [ 'mesos', 'mesos.native' ],
> 'package_dir': { '': 'src' },
> - 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ],
> + 'install_requires': [ 'mesos.executor == @PACKAGE_VERSION@',
> + 'mesos.scheduler == @PACKAGE_VERSION@'],
> 'license': 'Apache 2.0',
> 'keywords': 'mesos',
> - 'classifiers': [ ],
> - 'ext_modules': [ ext_modules.mesos_module ]
> + 'classifiers': [ ]
> }
>
> from setuptools import setup
> -
> setup(**config)
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/__init__.py
> ----------------------------------------------------------------------
> diff --git a/src/python/native/src/mesos/native/__init__.py
> b/src/python/native/src/mesos/native/__init__.py
> index 226f943..b581537 100644
> --- a/src/python/native/src/mesos/native/__init__.py
> +++ b/src/python/native/src/mesos/native/__init__.py
> @@ -14,8 +14,5 @@
> # See the License for the specific language governing permissions and
> # limitations under the License.
>
> -from ._mesos import MesosExecutorDriverImpl
> -from ._mesos import MesosSchedulerDriverImpl
> -
> -MesosExecutorDriver = MesosExecutorDriverImpl
> -MesosSchedulerDriver = MesosSchedulerDriverImpl
> +from mesos.executor import MesosExecutorDriver
> +from mesos.scheduler import MesosSchedulerDriver
> \ No newline at end of file
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> ----------------------------------------------------------------------
> diff --git
> a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> b/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> deleted file mode 100644
> index 7838a07..0000000
> --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> +++ /dev/null
> @@ -1,347 +0,0 @@
> -// Licensed to the Apache Software Foundation (ASF) under one
> -// or more contributor license agreements. See the NOTICE file
> -// distributed with this work for additional information
> -// regarding copyright ownership. The ASF licenses this file
> -// to you under the Apache License, Version 2.0 (the
> -// "License"); you may not use this file except in compliance
> -// with the License. You may obtain a copy of the License at
> -//
> -// http://www.apache.org/licenses/LICENSE-2.0
> -//
> -// Unless required by applicable law or agreed to in writing, software
> -// distributed under the License is distributed on an "AS IS" BASIS,
> -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> -// See the License for the specific language governing permissions and
> -// limitations under the License.
> -
> -// Python.h must be included before standard headers.
> -// See: http://docs.python.org/2/c-api/intro.html#include-files
> -#include <Python.h>
> -
> -#include <string>
> -
> -#include "mesos_executor_driver_impl.hpp"
> -#include "module.hpp"
> -#include "proxy_executor.hpp"
> -
> -using namespace mesos;
> -using namespace mesos::python;
> -
> -using std::cerr;
> -using std::endl;
> -using std::string;
> -using std::vector;
> -using std::map;
> -
> -
> -namespace mesos { namespace python {
> -
> -/**
> - * Python type object for MesosExecutorDriverImpl.
> - */
> -PyTypeObject MesosExecutorDriverImplType = {
> - PyObject_HEAD_INIT(NULL)
> - 0, /* ob_size */
> - "_mesos.MesosExecutorDriverImpl", /* tp_name */
> - sizeof(MesosExecutorDriverImpl), /* tp_basicsize */
> - 0, /* tp_itemsize */
> - (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */
> - 0, /* tp_print */
> - 0, /* tp_getattr */
> - 0, /* tp_setattr */
> - 0, /* tp_compare */
> - 0, /* tp_repr */
> - 0, /* tp_as_number */
> - 0, /* tp_as_sequence */
> - 0, /* tp_as_mapping */
> - 0, /* tp_hash */
> - 0, /* tp_call */
> - 0, /* tp_str */
> - 0, /* tp_getattro */
> - 0, /* tp_setattro */
> - 0, /* tp_as_buffer */
> - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> - "Private MesosExecutorDriver implementation", /* tp_doc */
> - (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */
> - (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */
> - 0, /* tp_richcompare */
> - 0, /* tp_weaklistoffset */
> - 0, /* tp_iter */
> - 0, /* tp_iternext */
> - MesosExecutorDriverImpl_methods, /* tp_methods */
> - 0, /* tp_members */
> - 0, /* tp_getset */
> - 0, /* tp_base */
> - 0, /* tp_dict */
> - 0, /* tp_descr_get */
> - 0, /* tp_descr_set */
> - 0, /* tp_dictoffset */
> - (initproc) MesosExecutorDriverImpl_init, /* tp_init */
> - 0, /* tp_alloc */
> - MesosExecutorDriverImpl_new, /* tp_new */
> -};
> -
> -
> -/**
> - * List of Python methods in MesosExecutorDriverImpl.
> - */
> -PyMethodDef MesosExecutorDriverImpl_methods[] = {
> - { "start",
> - (PyCFunction) MesosExecutorDriverImpl_start,
> - METH_NOARGS,
> - "Start the driver to connect to Mesos"
> - },
> - { "stop",
> - (PyCFunction) MesosExecutorDriverImpl_stop,
> - METH_NOARGS,
> - "Stop the driver, disconnecting from Mesos"
> - },
> - { "abort",
> - (PyCFunction) MesosExecutorDriverImpl_abort,
> - METH_NOARGS,
> - "Abort the driver, disallowing calls from and to the driver"
> - },
> - { "join",
> - (PyCFunction) MesosExecutorDriverImpl_join,
> - METH_NOARGS,
> - "Wait for a running driver to disconnect from Mesos"
> - },
> - { "run",
> - (PyCFunction) MesosExecutorDriverImpl_run,
> - METH_NOARGS,
> - "Start a driver and run it, returning when it disconnects from Mesos"
> - },
> - { "sendStatusUpdate",
> - (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate,
> - METH_VARARGS,
> - "Send a status update for a task"
> - },
> - { "sendFrameworkMessage",
> - (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage,
> - METH_VARARGS,
> - "Send a FrameworkMessage to a slave"
> - },
> - { NULL } /* Sentinel */
> -};
> -
> -
> -/**
> - * Create, but don't initialize, a new MesosExecutorDriverImpl
> - * (called by Python before init method).
> - */
> -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> - PyObject *args,
> - PyObject *kwds)
> -{
> - MesosExecutorDriverImpl *self;
> - self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0);
> - if (self != NULL) {
> - self->driver = NULL;
> - self->proxyExecutor = NULL;
> - self->pythonExecutor = NULL;
> - }
> - return (PyObject*) self;
> -}
> -
> -
> -/**
> - * Initialize a MesosExecutorDriverImpl with constructor arguments.
> - */
> -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> - PyObject *args,
> - PyObject *kwds)
> -{
> - PyObject *pythonExecutor = NULL;
> -
> - if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) {
> - return -1;
> - }
> -
> - if (pythonExecutor != NULL) {
> - PyObject* tmp = self->pythonExecutor;
> - Py_INCREF(pythonExecutor);
> - self->pythonExecutor = pythonExecutor;
> - Py_XDECREF(tmp);
> - }
> -
> - if (self->driver != NULL) {
> - delete self->driver;
> - self->driver = NULL;
> - }
> -
> - if (self->proxyExecutor != NULL) {
> - delete self->proxyExecutor;
> - self->proxyExecutor = NULL;
> - }
> -
> - self->proxyExecutor = new ProxyExecutor(self);
> - self->driver = new MesosExecutorDriver(self->proxyExecutor);
> -
> - return 0;
> -}
> -
> -
> -/**
> - * Free a MesosExecutorDriverImpl.
> - */
> -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver != NULL) {
> - // We need to wrap the driver destructor in an "allow threads"
> - // macro since the MesosExecutorDriver destructor waits for the
> - // ExecutorProcess to terminate and there might be a thread that
> - // is trying to acquire the GIL to call through the
> - // ProxyExecutor. It will only be after this thread executes that
> - // the ExecutorProcess might actually get a terminate.
> - Py_BEGIN_ALLOW_THREADS
> - delete self->driver;
> - Py_END_ALLOW_THREADS
> - self->driver = NULL;
> - }
> -
> - if (self->proxyExecutor != NULL) {
> - delete self->proxyExecutor;
> - self->proxyExecutor = NULL;
> - }
> -
> - MesosExecutorDriverImpl_clear(self);
> - self->ob_type->tp_free((PyObject*) self);
> -}
> -
> -
> -/**
> - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> - * See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> - visitproc visit,
> - void* arg)
> -{
> - Py_VISIT(self->pythonExecutor);
> - return 0;
> -}
> -
> -
> -/**
> - * Clear fields of a MesosExecutorDriverImpl that can participate in
> - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self)
> -{
> - Py_CLEAR(self->pythonExecutor);
> - return 0;
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->start();
> - return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->stop();
> - return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->abort();
> - return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->join();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->run();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> - MesosExecutorDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* statusObj = NULL;
> - TaskStatus taskStatus;
> - if (!PyArg_ParseTuple(args, "O", &statusObj)) {
> - return NULL;
> - }
> - if (!readPythonProtobuf(statusObj, &taskStatus)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python TaskStatus");
> - return NULL;
> - }
> -
> - Status status = self->driver->sendStatusUpdate(taskStatus);
> - return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> - MesosExecutorDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - const char* data;
> - int length;
> - if (!PyArg_ParseTuple(args, "s#", &data, &length)) {
> - return NULL;
> - }
> -
> - Status status = self->driver->sendFrameworkMessage(string(data,
> length));
> - return PyInt_FromLong(status); // Sets an exception if creating the int
> fails.
> -}
> -
> -} // namespace python {
> -} // namespace mesos {
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> ----------------------------------------------------------------------
> diff --git
> a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> b/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> deleted file mode 100644
> index 6e672f8..0000000
> --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> +++ /dev/null
> @@ -1,103 +0,0 @@
> -// Licensed to the Apache Software Foundation (ASF) under one
> -// or more contributor license agreements. See the NOTICE file
> -// distributed with this work for additional information
> -// regarding copyright ownership. The ASF licenses this file
> -// to you under the Apache License, Version 2.0 (the
> -// "License"); you may not use this file except in compliance
> -// with the License. You may obtain a copy of the License at
> -//
> -// http://www.apache.org/licenses/LICENSE-2.0
> -//
> -// Unless required by applicable law or agreed to in writing, software
> -// distributed under the License is distributed on an "AS IS" BASIS,
> -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> -// See the License for the specific language governing permissions and
> -// limitations under the License.
> -
> -#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP
> -#define MESOS_EXECUTOR_DRIVER_IMPL_HPP
> -
> -#include <mesos/executor.hpp>
> -
> -
> -namespace mesos { namespace python {
> -
> -class ProxyExecutor;
> -
> -/**
> - * Python object structure for MesosExecutorDriverImpl objects.
> - */
> -struct MesosExecutorDriverImpl {
> - PyObject_HEAD
> - /* Type-specific fields go here. */
> - MesosExecutorDriver* driver;
> - ProxyExecutor* proxyExecutor;
> - PyObject* pythonExecutor;
> -};
> -
> -/**
> - * Python type object for MesosExecutorDriverImpl.
> - */
> -extern PyTypeObject MesosExecutorDriverImplType;
> -
> -/**
> - * List of Python methods in MesosExecutorDriverImpl.
> - */
> -extern PyMethodDef MesosExecutorDriverImpl_methods[];
> -
> -/**
> - * Create, but don't initialize, a new MesosExecutorDriverImpl
> - * (called by Python before init method).
> - */
> -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> - PyObject *args,
> - PyObject *kwds);
> -
> -/**
> - * Initialize a MesosExecutorDriverImpl with constructor arguments.
> - */
> -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> - PyObject *args,
> - PyObject *kwds);
> -
> -/**
> - * Free a MesosExecutorDriverImpl.
> - */
> -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self);
> -
> -/**
> - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> - * See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> - visitproc visit,
> - void* arg);
> -/**
> - * Clear fields of a MesosExecutorDriverImpl that can participate in
> - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self);
> -
> -// MesosExecutorDriverImpl methods.
> -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> - MesosExecutorDriverImpl* self,
> - PyObject* args);
> -
> -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> - MesosExecutorDriverImpl* self,
> - PyObject* args);
> -
> -} // namespace python {
> -} // namespace mesos {
> -
> -#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> ----------------------------------------------------------------------
> diff --git
> a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> b/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> deleted file mode 100644
> index f8be49b..0000000
> --- a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> +++ /dev/null
> @@ -1,782 +0,0 @@
> -// Licensed to the Apache Software Foundation (ASF) under one
> -// or more contributor license agreements. See the NOTICE file
> -// distributed with this work for additional information
> -// regarding copyright ownership. The ASF licenses this file
> -// to you under the Apache License, Version 2.0 (the
> -// "License"); you may not use this file except in compliance
> -// with the License. You may obtain a copy of the License at
> -//
> -// http://www.apache.org/licenses/LICENSE-2.0
> -//
> -// Unless required by applicable law or agreed to in writing, software
> -// distributed under the License is distributed on an "AS IS" BASIS,
> -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> -// See the License for the specific language governing permissions and
> -// limitations under the License.
> -
> -// Python.h must be included before standard headers.
> -// See: http://docs.python.org/2/c-api/intro.html#include-files
> -#include <Python.h>
> -
> -#include <string>
> -
> -#include "mesos_scheduler_driver_impl.hpp"
> -#include "module.hpp"
> -#include "proxy_scheduler.hpp"
> -
> -using namespace mesos;
> -using namespace mesos::python;
> -
> -using std::cerr;
> -using std::endl;
> -using std::string;
> -using std::vector;
> -using std::map;
> -
> -namespace mesos {
> -namespace python {
> -
> -/**
> - * Python type object for MesosSchedulerDriverImpl.
> - */
> -PyTypeObject MesosSchedulerDriverImplType = {
> - PyObject_HEAD_INIT(NULL)
> - 0, /* ob_size */
> - "_mesos.MesosSchedulerDriverImpl", /* tp_name */
> - sizeof(MesosSchedulerDriverImpl), /* tp_basicsize */
> - 0, /* tp_itemsize */
> - (destructor) MesosSchedulerDriverImpl_dealloc, /* tp_dealloc */
> - 0, /* tp_print */
> - 0, /* tp_getattr */
> - 0, /* tp_setattr */
> - 0, /* tp_compare */
> - 0, /* tp_repr */
> - 0, /* tp_as_number */
> - 0, /* tp_as_sequence */
> - 0, /* tp_as_mapping */
> - 0, /* tp_hash */
> - 0, /* tp_call */
> - 0, /* tp_str */
> - 0, /* tp_getattro */
> - 0, /* tp_setattro */
> - 0, /* tp_as_buffer */
> - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> - "Private MesosSchedulerDriver implementation", /* tp_doc */
> - (traverseproc) MesosSchedulerDriverImpl_traverse, /* tp_traverse */
> - (inquiry) MesosSchedulerDriverImpl_clear, /* tp_clear */
> - 0, /* tp_richcompare */
> - 0, /* tp_weaklistoffset
> */
> - 0, /* tp_iter */
> - 0, /* tp_iternext */
> - MesosSchedulerDriverImpl_methods, /* tp_methods */
> - 0, /* tp_members */
> - 0, /* tp_getset */
> - 0, /* tp_base */
> - 0, /* tp_dict */
> - 0, /* tp_descr_get */
> - 0, /* tp_descr_set */
> - 0, /* tp_dictoffset */
> - (initproc) MesosSchedulerDriverImpl_init, /* tp_init */
> - 0, /* tp_alloc */
> - MesosSchedulerDriverImpl_new, /* tp_new */
> -};
> -
> -
> -/**
> - * List of Python methods in MesosSchedulerDriverImpl.
> - */
> -PyMethodDef MesosSchedulerDriverImpl_methods[] = {
> - { "start",
> - (PyCFunction) MesosSchedulerDriverImpl_start,
> - METH_NOARGS,
> - "Start the driver to connect to Mesos"
> - },
> - { "stop",
> - (PyCFunction) MesosSchedulerDriverImpl_stop,
> - METH_VARARGS,
> - "Stop the driver, disconnecting from Mesos"
> - },
> - { "abort",
> - (PyCFunction) MesosSchedulerDriverImpl_abort,
> - METH_NOARGS,
> - "Abort the driver, disabling calls from and to the driver"
> - },
> - { "join",
> - (PyCFunction) MesosSchedulerDriverImpl_join,
> - METH_NOARGS,
> - "Wait for a running driver to disconnect from Mesos"
> - },
> - { "run",
> - (PyCFunction) MesosSchedulerDriverImpl_run,
> - METH_NOARGS,
> - "Start a driver and run it, returning when it disconnects from Mesos"
> - },
> - { "requestResources",
> - (PyCFunction) MesosSchedulerDriverImpl_requestResources,
> - METH_VARARGS,
> - "Request resources from the Mesos allocator"
> - },
> - { "launchTasks",
> - (PyCFunction) MesosSchedulerDriverImpl_launchTasks,
> - METH_VARARGS,
> - "Reply to a Mesos offer with a list of tasks"
> - },
> - { "killTask",
> - (PyCFunction) MesosSchedulerDriverImpl_killTask,
> - METH_VARARGS,
> - "Kill the task with the given ID"
> - },
> - { "acceptOffers",
> - (PyCFunction) MesosSchedulerDriverImpl_acceptOffers,
> - METH_VARARGS,
> - "Reply to a Mesos offer with a list of offer operations"
> - },
> - { "declineOffer",
> - (PyCFunction) MesosSchedulerDriverImpl_declineOffer,
> - METH_VARARGS,
> - "Decline a Mesos offer"
> - },
> - { "reviveOffers",
> - (PyCFunction) MesosSchedulerDriverImpl_reviveOffers,
> - METH_NOARGS,
> - "Remove all filters and ask Mesos for new offers"
> - },
> - { "suppressOffers",
> - (PyCFunction) MesosSchedulerDriverImpl_suppressOffers,
> - METH_NOARGS,
> - "Set suppressed attribute as true for the Framework"
> - },
> - { "acknowledgeStatusUpdate",
> - (PyCFunction) MesosSchedulerDriverImpl_acknowledgeStatusUpdate,
> - METH_VARARGS,
> - "Acknowledge a status update"
> - },
> - { "sendFrameworkMessage",
> - (PyCFunction) MesosSchedulerDriverImpl_sendFrameworkMessage,
> - METH_VARARGS,
> - "Send a FrameworkMessage to a slave"
> - },
> - { "reconcileTasks",
> - (PyCFunction) MesosSchedulerDriverImpl_reconcileTasks,
> - METH_VARARGS,
> - "Master sends status updates if task status is different from
> expected"
> - },
> - { NULL } /* Sentinel */
> -};
> -
> -
> -/**
> - * Create, but don't initialize, a new MesosSchedulerDriverImpl
> - * (called by Python before init method).
> - */
> -PyObject* MesosSchedulerDriverImpl_new(PyTypeObject* type,
> - PyObject* args,
> - PyObject* kwds)
> -{
> - MesosSchedulerDriverImpl* self;
> - self = (MesosSchedulerDriverImpl*) type->tp_alloc(type, 0);
> - if (self != NULL) {
> - self->driver = NULL;
> - self->proxyScheduler = NULL;
> - self->pythonScheduler = NULL;
> - }
> - return (PyObject*) self;
> -}
> -
> -
> -/**
> - * Initialize a MesosSchedulerDriverImpl with constructor arguments.
> - */
> -int MesosSchedulerDriverImpl_init(MesosSchedulerDriverImpl* self,
> - PyObject* args,
> - PyObject* kwds)
> -{
> - // Note: We use an integer for 'implicitAcknoweldgements' because
> - // it is the recommended way to pass booleans through CPython.
> - PyObject* schedulerObj = NULL;
> - PyObject* frameworkObj = NULL;
> - const char* master;
> - int implicitAcknowledgements = 1; // Enabled by default.
> - PyObject* credentialObj = NULL;
> -
> - if (!PyArg_ParseTuple(
> - args,
> - "OOs|iO",
> - &schedulerObj,
> - &frameworkObj,
> - &master,
> - &implicitAcknowledgements,
> - &credentialObj)) {
> - return -1;
> - }
> -
> - if (schedulerObj != NULL) {
> - PyObject* tmp = self->pythonScheduler;
> - Py_INCREF(schedulerObj);
> - self->pythonScheduler = schedulerObj;
> - Py_XDECREF(tmp);
> - }
> -
> - FrameworkInfo framework;
> - if (frameworkObj != NULL) {
> - if (!readPythonProtobuf(frameworkObj, &framework)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python FrameworkInfo");
> - return -1;
> - }
> - }
> -
> - Credential credential;
> - if (credentialObj != NULL) {
> - if (!readPythonProtobuf(credentialObj, &credential)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> Credential");
> - return -1;
> - }
> - }
> -
> -
> - if (self->driver != NULL) {
> - delete self->driver;
> - self->driver = NULL;
> - }
> -
> - if (self->proxyScheduler != NULL) {
> - delete self->proxyScheduler;
> - self->proxyScheduler = NULL;
> - }
> -
> - self->proxyScheduler = new ProxyScheduler(self);
> -
> - if (credentialObj != NULL) {
> - self->driver = new MesosSchedulerDriver(
> - self->proxyScheduler,
> - framework,
> - master,
> - implicitAcknowledgements != 0,
> - credential);
> - } else {
> - self->driver = new MesosSchedulerDriver(
> - self->proxyScheduler,
> - framework,
> - master,
> - implicitAcknowledgements != 0);
> - }
> -
> - return 0;
> -}
> -
> -
> -/**
> - * Free a MesosSchedulerDriverImpl.
> - */
> -void MesosSchedulerDriverImpl_dealloc(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver != NULL) {
> - // We need to wrap the driver destructor in an "allow threads"
> - // macro since the MesosSchedulerDriver destructor waits for the
> - // SchedulerProcess to terminate and there might be a thread that
> - // is trying to acquire the GIL to call through the
> - // ProxyScheduler. It will only be after this thread executes that
> - // the SchedulerProcess might actually get a terminate.
> - Py_BEGIN_ALLOW_THREADS
> - delete self->driver;
> - Py_END_ALLOW_THREADS
> - self->driver = NULL;
> - }
> -
> - if (self->proxyScheduler != NULL) {
> - delete self->proxyScheduler;
> - self->proxyScheduler = NULL;
> - }
> -
> - MesosSchedulerDriverImpl_clear(self);
> - self->ob_type->tp_free((PyObject*) self);
> -}
> -
> -
> -/**
> - * Traverse fields of a MesosSchedulerDriverImpl on a cyclic GC search.
> - * See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosSchedulerDriverImpl_traverse(MesosSchedulerDriverImpl* self,
> - visitproc visit,
> - void* arg)
> -{
> - Py_VISIT(self->pythonScheduler);
> - return 0;
> -}
> -
> -
> -/**
> - * Clear fields of a MesosSchedulerDriverImpl that can participate in
> - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosSchedulerDriverImpl_clear(MesosSchedulerDriverImpl* self)
> -{
> - Py_CLEAR(self->pythonScheduler);
> - return 0;
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_start(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->start();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_stop(MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - bool failover = false; // Should match default in mesos.py.
> -
> - if (!PyArg_ParseTuple(args, "|b", &failover)) {
> - return NULL;
> - }
> -
> - Status status = self->driver->stop(failover);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_abort(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->abort();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_join(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->join();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_run(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->run();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_requestResources(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* requestsObj = NULL;
> - vector<Request> requests;
> -
> - if (!PyArg_ParseTuple(args, "O", &requestsObj)) {
> - return NULL;
> - }
> -
> - if (!PyList_Check(requestsObj)) {
> - PyErr_Format(PyExc_Exception,
> - "Parameter 2 to requestsResources is not a list");
> - return NULL;
> - }
> - Py_ssize_t len = PyList_Size(requestsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* requestObj = PyList_GetItem(requestsObj, i);
> - if (requestObj == NULL) {
> - return NULL; // Exception will have been set by PyList_GetItem.
> - }
> - Request request;
> - if (!readPythonProtobuf(requestObj, &request)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> Request");
> - return NULL;
> - }
> - requests.push_back(request);
> - }
> -
> - Status status = self->driver->requestResources(requests);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_launchTasks(MesosSchedulerDriverImpl*
> self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* offerIdsObj = NULL;
> - PyObject* tasksObj = NULL;
> - PyObject* filtersObj = NULL;
> - vector<OfferID> offerIds;
> - vector<TaskInfo> tasks;
> - Filters filters;
> -
> - if (!PyArg_ParseTuple(args, "OO|O", &offerIdsObj, &tasksObj,
> &filtersObj)) {
> - return NULL;
> - }
> -
> - // Offer argument can be a list of offer ids or a single offer id (for
> - // backward compatibility).
> - if (!PyList_Check(offerIdsObj)) {
> - OfferID offerId;
> - if (!readPythonProtobuf(offerIdsObj, &offerId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> OfferID");
> - return NULL;
> - }
> - offerIds.push_back(offerId);
> - } else {
> - Py_ssize_t len = PyList_Size(offerIdsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* offerObj = PyList_GetItem(offerIdsObj, i);
> - if (offerObj == NULL) {
> - return NULL;
> - }
> - OfferID offerId;
> - if (!readPythonProtobuf(offerObj, &offerId)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python OfferID");
> - return NULL;
> - }
> - offerIds.push_back(offerId);
> - }
> - }
> -
> - if (!PyList_Check(tasksObj)) {
> - PyErr_Format(PyExc_Exception, "Parameter 2 to launchTasks is not a
> list");
> - return NULL;
> - }
> - Py_ssize_t len = PyList_Size(tasksObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* taskObj = PyList_GetItem(tasksObj, i);
> - if (taskObj == NULL) {
> - return NULL; // Exception will have been set by PyList_GetItem.
> - }
> - TaskInfo task;
> - if (!readPythonProtobuf(taskObj, &task)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python TaskInfo");
> - return NULL;
> - }
> - tasks.push_back(task);
> - }
> -
> - if (filtersObj != NULL) {
> - if (!readPythonProtobuf(filtersObj, &filters)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Filters");
> - return NULL;
> - }
> - }
> -
> - Status status = self->driver->launchTasks(offerIds, tasks, filters);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_killTask(MesosSchedulerDriverImpl*
> self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* tidObj = NULL;
> - TaskID tid;
> - if (!PyArg_ParseTuple(args, "O", &tidObj)) {
> - return NULL;
> - }
> - if (!readPythonProtobuf(tidObj, &tid)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python TaskID");
> - return NULL;
> - }
> -
> - Status status = self->driver->killTask(tid);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_acceptOffers(MesosSchedulerDriverImpl*
> self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* offerIdsObj = NULL;
> - PyObject* operationsObj = NULL;
> - PyObject* filtersObj = NULL;
> - Py_ssize_t len = 0;
> - vector<OfferID> offerIds;
> - vector<Offer::Operation> operations;
> - Filters filters;
> -
> - if (!PyArg_ParseTuple(args,
> - "OO|O",
> - &offerIdsObj,
> - &operationsObj,
> - &filtersObj)) {
> - return NULL;
> - }
> -
> - if (!PyList_Check(offerIdsObj)) {
> - PyErr_Format(PyExc_Exception, "Parameter 1 to acceptOffers is not a
> list");
> - return NULL;
> - }
> -
> - len = PyList_Size(offerIdsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* offerObj = PyList_GetItem(offerIdsObj, i);
> - if (offerObj == NULL) {
> - return NULL;
> - }
> -
> - OfferID offerId;
> - if (!readPythonProtobuf(offerObj, &offerId)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python OfferID");
> - return NULL;
> - }
> - offerIds.push_back(offerId);
> - }
> -
> - if (!PyList_Check(operationsObj)) {
> - PyErr_Format(PyExc_Exception, "Parameter 2 to acceptOffers is not a
> list");
> - return NULL;
> - }
> -
> - len = PyList_Size(operationsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* operationObj = PyList_GetItem(operationsObj, i);
> - if (operationObj == NULL) {
> - return NULL; // Exception will have been set by PyList_GetItem.
> - }
> -
> - Offer::Operation operation;
> - if (!readPythonProtobuf(operationObj, &operation)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Offer.Operation");
> - return NULL;
> - }
> - operations.push_back(operation);
> - }
> -
> - if (filtersObj != NULL) {
> - if (!readPythonProtobuf(filtersObj, &filters)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Filters");
> - return NULL;
> - }
> - }
> -
> - Status status = self->driver->acceptOffers(offerIds, operations,
> filters);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_declineOffer(MesosSchedulerDriverImpl*
> self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* offerIdObj = NULL;
> - PyObject* filtersObj = NULL;
> - OfferID offerId;
> - Filters filters;
> -
> - if (!PyArg_ParseTuple(args, "O|O", &offerIdObj, &filtersObj)) {
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(offerIdObj, &offerId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python OfferID");
> - return NULL;
> - }
> -
> - if (filtersObj != NULL) {
> - if (!readPythonProtobuf(filtersObj, &filters)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Filters");
> - return NULL;
> - }
> - }
> -
> - Status status = self->driver->declineOffer(offerId, filters);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_reviveOffers(MesosSchedulerDriverImpl*
> self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->reviveOffers();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_suppressOffers(
> - MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->suppressOffers();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_acknowledgeStatusUpdate(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* taskStatusObj = NULL;
> - TaskStatus taskStatus;
> -
> - if (!PyArg_ParseTuple(args, "O", &taskStatusObj)) {
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(taskStatusObj, &taskStatus)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> TaskStatus");
> - return NULL;
> - }
> -
> - Status status = self->driver->acknowledgeStatusUpdate(taskStatus);
> -
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_sendFrameworkMessage(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* slaveIdObj = NULL;
> - PyObject* executorIdObj = NULL;
> - SlaveID slaveId;
> - ExecutorID executorId;
> - const char* data;
> - int length;
> -
> - if (!PyArg_ParseTuple(
> - args, "OOs#", &executorIdObj, &slaveIdObj, &data, &length)) {
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(executorIdObj, &executorId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> ExecutorID");
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(slaveIdObj, &slaveId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python SlaveID");
> - return NULL;
> - }
> -
> - Status status = self->driver->sendFrameworkMessage(
> - executorId, slaveId, string(data, length));
> -
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_reconcileTasks(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> NULL");
> - return NULL;
> - }
> -
> - PyObject* statusesObj = NULL;
> - vector<TaskStatus> statuses;
> -
> - if (!PyArg_ParseTuple(args, "O", &statusesObj)) {
> - return NULL;
> - }
> -
> - if (!PyList_Check(statusesObj)) {
> - PyErr_Format(PyExc_Exception,
> - "Parameter 1 to reconcileTasks is not a list");
> -
> - return NULL;
> - }
> -
> - Py_ssize_t len = PyList_Size(statusesObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* statusObj = PyList_GetItem(statusesObj, i);
> - if (statusObj == NULL) {
> - return NULL;
> - }
> -
> - TaskStatus status;
> - if (!readPythonProtobuf(statusObj, &status)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python TaskStatus");
> - return NULL;
> - }
> - statuses.push_back(status);
> - }
> -
> - Status status = self->driver->reconcileTasks(statuses);
> - return PyInt_FromLong(status);
> -}
> -
> -} // namespace python {
> -} // namespace mesos {
>
>
Re: [3/3] mesos git commit: New python lib with only the executor driver.
Posted by haosdent <ha...@gmail.com>.
Guangya file a ticket https://issues.apache.org/jira/browse/MESOS-4924
related to this.
On Sat, Mar 12, 2016 at 10:55 AM, Benjamin Mahler <bm...@apache.org>
wrote:
> +vinod
>
> This breaks the build for me on OS X, it appears this line is the culprit:
>
> EXTRA_LINK_ARGS = ['-Wl,--as-needed']
>
> This leads to the following:
>
> clang++ -bundle -undefined dynamic_lookup -arch x86_64 -arch i386 -Wl,-F.
> -L/usr/local/opt/subversion/lib -O2 -O2 -Wno-unused-local-typedef
> -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -DGTEST_LANG_CXX11
> -Qunused-arguments -I/usr/local/opt/subversion/include/subversion-1
> -I/usr/include/apr-1 -I/usr/include/apr-1.0 -Qunused-arguments
>
> build/temp.macosx-10.11-intel-2.7/src/mesos/executor/mesos_executor_driver_impl.o
> build/temp.macosx-10.11-intel-2.7/src/mesos/executor/module.o
> build/temp.macosx-10.11-intel-2.7/src/mesos/executor/proxy_executor.o
> /Users/bmahler/git/mesos/build/src/.libs/libmesos_no_3rdparty.a
> /Users/bmahler/git/mesos/build/3rdparty/libprocess/.libs/libprocess.a
> /Users/bmahler/git/mesos/build/3rdparty/leveldb-1.4/libleveldb.a
>
> /Users/bmahler/git/mesos/build/3rdparty/zookeeper-3.4.5/src/c/.libs/libzookeeper_mt.a
>
> /Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/glog-0.3.3/.libs/libglog.a
>
> /Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/protobuf-2.5.0/src/.libs/libprotobuf.a
> -o build/lib.macosx-10.11-intel-2.7/mesos/executor/_executor.so
> -Wl,--as-needed -L/usr/local/opt/subversion/lib -levent_openssl -lcrypto
> -lssl -levent_pthreads -levent -lsasl2 -lsvn_delta-1 -lsvn_subr-1 -lapr-1
> -lcurl -lz
> ld: unknown option: --as-needed
> clang: error: linker command failed with exit code 1 (use -v to see
> invocation)
>
> On Fri, Mar 11, 2016 at 1:56 PM, <vi...@apache.org> wrote:
>
> > New python lib with only the executor driver.
> >
> > This patch produces a new python egg, mesos.executor, which contains only
> > the
> > code needed to create a MesosExecutorDriver. By doing so, the linker can
> > remove
> > unused code in libmesos_no_3rdparty.a, and therefor not include any
> > external
> > dependencies in the resulting _mesos.so.
> >
> > Review: https://reviews.apache.org/r/41049/
> >
> >
> > Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> > Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c81a52ec
> > Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c81a52ec
> > Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c81a52ec
> >
> > Branch: refs/heads/master
> > Commit: c81a52ec22266e1f2beb61b224c0f0d9be82521f
> > Parents: 482dc14
> > Author: Steve Niemitz <sn...@twitter.com>
> > Authored: Fri Mar 11 16:56:13 2016 -0500
> > Committer: Vinod Kone <vi...@gmail.com>
> > Committed: Fri Mar 11 16:56:13 2016 -0500
> >
> > ----------------------------------------------------------------------
> > configure.ac | 7 +-
> > src/Makefile.am | 33 +-
> > src/python/executor/setup.py.in | 39 +
> > src/python/executor/src/mesos/__init__.py | 10 +
> > .../executor/src/mesos/executor/__init__.py | 17 +
> > .../executor/mesos_executor_driver_impl.cpp | 347 ++++++++
> > .../executor/mesos_executor_driver_impl.hpp | 103 +++
> > .../executor/src/mesos/executor/module.cpp | 91 +++
> > .../src/mesos/executor/proxy_executor.cpp | 273 +++++++
> > .../src/mesos/executor/proxy_executor.hpp | 64 ++
> > src/python/native/ext_modules.py.in | 151 ----
> > src/python/native/setup.py.in | 9 +-
> > src/python/native/src/mesos/native/__init__.py | 7 +-
> > .../mesos/native/mesos_executor_driver_impl.cpp | 347 --------
> > .../mesos/native/mesos_executor_driver_impl.hpp | 103 ---
> > .../native/mesos_scheduler_driver_impl.cpp | 782
> -------------------
> > .../native/mesos_scheduler_driver_impl.hpp | 134 ----
> > src/python/native/src/mesos/native/module.cpp | 100 ---
> > src/python/native/src/mesos/native/module.hpp | 136 ----
> > .../native/src/mesos/native/proxy_executor.cpp | 273 -------
> > .../native/src/mesos/native/proxy_executor.hpp | 64 --
> > .../native/src/mesos/native/proxy_scheduler.cpp | 384 ---------
> > .../native/src/mesos/native/proxy_scheduler.hpp | 72 --
> > src/python/native_common/common.hpp | 136 ++++
> > src/python/native_common/ext_modules.py.in | 154 ++++
> > src/python/scheduler/setup.py.in | 39 +
> > src/python/scheduler/src/mesos/__init__.py | 10 +
> > .../scheduler/src/mesos/scheduler/__init__.py | 17 +
> > .../scheduler/mesos_scheduler_driver_impl.cpp | 782
> +++++++++++++++++++
> > .../scheduler/mesos_scheduler_driver_impl.hpp | 134 ++++
> > .../scheduler/src/mesos/scheduler/module.cpp | 91 +++
> > .../src/mesos/scheduler/proxy_scheduler.cpp | 384 +++++++++
> > .../src/mesos/scheduler/proxy_scheduler.hpp | 72 ++
> > 33 files changed, 2795 insertions(+), 2570 deletions(-)
> > ----------------------------------------------------------------------
> >
> >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/configure.ac
> > ----------------------------------------------------------------------
> > diff --git a/configure.ac b/configure.ac
> > index a20382e..8e4f035 100644
> > --- a/configure.ac
> > +++ b/configure.ac
> > @@ -1315,8 +1315,13 @@ There are two possible workarounds for this issue:
> > AC_CONFIG_FILES([src/python/setup.py])
> > AC_CONFIG_FILES([src/python/cli/setup.py])
> > AC_CONFIG_FILES([src/python/interface/setup.py])
> > - AC_CONFIG_FILES([src/python/native/ext_modules.py])
> > + AC_CONFIG_FILES([src/python/native_common/ext_modules.py])
> > + AC_CONFIG_FILES([src/python/executor/setup.py])
> > AC_CONFIG_FILES([src/python/native/setup.py])
> > + AC_CONFIG_FILES([src/python/scheduler/setup.py])
> > +
> > +
> >
> AC_CONFIG_LINKS([src/python/executor/ext_modules.py:src/python/native_common/ext_modules.py])
> > +
> >
> AC_CONFIG_LINKS([src/python/scheduler/ext_modules.py:src/python/native_common/ext_modules.py])
> >
> > # When clang is being used, make sure that the distutils python-
> > # config cflags extraction does not cause build errors (MESOS-1079).
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/Makefile.am
> > ----------------------------------------------------------------------
> > diff --git a/src/Makefile.am b/src/Makefile.am
> > index f59ae12..8abef3b 100644
> > --- a/src/Makefile.am
> > +++ b/src/Makefile.am
> > @@ -1467,18 +1467,23 @@ PYTHON_SOURCE =
> > \
> > python/interface/src/mesos/interface/__init__.py \
> > python/interface/src/mesos/v1/__init__.py \
> > python/interface/src/mesos/v1/interface/__init__.py \
> > + python/native_common/common.hpp \
> > python/native/src/mesos/__init__.py \
> > python/native/src/mesos/native/__init__.py \
> > - python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> > \
> > - python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> > \
> > - python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp \
> > - python/native/src/mesos/native/mesos_scheduler_driver_impl.hpp \
> > - python/native/src/mesos/native/module.cpp \
> > - python/native/src/mesos/native/module.hpp \
> > - python/native/src/mesos/native/proxy_executor.cpp \
> > - python/native/src/mesos/native/proxy_executor.hpp \
> > - python/native/src/mesos/native/proxy_scheduler.cpp \
> > - python/native/src/mesos/native/proxy_scheduler.hpp \
> > + python/executor/src/mesos/__init__.py
> > \
> > + python/executor/src/mesos/executor/__init__.py \
> > + python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp \
> > + python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp \
> > + python/executor/src/mesos/executor/module.cpp
> > \
> > + python/executor/src/mesos/executor/proxy_executor.cpp
> > \
> > + python/executor/src/mesos/executor/proxy_executor.hpp
> > \
> > + python/scheduler/src/mesos/__init__.py \
> > + python/scheduler/src/mesos/scheduler/__init__.py \
> > + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp \
> > + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.hpp \
> > + python/scheduler/src/mesos/scheduler/module.cpp \
> > + python/scheduler/src/mesos/scheduler/proxy_scheduler.cpp \
> > + python/scheduler/src/mesos/scheduler/proxy_scheduler.hpp \
> > python/src/mesos/__init__.py
> >
> > EXTRA_DIST += $(PYTHON_SOURCE)
> > @@ -1530,14 +1535,18 @@ $(PYTHON_SOURCE):
> > MESOS_EGGS =
> > \
> > python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg
> > \
> >
> > python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg
> > \
> > + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> > \
> >
> >
> python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg
> > \
> > - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> > + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> > \
> > +
> python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> >
> > MESOS_WHLS =
> > \
> > python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl
> > \
> >
> > python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl
> > \
> > + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> > \
> >
> >
> python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl
> > \
> > - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> > + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> > \
> > +
> python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> >
> > # The python source is in directories of the form: python/interface. The
> > make
> > # target is of the form: python/dist/mesos.interface-0.20.0-py2.7.egg.
> To
> > build
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/setup.py.in
> > ----------------------------------------------------------------------
> > diff --git a/src/python/executor/setup.py.in b/src/python/executor/
> > setup.py.in
> > new file mode 100644
> > index 0000000..76db97f
> > --- /dev/null
> > +++ b/src/python/executor/setup.py.in
> > @@ -0,0 +1,39 @@
> > +#!/usr/bin/env python
> > +
> > +# Licensed to the Apache Software Foundation (ASF) under one
> > +# or more contributor license agreements. See the NOTICE file
> > +# distributed with this work for additional information
> > +# regarding copyright ownership. The ASF licenses this file
> > +# to you under the Apache License, Version 2.0 (the
> > +# "License"); you may not use this file except in compliance
> > +# with the License. You may obtain a copy of the License at
> > +#
> > +# http://www.apache.org/licenses/LICENSE-2.0
> > +#
> > +# Unless required by applicable law or agreed to in writing, software
> > +# distributed under the License is distributed on an "AS IS" BASIS,
> > +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > +# See the License for the specific language governing permissions and
> > +# limitations under the License.
> > +
> > +import ext_modules
> > +
> > +config = {
> > + 'name': 'mesos.executor',
> > + 'version': '@PACKAGE_VERSION@',
> > + 'description': 'Mesos native executor driver implementation',
> > + 'author': 'Apache Mesos',
> > + 'author_email': 'dev@mesos.apache.org',
> > + 'url': 'http://pypi.python.org/pypi/mesos.executor',
> > + 'namespace_packages': [ 'mesos' ],
> > + 'packages': [ 'mesos', 'mesos.executor' ],
> > + 'package_dir': { '': 'src' },
> > + 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ],
> > + 'license': 'Apache 2.0',
> > + 'keywords': 'mesos',
> > + 'classifiers': [ ],
> > + 'ext_modules': [ ext_modules.executor_module ]
> > +}
> > +
> > +from setuptools import setup
> > +setup(**config)
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/__init__.py
> > ----------------------------------------------------------------------
> > diff --git a/src/python/executor/src/mesos/__init__.py
> > b/src/python/executor/src/mesos/__init__.py
> > new file mode 100644
> > index 0000000..3fcba01
> > --- /dev/null
> > +++ b/src/python/executor/src/mesos/__init__.py
> > @@ -0,0 +1,10 @@
> > +# See
> > http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
> > +# Because python does not normally allow the contents of a package to be
> > +# retrieved from more than one location, this code snippet ensures that
> > the
> > +# namespace package machinery is operating and that the current package
> is
> > +# registered as a namespace package.
> > +try:
> > + __import__('pkg_resources').declare_namespace(__name__)
> > +except ImportError:
> > + from pkgutil import extend_path
> > + __path__ = extend_path(__path__, __name__)
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/__init__.py
> > ----------------------------------------------------------------------
> > diff --git a/src/python/executor/src/mesos/executor/__init__.py
> > b/src/python/executor/src/mesos/executor/__init__.py
> > new file mode 100644
> > index 0000000..6808019
> > --- /dev/null
> > +++ b/src/python/executor/src/mesos/executor/__init__.py
> > @@ -0,0 +1,17 @@
> > +# Licensed to the Apache Software Foundation (ASF) under one
> > +# or more contributor license agreements. See the NOTICE file
> > +# distributed with this work for additional information
> > +# regarding copyright ownership. The ASF licenses this file
> > +# to you under the Apache License, Version 2.0 (the
> > +# "License"); you may not use this file except in compliance
> > +# with the License. You may obtain a copy of the License at
> > +#
> > +# http://www.apache.org/licenses/LICENSE-2.0
> > +#
> > +# Unless required by applicable law or agreed to in writing, software
> > +# distributed under the License is distributed on an "AS IS" BASIS,
> > +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > +# See the License for the specific language governing permissions and
> > +# limitations under the License.
> > +
> > +from ._executor import MesosExecutorDriverImpl as MesosExecutorDriver
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> > ----------------------------------------------------------------------
> > diff --git
> > a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> > b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> > new file mode 100644
> > index 0000000..4189411
> > --- /dev/null
> > +++
> b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> > @@ -0,0 +1,347 @@
> > +// Licensed to the Apache Software Foundation (ASF) under one
> > +// or more contributor license agreements. See the NOTICE file
> > +// distributed with this work for additional information
> > +// regarding copyright ownership. The ASF licenses this file
> > +// to you under the Apache License, Version 2.0 (the
> > +// "License"); you may not use this file except in compliance
> > +// with the License. You may obtain a copy of the License at
> > +//
> > +// http://www.apache.org/licenses/LICENSE-2.0
> > +//
> > +// Unless required by applicable law or agreed to in writing, software
> > +// distributed under the License is distributed on an "AS IS" BASIS,
> > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > +// See the License for the specific language governing permissions and
> > +// limitations under the License.
> > +
> > +// Python.h must be included before standard headers.
> > +// See: http://docs.python.org/2/c-api/intro.html#include-files
> > +#include <Python.h>
> > +
> > +#include <string>
> > +
> > +#include "common.hpp"
> > +#include "mesos_executor_driver_impl.hpp"
> > +#include "proxy_executor.hpp"
> > +
> > +using namespace mesos;
> > +using namespace mesos::python;
> > +
> > +using std::cerr;
> > +using std::endl;
> > +using std::string;
> > +using std::vector;
> > +using std::map;
> > +
> > +
> > +namespace mesos { namespace python {
> > +
> > +/**
> > + * Python type object for MesosExecutorDriverImpl.
> > + */
> > +PyTypeObject MesosExecutorDriverImplType = {
> > + PyObject_HEAD_INIT(NULL)
> > + 0, /* ob_size */
> > + "_mesos.MesosExecutorDriverImpl", /* tp_name */
> > + sizeof(MesosExecutorDriverImpl), /* tp_basicsize */
> > + 0, /* tp_itemsize */
> > + (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */
> > + 0, /* tp_print */
> > + 0, /* tp_getattr */
> > + 0, /* tp_setattr */
> > + 0, /* tp_compare */
> > + 0, /* tp_repr */
> > + 0, /* tp_as_number */
> > + 0, /* tp_as_sequence */
> > + 0, /* tp_as_mapping */
> > + 0, /* tp_hash */
> > + 0, /* tp_call */
> > + 0, /* tp_str */
> > + 0, /* tp_getattro */
> > + 0, /* tp_setattro */
> > + 0, /* tp_as_buffer */
> > + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> > + "Private MesosExecutorDriver implementation", /* tp_doc */
> > + (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */
> > + (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */
> > + 0, /* tp_richcompare */
> > + 0, /* tp_weaklistoffset
> */
> > + 0, /* tp_iter */
> > + 0, /* tp_iternext */
> > + MesosExecutorDriverImpl_methods, /* tp_methods */
> > + 0, /* tp_members */
> > + 0, /* tp_getset */
> > + 0, /* tp_base */
> > + 0, /* tp_dict */
> > + 0, /* tp_descr_get */
> > + 0, /* tp_descr_set */
> > + 0, /* tp_dictoffset */
> > + (initproc) MesosExecutorDriverImpl_init, /* tp_init */
> > + 0, /* tp_alloc */
> > + MesosExecutorDriverImpl_new, /* tp_new */
> > +};
> > +
> > +
> > +/**
> > + * List of Python methods in MesosExecutorDriverImpl.
> > + */
> > +PyMethodDef MesosExecutorDriverImpl_methods[] = {
> > + { "start",
> > + (PyCFunction) MesosExecutorDriverImpl_start,
> > + METH_NOARGS,
> > + "Start the driver to connect to Mesos"
> > + },
> > + { "stop",
> > + (PyCFunction) MesosExecutorDriverImpl_stop,
> > + METH_NOARGS,
> > + "Stop the driver, disconnecting from Mesos"
> > + },
> > + { "abort",
> > + (PyCFunction) MesosExecutorDriverImpl_abort,
> > + METH_NOARGS,
> > + "Abort the driver, disallowing calls from and to the driver"
> > + },
> > + { "join",
> > + (PyCFunction) MesosExecutorDriverImpl_join,
> > + METH_NOARGS,
> > + "Wait for a running driver to disconnect from Mesos"
> > + },
> > + { "run",
> > + (PyCFunction) MesosExecutorDriverImpl_run,
> > + METH_NOARGS,
> > + "Start a driver and run it, returning when it disconnects from
> Mesos"
> > + },
> > + { "sendStatusUpdate",
> > + (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate,
> > + METH_VARARGS,
> > + "Send a status update for a task"
> > + },
> > + { "sendFrameworkMessage",
> > + (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage,
> > + METH_VARARGS,
> > + "Send a FrameworkMessage to a slave"
> > + },
> > + { NULL } /* Sentinel */
> > +};
> > +
> > +
> > +/**
> > + * Create, but don't initialize, a new MesosExecutorDriverImpl
> > + * (called by Python before init method).
> > + */
> > +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> > + PyObject *args,
> > + PyObject *kwds)
> > +{
> > + MesosExecutorDriverImpl *self;
> > + self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0);
> > + if (self != NULL) {
> > + self->driver = NULL;
> > + self->proxyExecutor = NULL;
> > + self->pythonExecutor = NULL;
> > + }
> > + return (PyObject*) self;
> > +}
> > +
> > +
> > +/**
> > + * Initialize a MesosExecutorDriverImpl with constructor arguments.
> > + */
> > +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> > + PyObject *args,
> > + PyObject *kwds)
> > +{
> > + PyObject *pythonExecutor = NULL;
> > +
> > + if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) {
> > + return -1;
> > + }
> > +
> > + if (pythonExecutor != NULL) {
> > + PyObject* tmp = self->pythonExecutor;
> > + Py_INCREF(pythonExecutor);
> > + self->pythonExecutor = pythonExecutor;
> > + Py_XDECREF(tmp);
> > + }
> > +
> > + if (self->driver != NULL) {
> > + delete self->driver;
> > + self->driver = NULL;
> > + }
> > +
> > + if (self->proxyExecutor != NULL) {
> > + delete self->proxyExecutor;
> > + self->proxyExecutor = NULL;
> > + }
> > +
> > + self->proxyExecutor = new ProxyExecutor(self);
> > + self->driver = new MesosExecutorDriver(self->proxyExecutor);
> > +
> > + return 0;
> > +}
> > +
> > +
> > +/**
> > + * Free a MesosExecutorDriverImpl.
> > + */
> > +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self)
> > +{
> > + if (self->driver != NULL) {
> > + // We need to wrap the driver destructor in an "allow threads"
> > + // macro since the MesosExecutorDriver destructor waits for the
> > + // ExecutorProcess to terminate and there might be a thread that
> > + // is trying to acquire the GIL to call through the
> > + // ProxyExecutor. It will only be after this thread executes that
> > + // the ExecutorProcess might actually get a terminate.
> > + Py_BEGIN_ALLOW_THREADS
> > + delete self->driver;
> > + Py_END_ALLOW_THREADS
> > + self->driver = NULL;
> > + }
> > +
> > + if (self->proxyExecutor != NULL) {
> > + delete self->proxyExecutor;
> > + self->proxyExecutor = NULL;
> > + }
> > +
> > + MesosExecutorDriverImpl_clear(self);
> > + self->ob_type->tp_free((PyObject*) self);
> > +}
> > +
> > +
> > +/**
> > + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> > + * See http://docs.python.org/extending/newtypes.html.
> > + */
> > +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> > + visitproc visit,
> > + void* arg)
> > +{
> > + Py_VISIT(self->pythonExecutor);
> > + return 0;
> > +}
> > +
> > +
> > +/**
> > + * Clear fields of a MesosExecutorDriverImpl that can participate in
> > + * GC cycles. See http://docs.python.org/extending/newtypes.html.
> > + */
> > +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self)
> > +{
> > + Py_CLEAR(self->pythonExecutor);
> > + return 0;
> > +}
> > +
> > +
> > +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self)
> > +{
> > + if (self->driver == NULL) {
> > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > + return NULL;
> > + }
> > +
> > + Status status = self->driver->start();
> > + return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > +}
> > +
> > +
> > +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self)
> > +{
> > + if (self->driver == NULL) {
> > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > + return NULL;
> > + }
> > +
> > + Status status = self->driver->stop();
> > + return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > +}
> > +
> > +
> > +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self)
> > +{
> > + if (self->driver == NULL) {
> > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > + return NULL;
> > + }
> > +
> > + Status status = self->driver->abort();
> > + return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > +}
> > +
> > +
> > +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self)
> > +{
> > + if (self->driver == NULL) {
> > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > + return NULL;
> > + }
> > +
> > + Status status;
> > + Py_BEGIN_ALLOW_THREADS
> > + status = self->driver->join();
> > + Py_END_ALLOW_THREADS
> > + return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > +}
> > +
> > +
> > +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self)
> > +{
> > + if (self->driver == NULL) {
> > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > + return NULL;
> > + }
> > +
> > + Status status;
> > + Py_BEGIN_ALLOW_THREADS
> > + status = self->driver->run();
> > + Py_END_ALLOW_THREADS
> > + return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > +}
> > +
> > +
> > +PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> > + MesosExecutorDriverImpl* self,
> > + PyObject* args)
> > +{
> > + if (self->driver == NULL) {
> > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > + return NULL;
> > + }
> > +
> > + PyObject* statusObj = NULL;
> > + TaskStatus taskStatus;
> > + if (!PyArg_ParseTuple(args, "O", &statusObj)) {
> > + return NULL;
> > + }
> > + if (!readPythonProtobuf(statusObj, &taskStatus)) {
> > + PyErr_Format(PyExc_Exception,
> > + "Could not deserialize Python TaskStatus");
> > + return NULL;
> > + }
> > +
> > + Status status = self->driver->sendStatusUpdate(taskStatus);
> > + return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > +}
> > +
> > +
> > +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> > + MesosExecutorDriverImpl* self,
> > + PyObject* args)
> > +{
> > + if (self->driver == NULL) {
> > + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > + return NULL;
> > + }
> > +
> > + const char* data;
> > + int length;
> > + if (!PyArg_ParseTuple(args, "s#", &data, &length)) {
> > + return NULL;
> > + }
> > +
> > + Status status = self->driver->sendFrameworkMessage(string(data,
> > length));
> > + return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > +}
> > +
> > +} // namespace python {
> > +} // namespace mesos {
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> > ----------------------------------------------------------------------
> > diff --git
> > a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> > b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> > new file mode 100644
> > index 0000000..6e672f8
> > --- /dev/null
> > +++
> b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> > @@ -0,0 +1,103 @@
> > +// Licensed to the Apache Software Foundation (ASF) under one
> > +// or more contributor license agreements. See the NOTICE file
> > +// distributed with this work for additional information
> > +// regarding copyright ownership. The ASF licenses this file
> > +// to you under the Apache License, Version 2.0 (the
> > +// "License"); you may not use this file except in compliance
> > +// with the License. You may obtain a copy of the License at
> > +//
> > +// http://www.apache.org/licenses/LICENSE-2.0
> > +//
> > +// Unless required by applicable law or agreed to in writing, software
> > +// distributed under the License is distributed on an "AS IS" BASIS,
> > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > +// See the License for the specific language governing permissions and
> > +// limitations under the License.
> > +
> > +#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP
> > +#define MESOS_EXECUTOR_DRIVER_IMPL_HPP
> > +
> > +#include <mesos/executor.hpp>
> > +
> > +
> > +namespace mesos { namespace python {
> > +
> > +class ProxyExecutor;
> > +
> > +/**
> > + * Python object structure for MesosExecutorDriverImpl objects.
> > + */
> > +struct MesosExecutorDriverImpl {
> > + PyObject_HEAD
> > + /* Type-specific fields go here. */
> > + MesosExecutorDriver* driver;
> > + ProxyExecutor* proxyExecutor;
> > + PyObject* pythonExecutor;
> > +};
> > +
> > +/**
> > + * Python type object for MesosExecutorDriverImpl.
> > + */
> > +extern PyTypeObject MesosExecutorDriverImplType;
> > +
> > +/**
> > + * List of Python methods in MesosExecutorDriverImpl.
> > + */
> > +extern PyMethodDef MesosExecutorDriverImpl_methods[];
> > +
> > +/**
> > + * Create, but don't initialize, a new MesosExecutorDriverImpl
> > + * (called by Python before init method).
> > + */
> > +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> > + PyObject *args,
> > + PyObject *kwds);
> > +
> > +/**
> > + * Initialize a MesosExecutorDriverImpl with constructor arguments.
> > + */
> > +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> > + PyObject *args,
> > + PyObject *kwds);
> > +
> > +/**
> > + * Free a MesosExecutorDriverImpl.
> > + */
> > +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self);
> > +
> > +/**
> > + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> > + * See http://docs.python.org/extending/newtypes.html.
> > + */
> > +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> > + visitproc visit,
> > + void* arg);
> > +/**
> > + * Clear fields of a MesosExecutorDriverImpl that can participate in
> > + * GC cycles. See http://docs.python.org/extending/newtypes.html.
> > + */
> > +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self);
> > +
> > +// MesosExecutorDriverImpl methods.
> > +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self);
> > +
> > +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self);
> > +
> > +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self);
> > +
> > +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self);
> > +
> > +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self);
> > +
> > +PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> > + MesosExecutorDriverImpl* self,
> > + PyObject* args);
> > +
> > +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> > + MesosExecutorDriverImpl* self,
> > + PyObject* args);
> > +
> > +} // namespace python {
> > +} // namespace mesos {
> > +
> > +#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/module.cpp
> > ----------------------------------------------------------------------
> > diff --git a/src/python/executor/src/mesos/executor/module.cpp
> > b/src/python/executor/src/mesos/executor/module.cpp
> > new file mode 100644
> > index 0000000..f8c6382
> > --- /dev/null
> > +++ b/src/python/executor/src/mesos/executor/module.cpp
> > @@ -0,0 +1,91 @@
> > +// Licensed to the Apache Software Foundation (ASF) under one
> > +// or more contributor license agreements. See the NOTICE file
> > +// distributed with this work for additional information
> > +// regarding copyright ownership. The ASF licenses this file
> > +// to you under the Apache License, Version 2.0 (the
> > +// "License"); you may not use this file except in compliance
> > +// with the License. You may obtain a copy of the License at
> > +//
> > +// http://www.apache.org/licenses/LICENSE-2.0
> > +//
> > +// Unless required by applicable law or agreed to in writing, software
> > +// distributed under the License is distributed on an "AS IS" BASIS,
> > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > +// See the License for the specific language governing permissions and
> > +// limitations under the License.
> > +
> > +/**
> > + * This file defines the _mesos.so binary module used by the Mesos
> Python
> > API.
> > + * This module contains private implementations of MesosSchedulerDriver
> > and
> > + * MesosExecutorDriver as Python types that get called from the public
> > module
> > + * called mesos (in <root>/src/python/src/mesos.py). This design was
> > chosen
> > + * so that most of the API (e.g. the Scheduler and Executor interfaces)
> > can
> > + * be written in Python, and only the parts that need to call into C++
> are
> > + * in C++. Note that the mesos module also contains public classes
> called
> > + * MesosSchedulerDriver and MesosExecutorDriver. These call into the
> > private
> > + * _mesos.MesosSchedulerDriverImpl and _mesos.MesosExecutorDriverImpl.
> > + */
> > +
> > +// Python.h must be included before standard headers.
> > +// See: http://docs.python.org/2/c-api/intro.html#include-files
> > +#include <Python.h>
> > +
> > +#include <iostream>
> > +
> > +#include <mesos/executor.hpp>
> > +
> > +#include "common.hpp"
> > +#include "mesos_executor_driver_impl.hpp"
> > +#include "proxy_executor.hpp"
> > +
> > +using namespace mesos;
> > +using namespace mesos::python;
> > +
> > +using std::map;
> > +using std::string;
> > +using std::vector;
> > +
> > +
> > +/**
> > + * The Python module object for mesos_pb2 (which contains the protobuf
> > + * classes generated for Python).
> > + */
> > +PyObject* mesos::python::mesos_pb2 = NULL;
> > +
> > +
> > +namespace {
> > +
> > +/**
> > + * Method list for our Python module.
> > + */
> > +PyMethodDef MODULE_METHODS[] = {
> > + {NULL, NULL, 0, NULL} /* Sentinel */
> > +};
> > +
> > +} // namespace {
> > +
> > +
> > +/**
> > + * Entry point called by Python to initialize our module.
> > + */
> > +PyMODINIT_FUNC init_executor()
> > +{
> > + // Ensure that the interpreter's threading support is enabled.
> > + PyEval_InitThreads();
> > +
> > + // Import the mesos_pb2 module (on which we depend for protobuf
> classes)
> > + mesos_pb2 = PyImport_ImportModule("mesos.interface.mesos_pb2");
> > + if (mesos_pb2 == NULL)
> > + return;
> > +
> > + // Initialize our Python types.
> > + if (PyType_Ready(&MesosExecutorDriverImplType) < 0)
> > + return;
> > +
> > + // Create the _mesos module and add our types to it.
> > + PyObject* module = Py_InitModule("_executor", MODULE_METHODS);
> > + Py_INCREF(&MesosExecutorDriverImplType);
> > + PyModule_AddObject(module,
> > + "MesosExecutorDriverImpl",
> > + (PyObject*) &MesosExecutorDriverImplType);
> > +}
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.cpp
> > ----------------------------------------------------------------------
> > diff --git a/src/python/executor/src/mesos/executor/proxy_executor.cpp
> > b/src/python/executor/src/mesos/executor/proxy_executor.cpp
> > new file mode 100644
> > index 0000000..da1a49b
> > --- /dev/null
> > +++ b/src/python/executor/src/mesos/executor/proxy_executor.cpp
> > @@ -0,0 +1,273 @@
> > +// Licensed to the Apache Software Foundation (ASF) under one
> > +// or more contributor license agreements. See the NOTICE file
> > +// distributed with this work for additional information
> > +// regarding copyright ownership. The ASF licenses this file
> > +// to you under the Apache License, Version 2.0 (the
> > +// "License"); you may not use this file except in compliance
> > +// with the License. You may obtain a copy of the License at
> > +//
> > +// http://www.apache.org/licenses/LICENSE-2.0
> > +//
> > +// Unless required by applicable law or agreed to in writing, software
> > +// distributed under the License is distributed on an "AS IS" BASIS,
> > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > +// See the License for the specific language governing permissions and
> > +// limitations under the License.
> > +
> > +// Python.h must be included before standard headers.
> > +// See: http://docs.python.org/2/c-api/intro.html#include-files
> > +#include <Python.h>
> > +
> > +#include <iostream>
> > +
> > +#include "common.hpp"
> > +#include "mesos_executor_driver_impl.hpp"
> > +#include "proxy_executor.hpp"
> > +
> > +using namespace mesos;
> > +
> > +using std::cerr;
> > +using std::endl;
> > +using std::map;
> > +using std::string;
> > +using std::vector;
> > +
> > +namespace mesos {
> > +namespace python {
> > +
> > +void ProxyExecutor::registered(ExecutorDriver* driver,
> > + const ExecutorInfo& executorInfo,
> > + const FrameworkInfo& frameworkInfo,
> > + const SlaveInfo& slaveInfo)
> > +{
> > + InterpreterLock lock;
> > +
> > + PyObject* executorInfoObj = NULL;
> > + PyObject* frameworkInfoObj = NULL;
> > + PyObject* slaveInfoObj = NULL;
> > + PyObject* res = NULL;
> > +
> > + executorInfoObj = createPythonProtobuf(executorInfo, "ExecutorInfo");
> > + frameworkInfoObj = createPythonProtobuf(frameworkInfo,
> "FrameworkInfo");
> > + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo");
> > +
> > + if (executorInfoObj == NULL ||
> > + frameworkInfoObj == NULL ||
> > + slaveInfoObj == NULL) {
> > + goto cleanup; // createPythonProtobuf will have set an exception.
> > + }
> > +
> > + res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "registered",
> > + (char*) "OOOO",
> > + impl,
> > + executorInfoObj,
> > + frameworkInfoObj,
> > + slaveInfoObj);
> > + if (res == NULL) {
> > + cerr << "Failed to call executor registered" << endl;
> > + goto cleanup;
> > + }
> > +
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + driver->abort();
> > + }
> > + Py_XDECREF(executorInfoObj);
> > + Py_XDECREF(frameworkInfoObj);
> > + Py_XDECREF(slaveInfoObj);
> > + Py_XDECREF(res);
> > +}
> > +
> > +
> > +void ProxyExecutor::reregistered(ExecutorDriver* driver,
> > + const SlaveInfo& slaveInfo)
> > +{
> > + InterpreterLock lock;
> > +
> > + PyObject* slaveInfoObj = NULL;
> > + PyObject* res = NULL;
> > +
> > + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo");
> > +
> > + if (slaveInfoObj == NULL) {
> > + goto cleanup; // createPythonProtobuf will have set an exception.
> > + }
> > +
> > + res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "reregistered",
> > + (char*) "OO",
> > + impl,
> > + slaveInfoObj);
> > + if (res == NULL) {
> > + cerr << "Failed to call executor re-registered" << endl;
> > + goto cleanup;
> > + }
> > +
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + driver->abort();
> > + }
> > + Py_XDECREF(slaveInfoObj);
> > + Py_XDECREF(res);
> > +}
> > +
> > +
> > +void ProxyExecutor::disconnected(ExecutorDriver* driver)
> > +{
> > + InterpreterLock lock;
> > + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "disconnected",
> > + (char*) "O",
> > + impl);
> > + if (res == NULL) {
> > + cerr << "Failed to call executor's disconnected" << endl;
> > + goto cleanup;
> > + }
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + driver->abort();
> > + }
> > + Py_XDECREF(res);
> > +}
> > +
> > +
> > +void ProxyExecutor::launchTask(ExecutorDriver* driver,
> > + const TaskInfo& task)
> > +{
> > + InterpreterLock lock;
> > +
> > + PyObject* taskObj = NULL;
> > + PyObject* res = NULL;
> > +
> > + taskObj = createPythonProtobuf(task, "TaskInfo");
> > + if (taskObj == NULL) {
> > + goto cleanup; // createPythonProtobuf will have set an exception.
> > + }
> > +
> > + res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "launchTask",
> > + (char*) "OO",
> > + impl,
> > + taskObj);
> > + if (res == NULL) {
> > + cerr << "Failed to call executor's launchTask" << endl;
> > + goto cleanup;
> > + }
> > +
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + driver->abort();
> > + }
> > + Py_XDECREF(taskObj);
> > + Py_XDECREF(res);
> > +}
> > +
> > +
> > +void ProxyExecutor::killTask(ExecutorDriver* driver,
> > + const TaskID& taskId)
> > +{
> > + InterpreterLock lock;
> > +
> > + PyObject* taskIdObj = NULL;
> > + PyObject* res = NULL;
> > +
> > + taskIdObj = createPythonProtobuf(taskId, "TaskID");
> > + if (taskIdObj == NULL) {
> > + goto cleanup; // createPythonProtobuf will have set an exception.
> > + }
> > +
> > + res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "killTask",
> > + (char*) "OO",
> > + impl,
> > + taskIdObj);
> > + if (res == NULL) {
> > + cerr << "Failed to call executor's killTask" << endl;
> > + goto cleanup;
> > + }
> > +
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + driver->abort();
> > + }
> > + Py_XDECREF(taskIdObj);
> > + Py_XDECREF(res);
> > +}
> > +
> > +
> > +void ProxyExecutor::frameworkMessage(ExecutorDriver* driver,
> > + const string& data)
> > +{
> > + InterpreterLock lock;
> > +
> > + PyObject* res = NULL;
> > +
> > + res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "frameworkMessage",
> > + (char*) "Os#",
> > + impl,
> > + data.data(),
> > + data.length());
> > + if (res == NULL) {
> > + cerr << "Failed to call executor's frameworkMessage" << endl;
> > + goto cleanup;
> > + }
> > +
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + driver->abort();
> > + }
> > + Py_XDECREF(res);
> > +}
> > +
> > +
> > +void ProxyExecutor::shutdown(ExecutorDriver* driver)
> > +{
> > + InterpreterLock lock;
> > + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "shutdown",
> > + (char*) "O",
> > + impl);
> > + if (res == NULL) {
> > + cerr << "Failed to call executor's shutdown" << endl;
> > + goto cleanup;
> > + }
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + driver->abort();
> > + }
> > + Py_XDECREF(res);
> > +}
> > +
> > +
> > +void ProxyExecutor::error(ExecutorDriver* driver, const string& message)
> > +{
> > + InterpreterLock lock;
> > + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> > + (char*) "error",
> > + (char*) "Os#",
> > + impl,
> > + message.data(),
> > + message.length());
> > + if (res == NULL) {
> > + cerr << "Failed to call executor's error" << endl;
> > + goto cleanup;
> > + }
> > +cleanup:
> > + if (PyErr_Occurred()) {
> > + PyErr_Print();
> > + // No need for driver.stop(); it should stop itself.
> > + }
> > + Py_XDECREF(res);
> > +}
> > +
> > +} // namespace python {
> > +} // namespace mesos {
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.hpp
> > ----------------------------------------------------------------------
> > diff --git a/src/python/executor/src/mesos/executor/proxy_executor.hpp
> > b/src/python/executor/src/mesos/executor/proxy_executor.hpp
> > new file mode 100644
> > index 0000000..23d64fd
> > --- /dev/null
> > +++ b/src/python/executor/src/mesos/executor/proxy_executor.hpp
> > @@ -0,0 +1,64 @@
> > +// Licensed to the Apache Software Foundation (ASF) under one
> > +// or more contributor license agreements. See the NOTICE file
> > +// distributed with this work for additional information
> > +// regarding copyright ownership. The ASF licenses this file
> > +// to you under the Apache License, Version 2.0 (the
> > +// "License"); you may not use this file except in compliance
> > +// with the License. You may obtain a copy of the License at
> > +//
> > +// http://www.apache.org/licenses/LICENSE-2.0
> > +//
> > +// Unless required by applicable law or agreed to in writing, software
> > +// distributed under the License is distributed on an "AS IS" BASIS,
> > +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > +// See the License for the specific language governing permissions and
> > +// limitations under the License.
> > +
> > +#ifndef PROXY_EXECUTOR_HPP
> > +#define PROXY_EXECUTOR_HPP
> > +
> > +// Python.h must be included before standard headers.
> > +// See: http://docs.python.org/2/c-api/intro.html#include-files
> > +#include <Python.h>
> > +
> > +#include <string>
> > +#include <vector>
> > +
> > +#include <mesos/executor.hpp>
> > +
> > +namespace mesos {
> > +namespace python {
> > +
> > +struct MesosExecutorDriverImpl;
> > +
> > +/**
> > + * Proxy Executor implementation that will call into Python.
> > + */
> > +class ProxyExecutor : public Executor
> > +{
> > +public:
> > + explicit ProxyExecutor(MesosExecutorDriverImpl *_impl) : impl(_impl)
> {}
> > +
> > + virtual ~ProxyExecutor() {}
> > +
> > + virtual void registered(ExecutorDriver* driver,
> > + const ExecutorInfo& executorInfo,
> > + const FrameworkInfo& frameworkInfo,
> > + const SlaveInfo& slaveInfo);
> > + virtual void reregistered(ExecutorDriver* driver, const SlaveInfo&
> > slaveInfo);
> > + virtual void disconnected(ExecutorDriver* driver);
> > + virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task);
> > + virtual void killTask(ExecutorDriver* driver, const TaskID& taskId);
> > + virtual void frameworkMessage(ExecutorDriver* driver,
> > + const std::string& data);
> > + virtual void shutdown(ExecutorDriver* driver);
> > + virtual void error(ExecutorDriver* driver, const std::string&
> message);
> > +
> > +private:
> > + MesosExecutorDriverImpl *impl;
> > +};
> > +
> > +} // namespace python {
> > +} // namespace mesos {
> > +
> > +#endif // PROXY_EXECUTOR_HPP
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/ext_modules.py.in
> > ----------------------------------------------------------------------
> > diff --git a/src/python/native/ext_modules.py.in b/src/python/native/
> > ext_modules.py.in
> > deleted file mode 100644
> > index eb93864..0000000
> > --- a/src/python/native/ext_modules.py.in
> > +++ /dev/null
> > @@ -1,151 +0,0 @@
> > -# Licensed to the Apache Software Foundation (ASF) under one
> > -# or more contributor license agreements. See the NOTICE file
> > -# distributed with this work for additional information
> > -# regarding copyright ownership. The ASF licenses this file
> > -# to you under the Apache License, Version 2.0 (the
> > -# "License"); you may not use this file except in compliance
> > -# with the License. You may obtain a copy of the License at
> > -#
> > -# http://www.apache.org/licenses/LICENSE-2.0
> > -#
> > -# Unless required by applicable law or agreed to in writing, software
> > -# distributed under the License is distributed on an "AS IS" BASIS,
> > -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > -# See the License for the specific language governing permissions and
> > -# limitations under the License.
> > -
> > -import errno
> > -import glob
> > -import os
> > -import shutil
> > -
> > -from setuptools import Extension
> > -
> > -abs_top_srcdir = '@abs_top_srcdir@'
> > -abs_top_builddir = '@abs_top_builddir@'
> > -
> > -src_python_native = os.path.join(
> > - 'src', 'python', 'native', 'src', 'mesos', 'native')
> > -
> > -leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> > -zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.5', 'src', 'c')
> > -libprocess = os.path.join('3rdparty', 'libprocess')
> > -
> > -# Even though a statically compiled libprocess should include glog,
> > -# libev, gperftools, and protobuf before installation this isn't the
> > -# case, so while a libtool managed build will correctly pull in these
> > -# libraries when building the final result, we need to explicitly
> > -# include them here (or more precisely, down where we actually include
> > -# libev.a and libprofiler.a).
> > -glog = os.path.join(libprocess, '3rdparty', 'glog-0.3.3')
> > -gperftools = os.path.join(libprocess, '3rdparty', 'gperftools-2.0')
> > -protobuf = os.path.join(libprocess, '3rdparty', 'protobuf-2.5.0')
> > -
> > -# Build the list of source files. Note that each source must be
> > -# relative to our current directory (where this script lives).
> > -SOURCES = [
> > - os.path.join('src', 'mesos', 'native', file)
> > - for file in os.listdir(os.path.join(abs_top_srcdir,
> > src_python_native))
> > - if file.endswith('.cpp')
> > -]
> > -
> > -INCLUDE_DIRS = [
> > - os.path.join(abs_top_srcdir, 'include'),
> > - os.path.join(abs_top_builddir, 'include'),
> > - # Needed for the *.pb.h protobuf includes.
> > - os.path.join(abs_top_builddir, 'include', 'mesos'),
> > - os.path.join(abs_top_builddir, 'src'),
> > - os.path.join(abs_top_builddir, src_python_native),
> > - os.path.join(abs_top_builddir, protobuf, 'src'),
> > -]
> > -
> > -LIBRARY_DIRS = []
> > -
> > -EXTRA_OBJECTS = [
> > - os.path.join(abs_top_builddir, 'src', '.libs',
> > 'libmesos_no_3rdparty.a'),
> > - os.path.join(abs_top_builddir, libprocess, '.libs', 'libprocess.a')
> > -]
> > -
> > -# For leveldb, we need to check for the presence of libleveldb.a, since
> > -# it is possible to disable leveldb inside mesos.
> > -libglog = os.path.join(abs_top_builddir, glog, '.libs', 'libglog.a')
> > -libleveldb = os.path.join(abs_top_builddir, leveldb, 'libleveldb.a')
> > -libzookeeper = os.path.join(
> > - abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> > -libprotobuf = os.path.join(
> > - abs_top_builddir, protobuf, 'src', '.libs', 'libprotobuf.a')
> > -
> > -if os.path.exists(libleveldb):
> > - EXTRA_OBJECTS.append(libleveldb)
> > -else:
> > - EXTRA_OBJECTS.append('-lleveldb')
> > -
> > -if os.path.exists(libzookeeper):
> > - EXTRA_OBJECTS.append(libzookeeper)
> > -else:
> > - EXTRA_OBJECTS.append('-lzookeeper_mt')
> > -
> > -if os.path.exists(libglog):
> > - EXTRA_OBJECTS.append(libglog)
> > -else:
> > - EXTRA_OBJECTS.append('-lglog')
> > -
> > -if os.path.exists(libprotobuf):
> > - EXTRA_OBJECTS.append(libprotobuf)
> > -else:
> > - EXTRA_OBJECTS.append('-lprotobuf')
> > -
> > -
> > -# libev is a special case because it needs to be enabled only when
> > -# libevent *is not* enabled through the top level ./configure.
> > -#
> > -# TODO(hartem): this entire block MUST be removed once libev is
> deprecated
> > -# in favor of libevent.
> > -if '@ENABLE_LIBEVENT_TRUE@' == '#':
> > - libev = os.path.join(libprocess, '3rdparty', 'libev-4.15')
> > - libev = os.path.join(abs_top_builddir, libev, '.libs', 'libev.a')
> > -
> > - if os.path.exists(libev):
> > - EXTRA_OBJECTS.append(libev)
> > - else:
> > - EXTRA_OBJECTS.append('-lev')
> > -
> > -
> > -# For gperftools, we need to check for the presence of libprofiler.a,
> > since
> > -# it is possible to disable perftools inside libprocess.
> > -libprofiler = os.path.join(
> > - abs_top_builddir, gperftools, '.libs', 'libprofiler.a')
> > -
> > -if os.path.exists(libprofiler):
> > - EXTRA_OBJECTS.append(libprofiler)
> > -
> > -EXTRA_LINK_ARGS = []
> > -
> > -# Add any flags from LDFLAGS.
> > -if 'LDFLAGS' in os.environ:
> > - for flag in os.environ['LDFLAGS'].split():
> > - EXTRA_LINK_ARGS.append(flag)
> > -
> > -# Add any libraries from LIBS.
> > -if 'LIBS' in os.environ:
> > - for library in os.environ['LIBS'].split():
> > - EXTRA_LINK_ARGS.append(library)
> > -
> > -DEPENDS = [
> > - os.path.join(abs_top_srcdir, 'src', 'python', source)
> > - for source in SOURCES
> > -]
> > -
> > -# Note that we add EXTRA_OBJECTS to our dependency list to make sure
> > -# that we rebuild this module when one of them changes (e.g.,
> > -# libprocess).
> > -mesos_module = \
> > - Extension('mesos.native._mesos',
> > - sources = SOURCES,
> > - include_dirs = INCLUDE_DIRS,
> > - library_dirs = LIBRARY_DIRS,
> > - extra_objects = EXTRA_OBJECTS,
> > - extra_link_args = EXTRA_LINK_ARGS,
> > - depends = EXTRA_OBJECTS,
> > - language = 'c++',
> > - )
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/setup.py.in
> > ----------------------------------------------------------------------
> > diff --git a/src/python/native/setup.py.in b/src/python/native/
> setup.py.in
> > index 49ed612..10a5002 100644
> > --- a/src/python/native/setup.py.in
> > +++ b/src/python/native/setup.py.in
> > @@ -16,8 +16,6 @@
> > # See the License for the specific language governing permissions and
> > # limitations under the License.
> >
> > -import ext_modules
> > -
> > config = {
> > 'name': 'mesos.native',
> > 'version': '@PACKAGE_VERSION@',
> > @@ -28,13 +26,12 @@ config = {
> > 'namespace_packages': [ 'mesos' ],
> > 'packages': [ 'mesos', 'mesos.native' ],
> > 'package_dir': { '': 'src' },
> > - 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ],
> > + 'install_requires': [ 'mesos.executor == @PACKAGE_VERSION@',
> > + 'mesos.scheduler == @PACKAGE_VERSION@'],
> > 'license': 'Apache 2.0',
> > 'keywords': 'mesos',
> > - 'classifiers': [ ],
> > - 'ext_modules': [ ext_modules.mesos_module ]
> > + 'classifiers': [ ]
> > }
> >
> > from setuptools import setup
> > -
> > setup(**config)
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/__init__.py
> > ----------------------------------------------------------------------
> > diff --git a/src/python/native/src/mesos/native/__init__.py
> > b/src/python/native/src/mesos/native/__init__.py
> > index 226f943..b581537 100644
> > --- a/src/python/native/src/mesos/native/__init__.py
> > +++ b/src/python/native/src/mesos/native/__init__.py
> > @@ -14,8 +14,5 @@
> > # See the License for the specific language governing permissions and
> > # limitations under the License.
> >
> > -from ._mesos import MesosExecutorDriverImpl
> > -from ._mesos import MesosSchedulerDriverImpl
> > -
> > -MesosExecutorDriver = MesosExecutorDriverImpl
> > -MesosSchedulerDriver = MesosSchedulerDriverImpl
> > +from mesos.executor import MesosExecutorDriver
> > +from mesos.scheduler import MesosSchedulerDriver
> > \ No newline at end of file
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> > ----------------------------------------------------------------------
> > diff --git
> > a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> > b/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> > deleted file mode 100644
> > index 7838a07..0000000
> > --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> > +++ /dev/null
> > @@ -1,347 +0,0 @@
> > -// Licensed to the Apache Software Foundation (ASF) under one
> > -// or more contributor license agreements. See the NOTICE file
> > -// distributed with this work for additional information
> > -// regarding copyright ownership. The ASF licenses this file
> > -// to you under the Apache License, Version 2.0 (the
> > -// "License"); you may not use this file except in compliance
> > -// with the License. You may obtain a copy of the License at
> > -//
> > -// http://www.apache.org/licenses/LICENSE-2.0
> > -//
> > -// Unless required by applicable law or agreed to in writing, software
> > -// distributed under the License is distributed on an "AS IS" BASIS,
> > -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > -// See the License for the specific language governing permissions and
> > -// limitations under the License.
> > -
> > -// Python.h must be included before standard headers.
> > -// See: http://docs.python.org/2/c-api/intro.html#include-files
> > -#include <Python.h>
> > -
> > -#include <string>
> > -
> > -#include "mesos_executor_driver_impl.hpp"
> > -#include "module.hpp"
> > -#include "proxy_executor.hpp"
> > -
> > -using namespace mesos;
> > -using namespace mesos::python;
> > -
> > -using std::cerr;
> > -using std::endl;
> > -using std::string;
> > -using std::vector;
> > -using std::map;
> > -
> > -
> > -namespace mesos { namespace python {
> > -
> > -/**
> > - * Python type object for MesosExecutorDriverImpl.
> > - */
> > -PyTypeObject MesosExecutorDriverImplType = {
> > - PyObject_HEAD_INIT(NULL)
> > - 0, /* ob_size */
> > - "_mesos.MesosExecutorDriverImpl", /* tp_name */
> > - sizeof(MesosExecutorDriverImpl), /* tp_basicsize */
> > - 0, /* tp_itemsize */
> > - (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */
> > - 0, /* tp_print */
> > - 0, /* tp_getattr */
> > - 0, /* tp_setattr */
> > - 0, /* tp_compare */
> > - 0, /* tp_repr */
> > - 0, /* tp_as_number */
> > - 0, /* tp_as_sequence */
> > - 0, /* tp_as_mapping */
> > - 0, /* tp_hash */
> > - 0, /* tp_call */
> > - 0, /* tp_str */
> > - 0, /* tp_getattro */
> > - 0, /* tp_setattro */
> > - 0, /* tp_as_buffer */
> > - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> > - "Private MesosExecutorDriver implementation", /* tp_doc */
> > - (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */
> > - (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */
> > - 0, /* tp_richcompare */
> > - 0, /* tp_weaklistoffset
> */
> > - 0, /* tp_iter */
> > - 0, /* tp_iternext */
> > - MesosExecutorDriverImpl_methods, /* tp_methods */
> > - 0, /* tp_members */
> > - 0, /* tp_getset */
> > - 0, /* tp_base */
> > - 0, /* tp_dict */
> > - 0, /* tp_descr_get */
> > - 0, /* tp_descr_set */
> > - 0, /* tp_dictoffset */
> > - (initproc) MesosExecutorDriverImpl_init, /* tp_init */
> > - 0, /* tp_alloc */
> > - MesosExecutorDriverImpl_new, /* tp_new */
> > -};
> > -
> > -
> > -/**
> > - * List of Python methods in MesosExecutorDriverImpl.
> > - */
> > -PyMethodDef MesosExecutorDriverImpl_methods[] = {
> > - { "start",
> > - (PyCFunction) MesosExecutorDriverImpl_start,
> > - METH_NOARGS,
> > - "Start the driver to connect to Mesos"
> > - },
> > - { "stop",
> > - (PyCFunction) MesosExecutorDriverImpl_stop,
> > - METH_NOARGS,
> > - "Stop the driver, disconnecting from Mesos"
> > - },
> > - { "abort",
> > - (PyCFunction) MesosExecutorDriverImpl_abort,
> > - METH_NOARGS,
> > - "Abort the driver, disallowing calls from and to the driver"
> > - },
> > - { "join",
> > - (PyCFunction) MesosExecutorDriverImpl_join,
> > - METH_NOARGS,
> > - "Wait for a running driver to disconnect from Mesos"
> > - },
> > - { "run",
> > - (PyCFunction) MesosExecutorDriverImpl_run,
> > - METH_NOARGS,
> > - "Start a driver and run it, returning when it disconnects from
> Mesos"
> > - },
> > - { "sendStatusUpdate",
> > - (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate,
> > - METH_VARARGS,
> > - "Send a status update for a task"
> > - },
> > - { "sendFrameworkMessage",
> > - (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage,
> > - METH_VARARGS,
> > - "Send a FrameworkMessage to a slave"
> > - },
> > - { NULL } /* Sentinel */
> > -};
> > -
> > -
> > -/**
> > - * Create, but don't initialize, a new MesosExecutorDriverImpl
> > - * (called by Python before init method).
> > - */
> > -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> > - PyObject *args,
> > - PyObject *kwds)
> > -{
> > - MesosExecutorDriverImpl *self;
> > - self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0);
> > - if (self != NULL) {
> > - self->driver = NULL;
> > - self->proxyExecutor = NULL;
> > - self->pythonExecutor = NULL;
> > - }
> > - return (PyObject*) self;
> > -}
> > -
> > -
> > -/**
> > - * Initialize a MesosExecutorDriverImpl with constructor arguments.
> > - */
> > -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> > - PyObject *args,
> > - PyObject *kwds)
> > -{
> > - PyObject *pythonExecutor = NULL;
> > -
> > - if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) {
> > - return -1;
> > - }
> > -
> > - if (pythonExecutor != NULL) {
> > - PyObject* tmp = self->pythonExecutor;
> > - Py_INCREF(pythonExecutor);
> > - self->pythonExecutor = pythonExecutor;
> > - Py_XDECREF(tmp);
> > - }
> > -
> > - if (self->driver != NULL) {
> > - delete self->driver;
> > - self->driver = NULL;
> > - }
> > -
> > - if (self->proxyExecutor != NULL) {
> > - delete self->proxyExecutor;
> > - self->proxyExecutor = NULL;
> > - }
> > -
> > - self->proxyExecutor = new ProxyExecutor(self);
> > - self->driver = new MesosExecutorDriver(self->proxyExecutor);
> > -
> > - return 0;
> > -}
> > -
> > -
> > -/**
> > - * Free a MesosExecutorDriverImpl.
> > - */
> > -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self)
> > -{
> > - if (self->driver != NULL) {
> > - // We need to wrap the driver destructor in an "allow threads"
> > - // macro since the MesosExecutorDriver destructor waits for the
> > - // ExecutorProcess to terminate and there might be a thread that
> > - // is trying to acquire the GIL to call through the
> > - // ProxyExecutor. It will only be after this thread executes that
> > - // the ExecutorProcess might actually get a terminate.
> > - Py_BEGIN_ALLOW_THREADS
> > - delete self->driver;
> > - Py_END_ALLOW_THREADS
> > - self->driver = NULL;
> > - }
> > -
> > - if (self->proxyExecutor != NULL) {
> > - delete self->proxyExecutor;
> > - self->proxyExecutor = NULL;
> > - }
> > -
> > - MesosExecutorDriverImpl_clear(self);
> > - self->ob_type->tp_free((PyObject*) self);
> > -}
> > -
> > -
> > -/**
> > - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> > - * See http://docs.python.org/extending/newtypes.html.
> > - */
> > -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> > - visitproc visit,
> > - void* arg)
> > -{
> > - Py_VISIT(self->pythonExecutor);
> > - return 0;
> > -}
> > -
> > -
> > -/**
> > - * Clear fields of a MesosExecutorDriverImpl that can participate in
> > - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> > - */
> > -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self)
> > -{
> > - Py_CLEAR(self->pythonExecutor);
> > - return 0;
> > -}
> > -
> > -
> > -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->start();
> > - return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > -}
> > -
> > -
> > -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->stop();
> > - return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > -}
> > -
> > -
> > -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->abort();
> > - return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > -}
> > -
> > -
> > -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status;
> > - Py_BEGIN_ALLOW_THREADS
> > - status = self->driver->join();
> > - Py_END_ALLOW_THREADS
> > - return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > -}
> > -
> > -
> > -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status;
> > - Py_BEGIN_ALLOW_THREADS
> > - status = self->driver->run();
> > - Py_END_ALLOW_THREADS
> > - return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > -}
> > -
> > -
> > -PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> > - MesosExecutorDriverImpl* self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* statusObj = NULL;
> > - TaskStatus taskStatus;
> > - if (!PyArg_ParseTuple(args, "O", &statusObj)) {
> > - return NULL;
> > - }
> > - if (!readPythonProtobuf(statusObj, &taskStatus)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python TaskStatus");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->sendStatusUpdate(taskStatus);
> > - return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > -}
> > -
> > -
> > -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> > - MesosExecutorDriverImpl* self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - const char* data;
> > - int length;
> > - if (!PyArg_ParseTuple(args, "s#", &data, &length)) {
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->sendFrameworkMessage(string(data,
> > length));
> > - return PyInt_FromLong(status); // Sets an exception if creating the
> int
> > fails.
> > -}
> > -
> > -} // namespace python {
> > -} // namespace mesos {
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> > ----------------------------------------------------------------------
> > diff --git
> > a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> > b/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> > deleted file mode 100644
> > index 6e672f8..0000000
> > --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> > +++ /dev/null
> > @@ -1,103 +0,0 @@
> > -// Licensed to the Apache Software Foundation (ASF) under one
> > -// or more contributor license agreements. See the NOTICE file
> > -// distributed with this work for additional information
> > -// regarding copyright ownership. The ASF licenses this file
> > -// to you under the Apache License, Version 2.0 (the
> > -// "License"); you may not use this file except in compliance
> > -// with the License. You may obtain a copy of the License at
> > -//
> > -// http://www.apache.org/licenses/LICENSE-2.0
> > -//
> > -// Unless required by applicable law or agreed to in writing, software
> > -// distributed under the License is distributed on an "AS IS" BASIS,
> > -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > -// See the License for the specific language governing permissions and
> > -// limitations under the License.
> > -
> > -#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP
> > -#define MESOS_EXECUTOR_DRIVER_IMPL_HPP
> > -
> > -#include <mesos/executor.hpp>
> > -
> > -
> > -namespace mesos { namespace python {
> > -
> > -class ProxyExecutor;
> > -
> > -/**
> > - * Python object structure for MesosExecutorDriverImpl objects.
> > - */
> > -struct MesosExecutorDriverImpl {
> > - PyObject_HEAD
> > - /* Type-specific fields go here. */
> > - MesosExecutorDriver* driver;
> > - ProxyExecutor* proxyExecutor;
> > - PyObject* pythonExecutor;
> > -};
> > -
> > -/**
> > - * Python type object for MesosExecutorDriverImpl.
> > - */
> > -extern PyTypeObject MesosExecutorDriverImplType;
> > -
> > -/**
> > - * List of Python methods in MesosExecutorDriverImpl.
> > - */
> > -extern PyMethodDef MesosExecutorDriverImpl_methods[];
> > -
> > -/**
> > - * Create, but don't initialize, a new MesosExecutorDriverImpl
> > - * (called by Python before init method).
> > - */
> > -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> > - PyObject *args,
> > - PyObject *kwds);
> > -
> > -/**
> > - * Initialize a MesosExecutorDriverImpl with constructor arguments.
> > - */
> > -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> > - PyObject *args,
> > - PyObject *kwds);
> > -
> > -/**
> > - * Free a MesosExecutorDriverImpl.
> > - */
> > -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self);
> > -
> > -/**
> > - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> > - * See http://docs.python.org/extending/newtypes.html.
> > - */
> > -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> > - visitproc visit,
> > - void* arg);
> > -/**
> > - * Clear fields of a MesosExecutorDriverImpl that can participate in
> > - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> > - */
> > -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self);
> > -
> > -// MesosExecutorDriverImpl methods.
> > -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self);
> > -
> > -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self);
> > -
> > -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self);
> > -
> > -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self);
> > -
> > -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self);
> > -
> > -PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> > - MesosExecutorDriverImpl* self,
> > - PyObject* args);
> > -
> > -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> > - MesosExecutorDriverImpl* self,
> > - PyObject* args);
> > -
> > -} // namespace python {
> > -} // namespace mesos {
> > -
> > -#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> > ----------------------------------------------------------------------
> > diff --git
> > a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> > b/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> > deleted file mode 100644
> > index f8be49b..0000000
> > --- a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> > +++ /dev/null
> > @@ -1,782 +0,0 @@
> > -// Licensed to the Apache Software Foundation (ASF) under one
> > -// or more contributor license agreements. See the NOTICE file
> > -// distributed with this work for additional information
> > -// regarding copyright ownership. The ASF licenses this file
> > -// to you under the Apache License, Version 2.0 (the
> > -// "License"); you may not use this file except in compliance
> > -// with the License. You may obtain a copy of the License at
> > -//
> > -// http://www.apache.org/licenses/LICENSE-2.0
> > -//
> > -// Unless required by applicable law or agreed to in writing, software
> > -// distributed under the License is distributed on an "AS IS" BASIS,
> > -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > -// See the License for the specific language governing permissions and
> > -// limitations under the License.
> > -
> > -// Python.h must be included before standard headers.
> > -// See: http://docs.python.org/2/c-api/intro.html#include-files
> > -#include <Python.h>
> > -
> > -#include <string>
> > -
> > -#include "mesos_scheduler_driver_impl.hpp"
> > -#include "module.hpp"
> > -#include "proxy_scheduler.hpp"
> > -
> > -using namespace mesos;
> > -using namespace mesos::python;
> > -
> > -using std::cerr;
> > -using std::endl;
> > -using std::string;
> > -using std::vector;
> > -using std::map;
> > -
> > -namespace mesos {
> > -namespace python {
> > -
> > -/**
> > - * Python type object for MesosSchedulerDriverImpl.
> > - */
> > -PyTypeObject MesosSchedulerDriverImplType = {
> > - PyObject_HEAD_INIT(NULL)
> > - 0, /* ob_size */
> > - "_mesos.MesosSchedulerDriverImpl", /* tp_name */
> > - sizeof(MesosSchedulerDriverImpl), /* tp_basicsize */
> > - 0, /* tp_itemsize */
> > - (destructor) MesosSchedulerDriverImpl_dealloc, /* tp_dealloc */
> > - 0, /* tp_print */
> > - 0, /* tp_getattr */
> > - 0, /* tp_setattr */
> > - 0, /* tp_compare */
> > - 0, /* tp_repr */
> > - 0, /* tp_as_number */
> > - 0, /* tp_as_sequence */
> > - 0, /* tp_as_mapping */
> > - 0, /* tp_hash */
> > - 0, /* tp_call */
> > - 0, /* tp_str */
> > - 0, /* tp_getattro */
> > - 0, /* tp_setattro */
> > - 0, /* tp_as_buffer */
> > - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> > - "Private MesosSchedulerDriver implementation", /* tp_doc */
> > - (traverseproc) MesosSchedulerDriverImpl_traverse, /* tp_traverse */
> > - (inquiry) MesosSchedulerDriverImpl_clear, /* tp_clear */
> > - 0, /* tp_richcompare */
> > - 0, /* tp_weaklistoffset
> > */
> > - 0, /* tp_iter */
> > - 0, /* tp_iternext */
> > - MesosSchedulerDriverImpl_methods, /* tp_methods */
> > - 0, /* tp_members */
> > - 0, /* tp_getset */
> > - 0, /* tp_base */
> > - 0, /* tp_dict */
> > - 0, /* tp_descr_get */
> > - 0, /* tp_descr_set */
> > - 0, /* tp_dictoffset */
> > - (initproc) MesosSchedulerDriverImpl_init, /* tp_init */
> > - 0, /* tp_alloc */
> > - MesosSchedulerDriverImpl_new, /* tp_new */
> > -};
> > -
> > -
> > -/**
> > - * List of Python methods in MesosSchedulerDriverImpl.
> > - */
> > -PyMethodDef MesosSchedulerDriverImpl_methods[] = {
> > - { "start",
> > - (PyCFunction) MesosSchedulerDriverImpl_start,
> > - METH_NOARGS,
> > - "Start the driver to connect to Mesos"
> > - },
> > - { "stop",
> > - (PyCFunction) MesosSchedulerDriverImpl_stop,
> > - METH_VARARGS,
> > - "Stop the driver, disconnecting from Mesos"
> > - },
> > - { "abort",
> > - (PyCFunction) MesosSchedulerDriverImpl_abort,
> > - METH_NOARGS,
> > - "Abort the driver, disabling calls from and to the driver"
> > - },
> > - { "join",
> > - (PyCFunction) MesosSchedulerDriverImpl_join,
> > - METH_NOARGS,
> > - "Wait for a running driver to disconnect from Mesos"
> > - },
> > - { "run",
> > - (PyCFunction) MesosSchedulerDriverImpl_run,
> > - METH_NOARGS,
> > - "Start a driver and run it, returning when it disconnects from
> Mesos"
> > - },
> > - { "requestResources",
> > - (PyCFunction) MesosSchedulerDriverImpl_requestResources,
> > - METH_VARARGS,
> > - "Request resources from the Mesos allocator"
> > - },
> > - { "launchTasks",
> > - (PyCFunction) MesosSchedulerDriverImpl_launchTasks,
> > - METH_VARARGS,
> > - "Reply to a Mesos offer with a list of tasks"
> > - },
> > - { "killTask",
> > - (PyCFunction) MesosSchedulerDriverImpl_killTask,
> > - METH_VARARGS,
> > - "Kill the task with the given ID"
> > - },
> > - { "acceptOffers",
> > - (PyCFunction) MesosSchedulerDriverImpl_acceptOffers,
> > - METH_VARARGS,
> > - "Reply to a Mesos offer with a list of offer operations"
> > - },
> > - { "declineOffer",
> > - (PyCFunction) MesosSchedulerDriverImpl_declineOffer,
> > - METH_VARARGS,
> > - "Decline a Mesos offer"
> > - },
> > - { "reviveOffers",
> > - (PyCFunction) MesosSchedulerDriverImpl_reviveOffers,
> > - METH_NOARGS,
> > - "Remove all filters and ask Mesos for new offers"
> > - },
> > - { "suppressOffers",
> > - (PyCFunction) MesosSchedulerDriverImpl_suppressOffers,
> > - METH_NOARGS,
> > - "Set suppressed attribute as true for the Framework"
> > - },
> > - { "acknowledgeStatusUpdate",
> > - (PyCFunction) MesosSchedulerDriverImpl_acknowledgeStatusUpdate,
> > - METH_VARARGS,
> > - "Acknowledge a status update"
> > - },
> > - { "sendFrameworkMessage",
> > - (PyCFunction) MesosSchedulerDriverImpl_sendFrameworkMessage,
> > - METH_VARARGS,
> > - "Send a FrameworkMessage to a slave"
> > - },
> > - { "reconcileTasks",
> > - (PyCFunction) MesosSchedulerDriverImpl_reconcileTasks,
> > - METH_VARARGS,
> > - "Master sends status updates if task status is different from
> > expected"
> > - },
> > - { NULL } /* Sentinel */
> > -};
> > -
> > -
> > -/**
> > - * Create, but don't initialize, a new MesosSchedulerDriverImpl
> > - * (called by Python before init method).
> > - */
> > -PyObject* MesosSchedulerDriverImpl_new(PyTypeObject* type,
> > - PyObject* args,
> > - PyObject* kwds)
> > -{
> > - MesosSchedulerDriverImpl* self;
> > - self = (MesosSchedulerDriverImpl*) type->tp_alloc(type, 0);
> > - if (self != NULL) {
> > - self->driver = NULL;
> > - self->proxyScheduler = NULL;
> > - self->pythonScheduler = NULL;
> > - }
> > - return (PyObject*) self;
> > -}
> > -
> > -
> > -/**
> > - * Initialize a MesosSchedulerDriverImpl with constructor arguments.
> > - */
> > -int MesosSchedulerDriverImpl_init(MesosSchedulerDriverImpl* self,
> > - PyObject* args,
> > - PyObject* kwds)
> > -{
> > - // Note: We use an integer for 'implicitAcknoweldgements' because
> > - // it is the recommended way to pass booleans through CPython.
> > - PyObject* schedulerObj = NULL;
> > - PyObject* frameworkObj = NULL;
> > - const char* master;
> > - int implicitAcknowledgements = 1; // Enabled by default.
> > - PyObject* credentialObj = NULL;
> > -
> > - if (!PyArg_ParseTuple(
> > - args,
> > - "OOs|iO",
> > - &schedulerObj,
> > - &frameworkObj,
> > - &master,
> > - &implicitAcknowledgements,
> > - &credentialObj)) {
> > - return -1;
> > - }
> > -
> > - if (schedulerObj != NULL) {
> > - PyObject* tmp = self->pythonScheduler;
> > - Py_INCREF(schedulerObj);
> > - self->pythonScheduler = schedulerObj;
> > - Py_XDECREF(tmp);
> > - }
> > -
> > - FrameworkInfo framework;
> > - if (frameworkObj != NULL) {
> > - if (!readPythonProtobuf(frameworkObj, &framework)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python FrameworkInfo");
> > - return -1;
> > - }
> > - }
> > -
> > - Credential credential;
> > - if (credentialObj != NULL) {
> > - if (!readPythonProtobuf(credentialObj, &credential)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> > Credential");
> > - return -1;
> > - }
> > - }
> > -
> > -
> > - if (self->driver != NULL) {
> > - delete self->driver;
> > - self->driver = NULL;
> > - }
> > -
> > - if (self->proxyScheduler != NULL) {
> > - delete self->proxyScheduler;
> > - self->proxyScheduler = NULL;
> > - }
> > -
> > - self->proxyScheduler = new ProxyScheduler(self);
> > -
> > - if (credentialObj != NULL) {
> > - self->driver = new MesosSchedulerDriver(
> > - self->proxyScheduler,
> > - framework,
> > - master,
> > - implicitAcknowledgements != 0,
> > - credential);
> > - } else {
> > - self->driver = new MesosSchedulerDriver(
> > - self->proxyScheduler,
> > - framework,
> > - master,
> > - implicitAcknowledgements != 0);
> > - }
> > -
> > - return 0;
> > -}
> > -
> > -
> > -/**
> > - * Free a MesosSchedulerDriverImpl.
> > - */
> > -void MesosSchedulerDriverImpl_dealloc(MesosSchedulerDriverImpl* self)
> > -{
> > - if (self->driver != NULL) {
> > - // We need to wrap the driver destructor in an "allow threads"
> > - // macro since the MesosSchedulerDriver destructor waits for the
> > - // SchedulerProcess to terminate and there might be a thread that
> > - // is trying to acquire the GIL to call through the
> > - // ProxyScheduler. It will only be after this thread executes that
> > - // the SchedulerProcess might actually get a terminate.
> > - Py_BEGIN_ALLOW_THREADS
> > - delete self->driver;
> > - Py_END_ALLOW_THREADS
> > - self->driver = NULL;
> > - }
> > -
> > - if (self->proxyScheduler != NULL) {
> > - delete self->proxyScheduler;
> > - self->proxyScheduler = NULL;
> > - }
> > -
> > - MesosSchedulerDriverImpl_clear(self);
> > - self->ob_type->tp_free((PyObject*) self);
> > -}
> > -
> > -
> > -/**
> > - * Traverse fields of a MesosSchedulerDriverImpl on a cyclic GC search.
> > - * See http://docs.python.org/extending/newtypes.html.
> > - */
> > -int MesosSchedulerDriverImpl_traverse(MesosSchedulerDriverImpl* self,
> > - visitproc visit,
> > - void* arg)
> > -{
> > - Py_VISIT(self->pythonScheduler);
> > - return 0;
> > -}
> > -
> > -
> > -/**
> > - * Clear fields of a MesosSchedulerDriverImpl that can participate in
> > - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> > - */
> > -int MesosSchedulerDriverImpl_clear(MesosSchedulerDriverImpl* self)
> > -{
> > - Py_CLEAR(self->pythonScheduler);
> > - return 0;
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_start(MesosSchedulerDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->start();
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_stop(MesosSchedulerDriverImpl* self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - bool failover = false; // Should match default in mesos.py.
> > -
> > - if (!PyArg_ParseTuple(args, "|b", &failover)) {
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->stop(failover);
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_abort(MesosSchedulerDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->abort();
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_join(MesosSchedulerDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status;
> > - Py_BEGIN_ALLOW_THREADS
> > - status = self->driver->join();
> > - Py_END_ALLOW_THREADS
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_run(MesosSchedulerDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status;
> > - Py_BEGIN_ALLOW_THREADS
> > - status = self->driver->run();
> > - Py_END_ALLOW_THREADS
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_requestResources(
> > - MesosSchedulerDriverImpl* self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* requestsObj = NULL;
> > - vector<Request> requests;
> > -
> > - if (!PyArg_ParseTuple(args, "O", &requestsObj)) {
> > - return NULL;
> > - }
> > -
> > - if (!PyList_Check(requestsObj)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Parameter 2 to requestsResources is not a list");
> > - return NULL;
> > - }
> > - Py_ssize_t len = PyList_Size(requestsObj);
> > - for (int i = 0; i < len; i++) {
> > - PyObject* requestObj = PyList_GetItem(requestsObj, i);
> > - if (requestObj == NULL) {
> > - return NULL; // Exception will have been set by PyList_GetItem.
> > - }
> > - Request request;
> > - if (!readPythonProtobuf(requestObj, &request)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> > Request");
> > - return NULL;
> > - }
> > - requests.push_back(request);
> > - }
> > -
> > - Status status = self->driver->requestResources(requests);
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_launchTasks(MesosSchedulerDriverImpl*
> > self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* offerIdsObj = NULL;
> > - PyObject* tasksObj = NULL;
> > - PyObject* filtersObj = NULL;
> > - vector<OfferID> offerIds;
> > - vector<TaskInfo> tasks;
> > - Filters filters;
> > -
> > - if (!PyArg_ParseTuple(args, "OO|O", &offerIdsObj, &tasksObj,
> > &filtersObj)) {
> > - return NULL;
> > - }
> > -
> > - // Offer argument can be a list of offer ids or a single offer id (for
> > - // backward compatibility).
> > - if (!PyList_Check(offerIdsObj)) {
> > - OfferID offerId;
> > - if (!readPythonProtobuf(offerIdsObj, &offerId)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> > OfferID");
> > - return NULL;
> > - }
> > - offerIds.push_back(offerId);
> > - } else {
> > - Py_ssize_t len = PyList_Size(offerIdsObj);
> > - for (int i = 0; i < len; i++) {
> > - PyObject* offerObj = PyList_GetItem(offerIdsObj, i);
> > - if (offerObj == NULL) {
> > - return NULL;
> > - }
> > - OfferID offerId;
> > - if (!readPythonProtobuf(offerObj, &offerId)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python OfferID");
> > - return NULL;
> > - }
> > - offerIds.push_back(offerId);
> > - }
> > - }
> > -
> > - if (!PyList_Check(tasksObj)) {
> > - PyErr_Format(PyExc_Exception, "Parameter 2 to launchTasks is not a
> > list");
> > - return NULL;
> > - }
> > - Py_ssize_t len = PyList_Size(tasksObj);
> > - for (int i = 0; i < len; i++) {
> > - PyObject* taskObj = PyList_GetItem(tasksObj, i);
> > - if (taskObj == NULL) {
> > - return NULL; // Exception will have been set by PyList_GetItem.
> > - }
> > - TaskInfo task;
> > - if (!readPythonProtobuf(taskObj, &task)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python TaskInfo");
> > - return NULL;
> > - }
> > - tasks.push_back(task);
> > - }
> > -
> > - if (filtersObj != NULL) {
> > - if (!readPythonProtobuf(filtersObj, &filters)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python Filters");
> > - return NULL;
> > - }
> > - }
> > -
> > - Status status = self->driver->launchTasks(offerIds, tasks, filters);
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_killTask(MesosSchedulerDriverImpl*
> > self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* tidObj = NULL;
> > - TaskID tid;
> > - if (!PyArg_ParseTuple(args, "O", &tidObj)) {
> > - return NULL;
> > - }
> > - if (!readPythonProtobuf(tidObj, &tid)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> TaskID");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->killTask(tid);
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject*
> MesosSchedulerDriverImpl_acceptOffers(MesosSchedulerDriverImpl*
> > self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* offerIdsObj = NULL;
> > - PyObject* operationsObj = NULL;
> > - PyObject* filtersObj = NULL;
> > - Py_ssize_t len = 0;
> > - vector<OfferID> offerIds;
> > - vector<Offer::Operation> operations;
> > - Filters filters;
> > -
> > - if (!PyArg_ParseTuple(args,
> > - "OO|O",
> > - &offerIdsObj,
> > - &operationsObj,
> > - &filtersObj)) {
> > - return NULL;
> > - }
> > -
> > - if (!PyList_Check(offerIdsObj)) {
> > - PyErr_Format(PyExc_Exception, "Parameter 1 to acceptOffers is not a
> > list");
> > - return NULL;
> > - }
> > -
> > - len = PyList_Size(offerIdsObj);
> > - for (int i = 0; i < len; i++) {
> > - PyObject* offerObj = PyList_GetItem(offerIdsObj, i);
> > - if (offerObj == NULL) {
> > - return NULL;
> > - }
> > -
> > - OfferID offerId;
> > - if (!readPythonProtobuf(offerObj, &offerId)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python OfferID");
> > - return NULL;
> > - }
> > - offerIds.push_back(offerId);
> > - }
> > -
> > - if (!PyList_Check(operationsObj)) {
> > - PyErr_Format(PyExc_Exception, "Parameter 2 to acceptOffers is not a
> > list");
> > - return NULL;
> > - }
> > -
> > - len = PyList_Size(operationsObj);
> > - for (int i = 0; i < len; i++) {
> > - PyObject* operationObj = PyList_GetItem(operationsObj, i);
> > - if (operationObj == NULL) {
> > - return NULL; // Exception will have been set by PyList_GetItem.
> > - }
> > -
> > - Offer::Operation operation;
> > - if (!readPythonProtobuf(operationObj, &operation)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python Offer.Operation");
> > - return NULL;
> > - }
> > - operations.push_back(operation);
> > - }
> > -
> > - if (filtersObj != NULL) {
> > - if (!readPythonProtobuf(filtersObj, &filters)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python Filters");
> > - return NULL;
> > - }
> > - }
> > -
> > - Status status = self->driver->acceptOffers(offerIds, operations,
> > filters);
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject*
> MesosSchedulerDriverImpl_declineOffer(MesosSchedulerDriverImpl*
> > self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* offerIdObj = NULL;
> > - PyObject* filtersObj = NULL;
> > - OfferID offerId;
> > - Filters filters;
> > -
> > - if (!PyArg_ParseTuple(args, "O|O", &offerIdObj, &filtersObj)) {
> > - return NULL;
> > - }
> > -
> > - if (!readPythonProtobuf(offerIdObj, &offerId)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> OfferID");
> > - return NULL;
> > - }
> > -
> > - if (filtersObj != NULL) {
> > - if (!readPythonProtobuf(filtersObj, &filters)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python Filters");
> > - return NULL;
> > - }
> > - }
> > -
> > - Status status = self->driver->declineOffer(offerId, filters);
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject*
> MesosSchedulerDriverImpl_reviveOffers(MesosSchedulerDriverImpl*
> > self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->reviveOffers();
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_suppressOffers(
> > - MesosSchedulerDriverImpl* self)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->suppressOffers();
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_acknowledgeStatusUpdate(
> > - MesosSchedulerDriverImpl* self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* taskStatusObj = NULL;
> > - TaskStatus taskStatus;
> > -
> > - if (!PyArg_ParseTuple(args, "O", &taskStatusObj)) {
> > - return NULL;
> > - }
> > -
> > - if (!readPythonProtobuf(taskStatusObj, &taskStatus)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> > TaskStatus");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->acknowledgeStatusUpdate(taskStatus);
> > -
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_sendFrameworkMessage(
> > - MesosSchedulerDriverImpl* self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* slaveIdObj = NULL;
> > - PyObject* executorIdObj = NULL;
> > - SlaveID slaveId;
> > - ExecutorID executorId;
> > - const char* data;
> > - int length;
> > -
> > - if (!PyArg_ParseTuple(
> > - args, "OOs#", &executorIdObj, &slaveIdObj, &data, &length)) {
> > - return NULL;
> > - }
> > -
> > - if (!readPythonProtobuf(executorIdObj, &executorId)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> > ExecutorID");
> > - return NULL;
> > - }
> > -
> > - if (!readPythonProtobuf(slaveIdObj, &slaveId)) {
> > - PyErr_Format(PyExc_Exception, "Could not deserialize Python
> SlaveID");
> > - return NULL;
> > - }
> > -
> > - Status status = self->driver->sendFrameworkMessage(
> > - executorId, slaveId, string(data, length));
> > -
> > - return PyInt_FromLong(status); // Sets exception if creating long
> fails.
> > -}
> > -
> > -
> > -PyObject* MesosSchedulerDriverImpl_reconcileTasks(
> > - MesosSchedulerDriverImpl* self,
> > - PyObject* args)
> > -{
> > - if (self->driver == NULL) {
> > - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is
> > NULL");
> > - return NULL;
> > - }
> > -
> > - PyObject* statusesObj = NULL;
> > - vector<TaskStatus> statuses;
> > -
> > - if (!PyArg_ParseTuple(args, "O", &statusesObj)) {
> > - return NULL;
> > - }
> > -
> > - if (!PyList_Check(statusesObj)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Parameter 1 to reconcileTasks is not a list");
> > -
> > - return NULL;
> > - }
> > -
> > - Py_ssize_t len = PyList_Size(statusesObj);
> > - for (int i = 0; i < len; i++) {
> > - PyObject* statusObj = PyList_GetItem(statusesObj, i);
> > - if (statusObj == NULL) {
> > - return NULL;
> > - }
> > -
> > - TaskStatus status;
> > - if (!readPythonProtobuf(statusObj, &status)) {
> > - PyErr_Format(PyExc_Exception,
> > - "Could not deserialize Python TaskStatus");
> > - return NULL;
> > - }
> > - statuses.push_back(status);
> > - }
> > -
> > - Status status = self->driver->reconcileTasks(statuses);
> > - return PyInt_FromLong(status);
> > -}
> > -
> > -} // namespace python {
> > -} // namespace mesos {
> >
> >
>
--
Best Regards,
Haosdent Huang
Re: [3/3] mesos git commit: New python lib with only the executor driver.
Posted by James Peach <jo...@gmail.com>.
FWIW I bet it would be helpful to have a general-purpose minimal executor library.
> On Mar 11, 2016, at 6:55 PM, Benjamin Mahler <bm...@apache.org> wrote:
>
> +vinod
>
> This breaks the build for me on OS X, it appears this line is the culprit:
>
> EXTRA_LINK_ARGS = ['-Wl,--as-needed']
>
> This leads to the following:
>
> clang++ -bundle -undefined dynamic_lookup -arch x86_64 -arch i386 -Wl,-F. -L/usr/local/opt/subversion/lib -O2 -O2 -Wno-unused-local-typedef -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -DGTEST_LANG_CXX11 -Qunused-arguments -I/usr/local/opt/subversion/include/subversion-1 -I/usr/include/apr-1 -I/usr/include/apr-1.0 -Qunused-arguments build/temp.macosx-10.11-intel-2.7/src/mesos/executor/mesos_executor_driver_impl.o build/temp.macosx-10.11-intel-2.7/src/mesos/executor/module.o build/temp.macosx-10.11-intel-2.7/src/mesos/executor/proxy_executor.o /Users/bmahler/git/mesos/build/src/.libs/libmesos_no_3rdparty.a /Users/bmahler/git/mesos/build/3rdparty/libprocess/.libs/libprocess.a /Users/bmahler/git/mesos/build/3rdparty/leveldb-1.4/libleveldb.a /Users/bmahler/git/mesos/build/3rdparty/zookeeper-3.4.5/src/c/.libs/libzookeeper_mt.a /Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/glog-0.3.3/.libs/libglog.a /Users/bmahler/git/mesos/build/3rdparty/libprocess/3rdparty/protobuf-2.5.0/src/.libs/libprotobuf.a -o build/lib.macosx-10.11-intel-2.7/mesos/executor/_executor.so -Wl,--as-needed -L/usr/local/opt/subversion/lib -levent_openssl -lcrypto -lssl -levent_pthreads -levent -lsasl2 -lsvn_delta-1 -lsvn_subr-1 -lapr-1 -lcurl -lz
> ld: unknown option: --as-needed
> clang: error: linker command failed with exit code 1 (use -v to see invocation)
>
> On Fri, Mar 11, 2016 at 1:56 PM, <vi...@apache.org> wrote:
> New python lib with only the executor driver.
>
> This patch produces a new python egg, mesos.executor, which contains only the
> code needed to create a MesosExecutorDriver. By doing so, the linker can remove
> unused code in libmesos_no_3rdparty.a, and therefor not include any external
> dependencies in the resulting _mesos.so.
>
> Review: https://reviews.apache.org/r/41049/
>
>
> Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c81a52ec
> Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c81a52ec
> Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c81a52ec
>
> Branch: refs/heads/master
> Commit: c81a52ec22266e1f2beb61b224c0f0d9be82521f
> Parents: 482dc14
> Author: Steve Niemitz <sn...@twitter.com>
> Authored: Fri Mar 11 16:56:13 2016 -0500
> Committer: Vinod Kone <vi...@gmail.com>
> Committed: Fri Mar 11 16:56:13 2016 -0500
>
> ----------------------------------------------------------------------
> configure.ac | 7 +-
> src/Makefile.am | 33 +-
> src/python/executor/setup.py.in | 39 +
> src/python/executor/src/mesos/__init__.py | 10 +
> .../executor/src/mesos/executor/__init__.py | 17 +
> .../executor/mesos_executor_driver_impl.cpp | 347 ++++++++
> .../executor/mesos_executor_driver_impl.hpp | 103 +++
> .../executor/src/mesos/executor/module.cpp | 91 +++
> .../src/mesos/executor/proxy_executor.cpp | 273 +++++++
> .../src/mesos/executor/proxy_executor.hpp | 64 ++
> src/python/native/ext_modules.py.in | 151 ----
> src/python/native/setup.py.in | 9 +-
> src/python/native/src/mesos/native/__init__.py | 7 +-
> .../mesos/native/mesos_executor_driver_impl.cpp | 347 --------
> .../mesos/native/mesos_executor_driver_impl.hpp | 103 ---
> .../native/mesos_scheduler_driver_impl.cpp | 782 -------------------
> .../native/mesos_scheduler_driver_impl.hpp | 134 ----
> src/python/native/src/mesos/native/module.cpp | 100 ---
> src/python/native/src/mesos/native/module.hpp | 136 ----
> .../native/src/mesos/native/proxy_executor.cpp | 273 -------
> .../native/src/mesos/native/proxy_executor.hpp | 64 --
> .../native/src/mesos/native/proxy_scheduler.cpp | 384 ---------
> .../native/src/mesos/native/proxy_scheduler.hpp | 72 --
> src/python/native_common/common.hpp | 136 ++++
> src/python/native_common/ext_modules.py.in | 154 ++++
> src/python/scheduler/setup.py.in | 39 +
> src/python/scheduler/src/mesos/__init__.py | 10 +
> .../scheduler/src/mesos/scheduler/__init__.py | 17 +
> .../scheduler/mesos_scheduler_driver_impl.cpp | 782 +++++++++++++++++++
> .../scheduler/mesos_scheduler_driver_impl.hpp | 134 ++++
> .../scheduler/src/mesos/scheduler/module.cpp | 91 +++
> .../src/mesos/scheduler/proxy_scheduler.cpp | 384 +++++++++
> .../src/mesos/scheduler/proxy_scheduler.hpp | 72 ++
> 33 files changed, 2795 insertions(+), 2570 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/configure.ac
> ----------------------------------------------------------------------
> diff --git a/configure.ac b/configure.ac
> index a20382e..8e4f035 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -1315,8 +1315,13 @@ There are two possible workarounds for this issue:
> AC_CONFIG_FILES([src/python/setup.py])
> AC_CONFIG_FILES([src/python/cli/setup.py])
> AC_CONFIG_FILES([src/python/interface/setup.py])
> - AC_CONFIG_FILES([src/python/native/ext_modules.py])
> + AC_CONFIG_FILES([src/python/native_common/ext_modules.py])
> + AC_CONFIG_FILES([src/python/executor/setup.py])
> AC_CONFIG_FILES([src/python/native/setup.py])
> + AC_CONFIG_FILES([src/python/scheduler/setup.py])
> +
> + AC_CONFIG_LINKS([src/python/executor/ext_modules.py:src/python/native_common/ext_modules.py])
> + AC_CONFIG_LINKS([src/python/scheduler/ext_modules.py:src/python/native_common/ext_modules.py])
>
> # When clang is being used, make sure that the distutils python-
> # config cflags extraction does not cause build errors (MESOS-1079).
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/Makefile.am
> ----------------------------------------------------------------------
> diff --git a/src/Makefile.am b/src/Makefile.am
> index f59ae12..8abef3b 100644
> --- a/src/Makefile.am
> +++ b/src/Makefile.am
> @@ -1467,18 +1467,23 @@ PYTHON_SOURCE = \
> python/interface/src/mesos/interface/__init__.py \
> python/interface/src/mesos/v1/__init__.py \
> python/interface/src/mesos/v1/interface/__init__.py \
> + python/native_common/common.hpp \
> python/native/src/mesos/__init__.py \
> python/native/src/mesos/native/__init__.py \
> - python/native/src/mesos/native/mesos_executor_driver_impl.cpp \
> - python/native/src/mesos/native/mesos_executor_driver_impl.hpp \
> - python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp \
> - python/native/src/mesos/native/mesos_scheduler_driver_impl.hpp \
> - python/native/src/mesos/native/module.cpp \
> - python/native/src/mesos/native/module.hpp \
> - python/native/src/mesos/native/proxy_executor.cpp \
> - python/native/src/mesos/native/proxy_executor.hpp \
> - python/native/src/mesos/native/proxy_scheduler.cpp \
> - python/native/src/mesos/native/proxy_scheduler.hpp \
> + python/executor/src/mesos/__init__.py \
> + python/executor/src/mesos/executor/__init__.py \
> + python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp \
> + python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp \
> + python/executor/src/mesos/executor/module.cpp \
> + python/executor/src/mesos/executor/proxy_executor.cpp \
> + python/executor/src/mesos/executor/proxy_executor.hpp \
> + python/scheduler/src/mesos/__init__.py \
> + python/scheduler/src/mesos/scheduler/__init__.py \
> + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.cpp \
> + python/scheduler/src/mesos/scheduler/mesos_scheduler_driver_impl.hpp \
> + python/scheduler/src/mesos/scheduler/module.cpp \
> + python/scheduler/src/mesos/scheduler/proxy_scheduler.cpp \
> + python/scheduler/src/mesos/scheduler/proxy_scheduler.hpp \
> python/src/mesos/__init__.py
>
> EXTRA_DIST += $(PYTHON_SOURCE)
> @@ -1530,14 +1535,18 @@ $(PYTHON_SOURCE):
> MESOS_EGGS = \
> python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg \
> python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg \
> + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg \
> python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_EGG_PUREPY_POSTFIX).egg \
> - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
> + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg \
> + python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_EGG_POSTFIX).egg
>
> MESOS_WHLS = \
> python/dist/mesos-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl \
> python/dist/mesos.cli-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl \
> + python/dist/mesos.executor-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl \
> python/dist/mesos.interface-$(PACKAGE_VERSION)$(PYTHON_WHL_PUREPY_POSTFIX).whl \
> - python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
> + python/dist/mesos.native-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl \
> + python/dist/mesos.scheduler-$(PACKAGE_VERSION)$(PYTHON_WHL_POSTFIX).whl
>
> # The python source is in directories of the form: python/interface. The make
> # target is of the form: python/dist/mesos.interface-0.20.0-py2.7.egg. To build
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/setup.py.in
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/setup.py.in b/src/python/executor/setup.py.in
> new file mode 100644
> index 0000000..76db97f
> --- /dev/null
> +++ b/src/python/executor/setup.py.in
> @@ -0,0 +1,39 @@
> +#!/usr/bin/env python
> +
> +# Licensed to the Apache Software Foundation (ASF) under one
> +# or more contributor license agreements. See the NOTICE file
> +# distributed with this work for additional information
> +# regarding copyright ownership. The ASF licenses this file
> +# to you under the Apache License, Version 2.0 (the
> +# "License"); you may not use this file except in compliance
> +# with the License. You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +
> +import ext_modules
> +
> +config = {
> + 'name': 'mesos.executor',
> + 'version': '@PACKAGE_VERSION@',
> + 'description': 'Mesos native executor driver implementation',
> + 'author': 'Apache Mesos',
> + 'author_email': 'dev@mesos.apache.org',
> + 'url': 'http://pypi.python.org/pypi/mesos.executor',
> + 'namespace_packages': [ 'mesos' ],
> + 'packages': [ 'mesos', 'mesos.executor' ],
> + 'package_dir': { '': 'src' },
> + 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ],
> + 'license': 'Apache 2.0',
> + 'keywords': 'mesos',
> + 'classifiers': [ ],
> + 'ext_modules': [ ext_modules.executor_module ]
> +}
> +
> +from setuptools import setup
> +setup(**config)
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/__init__.py
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/__init__.py b/src/python/executor/src/mesos/__init__.py
> new file mode 100644
> index 0000000..3fcba01
> --- /dev/null
> +++ b/src/python/executor/src/mesos/__init__.py
> @@ -0,0 +1,10 @@
> +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
> +# Because python does not normally allow the contents of a package to be
> +# retrieved from more than one location, this code snippet ensures that the
> +# namespace package machinery is operating and that the current package is
> +# registered as a namespace package.
> +try:
> + __import__('pkg_resources').declare_namespace(__name__)
> +except ImportError:
> + from pkgutil import extend_path
> + __path__ = extend_path(__path__, __name__)
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/__init__.py
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/__init__.py b/src/python/executor/src/mesos/executor/__init__.py
> new file mode 100644
> index 0000000..6808019
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/__init__.py
> @@ -0,0 +1,17 @@
> +# Licensed to the Apache Software Foundation (ASF) under one
> +# or more contributor license agreements. See the NOTICE file
> +# distributed with this work for additional information
> +# regarding copyright ownership. The ASF licenses this file
> +# to you under the Apache License, Version 2.0 (the
> +# "License"); you may not use this file except in compliance
> +# with the License. You may obtain a copy of the License at
> +#
> +# http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS,
> +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +# See the License for the specific language governing permissions and
> +# limitations under the License.
> +
> +from ._executor import MesosExecutorDriverImpl as MesosExecutorDriver
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> new file mode 100644
> index 0000000..4189411
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.cpp
> @@ -0,0 +1,347 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <string>
> +
> +#include "common.hpp"
> +#include "mesos_executor_driver_impl.hpp"
> +#include "proxy_executor.hpp"
> +
> +using namespace mesos;
> +using namespace mesos::python;
> +
> +using std::cerr;
> +using std::endl;
> +using std::string;
> +using std::vector;
> +using std::map;
> +
> +
> +namespace mesos { namespace python {
> +
> +/**
> + * Python type object for MesosExecutorDriverImpl.
> + */
> +PyTypeObject MesosExecutorDriverImplType = {
> + PyObject_HEAD_INIT(NULL)
> + 0, /* ob_size */
> + "_mesos.MesosExecutorDriverImpl", /* tp_name */
> + sizeof(MesosExecutorDriverImpl), /* tp_basicsize */
> + 0, /* tp_itemsize */
> + (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */
> + 0, /* tp_print */
> + 0, /* tp_getattr */
> + 0, /* tp_setattr */
> + 0, /* tp_compare */
> + 0, /* tp_repr */
> + 0, /* tp_as_number */
> + 0, /* tp_as_sequence */
> + 0, /* tp_as_mapping */
> + 0, /* tp_hash */
> + 0, /* tp_call */
> + 0, /* tp_str */
> + 0, /* tp_getattro */
> + 0, /* tp_setattro */
> + 0, /* tp_as_buffer */
> + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> + "Private MesosExecutorDriver implementation", /* tp_doc */
> + (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */
> + (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */
> + 0, /* tp_richcompare */
> + 0, /* tp_weaklistoffset */
> + 0, /* tp_iter */
> + 0, /* tp_iternext */
> + MesosExecutorDriverImpl_methods, /* tp_methods */
> + 0, /* tp_members */
> + 0, /* tp_getset */
> + 0, /* tp_base */
> + 0, /* tp_dict */
> + 0, /* tp_descr_get */
> + 0, /* tp_descr_set */
> + 0, /* tp_dictoffset */
> + (initproc) MesosExecutorDriverImpl_init, /* tp_init */
> + 0, /* tp_alloc */
> + MesosExecutorDriverImpl_new, /* tp_new */
> +};
> +
> +
> +/**
> + * List of Python methods in MesosExecutorDriverImpl.
> + */
> +PyMethodDef MesosExecutorDriverImpl_methods[] = {
> + { "start",
> + (PyCFunction) MesosExecutorDriverImpl_start,
> + METH_NOARGS,
> + "Start the driver to connect to Mesos"
> + },
> + { "stop",
> + (PyCFunction) MesosExecutorDriverImpl_stop,
> + METH_NOARGS,
> + "Stop the driver, disconnecting from Mesos"
> + },
> + { "abort",
> + (PyCFunction) MesosExecutorDriverImpl_abort,
> + METH_NOARGS,
> + "Abort the driver, disallowing calls from and to the driver"
> + },
> + { "join",
> + (PyCFunction) MesosExecutorDriverImpl_join,
> + METH_NOARGS,
> + "Wait for a running driver to disconnect from Mesos"
> + },
> + { "run",
> + (PyCFunction) MesosExecutorDriverImpl_run,
> + METH_NOARGS,
> + "Start a driver and run it, returning when it disconnects from Mesos"
> + },
> + { "sendStatusUpdate",
> + (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate,
> + METH_VARARGS,
> + "Send a status update for a task"
> + },
> + { "sendFrameworkMessage",
> + (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage,
> + METH_VARARGS,
> + "Send a FrameworkMessage to a slave"
> + },
> + { NULL } /* Sentinel */
> +};
> +
> +
> +/**
> + * Create, but don't initialize, a new MesosExecutorDriverImpl
> + * (called by Python before init method).
> + */
> +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> + PyObject *args,
> + PyObject *kwds)
> +{
> + MesosExecutorDriverImpl *self;
> + self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0);
> + if (self != NULL) {
> + self->driver = NULL;
> + self->proxyExecutor = NULL;
> + self->pythonExecutor = NULL;
> + }
> + return (PyObject*) self;
> +}
> +
> +
> +/**
> + * Initialize a MesosExecutorDriverImpl with constructor arguments.
> + */
> +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> + PyObject *args,
> + PyObject *kwds)
> +{
> + PyObject *pythonExecutor = NULL;
> +
> + if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) {
> + return -1;
> + }
> +
> + if (pythonExecutor != NULL) {
> + PyObject* tmp = self->pythonExecutor;
> + Py_INCREF(pythonExecutor);
> + self->pythonExecutor = pythonExecutor;
> + Py_XDECREF(tmp);
> + }
> +
> + if (self->driver != NULL) {
> + delete self->driver;
> + self->driver = NULL;
> + }
> +
> + if (self->proxyExecutor != NULL) {
> + delete self->proxyExecutor;
> + self->proxyExecutor = NULL;
> + }
> +
> + self->proxyExecutor = new ProxyExecutor(self);
> + self->driver = new MesosExecutorDriver(self->proxyExecutor);
> +
> + return 0;
> +}
> +
> +
> +/**
> + * Free a MesosExecutorDriverImpl.
> + */
> +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver != NULL) {
> + // We need to wrap the driver destructor in an "allow threads"
> + // macro since the MesosExecutorDriver destructor waits for the
> + // ExecutorProcess to terminate and there might be a thread that
> + // is trying to acquire the GIL to call through the
> + // ProxyExecutor. It will only be after this thread executes that
> + // the ExecutorProcess might actually get a terminate.
> + Py_BEGIN_ALLOW_THREADS
> + delete self->driver;
> + Py_END_ALLOW_THREADS
> + self->driver = NULL;
> + }
> +
> + if (self->proxyExecutor != NULL) {
> + delete self->proxyExecutor;
> + self->proxyExecutor = NULL;
> + }
> +
> + MesosExecutorDriverImpl_clear(self);
> + self->ob_type->tp_free((PyObject*) self);
> +}
> +
> +
> +/**
> + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> + * See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> + visitproc visit,
> + void* arg)
> +{
> + Py_VISIT(self->pythonExecutor);
> + return 0;
> +}
> +
> +
> +/**
> + * Clear fields of a MesosExecutorDriverImpl that can participate in
> + * GC cycles. See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self)
> +{
> + Py_CLEAR(self->pythonExecutor);
> + return 0;
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> + return NULL;
> + }
> +
> + Status status = self->driver->start();
> + return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> + return NULL;
> + }
> +
> + Status status = self->driver->stop();
> + return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> + return NULL;
> + }
> +
> + Status status = self->driver->abort();
> + return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> + return NULL;
> + }
> +
> + Status status;
> + Py_BEGIN_ALLOW_THREADS
> + status = self->driver->join();
> + Py_END_ALLOW_THREADS
> + return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> + return NULL;
> + }
> +
> + Status status;
> + Py_BEGIN_ALLOW_THREADS
> + status = self->driver->run();
> + Py_END_ALLOW_THREADS
> + return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> + MesosExecutorDriverImpl* self,
> + PyObject* args)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> + return NULL;
> + }
> +
> + PyObject* statusObj = NULL;
> + TaskStatus taskStatus;
> + if (!PyArg_ParseTuple(args, "O", &statusObj)) {
> + return NULL;
> + }
> + if (!readPythonProtobuf(statusObj, &taskStatus)) {
> + PyErr_Format(PyExc_Exception,
> + "Could not deserialize Python TaskStatus");
> + return NULL;
> + }
> +
> + Status status = self->driver->sendStatusUpdate(taskStatus);
> + return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> +}
> +
> +
> +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> + MesosExecutorDriverImpl* self,
> + PyObject* args)
> +{
> + if (self->driver == NULL) {
> + PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> + return NULL;
> + }
> +
> + const char* data;
> + int length;
> + if (!PyArg_ParseTuple(args, "s#", &data, &length)) {
> + return NULL;
> + }
> +
> + Status status = self->driver->sendFrameworkMessage(string(data, length));
> + return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> +}
> +
> +} // namespace python {
> +} // namespace mesos {
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> new file mode 100644
> index 0000000..6e672f8
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/mesos_executor_driver_impl.hpp
> @@ -0,0 +1,103 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP
> +#define MESOS_EXECUTOR_DRIVER_IMPL_HPP
> +
> +#include <mesos/executor.hpp>
> +
> +
> +namespace mesos { namespace python {
> +
> +class ProxyExecutor;
> +
> +/**
> + * Python object structure for MesosExecutorDriverImpl objects.
> + */
> +struct MesosExecutorDriverImpl {
> + PyObject_HEAD
> + /* Type-specific fields go here. */
> + MesosExecutorDriver* driver;
> + ProxyExecutor* proxyExecutor;
> + PyObject* pythonExecutor;
> +};
> +
> +/**
> + * Python type object for MesosExecutorDriverImpl.
> + */
> +extern PyTypeObject MesosExecutorDriverImplType;
> +
> +/**
> + * List of Python methods in MesosExecutorDriverImpl.
> + */
> +extern PyMethodDef MesosExecutorDriverImpl_methods[];
> +
> +/**
> + * Create, but don't initialize, a new MesosExecutorDriverImpl
> + * (called by Python before init method).
> + */
> +PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> + PyObject *args,
> + PyObject *kwds);
> +
> +/**
> + * Initialize a MesosExecutorDriverImpl with constructor arguments.
> + */
> +int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> + PyObject *args,
> + PyObject *kwds);
> +
> +/**
> + * Free a MesosExecutorDriverImpl.
> + */
> +void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self);
> +
> +/**
> + * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> + * See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> + visitproc visit,
> + void* arg);
> +/**
> + * Clear fields of a MesosExecutorDriverImpl that can participate in
> + * GC cycles. See http://docs.python.org/extending/newtypes.html.
> + */
> +int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self);
> +
> +// MesosExecutorDriverImpl methods.
> +PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self);
> +
> +PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> + MesosExecutorDriverImpl* self,
> + PyObject* args);
> +
> +PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> + MesosExecutorDriverImpl* self,
> + PyObject* args);
> +
> +} // namespace python {
> +} // namespace mesos {
> +
> +#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/module.cpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/module.cpp b/src/python/executor/src/mesos/executor/module.cpp
> new file mode 100644
> index 0000000..f8c6382
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/module.cpp
> @@ -0,0 +1,91 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +/**
> + * This file defines the _mesos.so binary module used by the Mesos Python API.
> + * This module contains private implementations of MesosSchedulerDriver and
> + * MesosExecutorDriver as Python types that get called from the public module
> + * called mesos (in <root>/src/python/src/mesos.py). This design was chosen
> + * so that most of the API (e.g. the Scheduler and Executor interfaces) can
> + * be written in Python, and only the parts that need to call into C++ are
> + * in C++. Note that the mesos module also contains public classes called
> + * MesosSchedulerDriver and MesosExecutorDriver. These call into the private
> + * _mesos.MesosSchedulerDriverImpl and _mesos.MesosExecutorDriverImpl.
> + */
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <iostream>
> +
> +#include <mesos/executor.hpp>
> +
> +#include "common.hpp"
> +#include "mesos_executor_driver_impl.hpp"
> +#include "proxy_executor.hpp"
> +
> +using namespace mesos;
> +using namespace mesos::python;
> +
> +using std::map;
> +using std::string;
> +using std::vector;
> +
> +
> +/**
> + * The Python module object for mesos_pb2 (which contains the protobuf
> + * classes generated for Python).
> + */
> +PyObject* mesos::python::mesos_pb2 = NULL;
> +
> +
> +namespace {
> +
> +/**
> + * Method list for our Python module.
> + */
> +PyMethodDef MODULE_METHODS[] = {
> + {NULL, NULL, 0, NULL} /* Sentinel */
> +};
> +
> +} // namespace {
> +
> +
> +/**
> + * Entry point called by Python to initialize our module.
> + */
> +PyMODINIT_FUNC init_executor()
> +{
> + // Ensure that the interpreter's threading support is enabled.
> + PyEval_InitThreads();
> +
> + // Import the mesos_pb2 module (on which we depend for protobuf classes)
> + mesos_pb2 = PyImport_ImportModule("mesos.interface.mesos_pb2");
> + if (mesos_pb2 == NULL)
> + return;
> +
> + // Initialize our Python types.
> + if (PyType_Ready(&MesosExecutorDriverImplType) < 0)
> + return;
> +
> + // Create the _mesos module and add our types to it.
> + PyObject* module = Py_InitModule("_executor", MODULE_METHODS);
> + Py_INCREF(&MesosExecutorDriverImplType);
> + PyModule_AddObject(module,
> + "MesosExecutorDriverImpl",
> + (PyObject*) &MesosExecutorDriverImplType);
> +}
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.cpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/proxy_executor.cpp b/src/python/executor/src/mesos/executor/proxy_executor.cpp
> new file mode 100644
> index 0000000..da1a49b
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/proxy_executor.cpp
> @@ -0,0 +1,273 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <iostream>
> +
> +#include "common.hpp"
> +#include "mesos_executor_driver_impl.hpp"
> +#include "proxy_executor.hpp"
> +
> +using namespace mesos;
> +
> +using std::cerr;
> +using std::endl;
> +using std::map;
> +using std::string;
> +using std::vector;
> +
> +namespace mesos {
> +namespace python {
> +
> +void ProxyExecutor::registered(ExecutorDriver* driver,
> + const ExecutorInfo& executorInfo,
> + const FrameworkInfo& frameworkInfo,
> + const SlaveInfo& slaveInfo)
> +{
> + InterpreterLock lock;
> +
> + PyObject* executorInfoObj = NULL;
> + PyObject* frameworkInfoObj = NULL;
> + PyObject* slaveInfoObj = NULL;
> + PyObject* res = NULL;
> +
> + executorInfoObj = createPythonProtobuf(executorInfo, "ExecutorInfo");
> + frameworkInfoObj = createPythonProtobuf(frameworkInfo, "FrameworkInfo");
> + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo");
> +
> + if (executorInfoObj == NULL ||
> + frameworkInfoObj == NULL ||
> + slaveInfoObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "registered",
> + (char*) "OOOO",
> + impl,
> + executorInfoObj,
> + frameworkInfoObj,
> + slaveInfoObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor registered" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(executorInfoObj);
> + Py_XDECREF(frameworkInfoObj);
> + Py_XDECREF(slaveInfoObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::reregistered(ExecutorDriver* driver,
> + const SlaveInfo& slaveInfo)
> +{
> + InterpreterLock lock;
> +
> + PyObject* slaveInfoObj = NULL;
> + PyObject* res = NULL;
> +
> + slaveInfoObj = createPythonProtobuf(slaveInfo, "SlaveInfo");
> +
> + if (slaveInfoObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "reregistered",
> + (char*) "OO",
> + impl,
> + slaveInfoObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor re-registered" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(slaveInfoObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::disconnected(ExecutorDriver* driver)
> +{
> + InterpreterLock lock;
> + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "disconnected",
> + (char*) "O",
> + impl);
> + if (res == NULL) {
> + cerr << "Failed to call executor's disconnected" << endl;
> + goto cleanup;
> + }
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::launchTask(ExecutorDriver* driver,
> + const TaskInfo& task)
> +{
> + InterpreterLock lock;
> +
> + PyObject* taskObj = NULL;
> + PyObject* res = NULL;
> +
> + taskObj = createPythonProtobuf(task, "TaskInfo");
> + if (taskObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "launchTask",
> + (char*) "OO",
> + impl,
> + taskObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor's launchTask" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(taskObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::killTask(ExecutorDriver* driver,
> + const TaskID& taskId)
> +{
> + InterpreterLock lock;
> +
> + PyObject* taskIdObj = NULL;
> + PyObject* res = NULL;
> +
> + taskIdObj = createPythonProtobuf(taskId, "TaskID");
> + if (taskIdObj == NULL) {
> + goto cleanup; // createPythonProtobuf will have set an exception.
> + }
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "killTask",
> + (char*) "OO",
> + impl,
> + taskIdObj);
> + if (res == NULL) {
> + cerr << "Failed to call executor's killTask" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(taskIdObj);
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::frameworkMessage(ExecutorDriver* driver,
> + const string& data)
> +{
> + InterpreterLock lock;
> +
> + PyObject* res = NULL;
> +
> + res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "frameworkMessage",
> + (char*) "Os#",
> + impl,
> + data.data(),
> + data.length());
> + if (res == NULL) {
> + cerr << "Failed to call executor's frameworkMessage" << endl;
> + goto cleanup;
> + }
> +
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::shutdown(ExecutorDriver* driver)
> +{
> + InterpreterLock lock;
> + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "shutdown",
> + (char*) "O",
> + impl);
> + if (res == NULL) {
> + cerr << "Failed to call executor's shutdown" << endl;
> + goto cleanup;
> + }
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + driver->abort();
> + }
> + Py_XDECREF(res);
> +}
> +
> +
> +void ProxyExecutor::error(ExecutorDriver* driver, const string& message)
> +{
> + InterpreterLock lock;
> + PyObject* res = PyObject_CallMethod(impl->pythonExecutor,
> + (char*) "error",
> + (char*) "Os#",
> + impl,
> + message.data(),
> + message.length());
> + if (res == NULL) {
> + cerr << "Failed to call executor's error" << endl;
> + goto cleanup;
> + }
> +cleanup:
> + if (PyErr_Occurred()) {
> + PyErr_Print();
> + // No need for driver.stop(); it should stop itself.
> + }
> + Py_XDECREF(res);
> +}
> +
> +} // namespace python {
> +} // namespace mesos {
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/executor/src/mesos/executor/proxy_executor.hpp
> ----------------------------------------------------------------------
> diff --git a/src/python/executor/src/mesos/executor/proxy_executor.hpp b/src/python/executor/src/mesos/executor/proxy_executor.hpp
> new file mode 100644
> index 0000000..23d64fd
> --- /dev/null
> +++ b/src/python/executor/src/mesos/executor/proxy_executor.hpp
> @@ -0,0 +1,64 @@
> +// Licensed to the Apache Software Foundation (ASF) under one
> +// or more contributor license agreements. See the NOTICE file
> +// distributed with this work for additional information
> +// regarding copyright ownership. The ASF licenses this file
> +// to you under the Apache License, Version 2.0 (the
> +// "License"); you may not use this file except in compliance
> +// with the License. You may obtain a copy of the License at
> +//
> +// http://www.apache.org/licenses/LICENSE-2.0
> +//
> +// Unless required by applicable law or agreed to in writing, software
> +// distributed under the License is distributed on an "AS IS" BASIS,
> +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> +// See the License for the specific language governing permissions and
> +// limitations under the License.
> +
> +#ifndef PROXY_EXECUTOR_HPP
> +#define PROXY_EXECUTOR_HPP
> +
> +// Python.h must be included before standard headers.
> +// See: http://docs.python.org/2/c-api/intro.html#include-files
> +#include <Python.h>
> +
> +#include <string>
> +#include <vector>
> +
> +#include <mesos/executor.hpp>
> +
> +namespace mesos {
> +namespace python {
> +
> +struct MesosExecutorDriverImpl;
> +
> +/**
> + * Proxy Executor implementation that will call into Python.
> + */
> +class ProxyExecutor : public Executor
> +{
> +public:
> + explicit ProxyExecutor(MesosExecutorDriverImpl *_impl) : impl(_impl) {}
> +
> + virtual ~ProxyExecutor() {}
> +
> + virtual void registered(ExecutorDriver* driver,
> + const ExecutorInfo& executorInfo,
> + const FrameworkInfo& frameworkInfo,
> + const SlaveInfo& slaveInfo);
> + virtual void reregistered(ExecutorDriver* driver, const SlaveInfo& slaveInfo);
> + virtual void disconnected(ExecutorDriver* driver);
> + virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task);
> + virtual void killTask(ExecutorDriver* driver, const TaskID& taskId);
> + virtual void frameworkMessage(ExecutorDriver* driver,
> + const std::string& data);
> + virtual void shutdown(ExecutorDriver* driver);
> + virtual void error(ExecutorDriver* driver, const std::string& message);
> +
> +private:
> + MesosExecutorDriverImpl *impl;
> +};
> +
> +} // namespace python {
> +} // namespace mesos {
> +
> +#endif // PROXY_EXECUTOR_HPP
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/ext_modules.py.in
> ----------------------------------------------------------------------
> diff --git a/src/python/native/ext_modules.py.in b/src/python/native/ext_modules.py.in
> deleted file mode 100644
> index eb93864..0000000
> --- a/src/python/native/ext_modules.py.in
> +++ /dev/null
> @@ -1,151 +0,0 @@
> -# Licensed to the Apache Software Foundation (ASF) under one
> -# or more contributor license agreements. See the NOTICE file
> -# distributed with this work for additional information
> -# regarding copyright ownership. The ASF licenses this file
> -# to you under the Apache License, Version 2.0 (the
> -# "License"); you may not use this file except in compliance
> -# with the License. You may obtain a copy of the License at
> -#
> -# http://www.apache.org/licenses/LICENSE-2.0
> -#
> -# Unless required by applicable law or agreed to in writing, software
> -# distributed under the License is distributed on an "AS IS" BASIS,
> -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> -# See the License for the specific language governing permissions and
> -# limitations under the License.
> -
> -import errno
> -import glob
> -import os
> -import shutil
> -
> -from setuptools import Extension
> -
> -abs_top_srcdir = '@abs_top_srcdir@'
> -abs_top_builddir = '@abs_top_builddir@'
> -
> -src_python_native = os.path.join(
> - 'src', 'python', 'native', 'src', 'mesos', 'native')
> -
> -leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> -zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.5', 'src', 'c')
> -libprocess = os.path.join('3rdparty', 'libprocess')
> -
> -# Even though a statically compiled libprocess should include glog,
> -# libev, gperftools, and protobuf before installation this isn't the
> -# case, so while a libtool managed build will correctly pull in these
> -# libraries when building the final result, we need to explicitly
> -# include them here (or more precisely, down where we actually include
> -# libev.a and libprofiler.a).
> -glog = os.path.join(libprocess, '3rdparty', 'glog-0.3.3')
> -gperftools = os.path.join(libprocess, '3rdparty', 'gperftools-2.0')
> -protobuf = os.path.join(libprocess, '3rdparty', 'protobuf-2.5.0')
> -
> -# Build the list of source files. Note that each source must be
> -# relative to our current directory (where this script lives).
> -SOURCES = [
> - os.path.join('src', 'mesos', 'native', file)
> - for file in os.listdir(os.path.join(abs_top_srcdir, src_python_native))
> - if file.endswith('.cpp')
> -]
> -
> -INCLUDE_DIRS = [
> - os.path.join(abs_top_srcdir, 'include'),
> - os.path.join(abs_top_builddir, 'include'),
> - # Needed for the *.pb.h protobuf includes.
> - os.path.join(abs_top_builddir, 'include', 'mesos'),
> - os.path.join(abs_top_builddir, 'src'),
> - os.path.join(abs_top_builddir, src_python_native),
> - os.path.join(abs_top_builddir, protobuf, 'src'),
> -]
> -
> -LIBRARY_DIRS = []
> -
> -EXTRA_OBJECTS = [
> - os.path.join(abs_top_builddir, 'src', '.libs', 'libmesos_no_3rdparty.a'),
> - os.path.join(abs_top_builddir, libprocess, '.libs', 'libprocess.a')
> -]
> -
> -# For leveldb, we need to check for the presence of libleveldb.a, since
> -# it is possible to disable leveldb inside mesos.
> -libglog = os.path.join(abs_top_builddir, glog, '.libs', 'libglog.a')
> -libleveldb = os.path.join(abs_top_builddir, leveldb, 'libleveldb.a')
> -libzookeeper = os.path.join(
> - abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> -libprotobuf = os.path.join(
> - abs_top_builddir, protobuf, 'src', '.libs', 'libprotobuf.a')
> -
> -if os.path.exists(libleveldb):
> - EXTRA_OBJECTS.append(libleveldb)
> -else:
> - EXTRA_OBJECTS.append('-lleveldb')
> -
> -if os.path.exists(libzookeeper):
> - EXTRA_OBJECTS.append(libzookeeper)
> -else:
> - EXTRA_OBJECTS.append('-lzookeeper_mt')
> -
> -if os.path.exists(libglog):
> - EXTRA_OBJECTS.append(libglog)
> -else:
> - EXTRA_OBJECTS.append('-lglog')
> -
> -if os.path.exists(libprotobuf):
> - EXTRA_OBJECTS.append(libprotobuf)
> -else:
> - EXTRA_OBJECTS.append('-lprotobuf')
> -
> -
> -# libev is a special case because it needs to be enabled only when
> -# libevent *is not* enabled through the top level ./configure.
> -#
> -# TODO(hartem): this entire block MUST be removed once libev is deprecated
> -# in favor of libevent.
> -if '@ENABLE_LIBEVENT_TRUE@' == '#':
> - libev = os.path.join(libprocess, '3rdparty', 'libev-4.15')
> - libev = os.path.join(abs_top_builddir, libev, '.libs', 'libev.a')
> -
> - if os.path.exists(libev):
> - EXTRA_OBJECTS.append(libev)
> - else:
> - EXTRA_OBJECTS.append('-lev')
> -
> -
> -# For gperftools, we need to check for the presence of libprofiler.a, since
> -# it is possible to disable perftools inside libprocess.
> -libprofiler = os.path.join(
> - abs_top_builddir, gperftools, '.libs', 'libprofiler.a')
> -
> -if os.path.exists(libprofiler):
> - EXTRA_OBJECTS.append(libprofiler)
> -
> -EXTRA_LINK_ARGS = []
> -
> -# Add any flags from LDFLAGS.
> -if 'LDFLAGS' in os.environ:
> - for flag in os.environ['LDFLAGS'].split():
> - EXTRA_LINK_ARGS.append(flag)
> -
> -# Add any libraries from LIBS.
> -if 'LIBS' in os.environ:
> - for library in os.environ['LIBS'].split():
> - EXTRA_LINK_ARGS.append(library)
> -
> -DEPENDS = [
> - os.path.join(abs_top_srcdir, 'src', 'python', source)
> - for source in SOURCES
> -]
> -
> -# Note that we add EXTRA_OBJECTS to our dependency list to make sure
> -# that we rebuild this module when one of them changes (e.g.,
> -# libprocess).
> -mesos_module = \
> - Extension('mesos.native._mesos',
> - sources = SOURCES,
> - include_dirs = INCLUDE_DIRS,
> - library_dirs = LIBRARY_DIRS,
> - extra_objects = EXTRA_OBJECTS,
> - extra_link_args = EXTRA_LINK_ARGS,
> - depends = EXTRA_OBJECTS,
> - language = 'c++',
> - )
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/setup.py.in
> ----------------------------------------------------------------------
> diff --git a/src/python/native/setup.py.in b/src/python/native/setup.py.in
> index 49ed612..10a5002 100644
> --- a/src/python/native/setup.py.in
> +++ b/src/python/native/setup.py.in
> @@ -16,8 +16,6 @@
> # See the License for the specific language governing permissions and
> # limitations under the License.
>
> -import ext_modules
> -
> config = {
> 'name': 'mesos.native',
> 'version': '@PACKAGE_VERSION@',
> @@ -28,13 +26,12 @@ config = {
> 'namespace_packages': [ 'mesos' ],
> 'packages': [ 'mesos', 'mesos.native' ],
> 'package_dir': { '': 'src' },
> - 'install_requires': [ 'mesos.interface == @PACKAGE_VERSION@' ],
> + 'install_requires': [ 'mesos.executor == @PACKAGE_VERSION@',
> + 'mesos.scheduler == @PACKAGE_VERSION@'],
> 'license': 'Apache 2.0',
> 'keywords': 'mesos',
> - 'classifiers': [ ],
> - 'ext_modules': [ ext_modules.mesos_module ]
> + 'classifiers': [ ]
> }
>
> from setuptools import setup
> -
> setup(**config)
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/__init__.py
> ----------------------------------------------------------------------
> diff --git a/src/python/native/src/mesos/native/__init__.py b/src/python/native/src/mesos/native/__init__.py
> index 226f943..b581537 100644
> --- a/src/python/native/src/mesos/native/__init__.py
> +++ b/src/python/native/src/mesos/native/__init__.py
> @@ -14,8 +14,5 @@
> # See the License for the specific language governing permissions and
> # limitations under the License.
>
> -from ._mesos import MesosExecutorDriverImpl
> -from ._mesos import MesosSchedulerDriverImpl
> -
> -MesosExecutorDriver = MesosExecutorDriverImpl
> -MesosSchedulerDriver = MesosSchedulerDriverImpl
> +from mesos.executor import MesosExecutorDriver
> +from mesos.scheduler import MesosSchedulerDriver
> \ No newline at end of file
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> ----------------------------------------------------------------------
> diff --git a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp b/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> deleted file mode 100644
> index 7838a07..0000000
> --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.cpp
> +++ /dev/null
> @@ -1,347 +0,0 @@
> -// Licensed to the Apache Software Foundation (ASF) under one
> -// or more contributor license agreements. See the NOTICE file
> -// distributed with this work for additional information
> -// regarding copyright ownership. The ASF licenses this file
> -// to you under the Apache License, Version 2.0 (the
> -// "License"); you may not use this file except in compliance
> -// with the License. You may obtain a copy of the License at
> -//
> -// http://www.apache.org/licenses/LICENSE-2.0
> -//
> -// Unless required by applicable law or agreed to in writing, software
> -// distributed under the License is distributed on an "AS IS" BASIS,
> -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> -// See the License for the specific language governing permissions and
> -// limitations under the License.
> -
> -// Python.h must be included before standard headers.
> -// See: http://docs.python.org/2/c-api/intro.html#include-files
> -#include <Python.h>
> -
> -#include <string>
> -
> -#include "mesos_executor_driver_impl.hpp"
> -#include "module.hpp"
> -#include "proxy_executor.hpp"
> -
> -using namespace mesos;
> -using namespace mesos::python;
> -
> -using std::cerr;
> -using std::endl;
> -using std::string;
> -using std::vector;
> -using std::map;
> -
> -
> -namespace mesos { namespace python {
> -
> -/**
> - * Python type object for MesosExecutorDriverImpl.
> - */
> -PyTypeObject MesosExecutorDriverImplType = {
> - PyObject_HEAD_INIT(NULL)
> - 0, /* ob_size */
> - "_mesos.MesosExecutorDriverImpl", /* tp_name */
> - sizeof(MesosExecutorDriverImpl), /* tp_basicsize */
> - 0, /* tp_itemsize */
> - (destructor) MesosExecutorDriverImpl_dealloc, /* tp_dealloc */
> - 0, /* tp_print */
> - 0, /* tp_getattr */
> - 0, /* tp_setattr */
> - 0, /* tp_compare */
> - 0, /* tp_repr */
> - 0, /* tp_as_number */
> - 0, /* tp_as_sequence */
> - 0, /* tp_as_mapping */
> - 0, /* tp_hash */
> - 0, /* tp_call */
> - 0, /* tp_str */
> - 0, /* tp_getattro */
> - 0, /* tp_setattro */
> - 0, /* tp_as_buffer */
> - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> - "Private MesosExecutorDriver implementation", /* tp_doc */
> - (traverseproc) MesosExecutorDriverImpl_traverse, /* tp_traverse */
> - (inquiry) MesosExecutorDriverImpl_clear, /* tp_clear */
> - 0, /* tp_richcompare */
> - 0, /* tp_weaklistoffset */
> - 0, /* tp_iter */
> - 0, /* tp_iternext */
> - MesosExecutorDriverImpl_methods, /* tp_methods */
> - 0, /* tp_members */
> - 0, /* tp_getset */
> - 0, /* tp_base */
> - 0, /* tp_dict */
> - 0, /* tp_descr_get */
> - 0, /* tp_descr_set */
> - 0, /* tp_dictoffset */
> - (initproc) MesosExecutorDriverImpl_init, /* tp_init */
> - 0, /* tp_alloc */
> - MesosExecutorDriverImpl_new, /* tp_new */
> -};
> -
> -
> -/**
> - * List of Python methods in MesosExecutorDriverImpl.
> - */
> -PyMethodDef MesosExecutorDriverImpl_methods[] = {
> - { "start",
> - (PyCFunction) MesosExecutorDriverImpl_start,
> - METH_NOARGS,
> - "Start the driver to connect to Mesos"
> - },
> - { "stop",
> - (PyCFunction) MesosExecutorDriverImpl_stop,
> - METH_NOARGS,
> - "Stop the driver, disconnecting from Mesos"
> - },
> - { "abort",
> - (PyCFunction) MesosExecutorDriverImpl_abort,
> - METH_NOARGS,
> - "Abort the driver, disallowing calls from and to the driver"
> - },
> - { "join",
> - (PyCFunction) MesosExecutorDriverImpl_join,
> - METH_NOARGS,
> - "Wait for a running driver to disconnect from Mesos"
> - },
> - { "run",
> - (PyCFunction) MesosExecutorDriverImpl_run,
> - METH_NOARGS,
> - "Start a driver and run it, returning when it disconnects from Mesos"
> - },
> - { "sendStatusUpdate",
> - (PyCFunction) MesosExecutorDriverImpl_sendStatusUpdate,
> - METH_VARARGS,
> - "Send a status update for a task"
> - },
> - { "sendFrameworkMessage",
> - (PyCFunction) MesosExecutorDriverImpl_sendFrameworkMessage,
> - METH_VARARGS,
> - "Send a FrameworkMessage to a slave"
> - },
> - { NULL } /* Sentinel */
> -};
> -
> -
> -/**
> - * Create, but don't initialize, a new MesosExecutorDriverImpl
> - * (called by Python before init method).
> - */
> -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> - PyObject *args,
> - PyObject *kwds)
> -{
> - MesosExecutorDriverImpl *self;
> - self = (MesosExecutorDriverImpl *) type->tp_alloc(type, 0);
> - if (self != NULL) {
> - self->driver = NULL;
> - self->proxyExecutor = NULL;
> - self->pythonExecutor = NULL;
> - }
> - return (PyObject*) self;
> -}
> -
> -
> -/**
> - * Initialize a MesosExecutorDriverImpl with constructor arguments.
> - */
> -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> - PyObject *args,
> - PyObject *kwds)
> -{
> - PyObject *pythonExecutor = NULL;
> -
> - if (!PyArg_ParseTuple(args, "O", &pythonExecutor)) {
> - return -1;
> - }
> -
> - if (pythonExecutor != NULL) {
> - PyObject* tmp = self->pythonExecutor;
> - Py_INCREF(pythonExecutor);
> - self->pythonExecutor = pythonExecutor;
> - Py_XDECREF(tmp);
> - }
> -
> - if (self->driver != NULL) {
> - delete self->driver;
> - self->driver = NULL;
> - }
> -
> - if (self->proxyExecutor != NULL) {
> - delete self->proxyExecutor;
> - self->proxyExecutor = NULL;
> - }
> -
> - self->proxyExecutor = new ProxyExecutor(self);
> - self->driver = new MesosExecutorDriver(self->proxyExecutor);
> -
> - return 0;
> -}
> -
> -
> -/**
> - * Free a MesosExecutorDriverImpl.
> - */
> -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver != NULL) {
> - // We need to wrap the driver destructor in an "allow threads"
> - // macro since the MesosExecutorDriver destructor waits for the
> - // ExecutorProcess to terminate and there might be a thread that
> - // is trying to acquire the GIL to call through the
> - // ProxyExecutor. It will only be after this thread executes that
> - // the ExecutorProcess might actually get a terminate.
> - Py_BEGIN_ALLOW_THREADS
> - delete self->driver;
> - Py_END_ALLOW_THREADS
> - self->driver = NULL;
> - }
> -
> - if (self->proxyExecutor != NULL) {
> - delete self->proxyExecutor;
> - self->proxyExecutor = NULL;
> - }
> -
> - MesosExecutorDriverImpl_clear(self);
> - self->ob_type->tp_free((PyObject*) self);
> -}
> -
> -
> -/**
> - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> - * See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> - visitproc visit,
> - void* arg)
> -{
> - Py_VISIT(self->pythonExecutor);
> - return 0;
> -}
> -
> -
> -/**
> - * Clear fields of a MesosExecutorDriverImpl that can participate in
> - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self)
> -{
> - Py_CLEAR(self->pythonExecutor);
> - return 0;
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->start();
> - return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->stop();
> - return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->abort();
> - return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->join();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->run();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> - MesosExecutorDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* statusObj = NULL;
> - TaskStatus taskStatus;
> - if (!PyArg_ParseTuple(args, "O", &statusObj)) {
> - return NULL;
> - }
> - if (!readPythonProtobuf(statusObj, &taskStatus)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python TaskStatus");
> - return NULL;
> - }
> -
> - Status status = self->driver->sendStatusUpdate(taskStatus);
> - return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> -}
> -
> -
> -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> - MesosExecutorDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosExecutorDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - const char* data;
> - int length;
> - if (!PyArg_ParseTuple(args, "s#", &data, &length)) {
> - return NULL;
> - }
> -
> - Status status = self->driver->sendFrameworkMessage(string(data, length));
> - return PyInt_FromLong(status); // Sets an exception if creating the int fails.
> -}
> -
> -} // namespace python {
> -} // namespace mesos {
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> ----------------------------------------------------------------------
> diff --git a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp b/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> deleted file mode 100644
> index 6e672f8..0000000
> --- a/src/python/native/src/mesos/native/mesos_executor_driver_impl.hpp
> +++ /dev/null
> @@ -1,103 +0,0 @@
> -// Licensed to the Apache Software Foundation (ASF) under one
> -// or more contributor license agreements. See the NOTICE file
> -// distributed with this work for additional information
> -// regarding copyright ownership. The ASF licenses this file
> -// to you under the Apache License, Version 2.0 (the
> -// "License"); you may not use this file except in compliance
> -// with the License. You may obtain a copy of the License at
> -//
> -// http://www.apache.org/licenses/LICENSE-2.0
> -//
> -// Unless required by applicable law or agreed to in writing, software
> -// distributed under the License is distributed on an "AS IS" BASIS,
> -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> -// See the License for the specific language governing permissions and
> -// limitations under the License.
> -
> -#ifndef MESOS_EXECUTOR_DRIVER_IMPL_HPP
> -#define MESOS_EXECUTOR_DRIVER_IMPL_HPP
> -
> -#include <mesos/executor.hpp>
> -
> -
> -namespace mesos { namespace python {
> -
> -class ProxyExecutor;
> -
> -/**
> - * Python object structure for MesosExecutorDriverImpl objects.
> - */
> -struct MesosExecutorDriverImpl {
> - PyObject_HEAD
> - /* Type-specific fields go here. */
> - MesosExecutorDriver* driver;
> - ProxyExecutor* proxyExecutor;
> - PyObject* pythonExecutor;
> -};
> -
> -/**
> - * Python type object for MesosExecutorDriverImpl.
> - */
> -extern PyTypeObject MesosExecutorDriverImplType;
> -
> -/**
> - * List of Python methods in MesosExecutorDriverImpl.
> - */
> -extern PyMethodDef MesosExecutorDriverImpl_methods[];
> -
> -/**
> - * Create, but don't initialize, a new MesosExecutorDriverImpl
> - * (called by Python before init method).
> - */
> -PyObject* MesosExecutorDriverImpl_new(PyTypeObject *type,
> - PyObject *args,
> - PyObject *kwds);
> -
> -/**
> - * Initialize a MesosExecutorDriverImpl with constructor arguments.
> - */
> -int MesosExecutorDriverImpl_init(MesosExecutorDriverImpl *self,
> - PyObject *args,
> - PyObject *kwds);
> -
> -/**
> - * Free a MesosExecutorDriverImpl.
> - */
> -void MesosExecutorDriverImpl_dealloc(MesosExecutorDriverImpl* self);
> -
> -/**
> - * Traverse fields of a MesosExecutorDriverImpl on a cyclic GC search.
> - * See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_traverse(MesosExecutorDriverImpl* self,
> - visitproc visit,
> - void* arg);
> -/**
> - * Clear fields of a MesosExecutorDriverImpl that can participate in
> - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosExecutorDriverImpl_clear(MesosExecutorDriverImpl* self);
> -
> -// MesosExecutorDriverImpl methods.
> -PyObject* MesosExecutorDriverImpl_start(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_stop(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_abort(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_join(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_run(MesosExecutorDriverImpl* self);
> -
> -PyObject* MesosExecutorDriverImpl_sendStatusUpdate(
> - MesosExecutorDriverImpl* self,
> - PyObject* args);
> -
> -PyObject* MesosExecutorDriverImpl_sendFrameworkMessage(
> - MesosExecutorDriverImpl* self,
> - PyObject* args);
> -
> -} // namespace python {
> -} // namespace mesos {
> -
> -#endif /* MESOS_EXECUTOR_DRIVER_IMPL_HPP */
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/c81a52ec/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> ----------------------------------------------------------------------
> diff --git a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp b/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> deleted file mode 100644
> index f8be49b..0000000
> --- a/src/python/native/src/mesos/native/mesos_scheduler_driver_impl.cpp
> +++ /dev/null
> @@ -1,782 +0,0 @@
> -// Licensed to the Apache Software Foundation (ASF) under one
> -// or more contributor license agreements. See the NOTICE file
> -// distributed with this work for additional information
> -// regarding copyright ownership. The ASF licenses this file
> -// to you under the Apache License, Version 2.0 (the
> -// "License"); you may not use this file except in compliance
> -// with the License. You may obtain a copy of the License at
> -//
> -// http://www.apache.org/licenses/LICENSE-2.0
> -//
> -// Unless required by applicable law or agreed to in writing, software
> -// distributed under the License is distributed on an "AS IS" BASIS,
> -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> -// See the License for the specific language governing permissions and
> -// limitations under the License.
> -
> -// Python.h must be included before standard headers.
> -// See: http://docs.python.org/2/c-api/intro.html#include-files
> -#include <Python.h>
> -
> -#include <string>
> -
> -#include "mesos_scheduler_driver_impl.hpp"
> -#include "module.hpp"
> -#include "proxy_scheduler.hpp"
> -
> -using namespace mesos;
> -using namespace mesos::python;
> -
> -using std::cerr;
> -using std::endl;
> -using std::string;
> -using std::vector;
> -using std::map;
> -
> -namespace mesos {
> -namespace python {
> -
> -/**
> - * Python type object for MesosSchedulerDriverImpl.
> - */
> -PyTypeObject MesosSchedulerDriverImplType = {
> - PyObject_HEAD_INIT(NULL)
> - 0, /* ob_size */
> - "_mesos.MesosSchedulerDriverImpl", /* tp_name */
> - sizeof(MesosSchedulerDriverImpl), /* tp_basicsize */
> - 0, /* tp_itemsize */
> - (destructor) MesosSchedulerDriverImpl_dealloc, /* tp_dealloc */
> - 0, /* tp_print */
> - 0, /* tp_getattr */
> - 0, /* tp_setattr */
> - 0, /* tp_compare */
> - 0, /* tp_repr */
> - 0, /* tp_as_number */
> - 0, /* tp_as_sequence */
> - 0, /* tp_as_mapping */
> - 0, /* tp_hash */
> - 0, /* tp_call */
> - 0, /* tp_str */
> - 0, /* tp_getattro */
> - 0, /* tp_setattro */
> - 0, /* tp_as_buffer */
> - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
> - "Private MesosSchedulerDriver implementation", /* tp_doc */
> - (traverseproc) MesosSchedulerDriverImpl_traverse, /* tp_traverse */
> - (inquiry) MesosSchedulerDriverImpl_clear, /* tp_clear */
> - 0, /* tp_richcompare */
> - 0, /* tp_weaklistoffset */
> - 0, /* tp_iter */
> - 0, /* tp_iternext */
> - MesosSchedulerDriverImpl_methods, /* tp_methods */
> - 0, /* tp_members */
> - 0, /* tp_getset */
> - 0, /* tp_base */
> - 0, /* tp_dict */
> - 0, /* tp_descr_get */
> - 0, /* tp_descr_set */
> - 0, /* tp_dictoffset */
> - (initproc) MesosSchedulerDriverImpl_init, /* tp_init */
> - 0, /* tp_alloc */
> - MesosSchedulerDriverImpl_new, /* tp_new */
> -};
> -
> -
> -/**
> - * List of Python methods in MesosSchedulerDriverImpl.
> - */
> -PyMethodDef MesosSchedulerDriverImpl_methods[] = {
> - { "start",
> - (PyCFunction) MesosSchedulerDriverImpl_start,
> - METH_NOARGS,
> - "Start the driver to connect to Mesos"
> - },
> - { "stop",
> - (PyCFunction) MesosSchedulerDriverImpl_stop,
> - METH_VARARGS,
> - "Stop the driver, disconnecting from Mesos"
> - },
> - { "abort",
> - (PyCFunction) MesosSchedulerDriverImpl_abort,
> - METH_NOARGS,
> - "Abort the driver, disabling calls from and to the driver"
> - },
> - { "join",
> - (PyCFunction) MesosSchedulerDriverImpl_join,
> - METH_NOARGS,
> - "Wait for a running driver to disconnect from Mesos"
> - },
> - { "run",
> - (PyCFunction) MesosSchedulerDriverImpl_run,
> - METH_NOARGS,
> - "Start a driver and run it, returning when it disconnects from Mesos"
> - },
> - { "requestResources",
> - (PyCFunction) MesosSchedulerDriverImpl_requestResources,
> - METH_VARARGS,
> - "Request resources from the Mesos allocator"
> - },
> - { "launchTasks",
> - (PyCFunction) MesosSchedulerDriverImpl_launchTasks,
> - METH_VARARGS,
> - "Reply to a Mesos offer with a list of tasks"
> - },
> - { "killTask",
> - (PyCFunction) MesosSchedulerDriverImpl_killTask,
> - METH_VARARGS,
> - "Kill the task with the given ID"
> - },
> - { "acceptOffers",
> - (PyCFunction) MesosSchedulerDriverImpl_acceptOffers,
> - METH_VARARGS,
> - "Reply to a Mesos offer with a list of offer operations"
> - },
> - { "declineOffer",
> - (PyCFunction) MesosSchedulerDriverImpl_declineOffer,
> - METH_VARARGS,
> - "Decline a Mesos offer"
> - },
> - { "reviveOffers",
> - (PyCFunction) MesosSchedulerDriverImpl_reviveOffers,
> - METH_NOARGS,
> - "Remove all filters and ask Mesos for new offers"
> - },
> - { "suppressOffers",
> - (PyCFunction) MesosSchedulerDriverImpl_suppressOffers,
> - METH_NOARGS,
> - "Set suppressed attribute as true for the Framework"
> - },
> - { "acknowledgeStatusUpdate",
> - (PyCFunction) MesosSchedulerDriverImpl_acknowledgeStatusUpdate,
> - METH_VARARGS,
> - "Acknowledge a status update"
> - },
> - { "sendFrameworkMessage",
> - (PyCFunction) MesosSchedulerDriverImpl_sendFrameworkMessage,
> - METH_VARARGS,
> - "Send a FrameworkMessage to a slave"
> - },
> - { "reconcileTasks",
> - (PyCFunction) MesosSchedulerDriverImpl_reconcileTasks,
> - METH_VARARGS,
> - "Master sends status updates if task status is different from expected"
> - },
> - { NULL } /* Sentinel */
> -};
> -
> -
> -/**
> - * Create, but don't initialize, a new MesosSchedulerDriverImpl
> - * (called by Python before init method).
> - */
> -PyObject* MesosSchedulerDriverImpl_new(PyTypeObject* type,
> - PyObject* args,
> - PyObject* kwds)
> -{
> - MesosSchedulerDriverImpl* self;
> - self = (MesosSchedulerDriverImpl*) type->tp_alloc(type, 0);
> - if (self != NULL) {
> - self->driver = NULL;
> - self->proxyScheduler = NULL;
> - self->pythonScheduler = NULL;
> - }
> - return (PyObject*) self;
> -}
> -
> -
> -/**
> - * Initialize a MesosSchedulerDriverImpl with constructor arguments.
> - */
> -int MesosSchedulerDriverImpl_init(MesosSchedulerDriverImpl* self,
> - PyObject* args,
> - PyObject* kwds)
> -{
> - // Note: We use an integer for 'implicitAcknoweldgements' because
> - // it is the recommended way to pass booleans through CPython.
> - PyObject* schedulerObj = NULL;
> - PyObject* frameworkObj = NULL;
> - const char* master;
> - int implicitAcknowledgements = 1; // Enabled by default.
> - PyObject* credentialObj = NULL;
> -
> - if (!PyArg_ParseTuple(
> - args,
> - "OOs|iO",
> - &schedulerObj,
> - &frameworkObj,
> - &master,
> - &implicitAcknowledgements,
> - &credentialObj)) {
> - return -1;
> - }
> -
> - if (schedulerObj != NULL) {
> - PyObject* tmp = self->pythonScheduler;
> - Py_INCREF(schedulerObj);
> - self->pythonScheduler = schedulerObj;
> - Py_XDECREF(tmp);
> - }
> -
> - FrameworkInfo framework;
> - if (frameworkObj != NULL) {
> - if (!readPythonProtobuf(frameworkObj, &framework)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python FrameworkInfo");
> - return -1;
> - }
> - }
> -
> - Credential credential;
> - if (credentialObj != NULL) {
> - if (!readPythonProtobuf(credentialObj, &credential)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python Credential");
> - return -1;
> - }
> - }
> -
> -
> - if (self->driver != NULL) {
> - delete self->driver;
> - self->driver = NULL;
> - }
> -
> - if (self->proxyScheduler != NULL) {
> - delete self->proxyScheduler;
> - self->proxyScheduler = NULL;
> - }
> -
> - self->proxyScheduler = new ProxyScheduler(self);
> -
> - if (credentialObj != NULL) {
> - self->driver = new MesosSchedulerDriver(
> - self->proxyScheduler,
> - framework,
> - master,
> - implicitAcknowledgements != 0,
> - credential);
> - } else {
> - self->driver = new MesosSchedulerDriver(
> - self->proxyScheduler,
> - framework,
> - master,
> - implicitAcknowledgements != 0);
> - }
> -
> - return 0;
> -}
> -
> -
> -/**
> - * Free a MesosSchedulerDriverImpl.
> - */
> -void MesosSchedulerDriverImpl_dealloc(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver != NULL) {
> - // We need to wrap the driver destructor in an "allow threads"
> - // macro since the MesosSchedulerDriver destructor waits for the
> - // SchedulerProcess to terminate and there might be a thread that
> - // is trying to acquire the GIL to call through the
> - // ProxyScheduler. It will only be after this thread executes that
> - // the SchedulerProcess might actually get a terminate.
> - Py_BEGIN_ALLOW_THREADS
> - delete self->driver;
> - Py_END_ALLOW_THREADS
> - self->driver = NULL;
> - }
> -
> - if (self->proxyScheduler != NULL) {
> - delete self->proxyScheduler;
> - self->proxyScheduler = NULL;
> - }
> -
> - MesosSchedulerDriverImpl_clear(self);
> - self->ob_type->tp_free((PyObject*) self);
> -}
> -
> -
> -/**
> - * Traverse fields of a MesosSchedulerDriverImpl on a cyclic GC search.
> - * See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosSchedulerDriverImpl_traverse(MesosSchedulerDriverImpl* self,
> - visitproc visit,
> - void* arg)
> -{
> - Py_VISIT(self->pythonScheduler);
> - return 0;
> -}
> -
> -
> -/**
> - * Clear fields of a MesosSchedulerDriverImpl that can participate in
> - * GC cycles. See http://docs.python.org/extending/newtypes.html.
> - */
> -int MesosSchedulerDriverImpl_clear(MesosSchedulerDriverImpl* self)
> -{
> - Py_CLEAR(self->pythonScheduler);
> - return 0;
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_start(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->start();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_stop(MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - bool failover = false; // Should match default in mesos.py.
> -
> - if (!PyArg_ParseTuple(args, "|b", &failover)) {
> - return NULL;
> - }
> -
> - Status status = self->driver->stop(failover);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_abort(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->abort();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_join(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->join();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_run(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status;
> - Py_BEGIN_ALLOW_THREADS
> - status = self->driver->run();
> - Py_END_ALLOW_THREADS
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_requestResources(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* requestsObj = NULL;
> - vector<Request> requests;
> -
> - if (!PyArg_ParseTuple(args, "O", &requestsObj)) {
> - return NULL;
> - }
> -
> - if (!PyList_Check(requestsObj)) {
> - PyErr_Format(PyExc_Exception,
> - "Parameter 2 to requestsResources is not a list");
> - return NULL;
> - }
> - Py_ssize_t len = PyList_Size(requestsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* requestObj = PyList_GetItem(requestsObj, i);
> - if (requestObj == NULL) {
> - return NULL; // Exception will have been set by PyList_GetItem.
> - }
> - Request request;
> - if (!readPythonProtobuf(requestObj, &request)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python Request");
> - return NULL;
> - }
> - requests.push_back(request);
> - }
> -
> - Status status = self->driver->requestResources(requests);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_launchTasks(MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* offerIdsObj = NULL;
> - PyObject* tasksObj = NULL;
> - PyObject* filtersObj = NULL;
> - vector<OfferID> offerIds;
> - vector<TaskInfo> tasks;
> - Filters filters;
> -
> - if (!PyArg_ParseTuple(args, "OO|O", &offerIdsObj, &tasksObj, &filtersObj)) {
> - return NULL;
> - }
> -
> - // Offer argument can be a list of offer ids or a single offer id (for
> - // backward compatibility).
> - if (!PyList_Check(offerIdsObj)) {
> - OfferID offerId;
> - if (!readPythonProtobuf(offerIdsObj, &offerId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python OfferID");
> - return NULL;
> - }
> - offerIds.push_back(offerId);
> - } else {
> - Py_ssize_t len = PyList_Size(offerIdsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* offerObj = PyList_GetItem(offerIdsObj, i);
> - if (offerObj == NULL) {
> - return NULL;
> - }
> - OfferID offerId;
> - if (!readPythonProtobuf(offerObj, &offerId)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python OfferID");
> - return NULL;
> - }
> - offerIds.push_back(offerId);
> - }
> - }
> -
> - if (!PyList_Check(tasksObj)) {
> - PyErr_Format(PyExc_Exception, "Parameter 2 to launchTasks is not a list");
> - return NULL;
> - }
> - Py_ssize_t len = PyList_Size(tasksObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* taskObj = PyList_GetItem(tasksObj, i);
> - if (taskObj == NULL) {
> - return NULL; // Exception will have been set by PyList_GetItem.
> - }
> - TaskInfo task;
> - if (!readPythonProtobuf(taskObj, &task)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python TaskInfo");
> - return NULL;
> - }
> - tasks.push_back(task);
> - }
> -
> - if (filtersObj != NULL) {
> - if (!readPythonProtobuf(filtersObj, &filters)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Filters");
> - return NULL;
> - }
> - }
> -
> - Status status = self->driver->launchTasks(offerIds, tasks, filters);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_killTask(MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* tidObj = NULL;
> - TaskID tid;
> - if (!PyArg_ParseTuple(args, "O", &tidObj)) {
> - return NULL;
> - }
> - if (!readPythonProtobuf(tidObj, &tid)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python TaskID");
> - return NULL;
> - }
> -
> - Status status = self->driver->killTask(tid);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_acceptOffers(MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* offerIdsObj = NULL;
> - PyObject* operationsObj = NULL;
> - PyObject* filtersObj = NULL;
> - Py_ssize_t len = 0;
> - vector<OfferID> offerIds;
> - vector<Offer::Operation> operations;
> - Filters filters;
> -
> - if (!PyArg_ParseTuple(args,
> - "OO|O",
> - &offerIdsObj,
> - &operationsObj,
> - &filtersObj)) {
> - return NULL;
> - }
> -
> - if (!PyList_Check(offerIdsObj)) {
> - PyErr_Format(PyExc_Exception, "Parameter 1 to acceptOffers is not a list");
> - return NULL;
> - }
> -
> - len = PyList_Size(offerIdsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* offerObj = PyList_GetItem(offerIdsObj, i);
> - if (offerObj == NULL) {
> - return NULL;
> - }
> -
> - OfferID offerId;
> - if (!readPythonProtobuf(offerObj, &offerId)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python OfferID");
> - return NULL;
> - }
> - offerIds.push_back(offerId);
> - }
> -
> - if (!PyList_Check(operationsObj)) {
> - PyErr_Format(PyExc_Exception, "Parameter 2 to acceptOffers is not a list");
> - return NULL;
> - }
> -
> - len = PyList_Size(operationsObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* operationObj = PyList_GetItem(operationsObj, i);
> - if (operationObj == NULL) {
> - return NULL; // Exception will have been set by PyList_GetItem.
> - }
> -
> - Offer::Operation operation;
> - if (!readPythonProtobuf(operationObj, &operation)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Offer.Operation");
> - return NULL;
> - }
> - operations.push_back(operation);
> - }
> -
> - if (filtersObj != NULL) {
> - if (!readPythonProtobuf(filtersObj, &filters)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Filters");
> - return NULL;
> - }
> - }
> -
> - Status status = self->driver->acceptOffers(offerIds, operations, filters);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_declineOffer(MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* offerIdObj = NULL;
> - PyObject* filtersObj = NULL;
> - OfferID offerId;
> - Filters filters;
> -
> - if (!PyArg_ParseTuple(args, "O|O", &offerIdObj, &filtersObj)) {
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(offerIdObj, &offerId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python OfferID");
> - return NULL;
> - }
> -
> - if (filtersObj != NULL) {
> - if (!readPythonProtobuf(filtersObj, &filters)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python Filters");
> - return NULL;
> - }
> - }
> -
> - Status status = self->driver->declineOffer(offerId, filters);
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_reviveOffers(MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->reviveOffers();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_suppressOffers(
> - MesosSchedulerDriverImpl* self)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - Status status = self->driver->suppressOffers();
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_acknowledgeStatusUpdate(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* taskStatusObj = NULL;
> - TaskStatus taskStatus;
> -
> - if (!PyArg_ParseTuple(args, "O", &taskStatusObj)) {
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(taskStatusObj, &taskStatus)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python TaskStatus");
> - return NULL;
> - }
> -
> - Status status = self->driver->acknowledgeStatusUpdate(taskStatus);
> -
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_sendFrameworkMessage(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* slaveIdObj = NULL;
> - PyObject* executorIdObj = NULL;
> - SlaveID slaveId;
> - ExecutorID executorId;
> - const char* data;
> - int length;
> -
> - if (!PyArg_ParseTuple(
> - args, "OOs#", &executorIdObj, &slaveIdObj, &data, &length)) {
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(executorIdObj, &executorId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python ExecutorID");
> - return NULL;
> - }
> -
> - if (!readPythonProtobuf(slaveIdObj, &slaveId)) {
> - PyErr_Format(PyExc_Exception, "Could not deserialize Python SlaveID");
> - return NULL;
> - }
> -
> - Status status = self->driver->sendFrameworkMessage(
> - executorId, slaveId, string(data, length));
> -
> - return PyInt_FromLong(status); // Sets exception if creating long fails.
> -}
> -
> -
> -PyObject* MesosSchedulerDriverImpl_reconcileTasks(
> - MesosSchedulerDriverImpl* self,
> - PyObject* args)
> -{
> - if (self->driver == NULL) {
> - PyErr_Format(PyExc_Exception, "MesosSchedulerDriverImpl.driver is NULL");
> - return NULL;
> - }
> -
> - PyObject* statusesObj = NULL;
> - vector<TaskStatus> statuses;
> -
> - if (!PyArg_ParseTuple(args, "O", &statusesObj)) {
> - return NULL;
> - }
> -
> - if (!PyList_Check(statusesObj)) {
> - PyErr_Format(PyExc_Exception,
> - "Parameter 1 to reconcileTasks is not a list");
> -
> - return NULL;
> - }
> -
> - Py_ssize_t len = PyList_Size(statusesObj);
> - for (int i = 0; i < len; i++) {
> - PyObject* statusObj = PyList_GetItem(statusesObj, i);
> - if (statusObj == NULL) {
> - return NULL;
> - }
> -
> - TaskStatus status;
> - if (!readPythonProtobuf(statusObj, &status)) {
> - PyErr_Format(PyExc_Exception,
> - "Could not deserialize Python TaskStatus");
> - return NULL;
> - }
> - statuses.push_back(status);
> - }
> -
> - Status status = self->driver->reconcileTasks(statuses);
> - return PyInt_FromLong(status);
> -}
> -
> -} // namespace python {
> -} // namespace mesos {
>
>