You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2013/03/21 19:13:38 UTC
svn commit: r1459435 - in /incubator/mesos/trunk: include/mesos/executor.hpp
include/mesos/scheduler.hpp src/python/src/mesos.py
Author: vinodkone
Date: Thu Mar 21 18:13:38 2013
New Revision: 1459435
URL: http://svn.apache.org/r1459435
Log:
Add pydocs to mesos.py for Executor/Scheduler and driver methods.
From: Brian Wickman <wi...@gmail.com>
Review: https://reviews.apache.org/r/9572
Modified:
incubator/mesos/trunk/include/mesos/executor.hpp
incubator/mesos/trunk/include/mesos/scheduler.hpp
incubator/mesos/trunk/src/python/src/mesos.py
Modified: incubator/mesos/trunk/include/mesos/executor.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/include/mesos/executor.hpp?rev=1459435&r1=1459434&r2=1459435&view=diff
==============================================================================
--- incubator/mesos/trunk/include/mesos/executor.hpp (original)
+++ incubator/mesos/trunk/include/mesos/executor.hpp Thu Mar 21 18:13:38 2013
@@ -81,7 +81,7 @@ public:
/**
* Invoked once the executor driver has been able to successfully
* connect with Mesos. In particular, a scheduler can pass some
- * data to it's executors through the FrameworkInfo.ExecutorInfo's
+ * data to its executors through the FrameworkInfo.ExecutorInfo's
* data field.
*/
virtual void registered(ExecutorDriver* driver,
@@ -125,11 +125,11 @@ public:
* executor. These messages are best effort; do not expect a
* framework message to be retransmitted in any reliable fashion.
*/
- virtual void frameworkMessage(ExecutorDriver* driver,
+ virtual void frameworkMessage(ExecutorDriver* driver,
const std::string& data) = 0;
/**
- * Invoked when the executor should terminate all of it's currently
+ * Invoked when the executor should terminate all of its currently
* running tasks. Note that after a Mesos has determined that an
* executor has terminated any tasks that the executor did not send
* terminal status updates for (e.g., TASK_KILLED, TASK_FINISHED,
Modified: incubator/mesos/trunk/include/mesos/scheduler.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/include/mesos/scheduler.hpp?rev=1459435&r1=1459434&r2=1459435&view=diff
==============================================================================
--- incubator/mesos/trunk/include/mesos/scheduler.hpp (original)
+++ incubator/mesos/trunk/include/mesos/scheduler.hpp Thu Mar 21 18:13:38 2013
@@ -193,7 +193,7 @@ public:
/**
* Stops the scheduler driver. If the 'failover' flag is set to
* false then it is expected that this framework will never
- * reconnect to Mesos and all of it's executors and tasks can be
+ * reconnect to Mesos and all of its executors and tasks can be
* terminated. Otherwise, all executors and tasks will remain
* running (for some framework specific failover timeout) allowing the
* scheduler to reconnect (possibly in the same process, or from a
Modified: incubator/mesos/trunk/src/python/src/mesos.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/python/src/mesos.py?rev=1459435&r1=1459434&r2=1459435&view=diff
==============================================================================
--- incubator/mesos/trunk/src/python/src/mesos.py (original)
+++ incubator/mesos/trunk/src/python/src/mesos.py Thu Mar 21 18:13:38 2013
@@ -18,81 +18,342 @@
# include/mesos/mesos.proto for more information documenting this
# interface.
+"""Python bindings for Mesos."""
+
+from __future__ import print_function
+
import sys
import _mesos
-# Base class for Mesos schedulers. Users' schedulers should extend
-# this class to get default implementations of methods they don't
-# override.
-class Scheduler:
- def registered(self, driver, frameworkId, masterInfo): pass
- def reregistered(self, driver, masterInfo): pass
- def disconnected(self, driver): pass
- def resourceOffers(self, driver, offers): pass
- def offerRescinded(self, driver, offerId): pass
- def statusUpdate(self, driver, status): pass
- def frameworkMessage(self, driver, executorId, slaveId, message): pass
- def slaveLost(self, driver, slaveId): pass
- def executorLost(self, driver, executorId, slaveId, status): pass
- # Default implementation of error() prints to stderr because we
- # can't make error() an abstract method in Python.
- def error(self, driver, message):
- print >> sys.stderr, "Error from Mesos: %s " % message
+# Alias the implementations from _mesos.
+# TODO(wickman): Make Mesos{Scheduler,Executor}DriverImpl inherit from the
+# superclasses defined here.
+MesosSchedulerDriver = _mesos.MesosSchedulerDriverImpl
+MesosExecutorDriver = _mesos.MesosExecutorDriverImpl
-# Interface for Mesos scheduler drivers. Users may wish to extend this
-# class in mock objects for tests.
-class SchedulerDriver:
- def start(self): pass
- def stop(self, failover=False): pass
- def abort(self) : pass
- def join(self): pass
- def run(self): pass
- def requestResources(self, requests): pass
- def launchTasks(self, offerId, tasks, filters=None): pass
- def killTask(self, taskId): pass
- def declineOffer(self, offerId, filters=None): pass
- def reviveOffers(self): pass
- def sendFrameworkMessage(self, executorId, slaveId, data): pass
-
-
-# Base class for Mesos executors. Users' executors should extend this
-# class to get default implementations of methods they don't override.
-class Executor:
- def registered(self, driver, executorInfo, frameworkInfo, slaveInfo): pass
- def reregistered(self, driver, slaveInfo): pass
- def disconnected(self, driver): pass
- def launchTask(self, driver, task): pass
- def killTask(self, driver, taskId): pass
- def frameworkMessage(self, driver, message): pass
- def shutdown(self, driver): pass
- # Default implementation of error() prints to stderr because we
- # can't make error() an abstract method in Python.
+__all__ = (
+ 'Executor',
+ 'ExecutorDriver'
+ 'MesosExecutorDriver',
+ 'MesosSchedulerDriver',
+ 'Scheduler',
+ 'SchedulerDriver',
+)
+
+
+class Scheduler(object):
+ """
+ Base class for Mesos schedulers. Users' schedulers should extend this
+ class to get default implementations of methods they don't override.
+ """
+
+ def registered(self, driver, frameworkId, masterInfo):
+ """
+ Invoked when the scheduler successfully registers with a Mesos master.
+ It is called with the frameworkId, a unique ID generated by the
+ master, and the masterInfo which is information about the master
+ itself.
+ """
+
+ def reregistered(self, driver, masterInfo):
+ """
+ Invoked when the scheduler re-registers with a newly elected Mesos
+ master. This is only called when the scheduler has previously been
+ registered. masterInfo contains information about the newly elected
+ master.
+ """
+
+ def disconnected(self, driver):
+ """
+ Invoked when the scheduler becomes disconnected from the master, e.g.
+ the master fails and another is taking over.
+ """
+
+ def resourceOffers(self, driver, offers):
+ """
+ Invoked when resources have been offered to this framework. A single
+ offer will only contain resources from a single slave. Resources
+ associated with an offer will not be re-offered to _this_ framework
+ until either (a) this framework has rejected those resources (see
+ SchedulerDriver.launchTasks) or (b) those resources have been
+ rescinded (see Scheduler.offerRescinded). Note that resources may be
+ concurrently offered to more than one framework at a time (depending
+ on the allocator being used). In that case, the first framework to
+ launch tasks using those resources will be able to use them while the
+ other frameworks will have those resources rescinded (or if a
+ framework has already launched tasks with those resources then those
+ tasks will fail with a TASK_LOST status and a message saying as much).
+ """
+
+ def offerRescinded(self, driver, offerId):
+ """
+ Invoked when an offer is no longer valid (e.g., the slave was lost or
+ another framework used resources in the offer.) If for whatever reason
+ an offer is never rescinded (e.g., dropped message, failing over
+ framework, etc.), a framwork that attempts to launch tasks using an
+ invalid offer will receive TASK_LOST status updats for those tasks
+ (see Scheduler.resourceOffers).
+ """
+
+ def statusUpdate(self, driver, status):
+ """
+ Invoked when the status of a task has changed (e.g., a slave is lost
+ and so the task is lost, a task finishes and an executor sends a
+ status update saying so, etc.) Note that returning from this callback
+ acknowledges receipt of this status update. If for whatever reason
+ the scheduler aborts during this callback (or the process exits)
+ another status update will be delivered. Note, however, that this is
+ currently not true if the slave sending the status update is lost or
+ fails during that time.
+ """
+
+ def frameworkMessage(self, driver, executorId, slaveId, message):
+ """
+ Invoked when an executor sends a message. These messages are best
+ effort; do not expect a framework message to be retransmitted in any
+ reliable fashion.
+ """
+
+ def slaveLost(self, driver, slaveId):
+ """
+ Invoked when a slave has been determined unreachable (e.g., machine
+ failure, network partition.) Most frameworks will need to reschedule
+ any tasks launched on this slave on a new slave.
+ """
+
+ def executorLost(self, driver, executorId, slaveId, status):
+ """
+ Invoked when an executor has exited/terminated. Note that any tasks
+ running will have TASK_LOST status updates automatically generated.
+ """
+
def error(self, driver, message):
- print >> sys.stderr, "Error from Mesos: %s" % message
+ """
+ Invoked when there is an unrecoverable error in the scheduler or
+ scheduler driver. The driver will be aborted BEFORE invoking this
+ callback.
+ """
+ print("Error from Mesos: %s " % message, file=sys.stderr)
+
+
+class SchedulerDriver(object):
+ """
+ Interface for Mesos scheduler drivers. Users may wish to implement this
+ class in mock objects for tests.
+ """
+ def start(self):
+ """
+ Starts the scheduler driver. This needs to be called before any other
+ driver calls are made.
+ """
+
+ def stop(self, failover=False):
+ """
+ Stops the scheduler driver. If the 'failover' flag is set to False
+ then it is expected that this framework will never reconnect to Mesos
+ and all of its executors and tasks can be terminated. Otherwise, all
+ executors and tasks will remain running (for some framework specific
+ failover timeout) allowing the scheduler to reconnect (possibly in the
+ same process, or from a different process, for example, on a different
+ machine.)
+ """
+
+ def abort(self):
+ """
+ Aborts the driver so that no more callbacks can be made to the
+ scheduler. The semantics of abort and stop have deliberately been
+ separated so that code can detect an aborted driver (i.e., via the
+ return status of SchedulerDriver.join), and instantiate and start
+ another driver if desired (from within the same process.)
+ """
+
+ def join(self):
+ """
+ Waits for the driver to be stopped or aborted, possibly blocking the
+ current thread indefinitely. The return status of this function can
+ be used to determine if the driver was aborted (see mesos.proto for a
+ description of Status).
+ """
+
+ def run(self):
+ """
+ Starts and immediately joins (i.e., blocks on) the driver.
+ """
+
+ def requestResources(self, requests):
+ """
+ Requests resources from Mesos (see mesos.proto for a description of
+ Request and how, for example, to request resources from specific
+ slaves.) Any resources available are offered to the framework via
+ Scheduler.resourceOffers callback, asynchronously.
+ """
+
+ def launchTasks(self, offerId, tasks, filters=None):
+ """
+ Launches the given set of tasks. Any resources remaining (i.e., not
+ used by the tasks or their executors) will be considered declined.
+ The specified filters are applied on all unused resources (see
+ mesos.proto for a description of Filters.) Invoking this function with
+ an empty collection of tasks declines this offer in its entirety (see
+ Scheduler.declineOffer). Note that currently tasks can only be
+ launched per offer. In the future, frameworks will be allowed to
+ aggregate offers (resources) to launch their tasks.
+ """
+
+ def killTask(self, taskId):
+ """
+ Kills the specified task. Note that attempting to kill a task is
+ currently not reliable. If, for example, a scheduler fails over while
+ it was attempting to kill a task it will need to retry in the future
+ (these semantics may be changed in the future).
+ """
+
+ def declineOffer(self, offerId, filters=None):
+ """
+ Declines an offer in its entirety and applies the specified
+ filters on the resources (see mesos.proto for a description of
+ Filters). Note that this can be done at any time, it is not
+ necessary to do this within the Scheduler::resourceOffers
+ callback.
+ """
+
+ def reviveOffers(self):
+ """
+ Removes all filters previously set by the framework (via
+ launchTasks()). This enables the framework to receive offers from
+ those filtered slaves.
+ """
+
+ def sendFrameworkMessage(self, executorId, slaveId, data):
+ """
+ Sends a message from the framework to one of its executors. These
+ messages are best effort; do not expect a framework message to be
+ retransmitted in any reliable fashion.
+ """
+
+
+class Executor(object):
+ """
+ Base class for Mesos executors. Users' executors should extend this
+ class to get default implementations of methods they don't override.
+ """
+
+ def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
+ """
+ Invoked once the executor driver has been able to successfully connect
+ with Mesos. In particular, a scheduler can pass some data to its
+ executors through the FrameworkInfo.ExecutorInfo's data field.
+ """
+
+ def reregistered(self, driver, slaveInfo):
+ """
+ Invoked when the executor re-registers with a restarted slave.
+ """
+
+ def disconnected(self, driver):
+ """
+ Invoked when the executor becomes "disconnected" from the slave (e.g.,
+ the slave is being restarted due to an upgrade).
+ """
+
+ def launchTask(self, driver, task):
+ """
+ Invoked when a task has been launched on this executor (initiated via
+ Scheduler.launchTasks). Note that this task can be realized with a
+ thread, a process, or some simple computation, however, no other
+ callbacks will be invoked on this executor until this callback has
+ returned.
+ """
+
+ def killTask(self, driver, taskId):
+ """
+ Invoked when a task running within this executor has been killed (via
+ SchedulerDriver.killTask). Note that no status update will be sent on
+ behalf of the executor, the executor is responsible for creating a new
+ TaskStatus (i.e., with TASK_KILLED) and invoking ExecutorDriver's
+ sendStatusUpdate.
+ """
+
+ def frameworkMessage(self, driver, message):
+ """
+ Invoked when a framework message has arrived for this executor. These
+ messages are best effort; do not expect a framework message to be
+ retransmitted in any reliable fashion.
+ """
+
+ def shutdown(self, driver):
+ """
+ Invoked when the executor should terminate all of its currently
+ running tasks. Note that after Mesos has determined that an executor
+ has terminated any tasks that the executor did not send terminal
+ status updates for (e.g., TASK_KILLED, TASK_FINISHED, TASK_FAILED,
+ etc) a TASK_LOST status update will be created.
+ """
+ def error(self, driver, message):
+ """
+ Invoked when a fatal error has occured with the executor and/or
+ executor driver. The driver will be aborted BEFORE invoking this
+ callback.
+ """
+ print("Error from Mesos: %s" % message, file=sys.stderr)
-# Interface for Mesos executor drivers. Users may wish to extend this
-# class in mock objects for tests.
-class ExecutorDriver:
- def start(self): pass
- def stop(self): pass
- def abort(self): pass
- def join(self): pass
- def run(self): pass
- def sendStatusUpdate(self, status): pass
- def sendFrameworkMessage(self, data): pass
-
-
-# Alias the MesosSchedulerDriverImpl from _mesos. Ideally we would make this
-# class inherit from SchedulerDriver somehow, but this complicates the C++
-# code, and there seems to be no point in doing it in a dynamic language.
-MesosSchedulerDriver = _mesos.MesosSchedulerDriverImpl
-# Alias the MesosExecutorDriverImpl from _mesos. Ideally we would make this
-# class inherit from ExecutorDriver somehow, but this complicates the C++
-# code, and there seems to be no point in doing it in a dynamic language.
-MesosExecutorDriver = _mesos.MesosExecutorDriverImpl
+class ExecutorDriver(object):
+ """
+ Interface for Mesos executor drivers. Users may wish to extend this
+ class in mock objects for tests.
+ """
+ def start(self):
+ """
+ Starts the executor driver. This needs to be called before any other
+ driver calls are made.
+ """
+
+ def stop(self):
+ """
+ Stops the executor driver.
+ """
+
+ def abort(self):
+ """
+ Aborts the driver so that no more callbacks can be made to the
+ executor. The semantics of abort and stop have deliberately been
+ separated so that code can detect an aborted driver (i.e., via the
+ return status of ExecutorDriver.join), and instantiate and start
+ another driver if desired (from within the same process, although this
+ functionality is currently not supported for executors).
+ """
+
+ def join(self):
+ """
+ Waits for the driver to be stopped or aborted, possibly blocking the
+ current thread indefinitely. The return status of this function can
+ be used to determine if the driver was aborted (see mesos.proto for a
+ description of Status).
+ """
+
+ def run(self):
+ """
+ Starts and immediately joins (i.e., blocks on) the driver.
+ """
+
+ def sendStatusUpdate(self, status):
+ """
+ Sends a status update to the framework scheduler, retrying as
+ necessary until an acknowledgement has been received or the executor
+ is terminated (in which case, a TASK_LOST status update will be sent).
+ See Scheduler.statusUpdate for more information about status update
+ acknowledgements.
+ """
+
+ def sendFrameworkMessage(self, data):
+ """
+ Sends a message to the framework scheduler. These messages are best
+ effort; do not expect a framework message to be retransmitted in any
+ reliable fashion.
+ """