You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2013/03/21 19:13:38 UTC

svn commit: r1459435 - in /incubator/mesos/trunk: include/mesos/executor.hpp include/mesos/scheduler.hpp src/python/src/mesos.py

Author: vinodkone
Date: Thu Mar 21 18:13:38 2013
New Revision: 1459435

URL: http://svn.apache.org/r1459435
Log:
Add pydocs to mesos.py for Executor/Scheduler and driver methods.

From: Brian Wickman <wi...@gmail.com>
Review: https://reviews.apache.org/r/9572

Modified:
    incubator/mesos/trunk/include/mesos/executor.hpp
    incubator/mesos/trunk/include/mesos/scheduler.hpp
    incubator/mesos/trunk/src/python/src/mesos.py

Modified: incubator/mesos/trunk/include/mesos/executor.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/include/mesos/executor.hpp?rev=1459435&r1=1459434&r2=1459435&view=diff
==============================================================================
--- incubator/mesos/trunk/include/mesos/executor.hpp (original)
+++ incubator/mesos/trunk/include/mesos/executor.hpp Thu Mar 21 18:13:38 2013
@@ -81,7 +81,7 @@ public:
   /**
    * Invoked once the executor driver has been able to successfully
    * connect with Mesos. In particular, a scheduler can pass some
-   * data to it's executors through the FrameworkInfo.ExecutorInfo's
+   * data to its executors through the FrameworkInfo.ExecutorInfo's
    * data field.
    */
   virtual void registered(ExecutorDriver* driver,
@@ -125,11 +125,11 @@ public:
    * executor. These messages are best effort; do not expect a
    * framework message to be retransmitted in any reliable fashion.
    */
-    virtual void frameworkMessage(ExecutorDriver* driver,
+  virtual void frameworkMessage(ExecutorDriver* driver,
                                   const std::string& data) = 0;
 
   /**
-   * Invoked when the executor should terminate all of it's currently
+   * Invoked when the executor should terminate all of its currently
    * running tasks. Note that after a Mesos has determined that an
    * executor has terminated any tasks that the executor did not send
    * terminal status updates for (e.g., TASK_KILLED, TASK_FINISHED,

Modified: incubator/mesos/trunk/include/mesos/scheduler.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/include/mesos/scheduler.hpp?rev=1459435&r1=1459434&r2=1459435&view=diff
==============================================================================
--- incubator/mesos/trunk/include/mesos/scheduler.hpp (original)
+++ incubator/mesos/trunk/include/mesos/scheduler.hpp Thu Mar 21 18:13:38 2013
@@ -193,7 +193,7 @@ public:
   /**
    * Stops the scheduler driver. If the 'failover' flag is set to
    * false then it is expected that this framework will never
-   * reconnect to Mesos and all of it's executors and tasks can be
+   * reconnect to Mesos and all of its executors and tasks can be
    * terminated. Otherwise, all executors and tasks will remain
    * running (for some framework specific failover timeout) allowing the
    * scheduler to reconnect (possibly in the same process, or from a

Modified: incubator/mesos/trunk/src/python/src/mesos.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/python/src/mesos.py?rev=1459435&r1=1459434&r2=1459435&view=diff
==============================================================================
--- incubator/mesos/trunk/src/python/src/mesos.py (original)
+++ incubator/mesos/trunk/src/python/src/mesos.py Thu Mar 21 18:13:38 2013
@@ -18,81 +18,342 @@
 # include/mesos/mesos.proto for more information documenting this
 # interface.
 
+"""Python bindings for Mesos."""
+
+from __future__ import print_function
+
 import sys
 import _mesos
 
-# Base class for Mesos schedulers. Users' schedulers should extend
-# this class to get default implementations of methods they don't
-# override.
-class Scheduler:
-  def registered(self, driver, frameworkId, masterInfo): pass
-  def reregistered(self, driver, masterInfo): pass
-  def disconnected(self, driver): pass
-  def resourceOffers(self, driver, offers): pass
-  def offerRescinded(self, driver, offerId): pass
-  def statusUpdate(self, driver, status): pass
-  def frameworkMessage(self, driver, executorId, slaveId, message): pass
-  def slaveLost(self, driver, slaveId): pass
-  def executorLost(self, driver, executorId, slaveId, status): pass
 
-  # Default implementation of error() prints to stderr because we
-  # can't make error() an abstract method in Python.
-  def error(self, driver, message):
-    print >> sys.stderr, "Error from Mesos: %s " % message
+# Alias the implementations from _mesos.
 
+# TODO(wickman): Make Mesos{Scheduler,Executor}DriverImpl inherit from the
+# superclasses defined here.
+MesosSchedulerDriver = _mesos.MesosSchedulerDriverImpl
+MesosExecutorDriver = _mesos.MesosExecutorDriverImpl
 
-# Interface for Mesos scheduler drivers. Users may wish to extend this
-# class in mock objects for tests.
-class SchedulerDriver:
-  def start(self): pass
-  def stop(self, failover=False): pass
-  def abort(self) : pass
-  def join(self): pass
-  def run(self): pass
-  def requestResources(self, requests): pass
-  def launchTasks(self, offerId, tasks, filters=None): pass
-  def killTask(self, taskId): pass
-  def declineOffer(self, offerId, filters=None): pass
-  def reviveOffers(self): pass
-  def sendFrameworkMessage(self, executorId, slaveId, data): pass
-
-
-# Base class for Mesos executors. Users' executors should extend this
-# class to get default implementations of methods they don't override.
-class Executor:
-  def registered(self, driver, executorInfo, frameworkInfo, slaveInfo): pass
-  def reregistered(self, driver, slaveInfo): pass
-  def disconnected(self, driver): pass
-  def launchTask(self, driver, task): pass
-  def killTask(self, driver, taskId): pass
-  def frameworkMessage(self, driver, message): pass
-  def shutdown(self, driver): pass
 
-  # Default implementation of error() prints to stderr because we
-  # can't make error() an abstract method in Python.
+__all__ = (
+  'Executor',
+  'ExecutorDriver'
+  'MesosExecutorDriver',
+  'MesosSchedulerDriver',
+  'Scheduler',
+  'SchedulerDriver',
+)
+
+
+class Scheduler(object):
+  """
+    Base class for Mesos schedulers. Users' schedulers should extend this
+    class to get default implementations of methods they don't override.
+  """
+
+  def registered(self, driver, frameworkId, masterInfo):
+    """
+      Invoked when the scheduler successfully registers with a Mesos master.
+      It is called with the frameworkId, a unique ID generated by the
+      master, and the masterInfo which is information about the master
+      itself.
+    """
+
+  def reregistered(self, driver, masterInfo):
+    """
+      Invoked when the scheduler re-registers with a newly elected Mesos
+      master.  This is only called when the scheduler has previously been
+      registered.  masterInfo contains information about the newly elected
+      master.
+    """
+
+  def disconnected(self, driver):
+    """
+      Invoked when the scheduler becomes disconnected from the master, e.g.
+      the master fails and another is taking over.
+    """
+
+  def resourceOffers(self, driver, offers):
+    """
+      Invoked when resources have been offered to this framework. A single
+      offer will only contain resources from a single slave.  Resources
+      associated with an offer will not be re-offered to _this_ framework
+      until either (a) this framework has rejected those resources (see
+      SchedulerDriver.launchTasks) or (b) those resources have been
+      rescinded (see Scheduler.offerRescinded).  Note that resources may be
+      concurrently offered to more than one framework at a time (depending
+      on the allocator being used).  In that case, the first framework to
+      launch tasks using those resources will be able to use them while the
+      other frameworks will have those resources rescinded (or if a
+      framework has already launched tasks with those resources then those
+      tasks will fail with a TASK_LOST status and a message saying as much).
+    """
+
+  def offerRescinded(self, driver, offerId):
+    """
+      Invoked when an offer is no longer valid (e.g., the slave was lost or
+      another framework used resources in the offer.) If for whatever reason
+      an offer is never rescinded (e.g., dropped message, failing over
+      framework, etc.), a framwork that attempts to launch tasks using an
+      invalid offer will receive TASK_LOST status updats for those tasks
+      (see Scheduler.resourceOffers).
+    """
+
+  def statusUpdate(self, driver, status):
+    """
+      Invoked when the status of a task has changed (e.g., a slave is lost
+      and so the task is lost, a task finishes and an executor sends a
+      status update saying so, etc.) Note that returning from this callback
+      acknowledges receipt of this status update.  If for whatever reason
+      the scheduler aborts during this callback (or the process exits)
+      another status update will be delivered.  Note, however, that this is
+      currently not true if the slave sending the status update is lost or
+      fails during that time.
+    """
+
+  def frameworkMessage(self, driver, executorId, slaveId, message):
+    """
+      Invoked when an executor sends a message. These messages are best
+      effort; do not expect a framework message to be retransmitted in any
+      reliable fashion.
+    """
+
+  def slaveLost(self, driver, slaveId):
+    """
+      Invoked when a slave has been determined unreachable (e.g., machine
+      failure, network partition.) Most frameworks will need to reschedule
+      any tasks launched on this slave on a new slave.
+    """
+
+  def executorLost(self, driver, executorId, slaveId, status):
+    """
+      Invoked when an executor has exited/terminated. Note that any tasks
+      running will have TASK_LOST status updates automatically generated.
+    """
+
   def error(self, driver, message):
-    print >> sys.stderr, "Error from Mesos: %s" % message
+    """
+      Invoked when there is an unrecoverable error in the scheduler or
+      scheduler driver.  The driver will be aborted BEFORE invoking this
+      callback.
+    """
+    print("Error from Mesos: %s " % message, file=sys.stderr)
+
+
+class SchedulerDriver(object):
+  """
+    Interface for Mesos scheduler drivers. Users may wish to implement this
+    class in mock objects for tests.
+  """
+  def start(self):
+    """
+      Starts the scheduler driver.  This needs to be called before any other
+      driver calls are made.
+    """
+
+  def stop(self, failover=False):
+    """
+      Stops the scheduler driver. If the 'failover' flag is set to False
+      then it is expected that this framework will never reconnect to Mesos
+      and all of its executors and tasks can be terminated.  Otherwise, all
+      executors and tasks will remain running (for some framework specific
+      failover timeout) allowing the scheduler to reconnect (possibly in the
+      same process, or from a different process, for example, on a different
+      machine.)
+    """
+
+  def abort(self):
+    """
+      Aborts the driver so that no more callbacks can be made to the
+      scheduler.  The semantics of abort and stop have deliberately been
+      separated so that code can detect an aborted driver (i.e., via the
+      return status of SchedulerDriver.join), and instantiate and start
+      another driver if desired (from within the same process.)
+    """
+
+  def join(self):
+    """
+      Waits for the driver to be stopped or aborted, possibly blocking the
+      current thread indefinitely.  The return status of this function can
+      be used to determine if the driver was aborted (see mesos.proto for a
+      description of Status).
+    """
+
+  def run(self):
+    """
+      Starts and immediately joins (i.e., blocks on) the driver.
+    """
+
+  def requestResources(self, requests):
+    """
+      Requests resources from Mesos (see mesos.proto for a description of
+      Request and how, for example, to request resources from specific
+      slaves.)  Any resources available are offered to the framework via
+      Scheduler.resourceOffers callback, asynchronously.
+    """
+
+  def launchTasks(self, offerId, tasks, filters=None):
+    """
+      Launches the given set of tasks. Any resources remaining (i.e., not
+      used by the tasks or their executors) will be considered declined.
+      The specified filters are applied on all unused resources (see
+      mesos.proto for a description of Filters.) Invoking this function with
+      an empty collection of tasks declines this offer in its entirety (see
+      Scheduler.declineOffer).  Note that currently tasks can only be
+      launched per offer.  In the future, frameworks will be allowed to
+      aggregate offers (resources) to launch their tasks.
+    """
+
+  def killTask(self, taskId):
+    """
+      Kills the specified task. Note that attempting to kill a task is
+      currently not reliable.  If, for example, a scheduler fails over while
+      it was attempting to kill a task it will need to retry in the future
+      (these semantics may be changed in the future).
+    """
+
+  def declineOffer(self, offerId, filters=None):
+    """
+      Declines an offer in its entirety and applies the specified
+      filters on the resources (see mesos.proto for a description of
+      Filters). Note that this can be done at any time, it is not
+      necessary to do this within the Scheduler::resourceOffers
+      callback.
+    """
+
+  def reviveOffers(self):
+    """
+      Removes all filters previously set by the framework (via
+      launchTasks()).  This enables the framework to receive offers from
+      those filtered slaves.
+    """
+
+  def sendFrameworkMessage(self, executorId, slaveId, data):
+    """
+      Sends a message from the framework to one of its executors. These
+      messages are best effort; do not expect a framework message to be
+      retransmitted in any reliable fashion.
+    """
+
+
+class Executor(object):
+  """
+    Base class for Mesos executors. Users' executors should extend this
+    class to get default implementations of methods they don't override.
+  """
+
+  def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
+    """
+      Invoked once the executor driver has been able to successfully connect
+      with Mesos.  In particular, a scheduler can pass some data to its
+      executors through the FrameworkInfo.ExecutorInfo's data field.
+    """
+
+  def reregistered(self, driver, slaveInfo):
+    """
+      Invoked when the executor re-registers with a restarted slave.
+    """
+
+  def disconnected(self, driver):
+    """
+      Invoked when the executor becomes "disconnected" from the slave (e.g.,
+      the slave is being restarted due to an upgrade).
+    """
+
+  def launchTask(self, driver, task):
+    """
+      Invoked when a task has been launched on this executor (initiated via
+      Scheduler.launchTasks).  Note that this task can be realized with a
+      thread, a process, or some simple computation, however, no other
+      callbacks will be invoked on this executor until this callback has
+      returned.
+    """
+
+  def killTask(self, driver, taskId):
+    """
+      Invoked when a task running within this executor has been killed (via
+      SchedulerDriver.killTask).  Note that no status update will be sent on
+      behalf of the executor, the executor is responsible for creating a new
+      TaskStatus (i.e., with TASK_KILLED) and invoking ExecutorDriver's
+      sendStatusUpdate.
+    """
+
+  def frameworkMessage(self, driver, message):
+    """
+      Invoked when a framework message has arrived for this executor.  These
+      messages are best effort; do not expect a framework message to be
+      retransmitted in any reliable fashion.
+    """
+
+  def shutdown(self, driver):
+    """
+      Invoked when the executor should terminate all of its currently
+      running tasks.  Note that after Mesos has determined that an executor
+      has terminated any tasks that the executor did not send terminal
+      status updates for (e.g., TASK_KILLED, TASK_FINISHED, TASK_FAILED,
+      etc) a TASK_LOST status update will be created.
+    """
 
+  def error(self, driver, message):
+    """
+      Invoked when a fatal error has occured with the executor and/or
+      executor driver.  The driver will be aborted BEFORE invoking this
+      callback.
+    """
+    print("Error from Mesos: %s" % message, file=sys.stderr)
 
-# Interface for Mesos executor drivers. Users may wish to extend this
-# class in mock objects for tests.
-class ExecutorDriver:
-  def start(self): pass
-  def stop(self): pass
-  def abort(self): pass
-  def join(self): pass
-  def run(self): pass
-  def sendStatusUpdate(self, status): pass
-  def sendFrameworkMessage(self, data): pass
-
-
-# Alias the MesosSchedulerDriverImpl from _mesos. Ideally we would make this
-# class inherit from SchedulerDriver somehow, but this complicates the C++
-# code, and there seems to be no point in doing it in a dynamic language.
-MesosSchedulerDriver = _mesos.MesosSchedulerDriverImpl
 
 
-# Alias the MesosExecutorDriverImpl from _mesos. Ideally we would make this
-# class inherit from ExecutorDriver somehow, but this complicates the C++
-# code, and there seems to be no point in doing it in a dynamic language.
-MesosExecutorDriver = _mesos.MesosExecutorDriverImpl
+class ExecutorDriver(object):
+  """
+    Interface for Mesos executor drivers. Users may wish to extend this
+    class in mock objects for tests.
+  """
+  def start(self):
+    """
+      Starts the executor driver. This needs to be called before any other
+      driver calls are made.
+    """
+
+  def stop(self):
+    """
+      Stops the executor driver.
+    """
+
+  def abort(self):
+    """
+      Aborts the driver so that no more callbacks can be made to the
+      executor.  The semantics of abort and stop have deliberately been
+      separated so that code can detect an aborted driver (i.e., via the
+      return status of ExecutorDriver.join), and instantiate and start
+      another driver if desired (from within the same process, although this
+      functionality is currently not supported for executors).
+    """
+
+  def join(self):
+    """
+      Waits for the driver to be stopped or aborted, possibly blocking the
+      current thread indefinitely.  The return status of this function can
+      be used to determine if the driver was aborted (see mesos.proto for a
+      description of Status).
+    """
+
+  def run(self):
+    """
+      Starts and immediately joins (i.e., blocks on) the driver.
+    """
+
+  def sendStatusUpdate(self, status):
+    """
+      Sends a status update to the framework scheduler, retrying as
+      necessary until an acknowledgement has been received or the executor
+      is terminated (in which case, a TASK_LOST status update will be sent).
+      See Scheduler.statusUpdate for more information about status update
+      acknowledgements.
+    """
+
+  def sendFrameworkMessage(self, data):
+    """
+      Sends a message to the framework scheduler. These messages are best
+      effort; do not expect a framework message to be retransmitted in any
+      reliable fashion.
+    """