You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 10:29:53 UTC
svn commit: r1132054 - in /incubator/mesos/trunk/frameworks/haproxy+apache:
haproxy+apache haproxy+apache.py startapache.py startapache.sh
Author: benh
Date: Sun Jun 5 08:29:52 2011
New Revision: 1132054
URL: http://svn.apache.org/viewvc?rev=1132054&view=rev
Log:
Update to new Mesos alpha release API.
Modified:
incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache
incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache.py
incubator/mesos/trunk/frameworks/haproxy+apache/startapache.py
incubator/mesos/trunk/frameworks/haproxy+apache/startapache.sh
Modified: incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/haproxy%2Bapache/haproxy%2Bapache?rev=1132054&r1=1132053&r2=1132054&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache (original)
+++ incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache Sun Jun 5 08:29:52 2011
@@ -6,6 +6,6 @@ if [ "`uname`" == "SunOS" ]; then
PYTHON=python2.6
fi
-export PYTHONPATH=`dirname $0`/../../src/swig/python:$PYTHONPATH
+export PYTHONPATH=`dirname $0`/../../lib/python:$PYTHONPATH
$PYTHON ./haproxy+apache.py $@
Modified: incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/haproxy%2Bapache/haproxy%2Bapache.py?rev=1132054&r1=1132053&r2=1132054&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache.py (original)
+++ incubator/mesos/trunk/frameworks/haproxy+apache/haproxy+apache.py Sun Jun 5 08:29:52 2011
@@ -15,13 +15,12 @@ from socket import gethostname
MIN_SERVERS = 1
START_THRESHOLD = 25
KILL_THRESHOLD = 5
-#HAPROXY_EXE = "/root/haproxy-1.3.20/haproxy"
-HAPROXY_EXE = "/home/andyk/mesos/frameworks/haproxy+apache/haproxy-1.3.20/haproxy"
+HAPROXY_EXE = "/root/haproxy-1.3.20/haproxy" #EC2
+#HAPROXY_EXE = "/scratch/haproxy-1.3.25/haproxy" #Rcluster
-class Scheduler(mesos.Scheduler):
+class ApacheWebFWScheduler(mesos.Scheduler):
def __init__(self):
- mesos.Scheduler.__init__(self, "haproxy+apache",
- os.path.join(os.getcwd(), "startapache.sh"), "")
+ mesos.Scheduler.__init__(self)
self.lock = threading.RLock()
self.id = 0
self.haproxy = -1
@@ -29,7 +28,19 @@ class Scheduler(mesos.Scheduler):
self.servers = {}
self.overloaded = False
+ def registered(self, driver, fid):
+ print "Mesos haproxy+apache scheduler registered as framework #%s" % fid
+ self.driver = driver
+
+ def getFrameworkName(self, driver):
+ return "haproxy+apache"
+
+ def getExecutorInfo(self, driver):
+ execPath = os.path.join(os.getcwd(), "startapache.sh")
+ return mesos.ExecutorInfo(execPath, "")
+
def reconfigure(self):
+ print "reconfiguring haproxy"
name = "/tmp/haproxy.conf.%d" % self.reconfigs
with open(name, 'w') as config:
with open('haproxy.config.template', 'r') as template:
@@ -54,37 +65,72 @@ class Scheduler(mesos.Scheduler):
self.haproxy = Popen(cmd, shell = False)
self.reconfigs += 1
- def registered(self, fid):
- print "Mesos haproxy+apache scheduler registered as framework #%s" % fid
-
- def slotOffer(self, oid, slots):
- print "Got slot offer %d" % oid
+ def resourceOffer(self, driver, oid, slave_offers):
+ print "Got resource offer %s with %s slots." % (oid, len(slave_offers))
self.lock.acquire()
tasks = []
- for slot in slots:
- if not slot.host in self.servers.values() and self.overloaded or len(self.servers) < 1:
- params = "cpus=1\nmem=1024"
- td = mesos.TaskDescription(self.id, slot.slaveId, "server %s" % self.id, params, "")
+ for offer in slave_offers:
+ if offer.host in self.servers.values():
+ print "Rejecting slot on host " + offer.host + " because we've launched a server on that machine already."
+ #print "self.servers currently looks like: " + str(self.servers)
+ elif not self.overloaded and len(self.servers) > 0:
+ print "Rejecting slot because we've launched enough tasks."
+ elif int(offer.params['mem']) < 1024:
+ print "Rejecting offer because it doesn't contain enough memory (it has " + offer.params['mem'] + " and we need 1024mb."
+ elif int(offer.params['cpus']) < 1:
+ print "Rejecting offer because it doesn't contain enough CPUs."
+ else:
+ print "Offer is for " + offer.params['cpus'] + " CPUS and " + offer.params["mem"] + " MB on host " + offer.host
+ params = {"cpus": "1", "mem": "1024"}
+ td = mesos.TaskDescription(self.id, offer.slaveId, "server %s" % self.id, params, "")
+ print "Accepting task, id=" + str(self.id) + ", params: " + params['cpus'] + " CPUS, and " + params['mem'] + " MB, on node " + offer.host
tasks.append(td)
- self.servers[self.id] = slot.host
+ self.servers[self.id] = offer.host
self.id += 1
- else:
- print "Rejecting slot because we've launched enough tasks"
- self.replyToOffer(oid, tasks, "timeout=1")
- self.reconfigure()
+ self.overloaded = False
+ driver.replyToOffer(oid, tasks, {"timeout":"1"})
+ #driver.replyToOffer(oid, tasks, {})
+ print "done with resourceOffer()"
self.lock.release()
- def statusUpdate(self, status):
+ def statusUpdate(self, driver, status):
+ print "received status update from taskID " + str(status.taskId) + ", with state: " + str(status.state)
reconfigured = False
self.lock.acquire()
if status.taskId in self.servers.keys():
+ if status.state == mesos.TASK_STARTING:
+ print "Task " + str(status.taskId) + " reported that it is STARTING."
+ del self.servers[status.taskId]
+ self.reconfigure()
+ reconfigured = True
+ if status.state == mesos.TASK_RUNNING:
+ print "Task " + str(status.taskId) + " reported that it is RUNNING, reconfiguring haproxy to include it in webfarm now."
+ self.reconfigure()
+ reconfigured = True
if status.state == mesos.TASK_FINISHED:
del self.servers[status.taskId]
+ print "Task " + str(status.taskId) + " reported FINISHED (state " + status.state + ")."
+ self.reconfigure()
+ reconfigured = True
+ if status.state == mesos.TASK_FAILED:
+ print "Task " + str(status.taskId) + " reported that it FAILED!"
+ del self.servers[status.taskId]
+ self.reconfigure()
+ reconfigured = True
+ if status.state == mesos.TASK_KILLED:
+ print "Task " + str(status.taskId) + " reported that it was KILLED!"
+ del self.servers[status.taskId]
+ self.reconfigure()
+ reconfigured = True
+ if status.state == mesos.TASK_LOST:
+ print "Task " + str(status.taskId) + " reported was LOST!"
+ del self.servers[status.taskId]
self.reconfigure()
reconfigured = True
self.lock.release()
if reconfigured:
- self.reviveOffers()
+ None#driver.reviveOffers()
+ print "done in statusupdate"
def scaleUp(self):
print "SCALING UP"
@@ -102,20 +148,26 @@ class Scheduler(mesos.Scheduler):
kill = True
self.lock.release()
if kill:
- self.killTask(id)
+ self.driver.killTask(id)
def monitor(sched):
+ print "in MONITOR()"
while True:
time.sleep(1)
+ print "done sleeping"
try:
- conn = httplib.HTTPConnection("ec2-72-44-51-87.compute-1.amazonaws.com")
+ conn = httplib.HTTPConnection("r3.millennium.berkeley.edu:9001")
+ print "done creating connection"
conn.request("GET", "/stats;csv")
+ print "done with request()"
res = conn.getresponse()
+ print "testing response status"
if (res.status != 200):
print "response != 200"
continue
else:
+ print "got some stats"
data = res.read()
lines = data.split('\n')[2:-2]
@@ -139,7 +191,9 @@ def monitor(sched):
conn.close()
except Exception, e:
+ print "exception in monitor()"
continue
+ print "done in MONITOR()"
if __name__ == "__main__":
parser = OptionParser(usage = "Usage: %prog mesos_master")
@@ -150,12 +204,14 @@ if __name__ == "__main__":
print >> sys.stderr, "Use --help to show usage."
exit(2)
- sched = Scheduler()
-
- threading.Thread(target = monitor, args=[sched]).start()
+ print "sched = ApacheWebFWScheduler()"
+ sched = ApacheWebFWScheduler()
print "Connecting to mesos master %s" % args[0]
+ driver = mesos.MesosSchedulerDriver(sched, sys.argv[1])
+
+ threading.Thread(target = monitor, args=[sched]).start()
- sched.run(args[0])
+ driver.run()
- print "Finished!"
+ print "Scheduler finished!"
Modified: incubator/mesos/trunk/frameworks/haproxy+apache/startapache.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/haproxy%2Bapache/startapache.py?rev=1132054&r1=1132053&r2=1132054&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/haproxy+apache/startapache.py (original)
+++ incubator/mesos/trunk/frameworks/haproxy+apache/startapache.py Sun Jun 5 08:29:52 2011
@@ -7,10 +7,13 @@ import atexit
from subprocess import *
+APACHECTL = "/usr/apache2/2.2/bin/apachectl" #EC2
+#APACHECTL = "sudo /etc/init.d/apache2" #R Cluster
+
def cleanup():
try:
# TODO(*): This will kill ALL apaches...oops.
- os.waitpid(Popen("/usr/apache2/2.2/bin/apachectl stop", shell=True).pid, 0)
+ os.waitpid(Popen(APACHECTL + " stop", shell=True).pid, 0)
except Exception, e:
print e
None
@@ -22,7 +25,7 @@ class MyExecutor(mesos.Executor):
def launchTask(self, driver, task):
self.tid = task.taskId
- Popen("/usr/apache2/2.2/bin/apachectl start", shell=True)
+ Popen(APACHECTL + " start", shell=True)
def killTask(self, driver, tid):
if (tid != self.tid):
Modified: incubator/mesos/trunk/frameworks/haproxy+apache/startapache.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/haproxy%2Bapache/startapache.sh?rev=1132054&r1=1132053&r2=1132054&view=diff
==============================================================================
--- incubator/mesos/trunk/frameworks/haproxy+apache/startapache.sh (original)
+++ incubator/mesos/trunk/frameworks/haproxy+apache/startapache.sh Sun Jun 5 08:29:52 2011
@@ -6,6 +6,6 @@ if [ "`uname`" == "SunOS" ]; then
PYTHON=python2.6
fi
-export PYTHONPATH=`dirname $0`/../../src/swig/python:$PYTHONPATH
+export PYTHONPATH=`dirname $0`/../../lib/python:$PYTHONPATH
$PYTHON `dirname $0`/startapache.py $@