You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 07:27:42 UTC
svn commit: r1131770 - in /incubator/mesos/trunk/frameworks/deploy_jar: ./
README.txt daemon_executor.py daemon_executor.sh daemon_framework
daemon_scheduler.py haproxy.config.template hw.jar
Author: benh
Date: Sun Jun 5 05:27:42 2011
New Revision: 1131770
URL: http://svn.apache.org/viewvc?rev=1131770&view=rev
Log:
Adding a Java jar deployment FW.
Added:
incubator/mesos/trunk/frameworks/deploy_jar/
incubator/mesos/trunk/frameworks/deploy_jar/README.txt
incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.py (with props)
incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.sh (with props)
incubator/mesos/trunk/frameworks/deploy_jar/daemon_framework (with props)
incubator/mesos/trunk/frameworks/deploy_jar/daemon_scheduler.py (with props)
incubator/mesos/trunk/frameworks/deploy_jar/haproxy.config.template
incubator/mesos/trunk/frameworks/deploy_jar/hw.jar
Added: incubator/mesos/trunk/frameworks/deploy_jar/README.txt
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/deploy_jar/README.txt?rev=1131770&view=auto
==============================================================================
--- incubator/mesos/trunk/frameworks/deploy_jar/README.txt (added)
+++ incubator/mesos/trunk/frameworks/deploy_jar/README.txt Sun Jun 5 05:27:42 2011
@@ -0,0 +1,14 @@
+haproxy load balancer + Apache web server Nexus framework Readme
+----------------------------------------------------------------
+
+First you will need to be able to run apache on the slave nodes in your cluster.
+
+In ubuntu, you can run 'sudo apt-get install apache2'
+
+Then 'sudo /etc/init.d/apache2 restart'
+
+
+You need to have haproxy installed, currently it is assumed (in haproxy+apache.py) to be in /root/haproxy-1.3.20/haproxy.
+
+installation instructions are here: http://www.lastengine.com/99/installing-haproxy-load-balancing-for-http-and-https/
+
Added: incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.py?rev=1131770&view=auto
==============================================================================
--- incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.py (added)
+++ incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.py Sun Jun 5 05:27:42 2011
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+import nexus
+import sys
+import time
+import os
+
+from subprocess import *
+
+class MyExecutor(nexus.Executor):
+ def __init__(self):
+ nexus.Executor.__init__(self)
+
+ def init(self, driver, arg):
+ print "in daemon executor"
+
+ def launchTask(self, driver, task):
+ print "in launchTask"
+ self.tid = task.taskId
+ print "task id is " + str(task.taskId) + ", task.args is " + task.arg
+ self.args = task.arg.split("\t")
+ print "running: " + "java -cp " + self.args[0] + " " + self.args[1] + " " + self.args[2]
+ print Popen("java -cp " + self.args[0] + " " + self.args[1] + " " + self.args[2], shell=True, stdout=PIPE).stdout.readline()
+ update = nexus.TaskStatus(task.taskId, nexus.TASK_FINISHED, "")
+ driver.sendStatusUpdate(update)
+
+
+ def error(self, code, message):
+ print "Error: %s" % message
+
+if __name__ == "__main__":
+ print "starting daemon framework executor"
+ executor = MyExecutor()
+ nexus.NexusExecutorDriver(executor).run()
Propchange: incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.py
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.sh?rev=1131770&view=auto
==============================================================================
--- incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.sh (added)
+++ incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.sh Sun Jun 5 05:27:42 2011
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+PYTHON=python
+
+if [ "`uname`" == "SunOS" ]; then
+ PYTHON=python2.6
+fi
+
+export PYTHONPATH=`dirname $0`/../../src/swig/python:$PYTHONPATH
+
+$PYTHON `dirname $0`/daemon_executor.py $@
Propchange: incubator/mesos/trunk/frameworks/deploy_jar/daemon_executor.sh
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/mesos/trunk/frameworks/deploy_jar/daemon_framework
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/deploy_jar/daemon_framework?rev=1131770&view=auto
==============================================================================
--- incubator/mesos/trunk/frameworks/deploy_jar/daemon_framework (added)
+++ incubator/mesos/trunk/frameworks/deploy_jar/daemon_framework Sun Jun 5 05:27:42 2011
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+PYTHON=python
+
+if [ "`uname`" == "SunOS" ]; then
+ PYTHON=python2.6
+fi
+
+export PYTHONPATH=`dirname $0`/../../src/swig/python:$PYTHONPATH
+
+$PYTHON ./daemon_scheduler.py $@
Propchange: incubator/mesos/trunk/frameworks/deploy_jar/daemon_framework
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/mesos/trunk/frameworks/deploy_jar/daemon_scheduler.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/deploy_jar/daemon_scheduler.py?rev=1131770&view=auto
==============================================================================
--- incubator/mesos/trunk/frameworks/deploy_jar/daemon_scheduler.py (added)
+++ incubator/mesos/trunk/frameworks/deploy_jar/daemon_scheduler.py Sun Jun 5 05:27:42 2011
@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+
+import nexus
+import os
+import sys
+import time
+import httplib
+import Queue
+import threading
+
+from optparse import OptionParser
+from subprocess import *
+from socket import gethostname
+
+MIN_SERVERS = 1
+START_THRESHOLD = 25
+KILL_THRESHOLD = 5
+#HAPROXY_EXE = "/root/haproxy-1.3.20/haproxy"
+HAPROXY_EXE = "/home/andyk/nexus/frameworks/haproxy+apache/haproxy-1.3.20/haproxy"
+
+class MyScheduler(nexus.Scheduler):
+ def __init__(self, num_tasks, jar_url, jar_class, jar_args):
+ nexus.Scheduler.__init__(self)
+ self.lock = threading.RLock()
+ self.id = 0
+ self.haproxy = -1
+ self.reconfigs = 0
+ self.task_count = 0
+ self.overloaded = False
+ self.num_tasks = num_tasks
+ self.jar_url = jar_url
+ self.jar_class = jar_class
+ self.jar_args = jar_args
+
+# def reconfigure(self):
+# name = "/tmp/haproxy.conf.%d" % self.reconfigs
+# with open(name, 'w') as config:
+# with open('haproxy.config.template', 'r') as template:
+# for line in template:
+# config.write(line)
+# for id, host in self.servers.iteritems():
+# config.write(" ")
+# config.write("server %d %s:80 check\n" % (id, host))
+#
+# cmd = []
+# if self.haproxy != -1:
+# cmd = [HAPROXY_EXE,
+# "-f",
+# name,
+# "-sf",
+# str(self.haproxy.pid)]
+# else:
+# cmd = [HAPROXY_EXE,
+# "-f",
+# name]
+#
+# self.haproxy = Popen(cmd, shell = False)
+# self.reconfigs += 1
+
+ def getExecutorInfo(self, driver):
+ execPath = os.path.join(os.getcwd(), "daemon_executor.sh")
+ return nexus.ExecutorInfo(execPath, "")
+
+ def registered(self, driver, fid):
+ print "Nexus daemon scheduler registered as framework #%s" % fid
+
+ def resourceOffer(self, driver, oid, slave_offers):
+ print "Got slot offer %d" % oid
+ self.lock.acquire()
+ tasks = []
+ for offer in slave_offers:
+ if self.task_count < self.num_tasks and int(offer.params['mem']) >= 1073741824 and int(offer.params['cpus']) > 0:
+ print "accept slot here"
+ params = {"cpus": "1", "mem": "1073741824"}
+ task_args = self.jar_url + "\t" + self.jar_class + "\t" + self.jar_args
+ print "task args are: " + task_args
+ td = nexus.TaskDescription(
+ self.task_count, offer.slaveId, "task %d" % self.task_count, params, task_args)
+ tasks.append(td)
+ self.task_count += 1
+ else:
+ print "Rejecting slot because we've launched enough tasks"
+ driver.replyToOffer(oid, tasks, {"timeout": "1"})
+ #self.reconfigure()
+ self.lock.release()
+
+ #def statusUpdate(self, driver, status):
+ # self.lock.acquire()
+ # if status.taskId in self.servers.keys():
+ # if status.state == nexus.TASK_FINISHED:
+ # del self.servers[status.taskId]
+ # self.reconfigure()
+ # reconfigured = True
+ # self.lock.release()
+ # if reconfigured:
+ # self.reviveOffers()
+
+ #def scaleUp(self):
+ # print "SCALING UP"
+ # self.lock.acquire()
+ # self.overloaded = True
+ # self.lock.release()
+
+ #def scaleDown(self, id):
+ # print "SCALING DOWN (removing server %d)" % id
+ # kill = False
+ # self.lock.acquire()
+ # if self.overloaded:
+ # self.overloaded = False
+ # else:
+ # kill = True
+ # self.lock.release()
+ # if kill:
+ # self.killTask(id)
+
+
+def monitor(sched):
+ while True:
+ time.sleep(1)
+ try:
+ conn = httplib.HTTPConnection("ec2-72-44-51-87.compute-1.amazonaws.com")
+ conn.request("GET", "/stats;csv")
+ res = conn.getresponse()
+ if (res.status != 200):
+ print "response != 200"
+ continue
+ else:
+ data = res.read()
+ lines = data.split('\n')[2:-2]
+
+ data = data.split('\n')
+ data = data[1].split(',')
+
+ if int(data[33]) >= START_THRESHOLD:
+ sched.scaleUp()
+ elif int(data[4]) <= KILL_THRESHOLD:
+ minload, minid = (sys.maxint, 0)
+ for l in lines:
+ cols = l.split(',')
+ id = int(cols[1])
+ load = int(cols[4])
+ if load < minload:
+ minload = load
+ minid = id
+
+ if len(lines) > MIN_SERVERS and minload == 0:
+ sched.scaleDown(minid)
+
+ conn.close()
+ except Exception, e:
+ continue
+
+if __name__ == "__main__":
+ parser = OptionParser(usage = "Usage: daemon_framework <mesos-master> <num tasks> <URL of jar> <class name> <arguments>")
+
+ (options,args) = parser.parse_args()
+ if len(args) < 5:
+ print >> sys.stderr, "five parameters required. " + str(len(args))
+ print >> sys.stderr, "Use --help to show usage."
+ exit(2)
+
+ sched = MyScheduler(args[1], args[2], args[3], args[4])
+
+ #threading.Thread(target = monitor, args=[sched]).start()
+
+ print "Connecting to nexus master %s" % args[0]
+
+ nexus.NexusSchedulerDriver(sched, args[0]).run()
+
+ print "Finished!"
Propchange: incubator/mesos/trunk/frameworks/deploy_jar/daemon_scheduler.py
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/mesos/trunk/frameworks/deploy_jar/haproxy.config.template
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/deploy_jar/haproxy.config.template?rev=1131770&view=auto
==============================================================================
--- incubator/mesos/trunk/frameworks/deploy_jar/haproxy.config.template (added)
+++ incubator/mesos/trunk/frameworks/deploy_jar/haproxy.config.template Sun Jun 5 05:27:42 2011
@@ -0,0 +1,8 @@
+listen webfarm ec2-174-129-94-218.compute-1.amazonaws.com:80
+ timeout server 7500
+ timeout client 7500
+ timeout connect 7500
+ mode http
+ balance roundrobin
+ option httpchk HEAD /index.html HTTP/1.0
+ stats uri /stats
Added: incubator/mesos/trunk/frameworks/deploy_jar/hw.jar
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/frameworks/deploy_jar/hw.jar?rev=1131770&view=auto
==============================================================================
Files incubator/mesos/trunk/frameworks/deploy_jar/hw.jar (added) and incubator/mesos/trunk/frameworks/deploy_jar/hw.jar Sun Jun 5 05:27:42 2011 differ