You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 07:38:59 UTC
svn commit: r1131835 - in /incubator/mesos/trunk/src: Makefile.in
scheduled_memhog.cpp
Author: benh
Date: Sun Jun 5 05:38:59 2011
New Revision: 1131835
URL: http://svn.apache.org/viewvc?rev=1131835&view=rev
Log:
Added scheduled memhog framework
Added:
incubator/mesos/trunk/src/scheduled_memhog.cpp
Modified:
incubator/mesos/trunk/src/Makefile.in
Modified: incubator/mesos/trunk/src/Makefile.in
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/Makefile.in?rev=1131835&r1=1131834&r2=1131835&view=diff
==============================================================================
--- incubator/mesos/trunk/src/Makefile.in (original)
+++ incubator/mesos/trunk/src/Makefile.in Sun Jun 5 05:38:59 2011
@@ -93,7 +93,7 @@ LIBS += -lglog -lgtest -lprocess -lev -l
NEXUS_EXES = nexus-master nexus-slave nexus-local nexus-launcher \
test-framework test-executor cpp-test-framework cpp-test-executor \
- memhog memhog-executor
+ memhog memhog-executor scheduled-memhog
ifeq ($(OS_NAME),solaris)
NEXUS_EXES += nexus-projd
@@ -110,13 +110,11 @@ NEXUS_LIBS = $(SCHED_LIB) $(EXEC_LIB) $(
MASTER_OBJ = master.o allocator_factory.o simple_allocator.o
SLAVE_OBJ = slave.o launcher.o isolation_module_factory.o \
process_based_isolation_module.o
-COMMON_OBJ = fatal.o hash_pid.o messages.o lock.o master_detector.o \
- url_processor.o zookeeper.o configuration.o
+COMMON_OBJ = fatal.o hash_pid.o messages.o lock.o master_detector.o url_processor.o zookeeper.o
EXEC_LIB_OBJ = nexus_exec.o
SCHED_LIB_OBJ = nexus_sched.o nexus_local.o params.o
TEST_OBJ = tests/main.o tests/test_master.o tests/test_resources.o \
- tests/external_test.o tests/test_sample_frameworks.o \
- tests/test_configuration.o
+ tests/external_test.o tests/test_sample_frameworks.o
ifeq ($(OS_NAME),solaris)
SLAVE_OBJ += solaris_project_isolation_module.o
@@ -226,7 +224,10 @@ memhog: memhog.cpp $(NEXUS_LIB) third_pa
memhog-executor: memhog_executor.cpp $(NEXUS_LIB) third_party/libprocess/libprocess.a
$(CXX) $(CXXFLAGS) -o $@ $< -L. $(LDFLAGS) -lnexus++ $(LIBS)
-java: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestFramework.class swig/java/TestExecutor.class swig/java/TestExceptionFramework.class swig/java/TestExceptionExecutor.class
+scheduled-memhog: scheduled_memhog.cpp $(NEXUS_LIB) third_party/libprocess/libprocess.a
+ $(CXX) $(CXXFLAGS) -o $@ $< -L. $(LDFLAGS) -lnexus++ $(LIBS)
+
+java: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestFramework.class swig/java/TestExecutor.class
python: $(PYTHON_LIB)
@@ -235,8 +236,8 @@ ruby: $(RUBY_LIB)
swig/java/nexus.jar: $(JAVA_LIB)
ifdef JAVA_HOME
$(JAVA_HOME)/bin/javac -sourcepath swig/java -d swig/java swig/java/nexus/*.java
- patch -N swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch1 || echo -n
- patch swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch2 || echo -n
+# patch -N swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch1 || echo -n
+# patch swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch2 || echo -n
$(JAVA_HOME)/bin/jar cf $@ -C swig/java nexus
endif
@@ -258,16 +259,6 @@ ifdef JAVA_HOME
$(JAVA_HOME)/bin/javac -cp swig/java/nexus.jar -sourcepath swig/java -d swig/java swig/java/TestExecutor.java
endif
-swig/java/TestExceptionFramework.class: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestExceptionFramework.java
-ifdef JAVA_HOME
- $(JAVA_HOME)/bin/javac -cp swig/java/nexus.jar -sourcepath swig/java -d swig/java swig/java/TestExceptionFramework.java
-endif
-
-swig/java/TestExceptionExecutor.class: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestExceptionExecutor.java
-ifdef JAVA_HOME
- $(JAVA_HOME)/bin/javac -cp swig/java/nexus.jar -sourcepath swig/java -d swig/java swig/java/TestExceptionExecutor.java
-endif
-
$(PYTHON_LIB): swig/nexus.i $(NEXUS_LIB)
ifdef PYTHON_HEADERS
$(SWIG) -c++ -python -threads -I../include -o swig/python/nexus_wrap.cpp -outdir swig/python swig/nexus.i
Added: incubator/mesos/trunk/src/scheduled_memhog.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/scheduled_memhog.cpp?rev=1131835&view=auto
==============================================================================
--- incubator/mesos/trunk/src/scheduled_memhog.cpp (added)
+++ incubator/mesos/trunk/src/scheduled_memhog.cpp Sun Jun 5 05:38:59 2011
@@ -0,0 +1,146 @@
+#include <nexus_sched.hpp>
+
+#include <algorithm>
+#include <cstdlib>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include <boost/lexical_cast.hpp>
+#include <boost/unordered_map.hpp>
+
+#include "foreach.hpp"
+
+using namespace std;
+using namespace nexus;
+
+using boost::lexical_cast;
+
+
+struct Task {
+ double launchTime;
+ double duration;
+ bool launched;
+ bool finished;
+
+ Task(double launchTime_, double duration_)
+ : launchTime(launchTime_), duration(duration_),
+ launched(0), finished(0) {}
+};
+
+
+struct TaskComparator
+{
+ bool operator () (const Task& t1, const Task& t2) {
+ return t1.launchTime < t2.launchTime;
+ }
+};
+
+
+class MyScheduler : public Scheduler
+{
+ string executor;
+ double taskLen;
+ int threadsPerTask;
+ int64_t memToRequest;
+ int64_t memToHog;
+ string scheduleFile;
+ vector<Task> tasks;
+ time_t startTime;
+ int tasksLaunched;
+ int tasksFinished;
+
+public:
+ MyScheduler(const string& executor_, const string& scheduleFile_,
+ int threadsPerTask_, int64_t memToRequest_, int64_t memToHog_)
+ : executor(executor_), scheduleFile(scheduleFile_),
+ threadsPerTask(threadsPerTask_),
+ memToRequest(memToRequest_), memToHog(memToHog_),
+ tasksLaunched(0), tasksFinished(0)
+ {
+ ifstream in(scheduleFile.c_str());
+ double launchTime;
+ double duration;
+ while (in >> launchTime >> duration) {
+ tasks.push_back(Task(launchTime, duration));
+ }
+ in.close();
+ TaskComparator comp;
+ sort(tasks.begin(), tasks.end(), comp);
+ }
+
+ virtual ~MyScheduler() {}
+
+ virtual string getFrameworkName(SchedulerDriver*) {
+ return "Memory hog";
+ }
+
+ virtual ExecutorInfo getExecutorInfo(SchedulerDriver*) {
+ return ExecutorInfo(executor, "");
+ }
+
+ virtual void registered(SchedulerDriver*, FrameworkID fid) {
+ cout << "Registered!" << endl;
+ startTime = time(0);
+ }
+
+ virtual void resourceOffer(SchedulerDriver* d,
+ OfferID id,
+ const vector<SlaveOffer>& offers) {
+ time_t now = time(0);
+ double curTime = difftime(now, startTime);
+ vector<TaskDescription> toLaunch;
+ foreach (const SlaveOffer &offer, offers) {
+ // This is kind of ugly because operator[] isn't a const function
+ int32_t cpus = lexical_cast<int32_t>(offer.params.find("cpus")->second);
+ int64_t mem = lexical_cast<int64_t>(offer.params.find("mem")->second);
+ if ((tasksLaunched < tasks.size()) && (cpus >= 1 && mem >= memToRequest) &&
+ curTime >= tasks[tasksLaunched].launchTime) {
+ TaskID tid = tasksLaunched++;
+ cout << "Launcing task " << tid << " on " << offer.host << endl;
+ map<string, string> params;
+ params["cpus"] = "1";
+ params["mem"] = lexical_cast<string>(memToRequest);
+ ostringstream arg;
+ arg << memToHog << " " << tasks[tid].duration << " " << threadsPerTask;
+ TaskDescription desc(tid, offer.slaveId, "task", params, arg.str());
+ toLaunch.push_back(desc);
+ }
+ }
+ map<string, string> params;
+ d->replyToOffer(id, toLaunch, params);
+ }
+
+ virtual void statusUpdate(SchedulerDriver* d, const TaskStatus& status) {
+ cout << "Task " << status.taskId << " is in state " << status.state << endl;
+ if (status.state == TASK_FINISHED || status.state == TASK_FAILED ||
+ status.state == TASK_KILLED || status.state == TASK_LOST) {
+ tasks[status.taskId].finished = true;
+ tasksFinished++;
+ if (tasksFinished == tasks.size()) {
+ d->stop();
+ }
+ }
+ }
+};
+
+
+int main(int argc, char ** argv) {
+ if (argc != 6) {
+ cerr << "Usage: " << argv[0]
+ << " <master> <schedule_file> <threads_per_task>"
+ << " <MB_to_request> <MB_per_task>" << endl;
+ return -1;
+ }
+ char cwd[512];
+ getcwd(cwd, sizeof(cwd));
+ string executor = string(cwd) + "/memhog-executor";
+ MyScheduler sched(executor,
+ argv[2],
+ lexical_cast<int>(argv[3]),
+ lexical_cast<int64_t>(argv[4]) * 1024 * 1024,
+ lexical_cast<int64_t>(argv[5]) * 1024 * 1024);
+ NexusSchedulerDriver driver(&sched, argv[1]);
+ driver.run();
+ return 0;
+}