You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 07:38:59 UTC

svn commit: r1131835 - in /incubator/mesos/trunk/src: Makefile.in scheduled_memhog.cpp

Author: benh
Date: Sun Jun  5 05:38:59 2011
New Revision: 1131835

URL: http://svn.apache.org/viewvc?rev=1131835&view=rev
Log:
Added scheduled memhog framework

Added:
    incubator/mesos/trunk/src/scheduled_memhog.cpp
Modified:
    incubator/mesos/trunk/src/Makefile.in

Modified: incubator/mesos/trunk/src/Makefile.in
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/Makefile.in?rev=1131835&r1=1131834&r2=1131835&view=diff
==============================================================================
--- incubator/mesos/trunk/src/Makefile.in (original)
+++ incubator/mesos/trunk/src/Makefile.in Sun Jun  5 05:38:59 2011
@@ -93,7 +93,7 @@ LIBS += -lglog -lgtest -lprocess -lev -l
 
 NEXUS_EXES = nexus-master nexus-slave nexus-local nexus-launcher \
 	     test-framework test-executor cpp-test-framework cpp-test-executor \
-	     memhog memhog-executor
+	     memhog memhog-executor scheduled-memhog
 
 ifeq ($(OS_NAME),solaris)
   NEXUS_EXES += nexus-projd
@@ -110,13 +110,11 @@ NEXUS_LIBS = $(SCHED_LIB) $(EXEC_LIB) $(
 MASTER_OBJ = master.o allocator_factory.o simple_allocator.o
 SLAVE_OBJ = slave.o launcher.o isolation_module_factory.o \
 	    process_based_isolation_module.o
-COMMON_OBJ = fatal.o hash_pid.o messages.o lock.o master_detector.o \
-	     url_processor.o zookeeper.o configuration.o
+COMMON_OBJ = fatal.o hash_pid.o messages.o lock.o master_detector.o url_processor.o zookeeper.o
 EXEC_LIB_OBJ = nexus_exec.o
 SCHED_LIB_OBJ = nexus_sched.o nexus_local.o params.o
 TEST_OBJ = tests/main.o tests/test_master.o tests/test_resources.o \
-	   tests/external_test.o tests/test_sample_frameworks.o \
-	   tests/test_configuration.o
+	   tests/external_test.o tests/test_sample_frameworks.o
 
 ifeq ($(OS_NAME),solaris)
   SLAVE_OBJ += solaris_project_isolation_module.o
@@ -226,7 +224,10 @@ memhog: memhog.cpp $(NEXUS_LIB) third_pa
 memhog-executor: memhog_executor.cpp $(NEXUS_LIB) third_party/libprocess/libprocess.a
 	$(CXX) $(CXXFLAGS) -o $@ $< -L. $(LDFLAGS) -lnexus++ $(LIBS) 
 
-java: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestFramework.class swig/java/TestExecutor.class swig/java/TestExceptionFramework.class swig/java/TestExceptionExecutor.class
+scheduled-memhog: scheduled_memhog.cpp $(NEXUS_LIB) third_party/libprocess/libprocess.a
+	$(CXX) $(CXXFLAGS) -o $@ $< -L. $(LDFLAGS) -lnexus++ $(LIBS) 
+
+java: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestFramework.class swig/java/TestExecutor.class
 
 python: $(PYTHON_LIB)
 
@@ -235,8 +236,8 @@ ruby: $(RUBY_LIB)
 swig/java/nexus.jar: $(JAVA_LIB)
 ifdef JAVA_HOME
 	$(JAVA_HOME)/bin/javac -sourcepath swig/java -d swig/java swig/java/nexus/*.java
-	patch -N swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch1 || echo -n
-	patch swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch2 || echo -n
+#	patch -N swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch1 || echo -n
+#	patch swig/java/nexus/nexusJNI.java < swig/java/nexusJNI.java.patch2 || echo -n
 	$(JAVA_HOME)/bin/jar cf $@ -C swig/java nexus
 endif
 
@@ -258,16 +259,6 @@ ifdef JAVA_HOME
 	$(JAVA_HOME)/bin/javac -cp swig/java/nexus.jar -sourcepath swig/java -d swig/java swig/java/TestExecutor.java
 endif
 
-swig/java/TestExceptionFramework.class: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestExceptionFramework.java
-ifdef JAVA_HOME
-	$(JAVA_HOME)/bin/javac -cp swig/java/nexus.jar -sourcepath swig/java -d swig/java swig/java/TestExceptionFramework.java
-endif
-
-swig/java/TestExceptionExecutor.class: $(JAVA_LIB) swig/java/nexus.jar swig/java/TestExceptionExecutor.java
-ifdef JAVA_HOME
-	$(JAVA_HOME)/bin/javac -cp swig/java/nexus.jar -sourcepath swig/java -d swig/java swig/java/TestExceptionExecutor.java
-endif
-
 $(PYTHON_LIB): swig/nexus.i $(NEXUS_LIB)
 ifdef PYTHON_HEADERS
 	$(SWIG) -c++ -python -threads -I../include -o swig/python/nexus_wrap.cpp -outdir swig/python swig/nexus.i

Added: incubator/mesos/trunk/src/scheduled_memhog.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/scheduled_memhog.cpp?rev=1131835&view=auto
==============================================================================
--- incubator/mesos/trunk/src/scheduled_memhog.cpp (added)
+++ incubator/mesos/trunk/src/scheduled_memhog.cpp Sun Jun  5 05:38:59 2011
@@ -0,0 +1,146 @@
+#include <nexus_sched.hpp>
+
+#include <algorithm>
+#include <cstdlib>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include <boost/lexical_cast.hpp>
+#include <boost/unordered_map.hpp>
+
+#include "foreach.hpp"
+
+using namespace std;
+using namespace nexus;
+
+using boost::lexical_cast;
+
+
+struct Task {
+  double launchTime;
+  double duration;
+  bool launched;
+  bool finished;
+
+  Task(double launchTime_, double duration_)
+    : launchTime(launchTime_), duration(duration_),
+      launched(0), finished(0) {}
+};
+
+
+struct TaskComparator
+{
+  bool operator () (const Task& t1, const Task& t2) {
+    return t1.launchTime < t2.launchTime;
+  }
+};
+
+
+class MyScheduler : public Scheduler
+{
+  string executor;
+  double taskLen;
+  int threadsPerTask;
+  int64_t memToRequest;
+  int64_t memToHog;
+  string scheduleFile;
+  vector<Task> tasks;
+  time_t startTime;
+  int tasksLaunched;
+  int tasksFinished;
+
+public:
+  MyScheduler(const string& executor_, const string& scheduleFile_,
+      int threadsPerTask_, int64_t memToRequest_, int64_t memToHog_)
+    : executor(executor_), scheduleFile(scheduleFile_),
+      threadsPerTask(threadsPerTask_),
+      memToRequest(memToRequest_), memToHog(memToHog_),
+      tasksLaunched(0), tasksFinished(0)
+  {
+    ifstream in(scheduleFile.c_str());
+    double launchTime;
+    double duration;
+    while (in >> launchTime >> duration) {
+      tasks.push_back(Task(launchTime, duration));
+    }
+    in.close();
+    TaskComparator comp;
+    sort(tasks.begin(), tasks.end(), comp);
+  }
+
+  virtual ~MyScheduler() {}
+
+  virtual string getFrameworkName(SchedulerDriver*) {
+    return "Memory hog";
+  }
+
+  virtual ExecutorInfo getExecutorInfo(SchedulerDriver*) {
+    return ExecutorInfo(executor, "");
+  }
+
+  virtual void registered(SchedulerDriver*, FrameworkID fid) {
+    cout << "Registered!" << endl;
+    startTime = time(0);
+  }
+
+  virtual void resourceOffer(SchedulerDriver* d,
+                             OfferID id,
+                             const vector<SlaveOffer>& offers) {
+    time_t now = time(0);
+    double curTime = difftime(now, startTime);
+    vector<TaskDescription> toLaunch;
+    foreach (const SlaveOffer &offer, offers) {
+      // This is kind of ugly because operator[] isn't a const function
+      int32_t cpus = lexical_cast<int32_t>(offer.params.find("cpus")->second);
+      int64_t mem = lexical_cast<int64_t>(offer.params.find("mem")->second);
+      if ((tasksLaunched < tasks.size()) && (cpus >= 1 && mem >= memToRequest) &&
+          curTime >= tasks[tasksLaunched].launchTime) {
+        TaskID tid = tasksLaunched++;
+        cout << "Launcing task " << tid << " on " << offer.host << endl;
+        map<string, string> params;
+        params["cpus"] = "1";
+        params["mem"] = lexical_cast<string>(memToRequest);
+        ostringstream arg;
+        arg << memToHog << " " << tasks[tid].duration << " " << threadsPerTask;
+        TaskDescription desc(tid, offer.slaveId, "task", params, arg.str());
+        toLaunch.push_back(desc);
+      }
+    }
+    map<string, string> params;
+    d->replyToOffer(id, toLaunch, params);
+  }
+
+  virtual void statusUpdate(SchedulerDriver* d, const TaskStatus& status) {
+    cout << "Task " << status.taskId << " is in state " << status.state << endl;
+    if (status.state == TASK_FINISHED || status.state == TASK_FAILED ||
+        status.state == TASK_KILLED || status.state == TASK_LOST) {
+      tasks[status.taskId].finished = true;
+      tasksFinished++;
+      if (tasksFinished == tasks.size()) {
+        d->stop();
+      }
+    }
+  }
+};
+
+
+int main(int argc, char ** argv) {
+  if (argc != 6) {
+    cerr << "Usage: " << argv[0]
+         << " <master> <schedule_file> <threads_per_task>"
+         << " <MB_to_request> <MB_per_task>" << endl;
+    return -1;
+  }
+  char cwd[512];
+  getcwd(cwd, sizeof(cwd));
+  string executor = string(cwd) + "/memhog-executor";
+  MyScheduler sched(executor,
+                    argv[2],
+                    lexical_cast<int>(argv[3]),
+                    lexical_cast<int64_t>(argv[4]) * 1024 * 1024,
+                    lexical_cast<int64_t>(argv[5]) * 1024 * 1024);
+  NexusSchedulerDriver driver(&sched, argv[1]);
+  driver.run();
+  return 0;
+}