You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2012/03/28 19:50:48 UTC

svn commit: r1306509 - in /incubator/mesos/trunk: src/common/utils.hpp src/slave/slave.cpp third_party/libprocess/src/process.cpp

Author: benh
Date: Wed Mar 28 17:50:47 2012
New Revision: 1306509

URL: http://svn.apache.org/viewvc?rev=1306509&view=rev
Log:
Added auto detection of resources on a slave as well as auto detection of cores for libprocess.

Modified:
    incubator/mesos/trunk/src/common/utils.hpp
    incubator/mesos/trunk/src/slave/slave.cpp
    incubator/mesos/trunk/third_party/libprocess/src/process.cpp

Modified: incubator/mesos/trunk/src/common/utils.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/common/utils.hpp?rev=1306509&r1=1306508&r2=1306509&view=diff
==============================================================================
--- incubator/mesos/trunk/src/common/utils.hpp (original)
+++ incubator/mesos/trunk/src/common/utils.hpp Wed Mar 28 17:50:47 2012
@@ -42,6 +42,9 @@
 #include <glog/logging.h>
 
 #include <sys/stat.h>
+#ifdef __linux__
+#include <sys/sysinfo.h>
+#endif
 #include <sys/types.h>
 
 #include <google/protobuf/io/zero_copy_stream_impl.h>
@@ -489,6 +492,28 @@ inline int system(const std::string& com
   return ::system(command.c_str());
 }
 
+
+// Returns the total number of cpus (cores).
+inline Try<long> cpus()
+{
+  return sysconf(_SC_NPROCESSORS_ONLN);
+}
+
+
+// Returns the total size of main memory in bytes.
+inline Try<long> memory()
+{
+#ifdef __linux__
+  struct sysinfo info;
+  if (sysinfo(&info) != 0) {
+    return Try<long>::error(strerror(errno));
+  }
+  return info.totalram;
+#else
+  return Try<long>::error("Cannot determine the size of main memory");
+#endif
+}
+
 } // namespace os {
 
 

Modified: incubator/mesos/trunk/src/slave/slave.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/slave.cpp?rev=1306509&r1=1306508&r2=1306509&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/slave.cpp (original)
+++ incubator/mesos/trunk/src/slave/slave.cpp Wed Mar 28 17:50:47 2012
@@ -26,6 +26,7 @@
 
 #include "common/build.hpp"
 #include "common/option.hpp"
+#include "common/strings.hpp"
 #include "common/try.hpp"
 #include "common/type_utils.hpp"
 #include "common/utils.hpp"
@@ -89,8 +90,38 @@ Slave::Slave(const Configuration& _conf,
     local(_local),
     isolationModule(_isolationModule)
 {
+  Try<long> cpus = utils::os::cpus();
+  Try<long> mem = utils::os::memory();
+
+  if (!cpus.isSome()) {
+    LOG(WARNING) << "Failed to auto-detect the number of cpus to use,"
+                 << " defaulting to 1";
+    cpus = Try<long>::some(1);
+  }
+
+  if (!mem.isSome()) {
+    LOG(WARNING) << "Failed to auto-detect the size of main memory,"
+                 << " defaulting to 1024 MB";
+    mem = Try<long>::some(1024);
+  } else {
+    // Convert to MB.
+    mem = mem.get() / 1048576;
+
+    // Leave 1 GB free if we have more than 1 GB, otherwise, use all!
+    // TODO(benh): Have better default scheme (e.g., % of mem not
+    // greater than 1 GB?)
+    if (mem.get() > 1024) {
+      mem = Try<long>::some(mem.get() - 1024);
+    }
+  }
+
+  Try<string> defaults =
+    strings::format("cpus:%d;mem:%d", cpus.get(), mem.get());
+
+  CHECK(defaults.isSome());
+
   resources =
-    Resources::parse(conf.get<string>("resources", "cpus:1;mem:1024"));
+    Resources::parse(conf.get<string>("resources", defaults.get()));
 
   attributes =
     Attributes::parse(conf.get<string>("attributes", ""));

Modified: incubator/mesos/trunk/third_party/libprocess/src/process.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/src/process.cpp?rev=1306509&r1=1306508&r2=1306509&view=diff
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/src/process.cpp (original)
+++ incubator/mesos/trunk/third_party/libprocess/src/process.cpp Wed Mar 28 17:50:47 2012
@@ -414,8 +414,6 @@ static bool pending_timers = false;
 // Flag to indicate whether or to update the timer on async interrupt.
 static bool update_timer = false;
 
-const int NUMBER_OF_PROCESSING_THREADS = 4; // TODO(benh): Do 2x cores.
-
 
 // Thread local process pointer magic (constructed in
 // 'initialize'). We need the extra level of indirection from
@@ -1201,7 +1199,9 @@ void initialize(const string& delegate, 
   _process_ = new ThreadLocal<ProcessBase>(key);
 
   // Setup processing threads.
-  for (int i = 0; i < NUMBER_OF_PROCESSING_THREADS; i++) {
+  long cpus = sysconf(_SC_NPROCESSORS_ONLN);
+
+  for (int i = 0; i < cpus; i++) {
     pthread_t thread; // For now, not saving handles on our threads.
     if (pthread_create(&thread, NULL, schedule, NULL) != 0) {
       LOG(FATAL) << "Failed to initialize, pthread_create";
@@ -1343,7 +1343,8 @@ void initialize(const string& delegate, 
     PLOG(FATAL) << "Failed to initialize, inet_ntop";
   }
 
-  VLOG(1) << "libprocess is initialized on " << temp << ":" << __port__;
+  VLOG(1) << "libprocess is initialized on " << temp << ":" << __port__
+          << " for " << cpus << " cpus";
 }