You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2012/03/28 19:50:48 UTC
svn commit: r1306509 - in /incubator/mesos/trunk: src/common/utils.hpp
src/slave/slave.cpp third_party/libprocess/src/process.cpp
Author: benh
Date: Wed Mar 28 17:50:47 2012
New Revision: 1306509
URL: http://svn.apache.org/viewvc?rev=1306509&view=rev
Log:
Added auto detection of resources on a slave as well as auto detection of cores for libprocess.
Modified:
incubator/mesos/trunk/src/common/utils.hpp
incubator/mesos/trunk/src/slave/slave.cpp
incubator/mesos/trunk/third_party/libprocess/src/process.cpp
Modified: incubator/mesos/trunk/src/common/utils.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/common/utils.hpp?rev=1306509&r1=1306508&r2=1306509&view=diff
==============================================================================
--- incubator/mesos/trunk/src/common/utils.hpp (original)
+++ incubator/mesos/trunk/src/common/utils.hpp Wed Mar 28 17:50:47 2012
@@ -42,6 +42,9 @@
#include <glog/logging.h>
#include <sys/stat.h>
+#ifdef __linux__
+#include <sys/sysinfo.h>
+#endif
#include <sys/types.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
@@ -489,6 +492,28 @@ inline int system(const std::string& com
return ::system(command.c_str());
}
+
+// Returns the total number of cpus (cores).
+inline Try<long> cpus()
+{
+ return sysconf(_SC_NPROCESSORS_ONLN);
+}
+
+
+// Returns the total size of main memory in bytes.
+inline Try<long> memory()
+{
+#ifdef __linux__
+ struct sysinfo info;
+ if (sysinfo(&info) != 0) {
+ return Try<long>::error(strerror(errno));
+ }
+ return info.totalram;
+#else
+ return Try<long>::error("Cannot determine the size of main memory");
+#endif
+}
+
} // namespace os {
Modified: incubator/mesos/trunk/src/slave/slave.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave/slave.cpp?rev=1306509&r1=1306508&r2=1306509&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave/slave.cpp (original)
+++ incubator/mesos/trunk/src/slave/slave.cpp Wed Mar 28 17:50:47 2012
@@ -26,6 +26,7 @@
#include "common/build.hpp"
#include "common/option.hpp"
+#include "common/strings.hpp"
#include "common/try.hpp"
#include "common/type_utils.hpp"
#include "common/utils.hpp"
@@ -89,8 +90,38 @@ Slave::Slave(const Configuration& _conf,
local(_local),
isolationModule(_isolationModule)
{
+ Try<long> cpus = utils::os::cpus();
+ Try<long> mem = utils::os::memory();
+
+ if (!cpus.isSome()) {
+ LOG(WARNING) << "Failed to auto-detect the number of cpus to use,"
+ << " defaulting to 1";
+ cpus = Try<long>::some(1);
+ }
+
+ if (!mem.isSome()) {
+ LOG(WARNING) << "Failed to auto-detect the size of main memory,"
+ << " defaulting to 1024 MB";
+ mem = Try<long>::some(1024);
+ } else {
+ // Convert to MB.
+ mem = mem.get() / 1048576;
+
+ // Leave 1 GB free if we have more than 1 GB, otherwise, use all!
+ // TODO(benh): Have better default scheme (e.g., % of mem not
+ // greater than 1 GB?)
+ if (mem.get() > 1024) {
+ mem = Try<long>::some(mem.get() - 1024);
+ }
+ }
+
+ Try<string> defaults =
+ strings::format("cpus:%d;mem:%d", cpus.get(), mem.get());
+
+ CHECK(defaults.isSome());
+
resources =
- Resources::parse(conf.get<string>("resources", "cpus:1;mem:1024"));
+ Resources::parse(conf.get<string>("resources", defaults.get()));
attributes =
Attributes::parse(conf.get<string>("attributes", ""));
Modified: incubator/mesos/trunk/third_party/libprocess/src/process.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/src/process.cpp?rev=1306509&r1=1306508&r2=1306509&view=diff
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/src/process.cpp (original)
+++ incubator/mesos/trunk/third_party/libprocess/src/process.cpp Wed Mar 28 17:50:47 2012
@@ -414,8 +414,6 @@ static bool pending_timers = false;
// Flag to indicate whether or to update the timer on async interrupt.
static bool update_timer = false;
-const int NUMBER_OF_PROCESSING_THREADS = 4; // TODO(benh): Do 2x cores.
-
// Thread local process pointer magic (constructed in
// 'initialize'). We need the extra level of indirection from
@@ -1201,7 +1199,9 @@ void initialize(const string& delegate,
_process_ = new ThreadLocal<ProcessBase>(key);
// Setup processing threads.
- for (int i = 0; i < NUMBER_OF_PROCESSING_THREADS; i++) {
+ long cpus = sysconf(_SC_NPROCESSORS_ONLN);
+
+ for (int i = 0; i < cpus; i++) {
pthread_t thread; // For now, not saving handles on our threads.
if (pthread_create(&thread, NULL, schedule, NULL) != 0) {
LOG(FATAL) << "Failed to initialize, pthread_create";
@@ -1343,7 +1343,8 @@ void initialize(const string& delegate,
PLOG(FATAL) << "Failed to initialize, inet_ntop";
}
- VLOG(1) << "libprocess is initialized on " << temp << ":" << __port__;
+ VLOG(1) << "libprocess is initialized on " << temp << ":" << __port__
+ << " for " << cpus << " cpus";
}