You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by nn...@apache.org on 2014/11/24 21:46:15 UTC
mesos git commit: Extracted and genralized WhitelistWatcher.
Repository: mesos
Updated Branches:
refs/heads/master 560b4860f -> 4ee8740fe
Extracted and genralized WhitelistWatcher.
WhitelistWatcher can be used for tracking changes to all kind of
whitelists. In order to reuse the functionality, extract the code from
master sources.
Review: https://reviews.apache.org/r/28230
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4ee8740f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4ee8740f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4ee8740f
Branch: refs/heads/master
Commit: 4ee8740fe17f32470718df5676fee3b2d9e5be0c
Parents: 560b486
Author: Alexander Rukletsov <al...@mesosphere.io>
Authored: Mon Nov 24 11:53:16 2014 -0800
Committer: Niklas Q. Nielsen <ni...@mesosphere.io>
Committed: Mon Nov 24 11:53:18 2014 -0800
----------------------------------------------------------------------
src/Makefile.am | 2 +
src/master/hierarchical_allocator_process.hpp | 8 +-
src/master/master.cpp | 75 +++------------
src/master/master.hpp | 2 +-
src/watcher/whitelist_watcher.cpp | 103 +++++++++++++++++++++
src/watcher/whitelist_watcher.hpp | 60 ++++++++++++
6 files changed, 184 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4ee8740f/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 2448db8..86161fe 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -303,6 +303,7 @@ libmesos_no_3rdparty_la_SOURCES = \
slave/containerizer/mesos/launch.cpp \
slave/status_update_manager.cpp \
usage/usage.cpp \
+ watcher/whitelist_watcher.cpp \
zookeeper/contender.cpp \
zookeeper/detector.cpp \
zookeeper/zookeeper.cpp \
@@ -479,6 +480,7 @@ libmesos_no_3rdparty_la_SOURCES += \
tests/zookeeper.hpp \
tests/zookeeper_test_server.hpp \
usage/usage.hpp \
+ watcher/whitelist_watcher.hpp \
zookeeper/authentication.hpp \
zookeeper/contender.hpp \
zookeeper/detector.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/4ee8740f/src/master/hierarchical_allocator_process.hpp
----------------------------------------------------------------------
diff --git a/src/master/hierarchical_allocator_process.hpp b/src/master/hierarchical_allocator_process.hpp
index e631b60..fbaa23f 100644
--- a/src/master/hierarchical_allocator_process.hpp
+++ b/src/master/hierarchical_allocator_process.hpp
@@ -506,11 +506,17 @@ HierarchicalAllocatorProcess<RoleSorter, FrameworkSorter>::updateWhitelist(
whitelist = _whitelist;
if (whitelist.isSome()) {
- LOG(INFO) << "Updated slave white list: " << stringify(whitelist.get());
+ LOG(INFO) << "Updated slave whitelist: " << stringify(whitelist.get());
+
+ if (whitelist.get().empty()) {
+ LOG(WARNING) << "Whitelist is empty, no offers will be made!";
+ }
foreachkey (const SlaveID& slaveId, slaves) {
slaves[slaveId].whitelisted = isWhitelisted(slaveId);
}
+ } else {
+ LOG(INFO) << "Advertising offers for all slaves";
}
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/4ee8740f/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 5957db6..8fcda4b 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -47,7 +47,6 @@
#include <stout/nothing.hpp>
#include <stout/numify.hpp>
#include <stout/option.hpp>
-#include <stout/os.hpp>
#include <stout/path.hpp>
#include <stout/stringify.hpp>
#include <stout/utils.hpp>
@@ -75,6 +74,8 @@
#include "module/authenticator.hpp"
#include "module/manager.hpp"
+#include "watcher/whitelist_watcher.hpp"
+
using std::list;
using std::string;
using std::vector;
@@ -105,67 +106,6 @@ namespace master {
using allocator::Allocator;
-class WhitelistWatcher : public Process<WhitelistWatcher> {
-public:
- WhitelistWatcher(const string& _path, Allocator* _allocator)
- : ProcessBase(process::ID::generate("whitelist")),
- path(_path),
- allocator(_allocator) {}
-
-protected:
- virtual void initialize()
- {
- watch();
- }
-
- void watch()
- {
- // Get the list of white listed slaves.
- Option<hashset<string>> whitelist;
- if (path == "*") { // Accept all slaves.
- VLOG(1) << "No whitelist given. Advertising offers for all slaves";
- } else {
- // Read from local file.
- // TODO(vinod): Add support for reading from ZooKeeper.
- // TODO(vinod): Ensure this read is atomic w.r.t external
- // writes/updates to this file.
- Try<string> read = os::read(
- strings::remove(path, "file://", strings::PREFIX));
- if (read.isError()) {
- LOG(ERROR) << "Error reading whitelist file: " << read.error() << ". "
- << "Retrying";
- whitelist = lastWhitelist;
- } else if (read.get().empty()) {
- LOG(WARNING) << "Empty whitelist file " << path << ". "
- << "No offers will be made!";
- whitelist = hashset<string>();
- } else {
- hashset<string> hostnames;
- vector<string> lines = strings::tokenize(read.get(), "\n");
- foreach (const string& hostname, lines) {
- hostnames.insert(hostname);
- }
- whitelist = hostnames;
- }
- }
-
- // Send the whitelist to allocator, if necessary.
- if (whitelist != lastWhitelist) {
- allocator->updateWhitelist(whitelist);
- }
-
- // Check again.
- lastWhitelist = whitelist;
- delay(WHITELIST_WATCH_INTERVAL, self(), &WhitelistWatcher::watch);
- }
-
-private:
- const string path;
- Allocator* allocator;
- Option<hashset<string>> lastWhitelist;
-};
-
-
class SlaveObserver : public Process<SlaveObserver>
{
public:
@@ -518,8 +458,15 @@ void Master::initialize()
// Initialize the allocator.
allocator->initialize(flags, self(), roleInfos);
- // Parse the white list.
- whitelistWatcher = new WhitelistWatcher(flags.whitelist, allocator);
+ // Parse the whitelist. Passing allocator::updateWhitelist()
+ // callback is safe because we shut down the whitelistWatcher in
+ // Master::finalize(), while allocator lifetime is greater than
+ // masters. Therefore there is no risk of calling into an allocator
+ // that has been cleaned up.
+ whitelistWatcher = new WhitelistWatcher(
+ flags.whitelist,
+ WHITELIST_WATCH_INTERVAL,
+ lambda::bind(&Allocator::updateWhitelist, allocator, lambda::_1));
spawn(whitelistWatcher);
nextFrameworkId = 0;
http://git-wip-us.apache.org/repos/asf/mesos/blob/4ee8740f/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 6eabb07..79b9ba7 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -71,6 +71,7 @@ class Slaves;
class Authenticator;
class Authorizer;
+class WhitelistWatcher;
namespace master {
@@ -81,7 +82,6 @@ class Allocator;
class Repairer;
class SlaveObserver;
-class WhitelistWatcher;
struct Framework;
struct OfferVisitor;
http://git-wip-us.apache.org/repos/asf/mesos/blob/4ee8740f/src/watcher/whitelist_watcher.cpp
----------------------------------------------------------------------
diff --git a/src/watcher/whitelist_watcher.cpp b/src/watcher/whitelist_watcher.cpp
new file mode 100644
index 0000000..113ff15
--- /dev/null
+++ b/src/watcher/whitelist_watcher.cpp
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string>
+#include <vector>
+
+#include <glog/logging.h>
+
+#include <process/delay.hpp>
+#include <process/id.hpp>
+
+#include <stout/foreach.hpp>
+#include <stout/os.hpp>
+#include <stout/strings.hpp>
+#include <stout/try.hpp>
+
+#include "watcher/whitelist_watcher.hpp"
+
+namespace mesos {
+namespace internal {
+
+using std::string;
+using std::vector;
+
+using process::Process;
+
+using lambda::function;
+
+
+WhitelistWatcher::WhitelistWatcher(
+ const string& path,
+ const Duration& watchInterval,
+ const function<
+ void(const Option<hashset<string>>& whitelist)>& subscriber)
+ : ProcessBase(process::ID::generate("whitelist")),
+ path(path),
+ watchInterval(watchInterval),
+ subscriber(subscriber) {}
+
+
+void WhitelistWatcher::initialize()
+{
+ watch();
+}
+
+
+void WhitelistWatcher::watch()
+{
+ // Get the list of white listed nodes.
+ Option<hashset<string>> whitelist;
+ if (path == "*") { // Accept all nodes.
+ VLOG(1) << "No whitelist given";
+ } else {
+ // Read from local file.
+ // TODO(vinod): Add support for reading from ZooKeeper.
+ // TODO(vinod): Ensure this read is atomic w.r.t external
+ // writes/updates to this file.
+ Try<string> read = os::read(
+ strings::remove(path, "file://", strings::PREFIX));
+ if (read.isError()) {
+ LOG(ERROR) << "Error reading whitelist file: " << read.error() << ". "
+ << "Retrying";
+ whitelist = lastWhitelist;
+ } else if (read.get().empty()) {
+ VLOG(1) << "Empty whitelist file " << path;
+ whitelist = hashset<string>();
+ } else {
+ hashset<string> hostnames;
+ vector<string> lines = strings::tokenize(read.get(), "\n");
+ foreach (const string& hostname, lines) {
+ hostnames.insert(hostname);
+ }
+ whitelist = hostnames;
+ }
+ }
+
+ // Send the whitelist to subscriber, if necessary.
+ if (whitelist != lastWhitelist) {
+ subscriber(whitelist);
+ }
+
+ // Schedule the next check.
+ lastWhitelist = whitelist;
+ delay(watchInterval, self(), &WhitelistWatcher::watch);
+}
+
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/4ee8740f/src/watcher/whitelist_watcher.hpp
----------------------------------------------------------------------
diff --git a/src/watcher/whitelist_watcher.hpp b/src/watcher/whitelist_watcher.hpp
new file mode 100644
index 0000000..5838854
--- /dev/null
+++ b/src/watcher/whitelist_watcher.hpp
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __WHITELIST_WATCHER_HPP__
+#define __WHITELIST_WATCHER_HPP__
+
+#include <string>
+
+#include <process/process.hpp>
+
+#include <stout/duration.hpp>
+#include <stout/hashset.hpp>
+#include <stout/lambda.hpp>
+#include <stout/option.hpp>
+
+namespace mesos {
+namespace internal {
+
+class WhitelistWatcher : public process::Process<WhitelistWatcher>
+{
+public:
+ // NOTE: The caller should ensure a callback exists throughout
+ // WhitelistWatcher's lifetime.
+ WhitelistWatcher(
+ const std::string& path,
+ const Duration& watchInterval,
+ const lambda::function<
+ void(const Option<hashset<std::string>>& whitelist)>& subscriber);
+
+protected:
+ virtual void initialize();
+ void watch();
+
+private:
+ const std::string path;
+ const Duration watchInterval;
+ lambda::function<void(const Option<hashset<std::string>>& whitelist)>
+ subscriber;
+ Option<hashset<std::string>> lastWhitelist;
+};
+
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __WHITELIST_WATCHER_HPP__