You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/03/16 01:28:51 UTC
mesos git commit: Support the full CNI DNS specification.
Repository: mesos
Updated Branches:
refs/heads/master f08754d44 -> 83d1f9c1b
Support the full CNI DNS specification.
Add support for the full set of DNS resolver configuration items that a
CNI IPAM plugin can specify. This implements updating the container's
resolv.conf with the 'domain', 'search', and 'options' keywords.
Review: https://reviews.apache.org/r/55790/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/83d1f9c1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/83d1f9c1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/83d1f9c1
Branch: refs/heads/master
Commit: 83d1f9c1b842b59bf03c06d9392d4fb73b805622
Parents: f08754d
Author: James Peach <jp...@apache.org>
Authored: Wed Mar 15 18:04:29 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Mar 15 18:28:29 2017 -0700
----------------------------------------------------------------------
.../mesos/isolators/network/cni/cni.cpp | 37 +-
.../mesos/isolators/network/cni/spec.cpp | 34 ++
.../mesos/isolators/network/cni/spec.hpp | 5 +
src/tests/containerizer/cni_isolator_tests.cpp | 360 +++++++++++++++++--
4 files changed, 387 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/83d1f9c1/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
index 84dc157..ea12aa0 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
@@ -18,6 +18,8 @@
#include <list>
#include <set>
+#include <mesos/type_utils.hpp>
+
#include <process/io.hpp>
#include <process/pid.hpp>
#include <process/subprocess.hpp>
@@ -970,39 +972,34 @@ Future<Nothing> NetworkCniIsolatorProcess::_isolate(
hostsPath + "': " + write.error());
}
- // Update 'resolv.conf' with nameservers learned from IPAM. In case
- // IPAM has not specified a DNS then we set the container
- // 'resolv.conf' to be the same as the host 'resolv.conf'
- // ('/etc/resolv.conf').
- stringstream resolv;
- foreachvalue (const ContainerNetwork& network,
- info->containerNetworks) {
- if (network.cniNetworkInfo.isNone() || !network.cniNetworkInfo->has_dns()) {
- continue;
- }
+ cni::spec::DNS dns;
- foreach (const string& nameserver,
- network.cniNetworkInfo->dns().nameservers()) {
- resolv << "nameserver " << nameserver << endl;
+ // Collect all the DNS resolver specifications from the networks'
+ // IPAM plugins. Ordering is preserved and for single-value fields,
+ // the last network will win.
+ foreachvalue (const ContainerNetwork& network, info->containerNetworks) {
+ if (network.cniNetworkInfo.isSome() && network.cniNetworkInfo->has_dns()) {
+ dns.MergeFrom(network.cniNetworkInfo->dns());
}
}
- // If `resolv` does not have any nameserver set `resolvPath` to
- // '/etc/resolv.conf'.
- if (resolv.str().size() == 0) {
+ // If IPAM has not specified any DNS servers, then we set
+ // the container 'resolv.conf' to be the same as the host
+ // 'resolv.conf' ('/etc/resolv.conf').
+ if (dns.nameservers().empty()) {
if (!os::exists("/etc/resolv.conf")){
- return Failure("Cannot find host /etc/resolv.conf");
+ return Failure("Cannot find host's /etc/resolv.conf");
}
resolvPath = "/etc/resolv.conf";
LOG(INFO) << "Unable to find DNS nameservers for container "
- << containerId << ". Using host '/etc/resolv.conf'";
+ << containerId << ", using host '/etc/resolv.conf'";
} else {
LOG(INFO) << "DNS nameservers for container " << containerId
- << " are:\n" << resolv.str();
+ << " are " << dns.nameservers();
- write = os::write(resolvPath, resolv.str());
+ write = os::write(resolvPath, cni::spec::formatResolverConfig(dns));
if (write.isError()) {
return Failure(
"Failed to write 'resolv.conf' file at '" +
http://git-wip-us.apache.org/repos/asf/mesos/blob/83d1f9c1/src/slave/containerizer/mesos/isolators/network/cni/spec.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/spec.cpp b/src/slave/containerizer/mesos/isolators/network/cni/spec.cpp
index ac48159..ed5523e 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/spec.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/spec.cpp
@@ -59,6 +59,40 @@ Try<NetworkInfo> parseNetworkInfo(const string& s)
}
+string formatResolverConfig(const DNS& dns)
+{
+ std::stringstream resolv;
+
+ if (dns.has_domain()) {
+ resolv << "domain " << dns.domain() << std::endl;
+ }
+
+ if (!dns.search().empty()) {
+ resolv << "search";
+ foreach (const string& domain, dns.search()) {
+ resolv << " " << domain;
+ }
+ resolv << std::endl;
+ }
+
+ if (!dns.options().empty()) {
+ resolv << "options";
+ foreach (const string& opt, dns.options()) {
+ resolv << " " << opt;
+ }
+ resolv << std::endl;
+ }
+
+ if (!dns.nameservers().empty()) {
+ foreach (const string& nameserver, dns.nameservers()) {
+ resolv << "nameserver " << nameserver << std::endl;
+ }
+ }
+
+ return resolv.str();
+}
+
+
string error(const string& msg, uint32_t code)
{
spec::Error error;
http://git-wip-us.apache.org/repos/asf/mesos/blob/83d1f9c1/src/slave/containerizer/mesos/isolators/network/cni/spec.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/spec.hpp b/src/slave/containerizer/mesos/isolators/network/cni/spec.hpp
index ccd511e..faa37ba 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/spec.hpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/spec.hpp
@@ -47,6 +47,11 @@ Try<NetworkConfig> parseNetworkConfig(const std::string& s);
Try<NetworkInfo> parseNetworkInfo(const std::string& s);
+// Takes a DNS specification and returns a string containing
+// the equivalent resolv.conf(5) format.
+std::string formatResolverConfig(const DNS& dns);
+
+
// Creates a `spec::Error` object, with the `msg` and `code`
// specified. Returns a JSON string representation of the
// `spec::Error` object. See details in:
http://git-wip-us.apache.org/repos/asf/mesos/blob/83d1f9c1/src/tests/containerizer/cni_isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp b/src/tests/containerizer/cni_isolator_tests.cpp
index b770945..393c3e5 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -23,12 +23,14 @@
#include "slave/containerizer/fetcher.hpp"
#include "slave/containerizer/mesos/containerizer.hpp"
#include "slave/containerizer/mesos/isolators/network/cni/paths.hpp"
+#include "slave/containerizer/mesos/isolators/network/cni/spec.hpp"
#include "tests/mesos.hpp"
namespace master = mesos::internal::master;
namespace paths = mesos::internal::slave::cni::paths;
namespace slave = mesos::internal::slave;
+namespace spec = mesos::internal::slave::cni::spec;
using master::Master;
@@ -53,6 +55,40 @@ namespace mesos {
namespace internal {
namespace tests {
+TEST(CniSpecTest, GenerateResolverConfig)
+{
+ spec::DNS dns;
+
+ EXPECT_EQ("", spec::formatResolverConfig(dns));
+
+ dns.Clear();
+ dns.set_domain("m.a.org");
+ EXPECT_EQ("domain m.a.org\n", spec::formatResolverConfig(dns));
+
+ dns.Clear();
+ dns.add_nameservers("1.1.1.1");
+ dns.add_nameservers("2.2.2.2");
+ EXPECT_EQ(
+ "nameserver 1.1.1.1\n"
+ "nameserver 2.2.2.2\n",
+ spec::formatResolverConfig(dns));
+
+ dns.Clear();
+ dns.add_search("a.m.a.org");
+ dns.add_search("b.m.a.org");
+ EXPECT_EQ(
+ "search a.m.a.org b.m.a.org\n",
+ spec::formatResolverConfig(dns));
+
+ dns.Clear();
+ dns.add_options("debug");
+ dns.add_options("ndots:2");
+ EXPECT_EQ(
+ "options debug ndots:2\n",
+ spec::formatResolverConfig(dns));
+}
+
+
class CniIsolatorTest : public MesosTest
{
public:
@@ -60,20 +96,10 @@ public:
{
MesosTest::SetUp();
- Try<set<string>> links = net::links();
- ASSERT_SOME(links);
-
- Result<net::IPNetwork> hostIPNetwork = None();
- foreach (const string& link, links.get()) {
- hostIPNetwork = net::IPNetwork::fromLinkDevice(link, AF_INET);
- EXPECT_FALSE(hostIPNetwork.isError());
-
- if (hostIPNetwork.isSome() &&
- (hostIPNetwork.get() != net::IPNetwork::LOOPBACK_V4())) {
- break;
- }
- }
+ cniPluginDir = path::join(sandbox.get(), "plugins");
+ cniConfigDir = path::join(sandbox.get(), "configs");
+ Result<net::IPNetwork> hostIPNetwork = findHostNetwork();
ASSERT_SOME(hostIPNetwork);
// Get the first external name server.
@@ -96,13 +122,8 @@ public:
ASSERT_SOME(nameServer);
- // Generate the mock CNI plugin.
- cniPluginDir = path::join(sandbox.get(), "plugins");
- ASSERT_SOME(os::mkdir(cniPluginDir));
-
- // TODO(jieyu): Verify that Mesos metadata is set properly.
- Try<Nothing> write = os::write(
- path::join(cniPluginDir, "mockPlugin"),
+ // Set up the default CNI plugin.
+ Try<Nothing> result = setupMockPlugin(
strings::format(R"~(
#!/bin/sh
echo "{"
@@ -118,18 +139,12 @@ public:
hostIPNetwork->prefix(),
nameServer.get()).get());
- ASSERT_SOME(write);
-
- // Make sure the plugin has execution permission.
- ASSERT_SOME(os::chmod(
- path::join(cniPluginDir, "mockPlugin"),
- S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH));
+ ASSERT_SOME(result);
// Generate the mock CNI config.
- cniConfigDir = path::join(sandbox.get(), "configs");
ASSERT_SOME(os::mkdir(cniConfigDir));
- write = os::write(
+ result = os::write(
path::join(cniConfigDir, "mockConfig"),
R"~(
{
@@ -137,7 +152,59 @@ public:
"type": "mockPlugin"
})~");
- ASSERT_SOME(write);
+ ASSERT_SOME(result);
+ }
+
+ Try<net::IPNetwork> findHostNetwork()
+ {
+ Try<set<string>> links = net::links();
+
+ if (links.isError()) {
+ return Error("Failed to enumerate network interfaces: " + links.error());
+ }
+
+ Result<net::IPNetwork> hostIPNetwork = None();
+ foreach (const string& link, links.get()) {
+ hostIPNetwork = net::IPNetwork::fromLinkDevice(link, AF_INET);
+
+ if (hostIPNetwork.isError()) {
+ return Error("Failed to get address of " + link + ": " + links.error());
+ }
+
+ if (hostIPNetwork.isSome() &&
+ (hostIPNetwork.get() != net::IPNetwork::LOOPBACK_V4())) {
+ return hostIPNetwork.get();
+ }
+ }
+
+ return Error("Failed to find host network address");
+ }
+
+ // Generate the mock CNI plugin based on the given script.
+ Try<Nothing> setupMockPlugin(const string& pluginScript)
+ {
+ Try<Nothing> mkdir = os::mkdir(cniPluginDir);
+ if (mkdir.isError()) {
+ return Error("Failed to mkdir '" + cniPluginDir + "': " + mkdir.error());
+ }
+
+ string mockPlugin = path::join(cniPluginDir, "mockPlugin");
+
+ Try<Nothing> write = os::write(mockPlugin, pluginScript);
+ if (write.isError()) {
+ return Error("Failed to write '" + mockPlugin + "': " + write.error());
+ }
+
+ // Make sure the plugin has execution permission.
+ Try<Nothing> chmod = os::chmod(
+ mockPlugin,
+ S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+
+ if (chmod.isError()) {
+ return Error("Failed to chmod '" + mockPlugin + "': " + chmod.error());
+ }
+
+ return Nothing();
}
string cniPluginDir;
@@ -932,6 +999,241 @@ TEST_F(CniIsolatorTest, ROOT_OverrideHostname)
driver.join();
}
+
+// This test checks that a CNI DNS configuration ends up generating
+// the right settings in /etc/resolv.conf.
+TEST_F(CniIsolatorTest, ROOT_VerifyResolverConfig)
+{
+ Try<net::IPNetwork> hostIPNetwork = findHostNetwork();
+ ASSERT_SOME(hostIPNetwork);
+
+ Try<string> mockPlugin = strings::format(
+ R"~(
+ #!/bin/sh
+ echo '{'
+ echo ' "ip4": {'
+ echo ' "ip": "%s/%d"'
+ echo ' },'
+ echo ' "dns": {'
+ echo ' "nameservers": ['
+ echo ' "1.1.1.1",'
+ echo ' "1.1.1.2"'
+ echo ' ],'
+ echo ' "domain": "mesos.apache.org",'
+ echo ' "search": ['
+ echo ' "a.mesos.apache.org",'
+ echo ' "a.mesos.apache.org"'
+ echo ' ],'
+ echo ' "options":['
+ echo ' "option1",'
+ echo ' "option2"'
+ echo ' ]'
+ echo ' }'
+ echo '}'
+ )~",
+ hostIPNetwork.get().address(),
+ hostIPNetwork.get().prefix());
+
+ ASSERT_SOME(mockPlugin);
+
+ ASSERT_SOME(setupMockPlugin(mockPlugin.get()));
+
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.isolation = "network/cni";
+
+ flags.network_cni_plugins_dir = cniPluginDir;
+ flags.network_cni_config_dir = cniConfigDir;
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+
+ MesosSchedulerDriver driver(
+ &sched,
+ DEFAULT_FRAMEWORK_INFO,
+ master.get()->pid,
+ DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(_, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers->size());
+
+ const Offer& offer = offers.get()[0];
+
+ // Verify that /etc/resolv.conf was generated the way we expect.
+ // This is sensitive to changes in 'formatResolverConfig()'.
+ const string command =
+ "#! /bin/sh\n"
+ "set -x\n"
+ "cat > expected <<EOF\n"
+ "domain mesos.apache.org\n"
+ "search a.mesos.apache.org a.mesos.apache.org\n"
+ "options option1 option2\n"
+ "nameserver 1.1.1.1\n"
+ "nameserver 1.1.1.2\n"
+ "EOF\n"
+ "cat /etc/resolv.conf\n"
+ "exec diff -c /etc/resolv.conf expected\n";
+
+ TaskInfo task = createTask(offer, command);
+
+ ContainerInfo* container = task.mutable_container();
+ container->set_type(ContainerInfo::MESOS);
+
+ // Make sure the container joins the mock CNI network.
+ container->add_network_infos()->set_name("__MESOS_TEST__");
+
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusFinished;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusFinished));
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY(statusRunning);
+ EXPECT_EQ(task.task_id(), statusRunning->task_id());
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+
+ AWAIT_READY(statusFinished);
+ EXPECT_EQ(task.task_id(), statusFinished->task_id());
+ EXPECT_EQ(TASK_FINISHED, statusFinished->state());
+
+ driver.stop();
+ driver.join();
+}
+
+
+// This test verifies that we generate a /etc/resolv.conf
+// that glibc accepts by using it to ping a host.
+TEST_F(CniIsolatorTest, ROOT_INTERNET_VerifyResolverConfig)
+{
+ Try<net::IPNetwork> hostIPNetwork = findHostNetwork();
+ ASSERT_SOME(hostIPNetwork);
+
+ // Note: We set a dummy nameserver IP address followed by the
+ // Google anycast address. We also set the resolver timeout
+ // to 1sec so that ping doesn't time out waiting for DNS. Even
+ // so, this test is probably susceptible to network flakiness,
+ // especially in cloud providers.
+ Try<string> mockPlugin = strings::format(
+ R"~(
+ #!/bin/sh
+ echo '{'
+ echo ' "ip4": {'
+ echo ' "ip": "%s/%d"'
+ echo ' },'
+ echo ' "dns": {'
+ echo ' "nameservers": ['
+ echo ' "127.0.0.1",'
+ echo ' "8.8.8.8"'
+ echo ' ],'
+ echo ' "domain": "mesos.apache.org",'
+ echo ' "search": ['
+ echo ' "a.mesos.apache.org",'
+ echo ' "a.mesos.apache.org"'
+ echo ' ],'
+ echo ' "options":['
+ echo ' "timeout:1"'
+ echo ' ]'
+ echo ' }'
+ echo '}'
+ )~",
+ hostIPNetwork.get().address(),
+ hostIPNetwork.get().prefix());
+
+ ASSERT_SOME(mockPlugin);
+
+ ASSERT_SOME(setupMockPlugin(mockPlugin.get()));
+
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.isolation = "network/cni";
+
+ flags.network_cni_plugins_dir = cniPluginDir;
+ flags.network_cni_config_dir = cniConfigDir;
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+
+ MesosSchedulerDriver driver(
+ &sched,
+ DEFAULT_FRAMEWORK_INFO,
+ master.get()->pid,
+ DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(_, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers->size());
+
+ const Offer& offer = offers.get()[0];
+
+ // In the CNI config above, we configured the Google
+ // DNS servers as our second resolver. Verify that we
+ // generated a resolv.conf that libc accepts by using
+ // it to resolve www.google.com.
+ const string command = R"~(
+ #! /bin/sh
+ set -ex
+ exec ping -W 1 -c 2 www.google.com
+ )~";
+
+ TaskInfo task = createTask(offer, command);
+
+ ContainerInfo* container = task.mutable_container();
+ container->set_type(ContainerInfo::MESOS);
+
+ // Make sure the container joins the mock CNI network.
+ container->add_network_infos()->set_name("__MESOS_TEST__");
+
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusFinished;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusFinished));
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY(statusRunning);
+ EXPECT_EQ(task.task_id(), statusRunning->task_id());
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+
+ AWAIT_READY(statusFinished);
+ EXPECT_EQ(task.task_id(), statusFinished->task_id());
+ EXPECT_EQ(TASK_FINISHED, statusFinished->state());
+
+ driver.stop();
+ driver.join();
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {