You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/03/19 16:52:43 UTC
[4/6] mesos git commit: Added test for CNI port-mapper plugin.
Added test for CNI port-mapper plugin.
Review: https://reviews.apache.org/r/53264/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2467fa49
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2467fa49
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2467fa49
Branch: refs/heads/master
Commit: 2467fa4977f45f0350900b28b61f5dfb4ab2ae70
Parents: e7f756d
Author: Avinash sridharan <av...@mesosphere.io>
Authored: Sun Mar 19 09:09:17 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Sun Mar 19 09:09:17 2017 -0700
----------------------------------------------------------------------
src/tests/containerizer/cni_isolator_tests.cpp | 273 +++++++++++++++++---
1 file changed, 242 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/2467fa49/src/tests/containerizer/cni_isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/cni_isolator_tests.cpp b/src/tests/containerizer/cni_isolator_tests.cpp
index 393c3e5..d30f251 100644
--- a/src/tests/containerizer/cni_isolator_tests.cpp
+++ b/src/tests/containerizer/cni_isolator_tests.cpp
@@ -18,8 +18,6 @@
#include <process/clock.hpp>
-#include <stout/ip.hpp>
-
#include "slave/containerizer/fetcher.hpp"
#include "slave/containerizer/mesos/containerizer.hpp"
#include "slave/containerizer/mesos/isolators/network/cni/paths.hpp"
@@ -55,6 +53,11 @@ namespace mesos {
namespace internal {
namespace tests {
+constexpr char MESOS_CNI_PORT_MAPPER_NETWORK[] = "__MESOS_TEST__portMapper";
+constexpr char MESOS_MOCK_CNI_CONFIG[] = "mockConfig";
+constexpr char MESOS_TEST_PORT_MAPPER_CHAIN[] = "MESOS-TEST-PORT-MAPPER-CHAIN";
+
+
TEST(CniSpecTest, GenerateResolverConfig)
{
spec::DNS dns;
@@ -99,7 +102,8 @@ public:
cniPluginDir = path::join(sandbox.get(), "plugins");
cniConfigDir = path::join(sandbox.get(), "configs");
- Result<net::IPNetwork> hostIPNetwork = findHostNetwork();
+ Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP();
+
ASSERT_SOME(hostIPNetwork);
// Get the first external name server.
@@ -145,7 +149,7 @@ public:
ASSERT_SOME(os::mkdir(cniConfigDir));
result = os::write(
- path::join(cniConfigDir, "mockConfig"),
+ path::join(cniConfigDir, MESOS_MOCK_CNI_CONFIG),
R"~(
{
"name": "__MESOS_TEST__",
@@ -155,31 +159,6 @@ public:
ASSERT_SOME(result);
}
- Try<net::IPNetwork> findHostNetwork()
- {
- Try<set<string>> links = net::links();
-
- if (links.isError()) {
- return Error("Failed to enumerate network interfaces: " + links.error());
- }
-
- Result<net::IPNetwork> hostIPNetwork = None();
- foreach (const string& link, links.get()) {
- hostIPNetwork = net::IPNetwork::fromLinkDevice(link, AF_INET);
-
- if (hostIPNetwork.isError()) {
- return Error("Failed to get address of " + link + ": " + links.error());
- }
-
- if (hostIPNetwork.isSome() &&
- (hostIPNetwork.get() != net::IPNetwork::LOOPBACK_V4())) {
- return hostIPNetwork.get();
- }
- }
-
- return Error("Failed to find host network address");
- }
-
// Generate the mock CNI plugin based on the given script.
Try<Nothing> setupMockPlugin(const string& pluginScript)
{
@@ -212,6 +191,78 @@ public:
};
+class CniIsolatorPortMapperTest : public CniIsolatorTest
+{
+public:
+ virtual void SetUp()
+ {
+ CniIsolatorTest::SetUp();
+
+ Try<string> mockConfig = os::read(
+ path::join(cniConfigDir, MESOS_MOCK_CNI_CONFIG));
+
+ ASSERT_SOME(mockConfig);
+
+ // Create a CNI configuration to be used with the port-mapper plugin.
+ Try<string> portMapperConfig = strings::format(R"~(
+ {
+ "name": "%s",
+ "type": "mesos-cni-port-mapper",
+ "chain": "%s",
+ "delegate": %s
+ }
+ )~",
+ MESOS_CNI_PORT_MAPPER_NETWORK,
+ MESOS_TEST_PORT_MAPPER_CHAIN,
+ mockConfig.get());
+
+ ASSERT_SOME(portMapperConfig);
+
+ Try<Nothing> write = os::write(
+ path::join(cniConfigDir, "mockPortMapperConfig"),
+ portMapperConfig.get());
+
+ ASSERT_SOME(write);
+ }
+
+ virtual void TearDown()
+ {
+ // This is a best effort cleanup of the
+ // `MESOS_TEST_PORT_MAPPER_CHAIN`. We shouldn't fail and bail on
+ // rest of the `TearDown` if we are not able to clean up the
+ // chain.
+ string script = strings::format(
+ R"~(
+ #!/bin/sh
+ set -x
+
+ iptables -w -t nat --list %s
+
+ if [ $? -eq 0 ]; then
+ iptables -w -t nat -D OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j %s
+ iptables -w -t nat -D PREROUTING -m addrtype --dst-type LOCAL -j %s
+ iptables -w -t nat -F %s
+ iptables -w -t nat -X %s
+ fi)~",
+ stringify(MESOS_TEST_PORT_MAPPER_CHAIN),
+ stringify(MESOS_TEST_PORT_MAPPER_CHAIN),
+ stringify(MESOS_TEST_PORT_MAPPER_CHAIN),
+ stringify(MESOS_TEST_PORT_MAPPER_CHAIN),
+ stringify(MESOS_TEST_PORT_MAPPER_CHAIN),
+ stringify(MESOS_TEST_PORT_MAPPER_CHAIN)).get();
+
+ Try<string> result = os::shell(script);
+ if (result.isError()) {
+ LOG(ERROR) << "Unable to cleanup chain "
+ << stringify(MESOS_TEST_PORT_MAPPER_CHAIN)
+ << ": " << result.error();
+ }
+
+ CniIsolatorTest::TearDown();
+ }
+};
+
+
// This test verifies that a container is created and joins a mock CNI
// network, and a command task is executed in the container successfully.
TEST_F(CniIsolatorTest, ROOT_INTERNET_CURL_LaunchCommandTask)
@@ -1000,11 +1051,171 @@ TEST_F(CniIsolatorTest, ROOT_OverrideHostname)
}
+TEST_F(CniIsolatorPortMapperTest, ROOT_INETERNET_CURL_PortMapper)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.isolation = "docker/runtime,filesystem/linux";
+ flags.image_providers = "docker";
+ flags.docker_store_dir = path::join(sandbox.get(), "store");
+
+ // Augment the CNI plugins search path so that the `network/cni`
+ // isolator can find the port-mapper CNI plugin.
+ flags.network_cni_plugins_dir = cniPluginDir + ":" + getLauncherDir();
+
+ flags.network_cni_config_dir = cniConfigDir;
+
+ // Need to increase the registration timeout to give time for
+ // downloading and provisioning the "nginx:alpine" image.
+ flags.executor_registration_timeout = Minutes(5);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers->size());
+
+ const Offer& offer = offers.get()[0];
+
+ Resources resources(offers.get()[0].resources());
+
+ // Make sure we have a `ports` resource.
+ ASSERT_SOME(resources.ports());
+ ASSERT_LE(1u, resources.ports()->range().size());
+
+ // Select a random port from the offer.
+ std::srand(std::time(0));
+
+ Value::Range ports = resources.ports()->range(0);
+
+ uint16_t hostPort =
+ ports.begin() + std::rand() % (ports.end() - ports.begin() + 1);
+
+ CommandInfo command;
+ command.set_shell(false);
+
+ TaskInfo task = createTask(
+ offer.slave_id(),
+ Resources::parse(
+ "cpus:1;mem:128;"
+ "ports:[" + stringify(hostPort) + "," + stringify(hostPort) + "]")
+ .get(),
+ command);
+
+ ContainerInfo container = createContainerInfo("nginx:alpine");
+
+ // Make sure the container joins the test CNI port-mapper network.
+ NetworkInfo* networkInfo = container.add_network_infos();
+ networkInfo->set_name(MESOS_CNI_PORT_MAPPER_NETWORK);
+
+ NetworkInfo::PortMapping* portMapping = networkInfo->add_port_mappings();
+ portMapping->set_container_port(80);
+ portMapping->set_host_port(hostPort);
+
+ // Set the container for the task.
+ task.mutable_container()->CopyFrom(container);
+
+ Future<TaskStatus> statusRunning;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning));
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY_FOR(statusRunning, Seconds(300));
+ EXPECT_EQ(task.task_id(), statusRunning->task_id());
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+ ASSERT_TRUE(statusRunning->has_container_status());
+
+ ContainerID containerId = statusRunning->container_status().container_id();
+ ASSERT_EQ(1u, statusRunning->container_status().network_infos().size());
+
+ // Try connecting to the nginx server on port 80 through a
+ // non-loopback IP address on `hostPort`.
+ Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP();
+ ASSERT_SOME(hostIPNetwork);
+
+ // `TASK_RUNNING` does not guarantee that the service is running.
+ // Hence, we need to re-try the service multiple times.
+ Duration waited = Duration::zero();
+ do {
+ Try<string> connect = os::shell(
+ "curl -I http://" + stringify(hostIPNetwork->address()) +
+ ":" + stringify(hostPort));
+
+ if (connect.isSome()) {
+ LOG(INFO) << "Connection to nginx successful: " << connect.get();
+ break;
+ }
+
+ os::sleep(Milliseconds(100));
+ waited += Milliseconds(100);
+ } while (waited < Seconds(10));
+
+ EXPECT_LE(waited, Seconds(5));
+
+ // Kill the task.
+ Future<TaskStatus> statusKilled;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusKilled));
+
+ // Wait for the executor to exit. We are using 'gc.schedule' as a proxy event
+ // to monitor the exit of the executor.
+ Future<Nothing> gcSchedule = FUTURE_DISPATCH(
+ _, &slave::GarbageCollectorProcess::schedule);
+
+ driver.killTask(task.task_id());
+
+ AWAIT_READY(statusKilled);
+
+ // The executor would issue a SIGTERM to the container, followed by
+ // a SIGKILL (in case the container ignores the SIGTERM). The
+ // "nginx:alpine" container returns an "EXIT_STATUS" of 0 on
+ // receiving a SIGTERM making the executor send a `TASK_FINISHED`
+ // instead of a `TASK_KILLED`, hence checking for `TASK_FINISHED`
+ // instead of `TASK_KILLED`.
+ EXPECT_EQ(TASK_FINISHED, statusKilled.get().state());
+
+ AWAIT_READY(gcSchedule);
+
+ // Make sure the iptables chain `MESOS-TEST-PORT-MAPPER-CHAIN`
+ // doesn't have any iptable rules once the task is killed. The only
+ // rule that should exist in this chain is the
+ // `-N MESOS-TEST-PORT-MAPPER-CHAIN` rule.
+ Try<string> rules = os::shell(
+ "iptables -w -t nat -S " +
+ stringify(MESOS_TEST_PORT_MAPPER_CHAIN) + "| wc -l");
+
+ ASSERT_SOME(rules);
+ ASSERT_EQ("1", strings::trim(rules.get()));
+
+ driver.stop();
+ driver.join();
+}
+
+
// This test checks that a CNI DNS configuration ends up generating
// the right settings in /etc/resolv.conf.
TEST_F(CniIsolatorTest, ROOT_VerifyResolverConfig)
{
- Try<net::IPNetwork> hostIPNetwork = findHostNetwork();
+ Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP();
ASSERT_SOME(hostIPNetwork);
Try<string> mockPlugin = strings::format(
@@ -1122,7 +1333,7 @@ TEST_F(CniIsolatorTest, ROOT_VerifyResolverConfig)
// that glibc accepts by using it to ping a host.
TEST_F(CniIsolatorTest, ROOT_INTERNET_VerifyResolverConfig)
{
- Try<net::IPNetwork> hostIPNetwork = findHostNetwork();
+ Try<net::IPNetwork> hostIPNetwork = getNonLoopbackIP();
ASSERT_SOME(hostIPNetwork);
// Note: We set a dummy nameserver IP address followed by the