You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2017/07/07 00:07:04 UTC
[1/4] mesos git commit: Added an IPv6 address storage to UPID.
Repository: mesos
Updated Branches:
refs/heads/master 952a7422c -> 44494452c
Added an IPv6 address storage to UPID.
Review: https://reviews.apache.org/r/59131/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/60c472ec
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/60c472ec
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/60c472ec
Branch: refs/heads/master
Commit: 60c472ecac393710c354c729995167b3c13afaaf
Parents: 952a742
Author: Avinash sridharan <av...@mesosphere.io>
Authored: Thu Jul 6 17:01:12 2017 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Thu Jul 6 17:01:12 2017 -0700
----------------------------------------------------------------------
3rdparty/libprocess/include/process/pid.hpp | 27 +++++++++++++++++++++++-
3rdparty/libprocess/src/pid.cpp | 1 +
2 files changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/60c472ec/3rdparty/libprocess/include/process/pid.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/pid.hpp b/3rdparty/libprocess/include/process/pid.hpp
index b0c47ff..6ed936d 100644
--- a/3rdparty/libprocess/include/process/pid.hpp
+++ b/3rdparty/libprocess/include/process/pid.hpp
@@ -41,7 +41,7 @@ struct UPID
UPID() = default;
UPID(const UPID& that)
- : id(that.id), address(that.address) {}
+ : id(that.id), address(that.address), addresses(that.addresses) {}
UPID(const char* id_, const net::IP& ip_, uint16_t port_)
: id(id_), address(ip_, port_) {}
@@ -93,7 +93,31 @@ struct UPID
}
std::string id;
+
+ // TODO(asridharan): Ideally, the following `address` field should be of
+ // type `network::Address` so that the default address of the PID
+ // could be a unix domain socket or an IPv4/v6 address. This change
+ // however is disruptive at this point and should be done after we have
+ // introduced support for unix domain and IPv6 sockets into
+ // `libprocess`.
network::inet::Address address = network::inet4::Address::ANY_ANY();
+
+ // TODO(asridharan): Currently we are introducing only an `Optional`
+ // IPv6 address in the following `addresses` structure. This will
+ // help us initiate some basic IPv6 support for the
+ // `DockerContainerizer`. However, going forward, once we start
+ // supporting unix domain sockets and IPv4/IPv6 socket in
+ // `libprocess` we will add the following fields to this structure.
+ //
+ // Option<network::unix::Address> unix;
+ // Option<network::inet4::Address> v4;
+ //
+ // With the introduction of the above fields `libprocess` PID will
+ // be able to support unix, IPv4 and IPv6 sockets simultaneously.
+ struct
+ {
+ Option<network::inet6::Address> v6;
+ } addresses = {None()};
};
@@ -141,6 +165,7 @@ struct PID : UPID
PID<Base> pid;
pid.id = id;
pid.address = address;
+ pid.addresses = addresses;
return pid;
}
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/60c472ec/3rdparty/libprocess/src/pid.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/pid.cpp b/3rdparty/libprocess/src/pid.cpp
index 5191a9d..634ac44 100644
--- a/3rdparty/libprocess/src/pid.cpp
+++ b/3rdparty/libprocess/src/pid.cpp
@@ -62,6 +62,7 @@ UPID::UPID(const ProcessBase& process)
{
id = process.self().id;
address = process.self().address;
+ addresses = process.self().addresses;
}
[4/4] mesos git commit: Added a test for docker container running on
a v4/6 host network.
Posted by be...@apache.org.
Added a test for docker container running on a v4/6 host network.
Review: https://reviews.apache.org/r/60551/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/44494452
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/44494452
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/44494452
Branch: refs/heads/master
Commit: 44494452ca87d4f42b2a554ac820560c6384882f
Parents: 114bd98
Author: Avinash sridharan <av...@mesosphere.io>
Authored: Thu Jul 6 17:01:53 2017 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Thu Jul 6 17:01:53 2017 -0700
----------------------------------------------------------------------
.../docker_containerizer_tests.cpp | 219 +++++++++++++++++++
1 file changed, 219 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/44494452/src/tests/containerizer/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/docker_containerizer_tests.cpp b/src/tests/containerizer/docker_containerizer_tests.cpp
index cf94702..1e85a79 100644
--- a/src/tests/containerizer/docker_containerizer_tests.cpp
+++ b/src/tests/containerizer/docker_containerizer_tests.cpp
@@ -77,6 +77,19 @@ using testing::Eq;
using testing::Invoke;
using testing::Return;
+namespace process {
+
+// We need to reinitialize libprocess in order to test against
+// different configurations, such as when libprocess is initialized
+// with SSL or IPv6 enabled.
+void reinitialize(
+ const Option<string>& delegate,
+ const Option<string>& readonlyAuthenticationRealm,
+ const Option<string>& readwriteAuthenticationRealm);
+
+} // namespace process {
+
+
namespace mesos {
namespace internal {
namespace tests {
@@ -4338,6 +4351,212 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Non_Root_Sandbox)
EXPECT_TRUE(strings::contains(stdout.get(), "foo"));
}
+
+// Fixture for testing IPv6 support for docker containers on host network.
+//
+// TODO(asridharan): Currently in the `Setup` and `TearDown` methods
+// of this class we re-initialize libprocess to take an IPv6 address.
+// Ideally, we should be moving this into a more general test fixture
+// in mesos.hpp to be used by any other tests for IPv6. This might
+// need changes to `MesosTest` in to order to allow for multiple
+// inheritance.
+class DockerContainerizerIPv6Test : public DockerContainerizerTest
+{
+protected:
+ virtual void SetUp()
+ {
+ os::setenv("LIBPROCESS_IP6", "::1234");
+ process::reinitialize(
+ None(),
+ READWRITE_HTTP_AUTHENTICATION_REALM,
+ READONLY_HTTP_AUTHENTICATION_REALM);
+
+ DockerContainerizerTest::SetUp();
+ }
+
+ virtual void TearDown()
+ {
+ DockerContainerizerTest::TearDown();
+
+ os::unsetenv("LIBPROCESS_IP6");
+ process::reinitialize(
+ None(),
+ READWRITE_HTTP_AUTHENTICATION_REALM,
+ READONLY_HTTP_AUTHENTICATION_REALM);
+ }
+};
+
+
+// Launches a docker container on the host network. The host network
+// is assumed to have an IPv4 address and an IPv6 address. The test
+// passes if the Mesos state correctly exposes both the IPv4 and IPv6
+// address.
+TEST_F(DockerContainerizerIPv6Test, ROOT_DOCKER_LaunchIPv6HostNetwork)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockDocker* mockDocker =
+ new MockDocker(tests::flags.docker, tests::flags.docker_socket);
+
+ Shared<Docker> docker(mockDocker);
+
+ slave::Flags flags = CreateSlaveFlags();
+
+ Fetcher fetcher(flags);
+
+ Try<ContainerLogger*> logger =
+ ContainerLogger::create(flags.container_logger);
+
+ ASSERT_SOME(logger);
+
+ MockDockerContainerizer dockerContainerizer(
+ flags,
+ &fetcher,
+ Owned<ContainerLogger>(logger.get()),
+ docker);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave =
+ StartSlave(detector.get(), &dockerContainerizer, flags);
+
+ ASSERT_SOME(slave);
+
+ // Check if the slave has the IPv6 address stored in its PID.
+ EXPECT_SOME(slave.get()->pid.addresses.v6);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ Future<FrameworkID> frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureArg<1>(&frameworkId));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(frameworkId);
+
+ AWAIT_READY(offers);
+ ASSERT_NE(0u, offers->size());
+
+ const Offer& offer = offers.get()[0];
+
+ TaskInfo task = createTask(
+ offer.slave_id(),
+ offer.resources(),
+ SLEEP_COMMAND(10000));
+
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+
+ // TODO(tnachen): Use local image to test if possible.
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image("alpine");
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ task.mutable_container()->CopyFrom(containerInfo);
+
+ Future<ContainerID> containerId;
+ EXPECT_CALL(dockerContainerizer, launch(_, _, _, _))
+ .WillOnce(DoAll(FutureArg<0>(&containerId),
+ Invoke(&dockerContainerizer,
+ &MockDockerContainerizer::_launch)));
+
+ Future<TaskStatus> statusRunning;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillRepeatedly(DoDefault());
+
+ driver.launchTasks(offers.get()[0].id(), {task});
+
+ AWAIT_READY_FOR(containerId, Seconds(60));
+ AWAIT_READY_FOR(statusRunning, Seconds(60));
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+ ASSERT_TRUE(statusRunning->has_data());
+
+ Try<JSON::Array> array = JSON::parse<JSON::Array>(statusRunning->data());
+ ASSERT_SOME(array);
+
+ // Check if container information is exposed through master's state endpoint.
+ Future<http::Response> response = http::get(
+ master.get()->pid,
+ "state",
+ None(),
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(process::http::OK().status, response);
+
+ Try<JSON::Object> parse = JSON::parse<JSON::Object>(response->body);
+ ASSERT_SOME(parse);
+
+ Result<JSON::Value> find = parse->find<JSON::Value>(
+ "frameworks[0].tasks[0].container.docker.privileged");
+
+ EXPECT_SOME_FALSE(find);
+
+ // Check if container information is exposed through slave's state endpoint.
+ response = http::get(
+ slave.get()->pid,
+ "state",
+ None(),
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(process::http::OK().status, response);
+
+ parse = JSON::parse<JSON::Object>(response->body);
+ ASSERT_SOME(parse);
+
+ find = parse->find<JSON::Value>(
+ "frameworks[0].executors[0].tasks[0].container.docker.privileged");
+
+ EXPECT_SOME_FALSE(find);
+
+ // Now verify the ContainerStatus fields in the TaskStatus.
+ ASSERT_TRUE(statusRunning->has_container_status());
+ EXPECT_TRUE(statusRunning->container_status().has_container_id());
+ ASSERT_EQ(1, statusRunning->container_status().network_infos().size());
+ EXPECT_EQ(2, statusRunning->container_status().network_infos(0).ip_addresses().size()); // NOLINT(whitespace/line_length)
+
+ Option<string> containerIPv4 = None();
+ Option<string> containerIPv6 = None();
+
+ foreach(const NetworkInfo::IPAddress& ipAddress,
+ statusRunning->container_status().network_infos(0).ip_addresses()) {
+ if (ipAddress.protocol() == NetworkInfo::IPv4) {
+ containerIPv4 = ipAddress.ip_address();
+ }
+
+ if (ipAddress.protocol() == NetworkInfo::IPv6) {
+ containerIPv6 = ipAddress.ip_address();
+ }
+ }
+
+ EXPECT_SOME(containerIPv4);
+ ASSERT_SOME(containerIPv6);
+ EXPECT_EQ(containerIPv6.get(), "::1234");
+
+ ASSERT_TRUE(exists(docker, containerId.get()));
+
+ Future<Option<ContainerTermination>> termination =
+ dockerContainerizer.wait(containerId.get());
+
+ driver.stop();
+ driver.join();
+
+ AWAIT_READY(termination);
+ EXPECT_SOME(termination.get());
+
+ ASSERT_FALSE(
+ exists(docker, containerId.get(), ContainerState::RUNNING));
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
[2/4] mesos git commit: Added storage for IPv6 in a `libprocess`
process.
Posted by be...@apache.org.
Added storage for IPv6 in a `libprocess` process.
Review: https://reviews.apache.org/r/59130/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8d9929a5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8d9929a5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8d9929a5
Branch: refs/heads/master
Commit: 8d9929a5fe51c866aabac04c764d4e9ceca98d06
Parents: 60c472e
Author: Avinash sridharan <av...@mesosphere.io>
Authored: Thu Jul 6 17:01:23 2017 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Thu Jul 6 17:01:23 2017 -0700
----------------------------------------------------------------------
3rdparty/libprocess/src/process.cpp | 44 +++++++++++++++++++++++++++-----
1 file changed, 37 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8d9929a5/3rdparty/libprocess/src/process.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/process.cpp b/3rdparty/libprocess/src/process.cpp
index b3d5539..264298c 100644
--- a/3rdparty/libprocess/src/process.cpp
+++ b/3rdparty/libprocess/src/process.cpp
@@ -116,6 +116,10 @@
#include "gate.hpp"
#include "process_reference.hpp"
+namespace inet = process::network::inet;
+namespace inet4 = process::network::inet4;
+namespace inet6 = process::network::inet6;
+
using process::wait; // Necessary on some OS's to disambiguate.
using process::http::Accepted;
@@ -135,8 +139,6 @@ using process::http::authentication::AuthenticatorManager;
using process::http::authorization::AuthorizationCallbacks;
-namespace inet4 = process::network::inet4;
-
using process::network::inet::Address;
using process::network::inet::Socket;
@@ -169,7 +171,21 @@ struct Flags : public virtual flags::FlagsBase
"ip",
"The IP address for communication to and from libprocess.\n"
"If not specified, libprocess will attempt to reverse-DNS lookup\n"
- "the hostname and use that IP instead.");
+ "the hostname and use that IP instead.",
+ [](const Option<net::IP>& ip) -> Option<Error> {
+ if (ip.isSome() && ip->family() != AF_INET) {
+ return Error(
+ "Currently we allow only IPv4 address to be specified "
+ "with the `--ip` flag");
+ }
+
+ return None();
+ });
+
+ add(&Flags::ip6,
+ "ip6",
+ "The IPv6 address that `libprocess` will use in future to perform "
+ "communication of IPv6 sockets.\n");
add(&Flags::advertise_ip,
"advertise_ip",
@@ -230,6 +246,7 @@ struct Flags : public virtual flags::FlagsBase
}
Option<net::IP> ip;
+ Option<net::IPv6> ip6;
Option<net::IP> advertise_ip;
Option<int> port;
Option<int> advertise_port;
@@ -599,7 +616,10 @@ static std::mutex* socket_mutex = new std::mutex();
static Future<Socket> future_accept;
// Local socket address.
-static Address __address__ = inet4::Address::ANY_ANY();
+static inet::Address __address__ = inet4::Address::ANY_ANY();
+
+// Local IPv6 socket address.
+static Option<inet6::Address> __address6__ = None();
// Active SocketManager (eventually will probably be thread-local).
static SocketManager* socket_manager = nullptr;
@@ -1126,12 +1146,18 @@ bool initialize(
LOG(WARNING) << warning.message;
}
+ uint16_t port = 0;
+
+ if (libprocess_flags->port.isSome()) {
+ port = libprocess_flags->port.get();
+ }
+
if (libprocess_flags->ip.isSome()) {
- __address__.ip = libprocess_flags->ip.get();
+ __address__ = inet::Address(libprocess_flags->ip.get(), port);
}
- if (libprocess_flags->port.isSome()) {
- __address__.port = libprocess_flags->port.get();
+ if (libprocess_flags->ip6.isSome()) {
+ __address6__ = inet6::Address(libprocess_flags->ip6.get(), port);
}
// Create a "server" socket for communicating.
@@ -1360,6 +1386,9 @@ void finalize(bool finalize_wsa)
// cannot be cleared until after the `ProcessManager` is deleted.
__address__ = inet4::Address::ANY_ANY();
+ // Reset any IPv6 addresses set on the process.
+ __address6__ = None();
+
// Finally, reset the Flags to defaults.
*libprocess_flags = internal::Flags();
@@ -3682,6 +3711,7 @@ ProcessBase::ProcessBase(const string& id)
pid.id = id != "" ? id : ID::generate();
pid.address = __address__;
+ pid.addresses.v6 = __address6__;
// If using a manual clock, try and set current time of process
// using happens before relationship between creator (__process__)
[3/4] mesos git commit: Updated v6 address for containers running on
host network.
Posted by be...@apache.org.
Updated v6 address for containers running on host network.
Currently the agent is populating only the v4 address for containers
running on the host network. However, clusters running on a dual stack
network need to report v4 and v6 address for containers running on the
host network. This change allows v6 address specified by operators to be
advertised for containers running on the host network.
Review: https://reviews.apache.org/r/59233/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/114bd98a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/114bd98a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/114bd98a
Branch: refs/heads/master
Commit: 114bd98a04e05665c647c4a9c94e595ef104ec64
Parents: 8d9929a
Author: Avinash sridharan <av...@mesosphere.io>
Authored: Thu Jul 6 17:01:37 2017 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Thu Jul 6 17:01:37 2017 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 2 +-
include/mesos/v1/mesos.proto | 2 +-
src/slave/slave.cpp | 12 ++++++++++++
3 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/114bd98a/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index ab2a372..d70ac9e 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -2537,7 +2537,7 @@ message NetworkInfo {
// request the network isolator on the Agent to assign an IP address to the
// container being launched. If a specific IP address is specified in
// ip_address, this field should not be set.
- optional Protocol protocol = 1;
+ optional Protocol protocol = 1 [default = IPv4];
// Statically assigned IP provided by the Framework. This IP will be
// assigned to the container by the network isolator module on the Agent.
http://git-wip-us.apache.org/repos/asf/mesos/blob/114bd98a/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 5e92e5d..fee8b0c 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -2520,7 +2520,7 @@ message NetworkInfo {
// request the network isolator on the Agent to assign an IP address to the
// container being launched. If a specific IP address is specified in
// ip_address, this field should not be set.
- optional Protocol protocol = 1;
+ optional Protocol protocol = 1 [default = IPv4];
// Statically assigned IP provided by the Framework. This IP will be
// assigned to the container by the network isolator module on the Agent.
http://git-wip-us.apache.org/repos/asf/mesos/blob/114bd98a/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 0e24b8c..52f6736 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -4560,7 +4560,19 @@ void Slave::_statusUpdate(
if (containerStatus->network_infos().size() == 0) {
NetworkInfo* networkInfo = containerStatus->add_network_infos();
NetworkInfo::IPAddress* ipAddress = networkInfo->add_ip_addresses();
+
+ // Set up IPv4 address.
+ //
+ // NOTE: By default the protocol is set to IPv4 and therefore we
+ // don't explicitly set the protocol here.
ipAddress->set_ip_address(stringify(self().address.ip));
+
+ // Set up IPv6 address.
+ if (self().addresses.v6.isSome()) {
+ ipAddress = networkInfo->add_ip_addresses();
+ ipAddress->set_ip_address(stringify(self().addresses.v6->ip));
+ ipAddress->set_protocol(NetworkInfo::IPv6);
+ }
}
}