You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by al...@apache.org on 2017/04/26 12:41:17 UTC
[1/7] mesos git commit: Fixed issues in 1.1.2 changelog.
Repository: mesos
Updated Branches:
refs/heads/1.1.x d2b1888a5 -> 485b44319
Fixed issues in 1.1.2 changelog.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/645ad0f8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/645ad0f8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/645ad0f8
Branch: refs/heads/1.1.x
Commit: 645ad0f8b2826d93446e89e459c31692fc575232
Parents: d2b1888
Author: Alexander Rukletsov <al...@apache.org>
Authored: Wed Apr 26 14:38:51 2017 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 26 14:38:51 2017 +0200
----------------------------------------------------------------------
CHANGELOG | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/645ad0f8/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 2a8db87..ff9ee5e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -7,13 +7,14 @@ All Issues:
* [MESOS-2537] - AC_ARG_ENABLED checks are broken.
* [MESOS-5028] - Copy provisioner cannot replace directory with symlink.
* [MESOS-5172] - Registry puller cannot fetch blobs correctly from http Redirect 3xx urls.
+ * [MESOS-6327] - Large docker images causes container launch failures: Too many levels of symbolic links.
* [MESOS-7197] - Requesting tiny amount of CPU crashes master.
* [MESOS-7210] - HTTP health check doesn't work when mesos runs with --docker_mesos_image.
* [MESOS-7237] - Enabling cgroups_limit_swap can lead to "invalid argument" error.
* [MESOS-7265] - Containerizer startup may cause sensitive data to leak into sandbox logs.
+ * [MESOS-7350] - Failed to pull image from Nexus Registry due to signature missing.
* [MESOS-7366] - Agent sandbox gc could accidentally delete the entire persistent volume content.
* [MESOS-7383] - Docker executor logs possibly sensitive parameters.
- * [MESOS-7350] - Failed to pull image from Nexus Registry due to signature missing.
* [MESOS-7422] - Docker containerizer should not leak possibly sensitive data to agent log.
[6/7] mesos git commit: Modified the executor driver to always relink
on agent failover.
Posted by al...@apache.org.
Modified the executor driver to always relink on agent failover.
A relink is needed in cases where a netfilter module like iptables
can terminate the connection without notifying the executor. This
results in the executor still trying to reuse the stale "half-open"
connection upon receiving the reconnect message from the executor
leading to the erroneous behavior.
Review: https://reviews.apache.org/r/56568/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d9e3c8aa
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d9e3c8aa
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d9e3c8aa
Branch: refs/heads/1.1.x
Commit: d9e3c8aa4aedf1dac7716cc31a3e71f8db0242da
Parents: 044fafe
Author: Anand Mazumdar <an...@apache.org>
Authored: Fri Feb 10 17:10:49 2017 -0800
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 26 14:40:21 2017 +0200
----------------------------------------------------------------------
src/exec/exec.cpp | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/d9e3c8aa/src/exec/exec.cpp
----------------------------------------------------------------------
diff --git a/src/exec/exec.cpp b/src/exec/exec.cpp
index 1dc2039..2ac312d 100644
--- a/src/exec/exec.cpp
+++ b/src/exec/exec.cpp
@@ -284,7 +284,12 @@ protected:
// Update the slave link.
slave = from;
- link(slave);
+
+ // We force a reconnect here to avoid sending on a stale "half-open"
+ // socket. We do not detect a disconnection in some cases when the
+ // connection is terminated by a netfilter module e.g., iptables
+ // running on the agent (see MESOS-5332).
+ link(slave, RemoteConnection::RECONNECT);
// Re-register with slave.
ReregisterExecutorMessage message;
[3/7] mesos git commit: Added MESOS-7152 to 1.1.2 CHANGELOG.
Posted by al...@apache.org.
Added MESOS-7152 to 1.1.2 CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0ff757e9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0ff757e9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0ff757e9
Branch: refs/heads/1.1.x
Commit: 0ff757e9a78bbe4e0dea75d714f0165f3123a01d
Parents: 52baba7
Author: Alexander Rukletsov <al...@apache.org>
Authored: Mon Apr 24 14:18:53 2017 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 26 14:39:14 2017 +0200
----------------------------------------------------------------------
CHANGELOG | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0ff757e9/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index ff9ee5e..b5adc4b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -8,6 +8,7 @@ All Issues:
* [MESOS-5028] - Copy provisioner cannot replace directory with symlink.
* [MESOS-5172] - Registry puller cannot fetch blobs correctly from http Redirect 3xx urls.
* [MESOS-6327] - Large docker images causes container launch failures: Too many levels of symbolic links.
+ * [MESOS-7152] - The agent may be flapping after the machine reboots due to provisioner recover.
* [MESOS-7197] - Requesting tiny amount of CPU crashes master.
* [MESOS-7210] - HTTP health check doesn't work when mesos runs with --docker_mesos_image.
* [MESOS-7237] - Enabling cgroups_limit_swap can lead to "invalid argument" error.
[4/7] mesos git commit: Fixed a crash on the master upon receiving an
invalid inverse offer.
Posted by al...@apache.org.
Fixed a crash on the master upon receiving an invalid inverse offer.
The erroneous invariant check for `slaveId` can be trigerred when
the master accepts an invalid inverse offer or when the inverse offer
has been already rescinded.
Review: https://reviews.apache.org/r/56587/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/fabca9fa
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/fabca9fa
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/fabca9fa
Branch: refs/heads/1.1.x
Commit: fabca9fa50c3eaeb6fbad19a25e65804ec75fa2b
Parents: 0ff757e
Author: Anand Mazumdar <an...@apache.org>
Authored: Wed Feb 15 11:56:37 2017 -0800
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 26 14:39:40 2017 +0200
----------------------------------------------------------------------
src/master/master.cpp | 20 +++-------
src/tests/master_maintenance_tests.cpp | 57 +++++++++++++++++++++++++++++
2 files changed, 62 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/fabca9fa/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 66ccefa..ecd035a 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -4436,27 +4436,26 @@ void Master::acceptInverseOffers(
{
CHECK_NOTNULL(framework);
- Option<Error> error = None();
+ Option<Error> error;
if (accept.inverse_offer_ids().size() == 0) {
error = Error("No inverse offers specified");
} else {
+ LOG(INFO) << "Processing ACCEPT_INVERSE_OFFERS call for inverse offers: "
+ << accept.inverse_offer_ids() << " for framework " << *framework;
+
// Validate the inverse offers.
error = validation::offer::validateInverseOffers(
accept.inverse_offer_ids(),
this,
framework);
- Option<SlaveID> slaveId;
-
// Update each inverse offer in the allocator with the accept and
// filter.
+ // TODO(anand): Notify the framework if some of the offers were invalid.
foreach (const OfferID& offerId, accept.inverse_offer_ids()) {
InverseOffer* inverseOffer = getInverseOffer(offerId);
if (inverseOffer != nullptr) {
- CHECK(inverseOffer->has_slave_id());
- slaveId = inverseOffer->slave_id();
-
mesos::allocator::InverseOfferStatus status;
status.set_status(mesos::allocator::InverseOfferStatus::ACCEPT);
status.mutable_framework_id()->CopyFrom(inverseOffer->framework_id());
@@ -4480,15 +4479,6 @@ void Master::acceptInverseOffers(
LOG(WARNING) << "Ignoring accept of inverse offer " << offerId
<< " since it is no longer valid";
}
-
- CHECK_SOME(slaveId);
- Slave* slave = slaves.registered.get(slaveId.get());
- CHECK_NOTNULL(slave);
-
- LOG(INFO)
- << "Processing ACCEPT_INVERSE_OFFERS call for inverse offers: "
- << accept.inverse_offer_ids() << " on slave " << *slave
- << " for framework " << *framework;
}
if (error.isSome()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/fabca9fa/src/tests/master_maintenance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_maintenance_tests.cpp b/src/tests/master_maintenance_tests.cpp
index 6917272..1d8e950 100644
--- a/src/tests/master_maintenance_tests.cpp
+++ b/src/tests/master_maintenance_tests.cpp
@@ -72,9 +72,11 @@ using mesos::v1::scheduler::Mesos;
using process::Clock;
using process::Future;
+using process::Message;
using process::Owned;
using process::PID;
using process::Time;
+using process::UPID;
using process::http::BadRequest;
using process::http::OK;
@@ -90,6 +92,7 @@ using std::vector;
using testing::AtMost;
using testing::DoAll;
+using testing::Eq;
using testing::Not;
namespace mesos {
@@ -1828,6 +1831,60 @@ TEST_F(MasterMaintenanceTest, EndpointsBadAuthentication)
}
}
+
+// This test verifies that the Mesos master does not crash while
+// processing an invalid inverse offer (See MESOS-7119).
+TEST_F(MasterMaintenanceTest, AcceptInvalidInverseOffer)
+{
+ master::Flags masterFlags = CreateMasterFlags();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<Message> frameworkRegisteredMessage =
+ FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
+
+ driver.start();
+
+ AWAIT_READY(frameworkRegisteredMessage);
+ UPID frameworkPid = frameworkRegisteredMessage.get().to;
+
+ FrameworkRegisteredMessage message;
+ ASSERT_TRUE(message.ParseFromString(frameworkRegisteredMessage.get().body));
+
+ FrameworkID frameworkId = message.framework_id();
+
+ Future<mesos::scheduler::Call> acceptInverseOffersCall = FUTURE_CALL(
+ mesos::scheduler::Call(),
+ mesos::scheduler::Call::ACCEPT_INVERSE_OFFERS,
+ _,
+ _);
+
+ {
+ mesos::scheduler::Call call;
+ call.mutable_framework_id()->CopyFrom(frameworkId);
+ call.set_type(mesos::scheduler::Call::ACCEPT_INVERSE_OFFERS);
+
+ mesos::scheduler::Call::AcceptInverseOffers* accept =
+ call.mutable_accept_inverse_offers();
+
+ accept->add_inverse_offer_ids()->set_value("invalid-inverse-offer");
+
+ process::post(frameworkPid, master.get()->pid, call);
+ }
+
+ AWAIT_READY(acceptInverseOffersCall);
+
+ driver.stop();
+ driver.join();
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
[7/7] mesos git commit: Added MESOS-7057 to 1.1.2 CHANGELOG.
Posted by al...@apache.org.
Added MESOS-7057 to 1.1.2 CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/485b4431
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/485b4431
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/485b4431
Branch: refs/heads/1.1.x
Commit: 485b44319ac1629689b00649d85739acf9afc56e
Parents: d9e3c8a
Author: Alexander Rukletsov <al...@apache.org>
Authored: Mon Apr 24 14:33:47 2017 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 26 14:40:30 2017 +0200
----------------------------------------------------------------------
CHANGELOG | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/485b4431/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 5d172c4..5df2d08 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -8,6 +8,7 @@ All Issues:
* [MESOS-5028] - Copy provisioner cannot replace directory with symlink.
* [MESOS-5172] - Registry puller cannot fetch blobs correctly from http Redirect 3xx urls.
* [MESOS-6327] - Large docker images causes container launch failures: Too many levels of symbolic links.
+ * [MESOS-7057] - Consider using the relink functionality of libprocess in the executor driver.
* [MESOS-7119] - Mesos master crash while accepting inverse offer.
* [MESOS-7152] - The agent may be flapping after the machine reboots due to provisioner recover.
* [MESOS-7197] - Requesting tiny amount of CPU crashes master.
[5/7] mesos git commit: Added MESOS-7119 to 1.1.2 CHANGELOG.
Posted by al...@apache.org.
Added MESOS-7119 to 1.1.2 CHANGELOG.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/044fafe5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/044fafe5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/044fafe5
Branch: refs/heads/1.1.x
Commit: 044fafe5f402e586cca0f1527dae8cb06197deb3
Parents: fabca9f
Author: Alexander Rukletsov <al...@apache.org>
Authored: Mon Apr 24 14:28:01 2017 +0200
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 26 14:40:11 2017 +0200
----------------------------------------------------------------------
CHANGELOG | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/044fafe5/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index b5adc4b..5d172c4 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -8,6 +8,7 @@ All Issues:
* [MESOS-5028] - Copy provisioner cannot replace directory with symlink.
* [MESOS-5172] - Registry puller cannot fetch blobs correctly from http Redirect 3xx urls.
* [MESOS-6327] - Large docker images causes container launch failures: Too many levels of symbolic links.
+ * [MESOS-7119] - Mesos master crash while accepting inverse offer.
* [MESOS-7152] - The agent may be flapping after the machine reboots due to provisioner recover.
* [MESOS-7197] - Requesting tiny amount of CPU crashes master.
* [MESOS-7210] - HTTP health check doesn't work when mesos runs with --docker_mesos_image.
[2/7] mesos git commit: Fixed nested container agent flapping issue
after reboot.
Posted by al...@apache.org.
Fixed nested container agent flapping issue after reboot.
When recovering containers in provisioner, there is a particular case
that after the machine reboots the container runtime directory and
slave state is gone but the provisioner directory still exists since
it is under the agent work_dir(e.g., agent work_dir is /var/lib/mesos).
Then, all checkpointed containers will be cleaned up as unknown
containers in provisioner during recovery. However, the semantic that
a child container is always cleaned up before its parent container
cannot be guaranteed for this particular case. Ideally, we should
put the provisioner directory under the container runtime dir but this
is not backward compactible. It is an unfortunate that we have to
make the provisioner::destroy() to be recursive.
Review: https://reviews.apache.org/r/56808/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/52baba71
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/52baba71
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/52baba71
Branch: refs/heads/1.1.x
Commit: 52baba717e42907797b7cce65a51285593fd39ae
Parents: 645ad0f
Author: Gilbert Song <so...@gmail.com>
Authored: Wed Feb 22 09:10:58 2017 -0800
Committer: Alexander Rukletsov <al...@apache.org>
Committed: Wed Apr 26 14:39:08 2017 +0200
----------------------------------------------------------------------
.../mesos/provisioner/provisioner.cpp | 78 +++++++++++++++-----
.../mesos/provisioner/provisioner.hpp | 11 ++-
2 files changed, 70 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/52baba71/src/slave/containerizer/mesos/provisioner/provisioner.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.cpp b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
index db2d267..3b041a7 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.cpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
@@ -326,32 +326,68 @@ Future<bool> ProvisionerProcess::destroy(const ContainerID& containerId)
return false;
}
+ if (infos[containerId]->destroying) {
+ return infos[containerId]->termination.future();
+ }
+
+ infos[containerId]->destroying = true;
+
// Provisioner destroy can be invoked from:
// 1. Provisioner `recover` to destroy all unknown orphans.
// 2. Containerizer `recover` to destroy known orphans.
// 3. Containerizer `destroy` on one specific container.
//
- // In the above cases, we assume that the container being destroyed
- // has no corresponding child containers. We fail fast if this
- // condition is not satisfied.
+ // NOTE: For (2) and (3), we expect the container being destory
+ // has no any child contain remain running. However, for case (1),
+ // if the container runtime directory does not survive after the
+ // machine reboots and the provisioner directory under the agent
+ // work dir still exists, all containers will be regarded as
+ // unkown containers and will be destroyed. In this case, a parent
+ // container may be destoryed before its child containers are
+ // cleaned up. So we have to make `destroy()` recursively for
+ // this particular case.
//
- // NOTE: This check is expensive since it traverses the entire
- // `infos` hashmap. This is acceptable because we generally expect
- // the number of containers on a single agent to be on the order of
- // tens or hundreds of containers.
+ // TODO(gilbert): Move provisioner directory to the container
+ // runtime directory after a deprecation cycle to avoid
+ // making `provisioner::destroy()` being recursive.
+ list<Future<bool>> destroys;
+
foreachkey (const ContainerID& entry, infos) {
- if (entry.has_parent()) {
- CHECK(entry.parent() != containerId)
- << "Failed to destroy container "
- << containerId << " since its nested container "
- << entry << " has not been destroyed yet";
+ if (entry.has_parent() && entry.parent() == containerId) {
+ destroys.push_back(destroy(entry));
}
}
- // Unregister the container first. If destroy() fails, we can rely
- // on recover() to retry it later.
- Owned<Info> info = infos[containerId];
- infos.erase(containerId);
+ return await(destroys)
+ .then(defer(self(), &Self::_destroy, containerId, lambda::_1));
+}
+
+
+Future<bool> ProvisionerProcess::_destroy(
+ const ContainerID& containerId,
+ const list<Future<bool>>& destroys)
+{
+ CHECK(infos.contains(containerId));
+ CHECK(infos[containerId]->destroying);
+
+ vector<string> errors;
+ foreach (const Future<bool>& future, destroys) {
+ if (!future.isReady()) {
+ errors.push_back(future.isFailed()
+ ? future.failure()
+ : "discarded");
+ }
+ }
+
+ if (!errors.empty()) {
+ ++metrics.remove_container_errors;
+
+ return Failure(
+ "Failed to destory nested containers: " +
+ strings::join("; ", errors));
+ }
+
+ const Owned<Info>& info = infos[containerId];
list<Future<bool>> futures;
foreachkey (const string& backend, info->rootfses) {
@@ -381,12 +417,15 @@ Future<bool> ProvisionerProcess::destroy(const ContainerID& containerId)
// TODO(xujyan): Revisit the usefulness of this return value.
return collect(futures)
- .then(defer(self(), &ProvisionerProcess::_destroy, containerId));
+ .then(defer(self(), &ProvisionerProcess::__destroy, containerId));
}
-Future<bool> ProvisionerProcess::_destroy(const ContainerID& containerId)
+Future<bool> ProvisionerProcess::__destroy(const ContainerID& containerId)
{
+ CHECK(infos.contains(containerId));
+ CHECK(infos[containerId]->destroying);
+
// This should be fairly cheap as the directory should only
// contain a few empty sub-directories at this point.
//
@@ -406,6 +445,9 @@ Future<bool> ProvisionerProcess::_destroy(const ContainerID& containerId)
++metrics.remove_container_errors;
}
+ infos[containerId]->termination.set(true);
+ infos.erase(containerId);
+
return true;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/52baba71/src/slave/containerizer/mesos/provisioner/provisioner.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.hpp b/src/slave/containerizer/mesos/provisioner/provisioner.hpp
index 0a48617..a988025 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.hpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.hpp
@@ -134,7 +134,11 @@ private:
const Image& image,
const ImageInfo& imageInfo);
- process::Future<bool> _destroy(const ContainerID& containerId);
+ process::Future<bool> _destroy(
+ const ContainerID& containerId,
+ const std::list<process::Future<bool>>& destroys);
+
+ process::Future<bool> __destroy(const ContainerID& containerId);
const Flags flags;
@@ -152,6 +156,11 @@ private:
{
// Mappings: backend -> {rootfsId, ...}
hashmap<std::string, hashset<std::string>> rootfses;
+
+ process::Promise<bool> termination;
+
+ // The container status in provisioner.
+ bool destroying = false;
};
hashmap<ContainerID, process::Owned<Info>> infos;