You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2015/01/09 01:28:04 UTC
[1/3] incubator-twill git commit: (TWILL-110) added state guard to
prevent possible race-condition when shutting down zk client,
which causes deadlock.
Repository: incubator-twill
Updated Branches:
refs/heads/branch-0.4.1 [created] b6dea1985
Updated Tags: refs/tags/v0.4.1-incubating [created] 4eb137c18
(TWILL-110) added state guard to prevent possible race-condition when shutting down zk client, which causes deadlock.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/ae4ed40c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/ae4ed40c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/ae4ed40c
Branch: refs/heads/branch-0.4.1
Commit: ae4ed40c1ad1c4dbd9ffd37bfb5bf39bfa23ac86
Parents: 36577ae
Author: Terence Yim <ch...@apache.org>
Authored: Mon Dec 1 14:12:11 2014 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Thu Jan 8 16:27:38 2015 -0800
----------------------------------------------------------------------
.../zookeeper/DefaultZKClientService.java | 20 ++++++++++++--
.../apache/twill/zookeeper/ZKClientTest.java | 28 ++++++++++++++++++++
2 files changed, 46 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ae4ed40c/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
index 7c9bd08..73ca274 100644
--- a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
+++ b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
@@ -57,6 +57,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
@@ -371,6 +372,9 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
private final class ServiceDelegate extends AbstractService implements Watcher {
+ private final AtomicBoolean stopNotified = new AtomicBoolean(false);
+ private volatile boolean executorStopped;
+
@Override
protected void doStart() {
// A single thread executor
@@ -379,7 +383,12 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
@Override
protected void terminated() {
super.terminated();
- notifyStopped();
+
+ // Only call notifyStopped if the executor.shutdown() returned, otherwise deadlock (TWILL-110) can occur.
+ // Also, notifyStopped() should only be called once.
+ if (executorStopped && stopNotified.compareAndSet(false, true)) {
+ notifyStopped();
+ }
}
};
@@ -400,6 +409,13 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
notifyFailed(e);
} finally {
eventExecutor.shutdown();
+ executorStopped = true;
+
+ // If the executor state is terminated, meaning the terminate() method is triggered,
+ // call notifyStopped() if it hasn't been called yet.
+ if (eventExecutor.isTerminated() && stopNotified.compareAndSet(false, true)) {
+ notifyStopped();
+ }
}
}
}
@@ -408,7 +424,7 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
public void process(WatchedEvent event) {
try {
if (event.getState() == Event.KeeperState.SyncConnected && state() == State.STARTING) {
- LOG.info("Connected to ZooKeeper: " + zkStr);
+ LOG.debug("Connected to ZooKeeper: " + zkStr);
notifyStarted();
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ae4ed40c/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java b/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
index b0c7507..dc1b9eb 100644
--- a/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
+++ b/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
@@ -35,6 +35,8 @@ import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -54,6 +56,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
*/
public class ZKClientTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ZKClientTest.class);
+
@ClassRule
public static TemporaryFolder tmpFolder = new TemporaryFolder();
@@ -320,4 +324,28 @@ public class ZKClientTest {
zkServer.stopAndWait();
}
}
+
+ @Test (timeout = 120000L)
+ public void testDeadlock() throws IOException, InterruptedException {
+ // This is to test deadlock bug as described in (TWILL-110)
+ // This test has very high chance to get deadlock before the bug fix, hence failed with timeout.
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
+ zkServer.startAndWait();
+ try {
+ for (int i = 0; i < 5000; i++) {
+ final ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ zkClient.addConnectionWatcher(new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ LOG.debug("Connection event: {}", event);
+ }
+ });
+ zkClient.startAndWait();
+ zkClient.stopAndWait();
+ }
+
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
}
[2/3] incubator-twill git commit: (TWILL-108) Do not fail with
exception if same jar is added twice during bundle jar creation
Posted by ch...@apache.org.
(TWILL-108) Do not fail with exception if same jar is added twice during bundle jar creation
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/36577aee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/36577aee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/36577aee
Branch: refs/heads/branch-0.4.1
Commit: 36577aee4144d15f366107e992702331977856bb
Parents: 7da6e6a
Author: Mike Walch <mw...@gmail.com>
Authored: Wed Nov 5 11:13:49 2014 -0500
Committer: Terence Yim <ch...@apache.org>
Committed: Thu Jan 8 16:27:38 2015 -0800
----------------------------------------------------------------------
.../apache/twill/internal/ApplicationBundler.java | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/36577aee/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java b/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
index dcc4dbd..3f2d073 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
@@ -343,13 +343,15 @@ public final class ApplicationBundler {
String prefix = path.endsWith(".jar") ? SUBDIR_LIB : SUBDIR_RESOURCES;
path = prefix + path.substring(path.lastIndexOf('/') + 1);
- saveDirEntry(prefix, entries, jarOut);
- jarOut.putNextEntry(new JarEntry(path));
- InputStream is = url.openStream();
- try {
- ByteStreams.copy(is, jarOut);
- } finally {
- is.close();
+ if (entries.add(path)) {
+ saveDirEntry(prefix, entries, jarOut);
+ jarOut.putNextEntry(new JarEntry(path));
+ InputStream is = url.openStream();
+ try {
+ ByteStreams.copy(is, jarOut);
+ } finally {
+ is.close();
+ }
}
}
[3/3] incubator-twill git commit: Version bump for 0.4.1-incubating
release
Posted by ch...@apache.org.
Version bump for 0.4.1-incubating release
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/b6dea198
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/b6dea198
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/b6dea198
Branch: refs/heads/branch-0.4.1
Commit: b6dea198556e686827822b6dde09b269732c772b
Parents: ae4ed40
Author: Terence Yim <ch...@apache.org>
Authored: Thu Jan 8 16:28:47 2015 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Thu Jan 8 16:28:47 2015 -0800
----------------------------------------------------------------------
pom.xml | 2 +-
twill-api/pom.xml | 2 +-
twill-common/pom.xml | 2 +-
twill-core/pom.xml | 2 +-
twill-discovery-api/pom.xml | 2 +-
twill-discovery-core/pom.xml | 2 +-
twill-examples/echo/pom.xml | 2 +-
twill-examples/pom.xml | 2 +-
twill-examples/yarn/pom.xml | 2 +-
twill-ext/pom.xml | 2 +-
twill-yarn/pom.xml | 2 +-
twill-zookeeper/pom.xml | 2 +-
12 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e244e3d..d7eecbc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
<packaging>pom</packaging>
<name>Apache Twill</name>
<url>http://twill.incubator.apache.org</url>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-api/pom.xml
----------------------------------------------------------------------
diff --git a/twill-api/pom.xml b/twill-api/pom.xml
index 8535c25..e2096af 100644
--- a/twill-api/pom.xml
+++ b/twill-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<artifactId>twill-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-common/pom.xml
----------------------------------------------------------------------
diff --git a/twill-common/pom.xml b/twill-common/pom.xml
index 714d424..0d6b6cf 100644
--- a/twill-common/pom.xml
+++ b/twill-common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-core/pom.xml
----------------------------------------------------------------------
diff --git a/twill-core/pom.xml b/twill-core/pom.xml
index c9f6463..c326e8e 100644
--- a/twill-core/pom.xml
+++ b/twill-core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-discovery-api/pom.xml
----------------------------------------------------------------------
diff --git a/twill-discovery-api/pom.xml b/twill-discovery-api/pom.xml
index 2647b17..7db729b 100644
--- a/twill-discovery-api/pom.xml
+++ b/twill-discovery-api/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-discovery-core/pom.xml
----------------------------------------------------------------------
diff --git a/twill-discovery-core/pom.xml b/twill-discovery-core/pom.xml
index 8d78715..fe2c5cb 100644
--- a/twill-discovery-core/pom.xml
+++ b/twill-discovery-core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-examples/echo/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/echo/pom.xml b/twill-examples/echo/pom.xml
index 92a55a9..85a6219 100644
--- a/twill-examples/echo/pom.xml
+++ b/twill-examples/echo/pom.xml
@@ -25,7 +25,7 @@ limitations under the License.
<parent>
<artifactId>twill-examples</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<name>Apache Twill examples: Echo</name>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-examples/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/pom.xml b/twill-examples/pom.xml
index 9381b21..c1f6c45 100644
--- a/twill-examples/pom.xml
+++ b/twill-examples/pom.xml
@@ -25,7 +25,7 @@ limitations under the License.
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<artifactId>twill-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-examples/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/pom.xml b/twill-examples/yarn/pom.xml
index d0b8d20..f2aa401 100644
--- a/twill-examples/yarn/pom.xml
+++ b/twill-examples/yarn/pom.xml
@@ -24,7 +24,7 @@ limitations under the License.
<parent>
<artifactId>twill-examples</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<name>Apache Twill examples: YARN</name>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-ext/pom.xml
----------------------------------------------------------------------
diff --git a/twill-ext/pom.xml b/twill-ext/pom.xml
index f3998a1..e61762e 100644
--- a/twill-ext/pom.xml
+++ b/twill-ext/pom.xml
@@ -22,7 +22,7 @@ limitations under the License.
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/twill-yarn/pom.xml b/twill-yarn/pom.xml
index 7d2091e..2e0454b 100644
--- a/twill-yarn/pom.xml
+++ b/twill-yarn/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b6dea198/twill-zookeeper/pom.xml
----------------------------------------------------------------------
diff --git a/twill-zookeeper/pom.xml b/twill-zookeeper/pom.xml
index c2c0ce4..144b4b5 100644
--- a/twill-zookeeper/pom.xml
+++ b/twill-zookeeper/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.4.0-incubating</version>
+ <version>0.4.1-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>