You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2017/04/13 19:32:16 UTC
[1/7] storm git commit: STORM-1114: Handle race condition in
Storm/Trident transactional state when ZK nodes have already been
created/deleted
Repository: storm
Updated Branches:
refs/heads/1.0.x-branch 047166deb -> 8af1343cd
refs/heads/1.x-branch d185367fb -> c5ea9e355
refs/heads/master 1f5caca01 -> 1a17f7416
STORM-1114: Handle race condition in Storm/Trident transactional state when ZK nodes have already been created/deleted
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/41fdd98f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/41fdd98f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/41fdd98f
Branch: refs/heads/master
Commit: 41fdd98fffff52c346e5358ac8e704fc5b49ea66
Parents: d8368b0
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Apr 12 14:28:35 2017 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Apr 12 14:28:35 2017 -0400
----------------------------------------------------------------------
.../transactional/state/TransactionalState.java | 26 +++++++++++++++-----
.../topology/state/TransactionalState.java | 7 ++++--
2 files changed, 25 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/41fdd98f/storm-client/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/transactional/state/TransactionalState.java b/storm-client/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
index 419baaf..eb50522 100644
--- a/storm-client/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
+++ b/storm-client/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
@@ -33,8 +33,11 @@ import java.util.Map;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TransactionalState {
+ public static final Logger LOG = LoggerFactory.getLogger(TransactionalState.class);
CuratorFramework _curator;
KryoValuesSerializer _ser;
KryoValuesDeserializer _des;
@@ -82,11 +85,17 @@ public class TransactionalState {
}
}
- protected static String forPath(PathAndBytesable<String> builder,
+ protected static void forPath(PathAndBytesable<String> builder,
String path, byte[] data) throws Exception {
- return (data == null)
- ? builder.forPath(path)
- : builder.forPath(path, data);
+ try {
+ if (data == null) {
+ builder.forPath(path);
+ } else {
+ builder.forPath(path, data);
+ }
+ } catch (KeeperException.NodeExistsException e){
+ LOG.info("Path {} already exists.", path);
+ }
}
protected static void createNode(CuratorFramework curator, String path,
@@ -116,15 +125,20 @@ public class TransactionalState {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
- } catch(Exception e) {
+ } catch (KeeperException.NodeExistsException nee) {
+ LOG.warn("Path {} already exists.", path);
+ } catch (Exception e) {
throw new RuntimeException(e);
- }
+ }
}
public void delete(String path) {
path = "/" + path;
try {
_curator.delete().forPath(path);
+ } catch (KeeperException.NoNodeException nne){
+ // node was already deleted
+ LOG.info("Path {} has already been deleted.", path);
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/41fdd98f/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
index ae2f77b..7a3bf19 100644
--- a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
+++ b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
@@ -126,16 +126,19 @@ public class TransactionalState {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
- LOG.debug("Set [path = {}] => [data = {}]", path, asString(ser));
+ } catch (KeeperException.NodeExistsException nne){
+ LOG.warn("Node {} already created.", path);
} catch(Exception e) {
throw new RuntimeException(e);
- }
+ }
}
public void delete(String path) {
path = "/" + path;
try {
_curator.delete().forPath(path);
+ } catch (KeeperException.NoNodeException nne){
+ LOG.warn("Path {} already deleted.");
} catch (Exception e) {
throw new RuntimeException(e);
}
[7/7] storm git commit: add STORM-1114 to changelog
Posted by pt...@apache.org.
add STORM-1114 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8af1343c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8af1343c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8af1343c
Branch: refs/heads/1.0.x-branch
Commit: 8af1343cd4f912f2d68a708e1e8d23698aa7d68a
Parents: adb14cc
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Apr 13 15:31:59 2017 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Apr 13 15:31:59 2017 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8af1343c/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a35e499..3dcde7c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
\ufeff## 1.0.4
+ * STORM-1114: Race condition in trident zookeeper zk-node create/delete
* STORM-2429: Properly validate supervisor.scheduler.meta
* STORM-2194: Stop ignoring socket timeout error from executor
* STORM-2450: Write resources into correct local director
[5/7] storm git commit: add STORM-1114 to changelog
Posted by pt...@apache.org.
add STORM-1114 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c5ea9e35
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c5ea9e35
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c5ea9e35
Branch: refs/heads/1.x-branch
Commit: c5ea9e35516d056039c48698b1d0bdf83adac268
Parents: 9cf791b
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Apr 13 15:16:14 2017 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Apr 13 15:16:14 2017 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c5ea9e35/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 44a3a1e..a13dae1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
\ufeff## 1.1.1
+ * STORM-1114: Race condition in trident zookeeper zk-node create/delete
* STORM-2429: Properly validate supervisor.scheduler.meta
* STORM-2194: Stop ignoring socket timeout error from executor
* STORM-2451: windows storm.cmd does not set log4j2 config file correctly by default
[6/7] storm git commit: STORM-1114: Handle race condition in
Storm/Trident transactional state when ZK nodes have already been
created/deleted
Posted by pt...@apache.org.
STORM-1114: Handle race condition in Storm/Trident transactional state when ZK nodes have already been created/deleted
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/adb14ccb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/adb14ccb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/adb14ccb
Branch: refs/heads/1.0.x-branch
Commit: adb14ccbeb6f446eff9b4d71323ab8451b4b45d5
Parents: 047166d
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Apr 13 15:31:08 2017 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Apr 13 15:31:08 2017 -0400
----------------------------------------------------------------------
.../transactional/state/TransactionalState.java | 22 ++++++++++++++++----
.../topology/state/TransactionalState.java | 7 +++++++
2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/adb14ccb/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java b/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
index f05512e..bec5f66 100644
--- a/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
+++ b/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
@@ -34,8 +34,11 @@ import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TransactionalState {
+ public static final Logger LOG = LoggerFactory.getLogger(TransactionalState.class);
CuratorFramework _curator;
KryoValuesSerializer _ser;
KryoValuesDeserializer _des;
@@ -83,11 +86,17 @@ public class TransactionalState {
}
}
- protected static String forPath(PathAndBytesable<String> builder,
+ protected static void forPath(PathAndBytesable<String> builder,
String path, byte[] data) throws Exception {
- return (data == null)
- ? builder.forPath(path)
- : builder.forPath(path, data);
+ try {
+ if (data == null) {
+ builder.forPath(path);
+ } else {
+ builder.forPath(path, data);
+ }
+ } catch (KeeperException.NodeExistsException e){
+ LOG.info("Path {} already exists.", path);
+ }
}
protected static void createNode(CuratorFramework curator, String path,
@@ -117,6 +126,8 @@ public class TransactionalState {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
+ } catch (KeeperException.NodeExistsException nee) {
+ LOG.warn("Path {} already exists.", path);
} catch(Exception e) {
throw new RuntimeException(e);
}
@@ -126,6 +137,9 @@ public class TransactionalState {
path = "/" + path;
try {
_curator.delete().forPath(path);
+ } catch (KeeperException.NoNodeException nne){
+ // node was already deleted
+ LOG.info("Path {} has already been deleted.", path);
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/adb14ccb/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java b/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
index 906906d..5061590 100644
--- a/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
+++ b/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
@@ -37,8 +37,11 @@ import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.json.simple.JSONValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TransactionalState {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionalState.class);
CuratorFramework _curator;
List<ACL> _zkAcls = null;
@@ -115,6 +118,8 @@ public class TransactionalState {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
+ } catch (KeeperException.NodeExistsException nne){
+ LOG.warn("Node {} already created.", path);
} catch(Exception e) {
throw new RuntimeException(e);
}
@@ -124,6 +129,8 @@ public class TransactionalState {
path = "/" + path;
try {
_curator.delete().forPath(path);
+ } catch (KeeperException.NoNodeException nne){
+ LOG.warn("Path {} already deleted.");
} catch (Exception e) {
throw new RuntimeException(e);
}
[2/7] storm git commit: Merge branch 'STORM-1114' of
github.com:ptgoetz/storm
Posted by pt...@apache.org.
Merge branch 'STORM-1114' of github.com:ptgoetz/storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/66a9e772
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/66a9e772
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/66a9e772
Branch: refs/heads/master
Commit: 66a9e77294ef36bf1210d828cf9da1318d06677a
Parents: 1f5caca 41fdd98
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Apr 13 14:48:35 2017 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Apr 13 14:48:35 2017 -0400
----------------------------------------------------------------------
.../transactional/state/TransactionalState.java | 26 +++++++++++++++-----
.../topology/state/TransactionalState.java | 7 ++++--
2 files changed, 25 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[4/7] storm git commit: STORM-1114: Handle race condition in
Storm/Trident transactional state when ZK nodes have already been
created/deleted
Posted by pt...@apache.org.
STORM-1114: Handle race condition in Storm/Trident transactional state when ZK nodes have already been created/deleted
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9cf791b8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9cf791b8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9cf791b8
Branch: refs/heads/1.x-branch
Commit: 9cf791b876918bc9eb99757e180d7dfeca55ed87
Parents: d185367
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Apr 13 15:08:16 2017 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Apr 13 15:08:16 2017 -0400
----------------------------------------------------------------------
.../apache/storm/transactional/state/TransactionalState.java | 7 +++++++
.../storm/trident/topology/state/TransactionalState.java | 7 +++++--
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9cf791b8/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java b/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
index f05512e..7662daa 100644
--- a/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
+++ b/storm-core/src/jvm/org/apache/storm/transactional/state/TransactionalState.java
@@ -34,8 +34,11 @@ import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TransactionalState {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionalState.class);
CuratorFramework _curator;
KryoValuesSerializer _ser;
KryoValuesDeserializer _des;
@@ -117,6 +120,8 @@ public class TransactionalState {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
+ } catch (KeeperException.NodeExistsException nne){
+ LOG.warn("Node {} already created.", path);
} catch(Exception e) {
throw new RuntimeException(e);
}
@@ -126,6 +131,8 @@ public class TransactionalState {
path = "/" + path;
try {
_curator.delete().forPath(path);
+ } catch (KeeperException.NoNodeException nne){
+ LOG.warn("Path {} already deleted.");
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/9cf791b8/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java b/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
index 26ac404..71068dc 100644
--- a/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
+++ b/storm-core/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java
@@ -125,16 +125,19 @@ public class TransactionalState {
TransactionalState.createNode(_curator, path, ser, _zkAcls,
CreateMode.PERSISTENT);
}
- LOG.debug("Set [path = {}] => [data = {}]", path, asString(ser));
+ } catch (KeeperException.NodeExistsException nne){
+ LOG.warn("Node {} already created.", path);
} catch(Exception e) {
throw new RuntimeException(e);
- }
+ }
}
public void delete(String path) {
path = "/" + path;
try {
_curator.delete().forPath(path);
+ } catch (KeeperException.NoNodeException nne){
+ LOG.warn("Path {} already deleted.");
} catch (Exception e) {
throw new RuntimeException(e);
}
[3/7] storm git commit: add STORM-1114 to changelog
Posted by pt...@apache.org.
add STORM-1114 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1a17f741
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1a17f741
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1a17f741
Branch: refs/heads/master
Commit: 1a17f741657b723a8299b67c8cc1ae80fd5f0690
Parents: 66a9e77
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Apr 13 14:49:59 2017 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Apr 13 14:49:59 2017 -0400
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1a17f741/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1c70e68..edf384d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
\ufeff## 2.0.0
+ * STORM-1114: Race condition in trident zookeeper zk-node create/delete
* STORM-2469: fix integration-test dependencies issue
* STORM-2455: Expose the window start and end timestamp in TupleWindow
* STORM-2435: Logging levels and consistency with console.log etc