You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2017/02/24 18:16:10 UTC
[01/11] storm git commit: Fixing pacemaker delete-path bug.
Repository: storm
Updated Branches:
refs/heads/1.0.x-branch 38e4c64f2 -> 3a4773be8
refs/heads/1.x-branch ba9f0e038 -> 0c4fbacbd
refs/heads/master 03e6b8419 -> c73c17a3e
Fixing pacemaker delete-path bug.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ebed1c8b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ebed1c8b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ebed1c8b
Branch: refs/heads/1.x-branch
Commit: ebed1c8b01397b09f4083e66f574a25f9b7c585d
Parents: ab3ed3e
Author: Kyle Nusbaum <kn...@yahoo-inc.com>
Authored: Tue Feb 21 14:18:31 2017 -0600
Committer: Kyle Nusbaum <kn...@yahoo-inc.com>
Committed: Tue Feb 21 16:02:33 2017 -0600
----------------------------------------------------------------------
.../src/clj/org/apache/storm/pacemaker/pacemaker.clj | 2 +-
.../storm/pacemaker/pacemaker_state_factory.clj | 15 ++++++++++++---
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ebed1c8b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
index 70313e4..2b26a8a 100644
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
@@ -160,7 +160,7 @@
(defn delete-path [^String path heartbeats]
(let [prefix (if (= \/ (last path)) path (str path "/"))]
(doseq [k (.keySet heartbeats)
- :when (= (.indexOf k prefix) 0)]
+ :when (= (.indexOf (str k "/") prefix) 0)]
(delete-pulse-id k heartbeats)))
(HBMessage. HBServerMessageType/DELETE_PATH_RESPONSE nil))
http://git-wip-us.apache.org/repos/asf/storm/blob/ebed1c8b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
index 2bbeda4..3889dcf 100644
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
@@ -200,9 +200,18 @@
(pacemaker-retry-on-exception
max-retries
"delete-worker-hb"
- #(delete-worker-hb path (get-pacemaker-write-client conf servers pacemaker-client-pool))
- (fn delete_worker_hb_error [err]
- (shutdown-rotate servers pacemaker-client-pool))))
+ #(let [pacemaker-client-pool (makeClientPool conf pacemaker-client-pool servers)
+ results (map (fn [[host client]]
+ (try
+ (if (is-connection-ready client)
+ (delete-worker-hb path client)
+ :error)
+ (catch Exception e
+ :error)))
+ @pacemaker-client-pool)]
+ (when (every? (fn [result] (= :error result)) results)
+ (throw (HBExecutionException. "Cannot connect to any pacemaker servers"))))
+ nil))
;; aggregating worker heartbeat details
(get_worker_hb [this path watch?]
[10/11] storm git commit: Merge branch 'master' of
github.com:knusbaum/incubator-storm
Posted by pt...@apache.org.
Merge branch 'master' of github.com:knusbaum/incubator-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2c6ab5aa
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2c6ab5aa
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2c6ab5aa
Branch: refs/heads/master
Commit: 2c6ab5aa68b8ac008ed3a1a379a997e881f1a4bc
Parents: 03e6b84 469f025
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Feb 24 13:15:17 2017 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Feb 24 13:15:17 2017 -0500
----------------------------------------------------------------------
.../storm/cluster/PaceMakerStateStorage.java | 34 ++++++++++++++++----
.../org/apache/storm/pacemaker/Pacemaker.java | 3 +-
2 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[09/11] storm git commit: add STORM-2372 to changelog
Posted by pt...@apache.org.
add STORM-2372 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3a4773be
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3a4773be
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3a4773be
Branch: refs/heads/1.0.x-branch
Commit: 3a4773be8732d5788b6a6362de6a94a49e9f40a0
Parents: dbc4868
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Feb 24 13:13:59 2017 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Feb 24 13:13:59 2017 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3a4773be/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f8895ee..6e723be 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
\ufeff## 1.0.4
+ * STORM-2372: Pacemaker client doesn't clean up heartbeats properly
* STORM-2326: Upgrade log4j and slf4j
* STORM-2350: Storm-HDFS's listFilesByModificationTime is broken
[05/11] storm git commit: Kick Travis (empty)
Posted by pt...@apache.org.
Kick Travis (empty)
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/469f0256
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/469f0256
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/469f0256
Branch: refs/heads/master
Commit: 469f0256dc639c26b92cc8c1fc97789ca176ece0
Parents: 6839fec
Author: Kyle Nusbaum <kn...@yahoo-inc.com>
Authored: Thu Feb 23 10:48:34 2017 -0600
Committer: Kyle Nusbaum <kn...@yahoo-inc.com>
Committed: Thu Feb 23 10:48:34 2017 -0600
----------------------------------------------------------------------
----------------------------------------------------------------------
[04/11] storm git commit: Fixing minor issue.
Posted by pt...@apache.org.
Fixing minor issue.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6839fec1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6839fec1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6839fec1
Branch: refs/heads/master
Commit: 6839fec10b94a56f6d9614547f4cabafcd9b355a
Parents: a64f135
Author: Kyle Nusbaum <kn...@yahoo-inc.com>
Authored: Wed Feb 22 12:26:31 2017 -0600
Committer: Kyle Nusbaum <kn...@yahoo-inc.com>
Committed: Wed Feb 22 12:26:31 2017 -0600
----------------------------------------------------------------------
storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6839fec1/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java b/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
index c49e5a0..725f869 100644
--- a/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
+++ b/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
@@ -190,8 +190,8 @@ public class Pacemaker implements IServerMessageHandler {
private HBMessage deletePath(String path) {
String prefix = path.endsWith("/") ? path : (path + "/");
for (String key : heartbeats.keySet()) {
- key = key + "/";
- if (key.indexOf(prefix) == 0)
+ String checkKey = key + "/";
+ if (checkKey.indexOf(prefix) == 0)
deletePulseId(key);
}
return new HBMessage(HBServerMessageType.DELETE_PATH_RESPONSE, null);
[02/11] storm git commit: Fixing pacemaker delete-path bug.
Posted by pt...@apache.org.
Fixing pacemaker delete-path bug.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a6b9668c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a6b9668c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a6b9668c
Branch: refs/heads/1.0.x-branch
Commit: a6b9668c98b46a7acfb9d4f39c6e90342b2f41f7
Parents: 07410fe
Author: Kyle Nusbaum <kn...@yahoo-inc.com>
Authored: Tue Feb 21 14:18:31 2017 -0600
Committer: Kyle Nusbaum <kn...@yahoo-inc.com>
Committed: Tue Feb 21 16:06:24 2017 -0600
----------------------------------------------------------------------
.../org/apache/storm/pacemaker/pacemaker.clj | 2 +-
.../storm/pacemaker/pacemaker_state_factory.clj | 24 ++++++++++++--------
2 files changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a6b9668c/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
index 70313e4..2b26a8a 100644
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker.clj
@@ -160,7 +160,7 @@
(defn delete-path [^String path heartbeats]
(let [prefix (if (= \/ (last path)) path (str path "/"))]
(doseq [k (.keySet heartbeats)
- :when (= (.indexOf k prefix) 0)]
+ :when (= (.indexOf (str k "/") prefix) 0)]
(delete-pulse-id k heartbeats)))
(HBMessage. HBServerMessageType/DELETE_PATH_RESPONSE nil))
http://git-wip-us.apache.org/repos/asf/storm/blob/a6b9668c/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
index 80d4de5..21f3ee1 100644
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
@@ -86,15 +86,21 @@
(delete_worker_hb [this path]
(util/retry-on-exception
max-retries
- "delete_worker_hb"
- #(let [response
- (.send pacemaker-client
- (HBMessage. HBServerMessageType/DELETE_PATH
- (HBMessageData/path path)))]
- (if (= (.get_type response) HBServerMessageType/DELETE_PATH_RESPONSE)
- :ok
- (throw (HBExecutionException. "Invalid Response Type"))))))
-
+ "delete-worker-hb"
+ #(let [pacemaker-client-pool (makeClientPool conf pacemaker-client-pool servers)
+ results (map (fn [[host client]]
+ (try
+ (if (is-connection-ready client)
+ (delete-worker-hb path client)
+ :error)
+ (catch Exception e
+ :error)))
+ @pacemaker-client-pool)]
+ (when (every? (fn [result] (= :error result)) results)
+ (throw (HBExecutionException. "Cannot connect to any pacemaker servers"))))
+ nil))
+
+ ;; aggregating worker heartbeat details
(get_worker_hb [this path watch?]
(util/retry-on-exception
max-retries
[06/11] storm git commit: Merge branch '1.x-branch' of
github.com:knusbaum/incubator-storm into 1.x-branch
Posted by pt...@apache.org.
Merge branch '1.x-branch' of github.com:knusbaum/incubator-storm into 1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/65f33fdc
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/65f33fdc
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/65f33fdc
Branch: refs/heads/1.x-branch
Commit: 65f33fdcb9b342c869ca8ef494fd836be2f2ebb7
Parents: ba9f0e0 ebed1c8
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Feb 24 13:11:48 2017 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Feb 24 13:11:48 2017 -0500
----------------------------------------------------------------------
.../src/clj/org/apache/storm/pacemaker/pacemaker.clj | 2 +-
.../storm/pacemaker/pacemaker_state_factory.clj | 15 ++++++++++++---
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[11/11] storm git commit: add STORM-2372 to changelog
Posted by pt...@apache.org.
add STORM-2372 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c73c17a3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c73c17a3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c73c17a3
Branch: refs/heads/master
Commit: c73c17a3e13ed61c2add1f5701bd28566f72c6f0
Parents: 2c6ab5a
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Feb 24 13:15:54 2017 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Feb 24 13:15:54 2017 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c73c17a3/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index efd7962..9fd4e52 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
\ufeff## 2.0.0
+ * STORM-2372: Pacemaker client doesn't clean up heartbeats properly
* STORM-2334: Join Bolt implementation
* STORM-2367: Documentation for streams API
* STORM-2365: Support for specifying output stream in event hubs spout
[07/11] storm git commit: add STORM-2372 to changelog
Posted by pt...@apache.org.
add STORM-2372 to changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0c4fbacb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0c4fbacb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0c4fbacb
Branch: refs/heads/1.x-branch
Commit: 0c4fbacbd2aadbf79cecd3d3fe103e79ffcf0fd2
Parents: 65f33fd
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Feb 24 13:12:46 2017 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Feb 24 13:12:46 2017 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0c4fbacb/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7a6d18d..56a5e28 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
\ufeff## 1.1.0
+ * STORM-2372: Pacemaker client doesn't clean up heartbeats properly
* STORM-2326: Upgrade log4j and slf4j
* STORM-2334: Join Bolt implementation
* STORM-1363: TridentKafkaState should handle null values from TridentTupleToKafkaMapper.getMessageFromTuple()
[08/11] storm git commit: Merge branch '1.0.x-branch' of
github.com:knusbaum/incubator-storm into 1.0.x-branch
Posted by pt...@apache.org.
Merge branch '1.0.x-branch' of github.com:knusbaum/incubator-storm into 1.0.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/dbc48680
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/dbc48680
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/dbc48680
Branch: refs/heads/1.0.x-branch
Commit: dbc486803b72fa95d3267c693c912f01d4c12fef
Parents: 38e4c64 a6b9668
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Feb 24 13:13:38 2017 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Feb 24 13:13:38 2017 -0500
----------------------------------------------------------------------
.../org/apache/storm/pacemaker/pacemaker.clj | 2 +-
.../storm/pacemaker/pacemaker_state_factory.clj | 24 ++++++++++++--------
2 files changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
[03/11] storm git commit: Fixing pacemaker delete_pulse
Posted by pt...@apache.org.
Fixing pacemaker delete_pulse
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a64f135d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a64f135d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a64f135d
Branch: refs/heads/master
Commit: a64f135d2859d178e4645787e99638a84e1b192f
Parents: 15887ee
Author: Kyle Nusbaum <kn...@yahoo-inc.com>
Authored: Tue Feb 21 17:14:09 2017 -0600
Committer: Kyle Nusbaum <kn...@yahoo-inc.com>
Committed: Tue Feb 21 17:14:09 2017 -0600
----------------------------------------------------------------------
.../storm/cluster/PaceMakerStateStorage.java | 34 ++++++++++++++++----
.../org/apache/storm/pacemaker/Pacemaker.java | 1 +
2 files changed, 28 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a64f135d/storm-core/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java b/storm-core/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
index 3d7d402..e58746d 100644
--- a/storm-core/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
+++ b/storm-core/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
@@ -210,21 +210,41 @@ public class PaceMakerStateStorage implements IStateStorage {
@Override
public void delete_worker_hb(String path) {
int retry = maxRetries;
+ boolean someSucceeded;
while (true) {
+ someSucceeded = false;
try {
HBMessage message = new HBMessage(HBServerMessageType.DELETE_PATH, HBMessageData.path(path));
- HBMessage response = pacemakerClientPool.send(message);
- if (response.get_type() != HBServerMessageType.DELETE_PATH_RESPONSE) {
- throw new HBExecutionException("Invalid Response Type");
+ List<HBMessage> responses = pacemakerClientPool.sendAll(message);
+ boolean allSucceeded = true;
+ for(HBMessage response : responses) {
+ if (response.get_type() != HBServerMessageType.DELETE_PATH_RESPONSE) {
+ LOG.debug("Failed to delete heartbeat {}", response);
+ allSucceeded = false;
+ }
+ else {
+ someSucceeded = true;
+ }
+ }
+ if(allSucceeded) {
+ break;
+ }
+ else {
+ throw new HBExecutionException("Failed to delete from all pacemakers.");
}
- LOG.debug("Successful get_worker_hb");
- break;
} catch (Exception e) {
if (retry <= 0) {
- throw Utils.wrapInRuntime(e);
+ if(someSucceeded) {
+ LOG.warn("Unable to delete_worker_hb from every pacemaker.");
+ break;
+ }
+ else {
+ LOG.error("Unable to delete_worker_hb from any pacemaker.");
+ throw Utils.wrapInRuntime(e);
+ }
}
retry--;
- LOG.error("{} Failed to delete_worker_hb. Will make {} more attempts.", e.getMessage(), retry);
+ LOG.debug("{} Failed to delete_worker_hb. Will make {} more attempts.", e.getMessage(), retry);
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/a64f135d/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java b/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
index 0becf68..c49e5a0 100644
--- a/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
+++ b/storm-core/src/jvm/org/apache/storm/pacemaker/Pacemaker.java
@@ -190,6 +190,7 @@ public class Pacemaker implements IServerMessageHandler {
private HBMessage deletePath(String path) {
String prefix = path.endsWith("/") ? path : (path + "/");
for (String key : heartbeats.keySet()) {
+ key = key + "/";
if (key.indexOf(prefix) == 0)
deletePulseId(key);
}