You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by da...@apache.org on 2015/12/18 02:45:02 UTC
[1/3] storm git commit: Avoid supervisor crashing if nimbus is
unavailable
Repository: storm
Updated Branches:
refs/heads/master 47d0aabc1 -> 418792f84
Avoid supervisor crashing if nimbus is unavailable
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8cf9116e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8cf9116e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8cf9116e
Branch: refs/heads/master
Commit: 8cf9116ee3e635849bce2623f31ac42fde3918f7
Parents: b1d7a03
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Mon Dec 14 15:03:32 2015 -0600
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Mon Dec 14 15:37:22 2015 -0600
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 23 ++++++++---
.../jvm/backtype/storm/utils/NimbusClient.java | 9 +++--
.../utils/NimbusLeaderNotFoundException.java | 41 ++++++++++++++++++++
3 files changed, 65 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8cf9116e/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 3acc348..bbfcb7b 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -24,7 +24,7 @@
[org.apache.commons.io FileUtils])
(:use [backtype.storm config util log timer local-state])
(:import [backtype.storm.generated AuthorizationException KeyNotFoundException WorkerResources])
- (:import [backtype.storm.utils VersionInfo])
+ (:import [backtype.storm.utils NimbusLeaderNotFoundException VersionInfo])
(:import [java.nio.file Files StandardCopyOption])
(:import [backtype.storm Config])
(:import [backtype.storm.generated WorkerResources ProfileAction])
@@ -34,6 +34,7 @@
(:require [backtype.storm.daemon [worker :as worker]]
[backtype.storm [process-simulator :as psim] [cluster :as cluster] [event :as event]]
[clojure.set :as set])
+ (:import [org.apache.thrift.transport TTransportException])
(:import [org.apache.zookeeper data.ACL ZooDefs$Ids ZooDefs$Perms])
(:import [org.yaml.snakeyaml Yaml]
[org.yaml.snakeyaml.constructor SafeConstructor])
@@ -564,7 +565,13 @@
(when (and (not (downloaded-storm-ids storm-id))
(assigned-storm-ids storm-id))
(log-message "Downloading code for storm id " storm-id)
- (download-storm-code conf storm-id master-code-dir localizer)
+ (try-cause
+ (download-storm-code conf storm-id master-code-dir localizer)
+
+ (catch NimbusLeaderNotFoundException e
+ (log-warn-error e "Nimbus leader was not available."))
+ (catch TTransportException e
+ (log-warn-error e "There was a connection problem with nimbus.")))
(log-message "Finished downloading code for storm id " storm-id)))
(log-debug "Writing new assignment "
@@ -617,7 +624,7 @@
by a timer, created elsewhere."
[supervisor]
(fn []
- (try
+ (try-cause
(let [conf (:conf supervisor)
downloaded-storm-ids (set (read-downloaded-storm-ids conf))
new-assignment @(:curr-assignment supervisor)
@@ -627,8 +634,14 @@
(when (assigned-storm-ids topology-id)
(log-debug "Checking Blob updates for storm topology id " topology-id " With target_dir: " storm-root)
(update-blobs-for-topology! conf topology-id (:localizer supervisor))))))
- (catch Exception e
- (log-error e "Error updating blobs, will retry again later")))))
+ (catch TTransportException e
+ (log-error
+ e
+ "Network error while updating blobs, will retry again later"))
+ (catch NimbusLeaderNotFoundException e
+ (log-error
+ e
+ "Nimbus unavailable to update blobs, will retry again later")))))
(defn jvm-cmd [cmd]
(let [java-home (.get (System/getenv) "JAVA_HOME")]
http://git-wip-us.apache.org/repos/asf/storm/blob/8cf9116e/storm-core/src/jvm/backtype/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/NimbusClient.java b/storm-core/src/jvm/backtype/storm/utils/NimbusClient.java
index a9306f7..7e9c58a 100644
--- a/storm-core/src/jvm/backtype/storm/utils/NimbusClient.java
+++ b/storm-core/src/jvm/backtype/storm/utils/NimbusClient.java
@@ -82,12 +82,15 @@ public class NimbusClient extends ThriftClient {
}
}
}
- throw new RuntimeException("Found nimbuses " + nimbuses + " none of which is elected as leader, please try " +
+ throw new NimbusLeaderNotFoundException(
+ "Found nimbuses " + nimbuses + " none of which is elected as leader, please try " +
"again after some time.");
}
}
- throw new RuntimeException("Could not find leader nimbus from seed hosts " + seeds + ". " +
- "Did you specify a valid list of nimbus hosts for config " + Config.NIMBUS_SEEDS);
+ throw new NimbusLeaderNotFoundException(
+ "Could not find leader nimbus from seed hosts " + seeds + ". " +
+ "Did you specify a valid list of nimbus hosts for config " +
+ Config.NIMBUS_SEEDS + "?");
}
public NimbusClient(Map conf, String host, int port) throws TTransportException {
http://git-wip-us.apache.org/repos/asf/storm/blob/8cf9116e/storm-core/src/jvm/backtype/storm/utils/NimbusLeaderNotFoundException.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/NimbusLeaderNotFoundException.java b/storm-core/src/jvm/backtype/storm/utils/NimbusLeaderNotFoundException.java
new file mode 100644
index 0000000..cd0c07d
--- /dev/null
+++ b/storm-core/src/jvm/backtype/storm/utils/NimbusLeaderNotFoundException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.utils;
+
+/**
+ * This exception class is used to signify a problem with nimbus leader
+ * identification. It should not be used when connection failures happen, but
+ * only when successful operations result in the absence of an identified
+ * leader.
+ */
+public class NimbusLeaderNotFoundException extends RuntimeException {
+ public NimbusLeaderNotFoundException() {
+ super();
+ }
+ public NimbusLeaderNotFoundException(String msg) {
+ super(msg);
+ }
+
+ public NimbusLeaderNotFoundException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+ public NimbusLeaderNotFoundException(Throwable cause) {
+ super(cause);
+ }
+}
[3/3] storm git commit: adds STORM-1383
Posted by da...@apache.org.
adds STORM-1383
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/418792f8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/418792f8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/418792f8
Branch: refs/heads/master
Commit: 418792f84c95962a5f2e8d8f98aab73e8138c31a
Parents: 034a3eb
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Thu Dec 17 19:24:50 2015 -0600
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Thu Dec 17 19:24:50 2015 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/418792f8/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d3b092e..51c62bf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-1383: Supervisors should not crash if nimbus is unavailable
* STORM-1381: Client side topology submission hook.
* STORM-1376: Performance slowdown due excessive zk connections and log-debugging
* STORM-1395: Move JUnit dependency to top-level pom
[2/3] storm git commit: Merge branch
'storm-1383-nimbus-supvor-crash-loop' of https://github.com/d2r/storm into
STORM-1383-merge
Posted by da...@apache.org.
Merge branch 'storm-1383-nimbus-supvor-crash-loop' of https://github.com/d2r/storm into STORM-1383-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/034a3eb5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/034a3eb5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/034a3eb5
Branch: refs/heads/master
Commit: 034a3eb5be6b8b465ead68f4855e7520cc09e0a4
Parents: 47d0aab 8cf9116
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Thu Dec 17 19:24:12 2015 -0600
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Thu Dec 17 19:24:12 2015 -0600
----------------------------------------------------------------------
.../clj/backtype/storm/daemon/supervisor.clj | 23 ++++++++---
.../jvm/backtype/storm/utils/NimbusClient.java | 9 +++--
.../utils/NimbusLeaderNotFoundException.java | 41 ++++++++++++++++++++
3 files changed, 65 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/034a3eb5/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------