You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/04/24 15:45:28 UTC
[1/9] storm git commit: Nimbus should shutdown on OOM across any
thread
Repository: storm
Updated Branches:
refs/heads/master 37a7e8bb0 -> b4351ede1
Nimbus should shutdown on OOM across any thread
Conflicts:
storm-core/src/clj/backtype/storm/util.clj
storm-core/src/jvm/backtype/storm/utils/Utils.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/58da2016
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/58da2016
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/58da2016
Branch: refs/heads/master
Commit: 58da2016157c1c358f70cf0aceca8b9c56cffb0c
Parents: 1ea378d
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Thu Feb 19 16:51:40 2015 -0600
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Mar 18 12:01:24 2015 -0500
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/nimbus.clj | 1 +
storm-core/src/clj/backtype/storm/util.clj | 9 ++++
.../security/auth/SaslTransportPlugin.java | 16 ++++---
.../storm/utils/ExtendedThreadPoolExecutor.java | 47 ++++++++++++++++++++
.../src/jvm/backtype/storm/utils/Utils.java | 11 +++++
5 files changed, 79 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/58da2016/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index 5caf0a3..dd0d193 100644
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@ -1414,4 +1414,5 @@
))
(defn -main []
+ (setup-default-uncaught-exception-handler)
(-launch (standalone-nimbus)))
http://git-wip-us.apache.org/repos/asf/storm/blob/58da2016/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj
index 36dc62c..b1a9b2e 100644
--- a/storm-core/src/clj/backtype/storm/util.clj
+++ b/storm-core/src/clj/backtype/storm/util.clj
@@ -1031,3 +1031,12 @@
(defn hashmap-to-persistent [^HashMap m]
(zipmap (.keySet m) (.values m)))
+
+(defn setup-default-uncaught-exception-handler
+ "Set a default uncaught exception handler to handle exceptions not caught in other threads."
+ []
+ (Thread/setDefaultUncaughtExceptionHandler
+ (proxy [Thread$UncaughtExceptionHandler] []
+ (uncaughtException [thread thrown]
+ (Utils/handleUncaughtException thrown)))))
+
http://git-wip-us.apache.org/repos/asf/storm/blob/58da2016/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java b/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
index 532ebd7..91f1d69 100644
--- a/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
+++ b/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.net.Socket;
import java.security.Principal;
import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -28,6 +30,8 @@ import java.util.concurrent.TimeUnit;
import javax.security.auth.Subject;
import javax.security.auth.login.Configuration;
import javax.security.sasl.SaslServer;
+
+import backtype.storm.utils.ExtendedThreadPoolExecutor;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TBinaryProtocol;
@@ -78,11 +82,13 @@ public abstract class SaslTransportPlugin implements ITransportPlugin {
if (serverTransportFactory != null) {
server_args.transportFactory(serverTransportFactory);
}
-
- //if (queueSize != null) {
- // server_args.executorService(new ThreadPoolExecutor(numWorkerThreads, numWorkerThreads,
- // 60, TimeUnit.SECONDS, new ArrayBlockingQueue(queueSize)));
- //}
+ BlockingQueue workQueue = new SynchronousQueue();
+ if (queueSize != null) {
+ workQueue = new ArrayBlockingQueue(queueSize);
+ }
+ ThreadPoolExecutor executorService = new ExtendedThreadPoolExecutor(numWorkerThreads, numWorkerThreads,
+ 60, TimeUnit.SECONDS, workQueue);
+ server_args.executorService(executorService);
return new TThreadPoolServer(server_args);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/58da2016/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
new file mode 100644
index 0000000..36eeefa
--- /dev/null
+++ b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
@@ -0,0 +1,47 @@
+package backtype.storm.utils;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public class ExtendedThreadPoolExecutor extends ThreadPoolExecutor{
+
+ public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue) {
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
+ }
+
+ public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory) {
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory);
+ }
+
+ public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, RejectedExecutionHandler handler) {
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler);
+ }
+
+ public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory, RejectedExecutionHandler handler) {
+ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
+ }
+
+ protected void afterExecute(Runnable r, Throwable t) {
+ super.afterExecute(r, t);
+ if (t == null && r instanceof Future<?>) {
+ try {
+ Object result = ((Future<?>) r).get();
+ } catch (CancellationException ce) {
+ t = ce;
+ } catch (ExecutionException ee) {
+ t = ee.getCause();
+
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt(); // ignore/reset
+ }
+ }
+ Utils.handleUncaughtException(t);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/58da2016/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 6e8458a..6220bb4 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -533,4 +533,15 @@ public class Utils {
delegate.prepare(stormConf);
return delegate;
}
+
+ public static void handleUncaughtException(Throwable t) {
+ if(t!= null && t instanceof OutOfMemoryError) {
+ try {
+ System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName());
+ } catch (Throwable err) {
+ //Again we done want to exit because of logging issues.
+ }
+ Runtime.getRuntime().halt(-1);
+ }
+ }
}
[2/9] storm git commit: Adding license,
formatting and improving comments
Posted by bo...@apache.org.
Adding license, formatting and improving comments
Conflicts:
storm-core/src/jvm/backtype/storm/utils/Utils.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/720a93ee
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/720a93ee
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/720a93ee
Branch: refs/heads/master
Commit: 720a93ee92903d4ad4c4ab299042624fe4202b37
Parents: 58da201
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Fri Feb 20 17:43:19 2015 -0600
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Mar 18 12:02:31 2015 -0500
----------------------------------------------------------------------
.../security/auth/SaslTransportPlugin.java | 5 ++--
.../storm/utils/ExtendedThreadPoolExecutor.java | 24 +++++++++++++++++---
.../src/jvm/backtype/storm/utils/Utils.java | 1 -
3 files changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/720a93ee/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java b/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
index 91f1d69..7208a17 100644
--- a/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
+++ b/storm-core/src/jvm/backtype/storm/security/auth/SaslTransportPlugin.java
@@ -84,12 +84,11 @@ public abstract class SaslTransportPlugin implements ITransportPlugin {
}
BlockingQueue workQueue = new SynchronousQueue();
if (queueSize != null) {
- workQueue = new ArrayBlockingQueue(queueSize);
+ workQueue = new ArrayBlockingQueue(queueSize);
}
ThreadPoolExecutor executorService = new ExtendedThreadPoolExecutor(numWorkerThreads, numWorkerThreads,
- 60, TimeUnit.SECONDS, workQueue);
+ 60, TimeUnit.SECONDS, workQueue);
server_args.executorService(executorService);
-
return new TThreadPoolServer(server_args);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/720a93ee/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
index 36eeefa..6730780 100644
--- a/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
+++ b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
@@ -1,5 +1,23 @@
package backtype.storm.utils;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
@@ -27,6 +45,7 @@ public class ExtendedThreadPoolExecutor extends ThreadPoolExecutor{
super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
}
+ @Override
protected void afterExecute(Runnable r, Throwable t) {
super.afterExecute(r, t);
if (t == null && r instanceof Future<?>) {
@@ -36,12 +55,11 @@ public class ExtendedThreadPoolExecutor extends ThreadPoolExecutor{
t = ce;
} catch (ExecutionException ee) {
t = ee.getCause();
-
} catch (InterruptedException ie) {
- Thread.currentThread().interrupt(); // ignore/reset
+ // If future got interrupted exception, we want to main parent thread itself.
+ Thread.currentThread().interrupt();
}
}
Utils.handleUncaughtException(t);
}
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/720a93ee/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 6220bb4..927683b 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -543,5 +543,4 @@ public class Utils {
}
Runtime.getRuntime().halt(-1);
}
- }
}
[9/9] storm git commit: Added STORM-712 to Changelog
Posted by bo...@apache.org.
Added STORM-712 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b4351ede
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b4351ede
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b4351ede
Branch: refs/heads/master
Commit: b4351ede1a89b83888e9e3ace55227e5ae69c78b
Parents: 0cac820
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Apr 23 15:59:11 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Apr 23 15:59:11 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b4351ede/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a83ba5..fd140ea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-712: Storm daemons shutdown if OutOfMemoryError occurs in any thread
* STORM-730: remove extra curly brace
* STORM-735: [storm-redis] Upgrade Jedis to 2.7.0
* STORM-747: assignment-version-callback/info-with-version-callback are not fired when assignments change
[4/9] storm git commit: Daemons should shutdown on OOM across any
thread
Posted by bo...@apache.org.
Daemons should shutdown on OOM across any thread
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9ee87180
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9ee87180
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9ee87180
Branch: refs/heads/master
Commit: 9ee8718029b79b984e57f7624709d44e4703f400
Parents: 3235340
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Tue Mar 17 10:00:53 2015 -0500
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Mar 18 17:56:19 2015 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/drpc.clj | 1 +
.../src/clj/backtype/storm/daemon/logviewer.clj | 1 +
.../src/clj/backtype/storm/daemon/supervisor.clj | 1 +
storm-core/src/clj/backtype/storm/daemon/worker.clj | 1 +
storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java | 9 ++++++++-
storm-core/src/jvm/backtype/storm/utils/Utils.java | 14 ++++++++------
6 files changed, 20 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9ee87180/storm-core/src/clj/backtype/storm/daemon/drpc.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/drpc.clj b/storm-core/src/clj/backtype/storm/daemon/drpc.clj
index cb9b1d2..e470188 100644
--- a/storm-core/src/clj/backtype/storm/daemon/drpc.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/drpc.clj
@@ -240,4 +240,5 @@
(.serve handler-server)))))
(defn -main []
+ (setup-default-uncaught-exception-handler)
(launch-server!))
http://git-wip-us.apache.org/repos/asf/storm/blob/9ee87180/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
index 7af1d5d..dee7af2 100644
--- a/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
@@ -377,5 +377,6 @@ Note that if anything goes wrong, this will throw an Error and exit."
(defn -main []
(let [conf (read-storm-config)
log-root (log-root-dir (conf LOGVIEWER-APPENDER-NAME))]
+ (setup-default-uncaught-exception-handler)
(start-log-cleaner! conf log-root)
(start-logviewer! conf log-root)))
http://git-wip-us.apache.org/repos/asf/storm/blob/9ee87180/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index bc8b999..cced4f1 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -800,4 +800,5 @@
))))
(defn -main []
+ (setup-default-uncaught-exception-handler)
(-launch (standalone-supervisor)))
http://git-wip-us.apache.org/repos/asf/storm/blob/9ee87180/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/worker.clj b/storm-core/src/clj/backtype/storm/daemon/worker.clj
index dad9354..fbe7565 100644
--- a/storm-core/src/clj/backtype/storm/daemon/worker.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj
@@ -544,6 +544,7 @@
(defn -main [storm-id assignment-id port-str worker-id]
(let [conf (read-storm-config)]
+ (setup-default-uncaught-exception-handler)
(validate-distributed-mode! conf)
(let [worker (mk-worker conf nil storm-id assignment-id (Integer/parseInt port-str) worker-id)]
(add-shutdown-hook-with-force-kill-in-1-sec #(.shutdown worker)))))
http://git-wip-us.apache.org/repos/asf/storm/blob/9ee87180/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java b/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
index cbfb310..632f05a 100644
--- a/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
+++ b/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
@@ -28,6 +28,7 @@ import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
+import backtype.storm.utils.ExtendedThreadPoolExecutor;
import backtype.storm.utils.ServiceRegistry;
import backtype.storm.utils.Utils;
import java.util.ArrayList;
@@ -40,6 +41,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.Callable;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.thrift.TException;
@@ -129,7 +134,9 @@ public class DRPCSpout extends BaseRichSpout {
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
_collector = collector;
if(_local_drpc_id==null) {
- _backround = Executors.newCachedThreadPool();
+ _backround = new ExtendedThreadPoolExecutor(0, Integer.MAX_VALUE,
+ 60L, TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>());
_futures = new LinkedList<Future<Void>>();
int numTasks = context.getComponentTasks(context.getThisComponentId()).size();
http://git-wip-us.apache.org/repos/asf/storm/blob/9ee87180/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 927683b..94f9141 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -534,13 +534,15 @@ public class Utils {
return delegate;
}
- public static void handleUncaughtException(Throwable t) {
- if(t!= null && t instanceof OutOfMemoryError) {
- try {
- System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName());
- } catch (Throwable err) {
+ public static void handleUncaughtException(Throwable t) {
+ if(t!= null && t instanceof OutOfMemoryError) {
+ try {
+ System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName());
+ } catch (Throwable err) {
//Again we done want to exit because of logging issues.
+ }
+ Runtime.getRuntime().halt(-1);
}
- Runtime.getRuntime().halt(-1);
}
}
+
[8/9] storm git commit: Merge branch 'storm712' of
https://github.com/kishorvpatil/incubator-storm into STORM-712
Posted by bo...@apache.org.
Merge branch 'storm712' of https://github.com/kishorvpatil/incubator-storm into STORM-712
STORM-712: Storm daemons shutdown if OutOfMemoryError occurs in any thread
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0cac820f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0cac820f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0cac820f
Branch: refs/heads/master
Commit: 0cac820f5dc6ed1ec900f1f452167beeaa031fe2
Parents: 37a7e8b 849315b
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Apr 23 15:58:42 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Apr 23 15:58:42 2015 -0500
----------------------------------------------------------------------
.../src/clj/backtype/storm/daemon/drpc.clj | 1 +
.../src/clj/backtype/storm/daemon/logviewer.clj | 1 +
.../src/clj/backtype/storm/daemon/nimbus.clj | 1 +
.../clj/backtype/storm/daemon/supervisor.clj | 1 +
.../src/clj/backtype/storm/daemon/worker.clj | 1 +
storm-core/src/clj/backtype/storm/util.clj | 14 ++++
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 8 ++-
.../security/auth/SaslTransportPlugin.java | 17 +++--
.../storm/utils/ExtendedThreadPoolExecutor.java | 67 ++++++++++++++++++++
.../src/jvm/backtype/storm/utils/Utils.java | 17 +++++
10 files changed, 121 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0cac820f/storm-core/src/clj/backtype/storm/daemon/logviewer.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0cac820f/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0cac820f/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0cac820f/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0cac820f/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0cac820f/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
[7/9] storm git commit: Add null check on throwable before calling
unwantedExceptionHandler
Posted by bo...@apache.org.
Add null check on throwable before calling unwantedExceptionHandler
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/849315b4
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/849315b4
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/849315b4
Branch: refs/heads/master
Commit: 849315b495c3dce44f2fca584bb47db69890cf48
Parents: 85b87d9
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Apr 22 18:27:08 2015 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Apr 22 18:27:08 2015 +0000
----------------------------------------------------------------------
.../src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java | 4 +++-
storm-core/src/jvm/backtype/storm/utils/Utils.java | 2 --
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/849315b4/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
index ec2af1b..4614366 100644
--- a/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
+++ b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
@@ -60,6 +60,8 @@ public class ExtendedThreadPoolExecutor extends ThreadPoolExecutor{
Thread.currentThread().interrupt();
}
}
- Utils.handleUncaughtException(t);
+ if (t != null) {
+ Utils.handleUncaughtException(t);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/849315b4/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 47eba55..b804518 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -547,8 +547,6 @@ public class Utils {
//Running in daemon mode, we would pass Error to calling thread.
throw (Error) t;
}
- } else {
- LOG.error("Thread threw an Exception.", t);
}
}
}
[6/9] storm git commit: UncaughtExceptionHandler must not ignore any
errors
Posted by bo...@apache.org.
UncaughtExceptionHandler must not ignore any errors
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/85b87d92
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/85b87d92
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/85b87d92
Branch: refs/heads/master
Commit: 85b87d92219a10d458b78bdaab64cafae9d22b5a
Parents: 5d6b62b
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Apr 8 22:37:05 2015 +0000
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Apr 8 22:37:05 2015 +0000
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/util.clj | 7 +++++-
.../src/jvm/backtype/storm/utils/Utils.java | 23 +++++++++++++-------
2 files changed, 21 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/85b87d92/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/util.clj b/storm-core/src/clj/backtype/storm/util.clj
index b1a9b2e..41b900b 100644
--- a/storm-core/src/clj/backtype/storm/util.clj
+++ b/storm-core/src/clj/backtype/storm/util.clj
@@ -1038,5 +1038,10 @@
(Thread/setDefaultUncaughtExceptionHandler
(proxy [Thread$UncaughtExceptionHandler] []
(uncaughtException [thread thrown]
- (Utils/handleUncaughtException thrown)))))
+ (try
+ (Utils/handleUncaughtException thrown)
+ (catch Error err
+ (do
+ (log-error err "Received error in main thread.. terminating server...")
+ (.exit (Runtime/getRuntime) -2))))))))
http://git-wip-us.apache.org/repos/asf/storm/blob/85b87d92/storm-core/src/jvm/backtype/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/Utils.java b/storm-core/src/jvm/backtype/storm/utils/Utils.java
index 94f9141..47eba55 100644
--- a/storm-core/src/jvm/backtype/storm/utils/Utils.java
+++ b/storm-core/src/jvm/backtype/storm/utils/Utils.java
@@ -534,15 +534,22 @@ public class Utils {
return delegate;
}
- public static void handleUncaughtException(Throwable t) {
- if(t!= null && t instanceof OutOfMemoryError) {
- try {
- System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName());
- } catch (Throwable err) {
- //Again we done want to exit because of logging issues.
- }
- Runtime.getRuntime().halt(-1);
+ public static void handleUncaughtException(Throwable t) {
+ if (t != null && t instanceof Error) {
+ if (t instanceof OutOfMemoryError) {
+ try {
+ System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName());
+ } catch (Throwable err) {
+ //Again we don't want to exit because of logging issues.
+ }
+ Runtime.getRuntime().halt(-1);
+ } else {
+ //Running in daemon mode, we would pass Error to calling thread.
+ throw (Error) t;
}
+ } else {
+ LOG.error("Thread threw an Exception.", t);
}
+ }
}
[3/9] storm git commit: Minor comment fix
Posted by bo...@apache.org.
Minor comment fix
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/32353409
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/32353409
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/32353409
Branch: refs/heads/master
Commit: 323534097617d44e263c52c27dcb6da1b418816f
Parents: 720a93e
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Mon Feb 23 15:08:08 2015 -0600
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Mar 18 12:02:33 2015 -0500
----------------------------------------------------------------------
.../src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/32353409/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
index 6730780..ec2af1b 100644
--- a/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
+++ b/storm-core/src/jvm/backtype/storm/utils/ExtendedThreadPoolExecutor.java
@@ -56,7 +56,7 @@ public class ExtendedThreadPoolExecutor extends ThreadPoolExecutor{
} catch (ExecutionException ee) {
t = ee.getCause();
} catch (InterruptedException ie) {
- // If future got interrupted exception, we want to main parent thread itself.
+ // If future got interrupted exception, we want to interrupt parent thread itself.
Thread.currentThread().interrupt();
}
}
[5/9] storm git commit: Remove unwanted import from Utils.java
Posted by bo...@apache.org.
Remove unwanted import from Utils.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5d6b62ba
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5d6b62ba
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5d6b62ba
Branch: refs/heads/master
Commit: 5d6b62ba49609052426509d04ef8096b2c3dbd46
Parents: 9ee8718
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Mar 18 12:59:45 2015 -0500
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Mar 18 18:50:11 2015 +0000
----------------------------------------------------------------------
storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5d6b62ba/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java b/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
index 632f05a..c4a2275 100644
--- a/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
+++ b/storm-core/src/jvm/backtype/storm/drpc/DRPCSpout.java
@@ -42,7 +42,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.Callable;
import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;