You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2016/09/19 20:42:28 UTC
[01/11] storm git commit: STORM-1837: Fix complete-topology and
prevent message loss when running local clusters with no time simulation
Repository: storm
Updated Branches:
refs/heads/master 8e6552ce4 -> 4ce6f04e8
STORM-1837: Fix complete-topology and prevent message loss when running local clusters with no time simulation
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e7b12846
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e7b12846
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e7b12846
Branch: refs/heads/master
Commit: e7b128461366ba2597bb083529cb3e0c36ac3bd3
Parents: a536937
Author: Stig Rohde D�ssing <sd...@it-minds.dk>
Authored: Fri May 13 18:22:37 2016 +0200
Committer: Stig Rohde D�ssing <sd...@it-minds.dk>
Committed: Mon May 16 10:41:59 2016 +0200
----------------------------------------------------------------------
storm-core/src/clj/org/apache/storm/testing.clj | 3 +-
.../apache/storm/messaging/local/Context.java | 49 ++++++++++++++++++--
.../org/apache/storm/testing4j_test.clj | 25 ++++++----
3 files changed, 62 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e7b12846/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/testing.clj b/storm-core/src/clj/org/apache/storm/testing.clj
index 5b2e80d..d0234bc 100644
--- a/storm-core/src/clj/org/apache/storm/testing.clj
+++ b/storm-core/src/clj/org/apache/storm/testing.clj
@@ -601,7 +601,8 @@
(startup spout))
(submit-local-topology (:nimbus cluster-map) storm-name storm-conf topology)
- (advance-cluster-time cluster-map 11)
+ (when (Time/isSimulating)
+ (advance-cluster-time cluster-map 11))
(let [storm-id (StormCommon/getStormId state storm-name)]
;;Give the topology time to come up without using it to wait for the spouts to complete
http://git-wip-us.apache.org/repos/asf/storm/blob/e7b12846/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
index 4f0ba1f..fd68733 100644
--- a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
+++ b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
@@ -27,8 +27,12 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
-import org.apache.storm.Config;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
import org.apache.storm.grouping.Load;
import org.apache.storm.messaging.IConnection;
import org.apache.storm.messaging.TaskMessage;
@@ -82,31 +86,68 @@ public class Context implements IContext {
private static class LocalClient implements IConnection {
private final LocalServer _server;
+ //Messages sent before the server registered a callback
+ private final LinkedBlockingQueue<TaskMessage> _pendingDueToUnregisteredServer;
+ private final ScheduledExecutorService _pendingFlusher;
public LocalClient(LocalServer server) {
_server = server;
+ _pendingDueToUnregisteredServer = new LinkedBlockingQueue<>();
+ _pendingFlusher = Executors.newScheduledThreadPool(1, new ThreadFactory(){
+ @Override
+ public Thread newThread(Runnable runnable) {
+ Thread thread = new Thread(runnable);
+ thread.setName("LocalClientFlusher-" + thread.getId());
+ thread.setDaemon(true);
+ return thread;
+ }
+ });
+ _pendingFlusher.scheduleAtFixedRate(new Runnable(){
+ @Override
+ public void run(){
+ //Ensure messages are flushed even if no more sends are performed
+ flushPending();
+ }
+ }, 5, 5, TimeUnit.SECONDS);
}
@Override
public void registerRecv(IConnectionCallback cb) {
throw new IllegalArgumentException("SHOULD NOT HAPPEN");
}
-
+
+ private void flushPending(){
+ if (!_pendingDueToUnregisteredServer.isEmpty()) {
+ ArrayList<TaskMessage> ret = new ArrayList<>();
+ _pendingDueToUnregisteredServer.drainTo(ret);
+ _server._cb.recv(ret);
+ }
+ }
+
@Override
public void send(int taskId, byte[] payload) {
+ TaskMessage message = new TaskMessage(taskId, payload);
if (_server._cb != null) {
- _server._cb.recv(Arrays.asList(new TaskMessage(taskId, payload)));
+ flushPending();
+ _server._cb.recv(Arrays.asList(message));
+ } else {
+ _pendingDueToUnregisteredServer.add(message);
}
}
@Override
public void send(Iterator<TaskMessage> msgs) {
if (_server._cb != null) {
+ flushPending();
ArrayList<TaskMessage> ret = new ArrayList<>();
while (msgs.hasNext()) {
ret.add(msgs.next());
}
_server._cb.recv(ret);
+ } else {
+ while(msgs.hasNext()){
+ _pendingDueToUnregisteredServer.add(msgs.next());
+ }
}
}
@@ -122,7 +163,7 @@ public class Context implements IContext {
@Override
public void close() {
- //NOOP
+ _pendingFlusher.shutdown();
}
};
http://git-wip-us.apache.org/repos/asf/storm/blob/e7b12846/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
index 3b1a48b..342bba6 100644
--- a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
@@ -63,15 +63,8 @@
(is (Time/isSimulating)))))
(is (not (Time/isSimulating)))))
-(deftest test-complete-topology
- (doseq [zmq-on? [true false]
- :let [daemon-conf (doto (Config.)
- (.put STORM-LOCAL-MODE-ZMQ zmq-on?))
- mk-cluster-param (doto (MkClusterParam.)
- (.setSupervisors (int 4))
- (.setDaemonConf daemon-conf))]]
- (Testing/withSimulatedTimeLocalCluster
- (reify TestJob
+(def complete-topology-testjob
+ (reify TestJob
(^void run [this ^ILocalCluster cluster]
(let [topology (Thrift/buildTopology
{"1" (Thrift/prepareSpoutDetails (TestWordSpout. true) (Integer. 3))}
@@ -109,7 +102,19 @@
(Testing/readTuples results "3")))
(is (= [[1] [2] [3] [4]]
(Testing/readTuples results "4")))
- ))))))
+ ))))
+
+(deftest test-complete-topology
+ (doseq [zmq-on? [true false]
+ :let [daemon-conf (doto (Config.)
+ (.put STORM-LOCAL-MODE-ZMQ zmq-on?))
+ mk-cluster-param (doto (MkClusterParam.)
+ (.setSupervisors (int 4))
+ (.setDaemonConf daemon-conf))]]
+ (Testing/withSimulatedTimeLocalCluster
+ mk-cluster-param complete-topology-testjob )
+ (Testing/withLocalCluster
+ mk-cluster-param complete-topology-testjob)))
(deftest test-with-tracked-cluster
(Testing/withTrackedCluster
[04/11] storm git commit: Merge branch 'STORM-1837-2' of
github.com:srdo/storm into STORM-1837-2
Posted by bo...@apache.org.
Merge branch 'STORM-1837-2' of github.com:srdo/storm into STORM-1837-2
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/db1e9140
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/db1e9140
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/db1e9140
Branch: refs/heads/master
Commit: db1e91401480abd23657c7c7213565a518d54eef
Parents: 591618a 5be1340
Author: Stig Rohde D�ssing <sd...@it-minds.dk>
Authored: Mon Jul 25 09:39:53 2016 +0200
Committer: Stig Rohde D�ssing <sd...@it-minds.dk>
Committed: Mon Jul 25 09:39:53 2016 +0200
----------------------------------------------------------------------
----------------------------------------------------------------------
[06/11] storm git commit: Merge branch 'master' of
https://github.com/apache/storm into STORM-1837-2
Posted by bo...@apache.org.
Merge branch 'master' of https://github.com/apache/storm into STORM-1837-2
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3457a038
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3457a038
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3457a038
Branch: refs/heads/master
Commit: 3457a03884bba7976074b0a36c3c10c91a6ee074
Parents: e740c32 7e15fbe
Author: Stig Rohde D�ssing <sd...@it-minds.dk>
Authored: Mon Aug 29 09:48:37 2016 +0200
Committer: Stig Rohde D�ssing <sd...@it-minds.dk>
Committed: Mon Aug 29 09:48:37 2016 +0200
----------------------------------------------------------------------
.travis.yml | 1 -
CHANGELOG.md | 35 +-
README.markdown | 6 +-
bin/storm.py | 149 +-
docs/Command-line-client.md | 24 +-
docs/SECURITY.md | 4 +-
docs/STORM-UI-REST-API.md | 121 +-
docs/Troubleshooting.md | 68 -
docs/flux.md | 27 +-
docs/images/supervisor_page.png | Bin 0 -> 133290 bytes
docs/storm-sql.md | 63 +-
examples/storm-starter/README.markdown | 2 +-
examples/storm-starter/pom.xml | 4 +-
external/flux/README.md | 27 +-
external/flux/flux-examples/pom.xml | 2 +-
external/flux/pom.xml | 2 +-
external/sql/README.md | 67 +-
external/sql/storm-sql-core/pom.xml | 6 +
.../jvm/org/apache/storm/sql/StormSqlImpl.java | 19 +-
.../apache/storm/sql/compiler/ExprCompiler.java | 16 +-
.../sql/compiler/PostOrderRelNodeVisitor.java | 100 +-
.../backends/standalone/RelNodeCompiler.java | 14 +-
.../compiler/backends/trident/PlanCompiler.java | 222 +-
.../backends/trident/RelNodeCompiler.java | 116 -
.../trident/TridentLogicalPlanCompiler.java | 392 ++
.../storm/sql/compiler/TestCompilerUtils.java | 69 +-
.../storm/sql/compiler/TestExprCompiler.java | 2 +-
.../standalone/TestRelNodeCompiler.java | 7 +-
.../backends/trident/TestPlanCompiler.java | 99 +-
external/sql/storm-sql-kafka/pom.xml | 4 +-
external/sql/storm-sql-runtime/pom.xml | 83 +-
.../sql/runtime/trident/NumberConverter.java | 47 +
.../storm/sql/runtime/trident/TridentUtils.java | 35 +
.../trident/functions/EvaluationFilter.java | 62 +
.../trident/functions/EvaluationFunction.java | 64 +
.../trident/functions/ForwardFunction.java | 30 +
.../functions/UDAFWrappedAggregator.java | 154 +
.../sql/runtime/trident/operations/CountBy.java | 53 +
.../trident/operations/DivideForAverage.java | 45 +
.../sql/runtime/trident/operations/MaxBy.java | 68 +
.../sql/runtime/trident/operations/MinBy.java | 68 +
.../sql/runtime/trident/operations/SumBy.java | 60 +
.../test/org/apache/storm/sql/TestUtils.java | 74 +
external/storm-druid/README.md | 143 +
external/storm-druid/pom.xml | 83 +
.../apache/storm/druid/bolt/DruidBeamBolt.java | 110 +
.../storm/druid/bolt/DruidBeamFactory.java | 29 +
.../apache/storm/druid/bolt/DruidConfig.java | 104 +
.../druid/bolt/ITupleDruidEventMapper.java | 38 +
.../storm/druid/bolt/TupleDruidEventMapper.java | 44 +
.../storm/druid/trident/DruidBeamState.java | 96 +
.../druid/trident/DruidBeamStateFactory.java | 42 +
.../druid/trident/DruidBeamStateUpdater.java | 48 +
.../storm/druid/SampleDruidBeamFactoryImpl.java | 122 +
.../storm/druid/SampleDruidBoltTopology.java | 95 +
.../druid/SampleDruidBoltTridentTopology.java | 91 +
.../apache/storm/druid/SimpleBatchSpout.java | 95 +
.../org/apache/storm/druid/SimpleSpout.java | 68 +
.../apache/storm/hdfs/spout/TestHdfsSpout.java | 5 +-
external/storm-kafka-client/README.md | 29 +-
external/storm-kafka-client/pom.xml | 1 +
.../KafkaSpoutRetryExponentialBackoff.java | 2 +-
.../test/KafkaSpoutTopologyMainNamedTopics.java | 6 +-
external/storm-kafka-monitor/pom.xml | 4 +-
.../storm/kafka/monitor/KafkaOffsetLagUtil.java | 4 +-
external/storm-kafka/README.md | 14 +-
external/storm-kafka/pom.xml | 8 +-
external/storm-kinesis/README.md | 140 +
external/storm-kinesis/pom.xml | 69 +
.../kinesis/spout/CredentialsProviderChain.java | 39 +
.../spout/ExponentialBackoffRetrier.java | 164 +
.../spout/FailedMessageRetryHandler.java | 48 +
.../storm/kinesis/spout/KinesisConfig.java | 133 +
.../storm/kinesis/spout/KinesisConnection.java | 108 +
.../kinesis/spout/KinesisConnectionInfo.java | 111 +
.../storm/kinesis/spout/KinesisMessageId.java | 73 +
.../kinesis/spout/KinesisRecordsManager.java | 449 +++
.../storm/kinesis/spout/KinesisSpout.java | 86 +
.../kinesis/spout/RecordToTupleMapper.java | 38 +
.../storm/kinesis/spout/ZKConnection.java | 95 +
.../org/apache/storm/kinesis/spout/ZkInfo.java | 118 +
.../kinesis/spout/test/KinesisBoltTest.java | 48 +
.../spout/test/KinesisSpoutTopology.java | 57 +
.../spout/test/TestRecordToTupleMapper.java | 58 +
external/storm-redis/pom.xml | 4 +
.../storm/redis/state/RedisKeyValueState.java | 18 +-
external/storm-submit-tools/pom.xml | 199 +
.../submit/command/DependencyResolverMain.java | 105 +
.../storm/submit/dependency/AetherUtils.java | 81 +
.../apache/storm/submit/dependency/Booter.java | 63 +
.../submit/dependency/DependencyResolver.java | 70 +
.../dependency/RepositorySystemFactory.java | 66 +
.../submit/dependency/AetherUtilsTest.java | 102 +
.../dependency/DependencyResolverTest.java | 78 +
pom.xml | 50 +-
.../src/clj/org/apache/storm/blobstore.clj | 28 -
.../clj/org/apache/storm/command/blobstore.clj | 163 -
.../clj/org/apache/storm/command/get_errors.clj | 55 -
.../clj/org/apache/storm/command/heartbeats.clj | 54 -
.../src/clj/org/apache/storm/daemon/common.clj | 18 +-
.../clj/org/apache/storm/daemon/executor.clj | 839 -----
.../org/apache/storm/daemon/local_executor.clj | 42 +
.../src/clj/org/apache/storm/daemon/nimbus.clj | 351 +-
.../src/clj/org/apache/storm/daemon/worker.clj | 70 +-
storm-core/src/clj/org/apache/storm/testing.clj | 16 +-
storm-core/src/clj/org/apache/storm/ui/core.clj | 94 +-
.../src/jvm/org/apache/storm/Constants.java | 24 +-
.../jvm/org/apache/storm/StormSubmitter.java | 116 +-
.../apache/storm/blobstore/BlobStoreUtils.java | 21 +-
.../apache/storm/blobstore/ClientBlobStore.java | 17 +
.../apache/storm/blobstore/NimbusBlobStore.java | 3 +
.../jvm/org/apache/storm/command/Blobstore.java | 348 ++
.../jvm/org/apache/storm/command/GetErrors.java | 80 +
.../org/apache/storm/command/Heartbeats.java | 89 +
.../org/apache/storm/daemon/StormCommon.java | 175 +-
.../src/jvm/org/apache/storm/daemon/Task.java | 76 +-
.../daemon/supervisor/SyncSupervisorEvent.java | 101 +-
.../workermanager/DefaultWorkerManager.java | 23 +-
.../dependency/DependencyPropertiesParser.java | 61 +
.../storm/dependency/DependencyUploader.java | 166 +
.../dependency/FileNotAvailableException.java | 33 +
.../jvm/org/apache/storm/executor/Executor.java | 574 +++
.../apache/storm/executor/ExecutorShutdown.java | 111 +
.../apache/storm/executor/ExecutorTransfer.java | 87 +
.../apache/storm/executor/IRunningExecutor.java | 31 +
.../org/apache/storm/executor/TupleInfo.java | 90 +
.../storm/executor/bolt/BoltExecutor.java | 139 +
.../executor/bolt/BoltOutputCollectorImpl.java | 171 +
.../storm/executor/error/IReportError.java | 22 +
.../storm/executor/error/ReportError.java | 76 +
.../storm/executor/error/ReportErrorAndDie.java | 47 +
.../storm/executor/spout/SpoutExecutor.java | 256 ++
.../spout/SpoutOutputCollectorImpl.java | 147 +
.../org/apache/storm/generated/Assignment.java | 244 +-
.../org/apache/storm/generated/BoltStats.java | 406 +-
.../apache/storm/generated/ClusterSummary.java | 108 +-
.../storm/generated/ClusterWorkerHeartbeat.java | 52 +-
.../storm/generated/ComponentPageInfo.java | 220 +-
.../org/apache/storm/generated/Credentials.java | 44 +-
.../apache/storm/generated/ExecutorStats.java | 168 +-
.../jvm/org/apache/storm/generated/HBNodes.java | 32 +-
.../org/apache/storm/generated/HBRecords.java | 36 +-
.../storm/generated/LSApprovedWorkers.java | 44 +-
.../generated/LSSupervisorAssignments.java | 48 +-
.../apache/storm/generated/LSTopoHistory.java | 64 +-
.../storm/generated/LSTopoHistoryList.java | 36 +-
.../storm/generated/LSWorkerHeartbeat.java | 36 +-
.../apache/storm/generated/ListBlobsResult.java | 32 +-
.../apache/storm/generated/LocalAssignment.java | 36 +-
.../apache/storm/generated/LocalStateData.java | 48 +-
.../org/apache/storm/generated/LogConfig.java | 48 +-
.../jvm/org/apache/storm/generated/Nimbus.java | 3486 ++++++++++++------
.../org/apache/storm/generated/NodeInfo.java | 32 +-
.../storm/generated/RebalanceOptions.java | 44 +-
.../storm/generated/SettableBlobMeta.java | 36 +-
.../org/apache/storm/generated/SpoutStats.java | 252 +-
.../org/apache/storm/generated/StormBase.java | 92 +-
.../apache/storm/generated/StormTopology.java | 428 ++-
.../apache/storm/generated/SupervisorInfo.java | 152 +-
.../storm/generated/SupervisorPageInfo.java | 624 ++++
.../storm/generated/SupervisorSummary.java | 44 +-
.../storm/generated/TopologyHistoryInfo.java | 32 +-
.../apache/storm/generated/TopologyInfo.java | 160 +-
.../storm/generated/TopologyPageInfo.java | 284 +-
.../apache/storm/generated/TopologyStats.java | 220 +-
.../apache/storm/generated/WorkerSummary.java | 1880 ++++++++++
.../messaging/netty/StormClientHandler.java | 2 +-
.../jvm/org/apache/storm/scheduler/Cluster.java | 217 +-
.../resource/ResourceAwareScheduler.java | 9 +
.../storm/security/auth/ThriftClient.java | 3 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 8 +-
.../apache/storm/stats/BoltExecutorStats.java | 1 +
.../jvm/org/apache/storm/stats/CommonStats.java | 5 +-
.../apache/storm/stats/SpoutExecutorStats.java | 1 +
.../jvm/org/apache/storm/stats/StatsUtil.java | 132 +
.../org/apache/storm/task/TopologyContext.java | 32 +-
.../src/jvm/org/apache/storm/tuple/Fields.java | 4 +-
.../jvm/org/apache/storm/utils/ConfigUtils.java | 11 +-
.../org/apache/storm/utils/NimbusClient.java | 5 +-
.../apache/storm/utils/ThriftTopologyUtils.java | 4 +
.../src/jvm/org/apache/storm/utils/Utils.java | 35 +-
.../storm/utils/WorkerBackpressureThread.java | 11 +-
storm-core/src/py/storm/Nimbus-remote | 7 +
storm-core/src/py/storm/Nimbus.py | 272 +-
storm-core/src/py/storm/ttypes.py | 2219 +++++++----
storm-core/src/storm.thrift | 27 +
storm-core/src/ui/public/component.html | 8 +
storm-core/src/ui/public/css/style.css | 20 +
storm-core/src/ui/public/js/script.js | 191 +
storm-core/src/ui/public/supervisor.html | 132 +
.../public/templates/index-page-template.html | 4 +-
.../templates/supervisor-page-template.html | 145 +
.../templates/topology-page-template.html | 208 +-
storm-core/src/ui/public/topology.html | 12 +-
.../org/apache/storm/integration_test.clj | 11 +-
.../test/clj/org/apache/storm/grouping_test.clj | 2 +-
.../test/clj/org/apache/storm/nimbus_test.clj | 106 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 75 -
.../authorizer/DRPCSimpleACLAuthorizer_test.clj | 241 --
.../serialization/SerializationFactory_test.clj | 54 -
.../clj/org/apache/storm/supervisor_test.clj | 18 +-
.../utils/ZookeeperServerCnxnFactory_test.clj | 35 -
.../DependencyPropertiesParserTest.java | 72 +
.../dependency/DependencyUploaderTest.java | 297 ++
.../auth/DefaultHttpCredentialsPluginTest.java | 90 +
.../authorizer/DRPCSimpleACLAuthorizerTest.java | 163 +
.../serialization/SerializationFactoryTest.java | 61 +
.../org/apache/storm/stats/TestStatsUtil.java | 266 ++
.../jvm/org/apache/storm/tuple/FieldsTest.java | 2 +-
.../storm/utils/ThriftTopologyUtilsTest.java | 12 +
.../utils/ZookeeperServerCnxnFactoryTest.java | 39 +
.../staticmocking/ConfigUtilsInstaller.java | 2 +
storm-dist/binary/src/main/assembly/binary.xml | 49 +-
213 files changed, 20220 insertions(+), 5844 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3457a038/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
[08/11] storm git commit: STORM-1837: Wrap throwable in
RuntimeException before throwing from Runnable
Posted by bo...@apache.org.
STORM-1837: Wrap throwable in RuntimeException before throwing from Runnable
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/70ecffc8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/70ecffc8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/70ecffc8
Branch: refs/heads/master
Commit: 70ecffc8b83ccdf8f99bd35dfb53e14ef3578fa6
Parents: 0264698
Author: Stig Rohde D�ssing <st...@gmail.com>
Authored: Mon Sep 19 18:28:55 2016 +0200
Committer: Stig Rohde D�ssing <st...@gmail.com>
Committed: Mon Sep 19 18:28:55 2016 +0200
----------------------------------------------------------------------
storm-core/src/jvm/org/apache/storm/messaging/local/Context.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/70ecffc8/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
index 30c7936..83a8a99 100644
--- a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
+++ b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
@@ -110,7 +110,7 @@ public class Context implements IContext {
flushPending();
} catch (Throwable t) {
LOG.error("Uncaught throwable in pending message flusher thread, messages may be lost", t);
- throw t;
+ throw new RuntimeException(t);
}
}
}, 5, 5, TimeUnit.SECONDS);
[02/11] storm git commit: Merge branch 'master' of
https://github.com/apache/storm into STORM-1837-2
Posted by bo...@apache.org.
Merge branch 'master' of https://github.com/apache/storm into STORM-1837-2
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5be13404
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5be13404
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5be13404
Branch: refs/heads/master
Commit: 5be13404fce05f313e050ad2b082b36a47c430e4
Parents: e7b1284 f325feb
Author: Stig Rohde D�ssing <st...@gmail.com>
Authored: Tue Jun 14 19:40:55 2016 +0200
Committer: Stig Rohde D�ssing <st...@gmail.com>
Committed: Tue Jun 14 19:40:55 2016 +0200
----------------------------------------------------------------------
CHANGELOG.md | 751 ++----------------
README.markdown | 8 +-
bin/storm.py | 36 +-
conf/defaults.yaml | 6 +-
conf/storm.yaml.example | 20 +
docs/Clojure-DSL.md | 4 +-
docs/Command-line-client.md | 163 ++++
docs/Common-patterns.md | 4 +-
docs/Distributed-RPC.md | 2 +-
docs/Metrics.md | 97 ++-
docs/Powered-By.md | 10 +
docs/SECURITY.md | 2 +-
docs/State-checkpointing.md | 3 +
docs/Transactional-topologies.md | 8 +-
docs/Trident-state.md | 4 +-
docs/Tutorial.md | 2 +-
docs/Windowing.md | 37 +-
examples/storm-starter/README.markdown | 12 +-
.../storm/starter/SlidingWindowTopology.java | 4 +-
.../storm/starter/spout/TwitterSampleSpout.java | 12 +-
.../starter/trident/TridentKafkaWordCount.java | 67 +-
.../java/org/apache/storm/flux/FluxBuilder.java | 8 +
external/flux/flux-examples/README.md | 18 +
.../flux/examples/StatefulWordCounter.java | 64 ++
.../resources/simple_stateful_wordcount.yaml | 60 ++
.../flux/wrappers/bolts/FluxShellBolt.java | 79 +-
.../flux/wrappers/spouts/FluxShellSpout.java | 77 +-
.../jvm/org/apache/storm/sql/StormSqlImpl.java | 25 +-
.../apache/storm/sql/compiler/ExprCompiler.java | 2 +-
.../standalone/BuiltinAggregateFunctions.java | 178 +++++
.../backends/standalone/PlanCompiler.java | 3 +-
.../backends/standalone/RelNodeCompiler.java | 227 +++++-
.../test/org/apache/storm/sql/TestStormSql.java | 151 ++++
.../sql/runtime/AbstractChannelHandler.java | 5 +
.../storm/sql/runtime/ChannelContext.java | 1 +
.../storm/sql/runtime/ChannelHandler.java | 2 +
.../org/apache/storm/sql/runtime/Channels.java | 15 +
.../test/org/apache/storm/sql/TestUtils.java | 36 +
.../storm/hbase/bolt/AbstractHBaseBolt.java | 1 -
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 38 +-
.../apache/storm/hdfs/spout/TextFileReader.java | 2 +-
.../org/apache/storm/hive/bolt/HiveBolt.java | 34 +-
.../apache/storm/hive/trident/HiveState.java | 8 +-
.../storm/hive/trident/TridentHiveTopology.java | 29 +-
external/storm-kafka-client/README.md | 110 ++-
.../KafkaSpoutRetryExponentialBackoff.java | 2 +-
external/storm-kafka/README.md | 55 +-
external/storm-kafka/pom.xml | 5 +
.../storm/kafka/DynamicBrokersReader.java | 2 +-
.../kafka/DynamicPartitionConnections.java | 2 +-
.../ExponentialBackoffMsgRetryManager.java | 38 +-
.../storm/kafka/FailedMsgRetryManager.java | 60 +-
.../jvm/org/apache/storm/kafka/KafkaConfig.java | 2 +-
.../jvm/org/apache/storm/kafka/KafkaSpout.java | 2 +-
.../jvm/org/apache/storm/kafka/KafkaUtils.java | 2 +-
.../jvm/org/apache/storm/kafka/Partition.java | 3 +-
.../apache/storm/kafka/PartitionManager.java | 57 +-
.../jvm/org/apache/storm/kafka/SpoutConfig.java | 8 +-
.../storm/kafka/StringMultiSchemeWithTopic.java | 3 +-
.../org/apache/storm/kafka/ZkCoordinator.java | 2 +-
.../src/jvm/org/apache/storm/kafka/ZkState.java | 2 +-
.../kafka/trident/TridentKafkaEmitter.java | 2 +-
.../storm/kafka/trident/ZkBrokerReader.java | 2 +-
.../ExponentialBackoffMsgRetryManagerTest.java | 99 ++-
.../org/apache/storm/kafka/KafkaTestBroker.java | 7 +-
.../storm/mongodb/bolt/MongoInsertBolt.java | 61 +-
external/storm-redis/pom.xml | 2 +-
.../storm/redis/bolt/RedisLookupBolt.java | 4 +
.../apache/storm/redis/bolt/RedisStoreBolt.java | 15 +-
.../redis/common/config/JedisPoolConfig.java | 3 +-
.../common/container/JedisClusterContainer.java | 7 +-
.../common/mapper/RedisDataTypeDescription.java | 7 +-
.../storm/redis/state/RedisKeyValueState.java | 6 +-
.../storm/solr/topology/SolrTopology.java | 6 +-
log4j2/cluster.xml | 14 +
pom.xml | 11 +-
.../clj/org/apache/storm/daemon/executor.clj | 6 +-
.../src/clj/org/apache/storm/daemon/nimbus.clj | 198 +++--
.../src/clj/org/apache/storm/daemon/worker.clj | 2 +-
storm-core/src/jvm/org/apache/storm/Config.java | 35 +-
.../jvm/org/apache/storm/StormSubmitter.java | 48 +-
.../storm/blobstore/BlobStoreAclHandler.java | 10 +
.../apache/storm/cluster/ZKStateStorage.java | 10 +-
.../org/apache/storm/daemon/StormCommon.java | 9 +-
.../apache/storm/generated/HBMessageData.java | 6 +-
.../storm/hooks/SubmitterHookException.java | 41 +
.../metric/ClusterMetricsConsumerExecutor.java | 70 ++
.../metric/LoggingClusterMetricsConsumer.java | 86 ++
.../storm/metric/MetricsConsumerBolt.java | 85 +-
.../org/apache/storm/metric/api/DataPoint.java | 43 +
.../metric/api/IClusterMetricsConsumer.java | 63 ++
.../storm/metric/api/IMetricsConsumer.java | 2 +
.../storm/metric/filter/FilterByMetricName.java | 110 +++
.../storm/metric/filter/MetricsFilter.java | 26 +
.../apache/storm/pacemaker/PacemakerServer.java | 9 +-
.../storm/pacemaker/codec/ThriftDecoder.java | 21 +-
.../jvm/org/apache/storm/scheduler/Cluster.java | 7 +
.../storm/scheduler/resource/RAS_Node.java | 11 +-
.../DefaultResourceAwareStrategy.java | 228 ++++--
.../storm/spout/SpoutOutputCollector.java | 20 +-
.../jvm/org/apache/storm/stats/StatsUtil.java | 62 +-
.../storm/topology/WindowedBoltExecutor.java | 26 +-
.../topology/base/BaseStatefulWindowedBolt.java | 8 +
.../storm/topology/base/BaseWindowedBolt.java | 101 +++
.../spout/PartitionedTridentSpoutExecutor.java | 16 +-
.../jvm/org/apache/storm/utils/BatchHelper.java | 92 +++
.../org/apache/storm/utils/DisruptorQueue.java | 16 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 14 +-
.../storm/validation/ConfigValidation.java | 16 +
storm-core/src/py/storm/ttypes.py | 422 +++++-----
storm-core/src/storm.thrift | 26 +-
.../metric/filter/FilterByMetricNameTest.java | 95 +++
.../apache/storm/nimbus/LocalNimbusTest.java | 132 ++++
.../resource/TestResourceAwareScheduler.java | 788 +------------------
.../TestUtilsForResourceAwareScheduler.java | 68 +-
.../TestDefaultResourceAwareStrategy.java | 333 ++++++++
.../topology/WindowedBoltExecutorTest.java | 87 +-
.../src/main/resources/resources/storm.js | 17 +-
118 files changed, 4119 insertions(+), 2262 deletions(-)
----------------------------------------------------------------------
[05/11] storm git commit: STORM-1837: Check for null callback before
flushing messages, ensure flusher thread doesn't swallow exceptions
Posted by bo...@apache.org.
STORM-1837: Check for null callback before flushing messages, ensure flusher thread doesn't swallow exceptions
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e740c32a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e740c32a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e740c32a
Branch: refs/heads/master
Commit: e740c32a0e19b71fda5e0a72ef6c55d1685a213b
Parents: db1e914
Author: Stig Rohde D�ssing <st...@gmail.com>
Authored: Fri Aug 26 16:09:39 2016 +0200
Committer: Stig Rohde D�ssing <sd...@it-minds.dk>
Committed: Mon Aug 29 09:48:18 2016 +0200
----------------------------------------------------------------------
.../jvm/org/apache/storm/messaging/local/Context.java | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e740c32a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
index fd68733..b53b86b 100644
--- a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
+++ b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
@@ -17,6 +17,7 @@
*/
package org.apache.storm.messaging.local;
+import java.lang.Thread.UncaughtExceptionHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +44,7 @@ public class Context implements IContext {
private static final Logger LOG = LoggerFactory.getLogger(Context.class);
private static class LocalServer implements IConnection {
- IConnectionCallback _cb;
+ volatile IConnectionCallback _cb;
final ConcurrentHashMap<Integer, Double> _load = new ConcurrentHashMap<>();
@Override
@@ -105,8 +106,12 @@ public class Context implements IContext {
_pendingFlusher.scheduleAtFixedRate(new Runnable(){
@Override
public void run(){
- //Ensure messages are flushed even if no more sends are performed
- flushPending();
+ try {
+ //Ensure messages are flushed even if no more sends are performed
+ flushPending();
+ } catch (Throwable t) {
+ LOG.error("Uncaught throwable in pending message flusher thread, messages may be lost", t);
+ }
}
}, 5, 5, TimeUnit.SECONDS);
}
@@ -117,7 +122,7 @@ public class Context implements IContext {
}
private void flushPending(){
- if (!_pendingDueToUnregisteredServer.isEmpty()) {
+ if (_server._cb != null && !_pendingDueToUnregisteredServer.isEmpty()) {
ArrayList<TaskMessage> ret = new ArrayList<>();
_pendingDueToUnregisteredServer.drainTo(ret);
_server._cb.recv(ret);
[03/11] storm git commit: Merge branch 'master' of
https://github.com/apache/storm into STORM-1837-2
Posted by bo...@apache.org.
Merge branch 'master' of https://github.com/apache/storm into STORM-1837-2
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/591618a7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/591618a7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/591618a7
Branch: refs/heads/master
Commit: 591618a73690c8fa727a385d107ad0168d62de00
Parents: e7b1284 8115ef5
Author: Stig Rohde D�ssing <sd...@it-minds.dk>
Authored: Mon Jul 25 09:26:25 2016 +0200
Committer: Stig Rohde D�ssing <sd...@it-minds.dk>
Committed: Mon Jul 25 09:26:25 2016 +0200
----------------------------------------------------------------------
CHANGELOG.md | 817 +++----------------
DEVELOPER.md | 14 +
README.markdown | 13 +-
SECURITY.md | 25 +-
bin/storm-kafka-monitor | 43 +
bin/storm.py | 44 +-
conf/defaults.yaml | 9 +-
conf/storm.yaml.example | 28 +
docs/Clojure-DSL.md | 4 +-
docs/Command-line-client.md | 165 ++++
docs/Common-patterns.md | 4 +-
docs/Distributed-RPC.md | 2 +-
docs/Metrics.md | 97 ++-
docs/Powered-By.md | 10 +
docs/Resource_Aware_Scheduler_overview.md | 54 +-
docs/SECURITY.md | 2 +-
docs/STORM-UI-REST-API.md | 276 +++++++
docs/State-checkpointing.md | 3 +
docs/Transactional-topologies.md | 8 +-
docs/Trident-state.md | 4 +-
docs/Tutorial.md | 2 +-
docs/Windowing.md | 37 +-
docs/distcache-blobstore.md | 10 +-
...ware_scheduler_default_eviction_strategy.png | Bin 0 -> 331891 bytes
...ware_scheduler_default_eviction_strategy.svg | 3 +
docs/images/transactional-batches.png | Bin 23293 -> 21797 bytes
docs/images/transactional-commit-flow.png | Bin 17725 -> 17179 bytes
docs/images/transactional-design-2.png | Bin 13537 -> 13352 bytes
examples/storm-starter/README.markdown | 12 +-
.../storm/starter/SlidingWindowTopology.java | 4 +-
.../storm/starter/spout/TwitterSampleSpout.java | 12 +-
.../starter/trident/TridentKafkaWordCount.java | 69 +-
.../storm/starter/trident/TridentWordCount.java | 13 +-
.../jvm/storm/starter/HdfsSpoutTopology.java | 22 +-
.../java/org/apache/storm/flux/FluxBuilder.java | 8 +
external/flux/flux-examples/README.md | 18 +
.../flux/examples/StatefulWordCounter.java | 64 ++
.../resources/simple_stateful_wordcount.yaml | 60 ++
.../flux/wrappers/bolts/FluxShellBolt.java | 79 +-
.../flux/wrappers/spouts/FluxShellSpout.java | 77 +-
.../jvm/org/apache/storm/sql/StormSqlImpl.java | 25 +-
.../apache/storm/sql/compiler/ExprCompiler.java | 2 +-
.../sql/compiler/PostOrderRelNodeVisitor.java | 6 +
.../standalone/BuiltinAggregateFunctions.java | 178 ++++
.../backends/standalone/PlanCompiler.java | 36 +-
.../backends/standalone/RelNodeCompiler.java | 349 +++++++-
.../test/org/apache/storm/sql/TestStormSql.java | 231 ++++++
.../sql/runtime/AbstractChannelHandler.java | 10 +
.../storm/sql/runtime/ChannelContext.java | 2 +
.../storm/sql/runtime/ChannelHandler.java | 4 +
.../org/apache/storm/sql/runtime/Channels.java | 31 +
.../test/org/apache/storm/sql/TestUtils.java | 77 ++
.../storm/hbase/bolt/AbstractHBaseBolt.java | 11 +-
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 38 +-
.../apache/storm/hbase/common/HBaseClient.java | 8 +-
.../hbase/topology/PersistentWordCount.java | 14 +-
external/storm-hdfs/README.md | 108 +--
external/storm-hdfs/pom.xml | 8 +
.../hdfs/bolt/format/SimpleFileNameFormat.java | 102 +++
.../org/apache/storm/hdfs/spout/HdfsSpout.java | 194 +++--
.../apache/storm/hdfs/spout/TextFileReader.java | 2 +-
.../trident/format/SimpleFileNameFormat.java | 97 +++
.../storm/hdfs/bolt/HdfsFileTopology.java | 4 +-
.../storm/hdfs/bolt/SequenceFileTopology.java | 21 +-
.../bolt/format/TestSimpleFileNameFormat.java | 79 ++
.../apache/storm/hdfs/spout/TestHdfsSpout.java | 142 ++--
.../storm/hdfs/trident/TridentFileTopology.java | 27 +-
.../hdfs/trident/TridentSequenceTopology.java | 24 +-
.../format/TestSimpleFileNameFormat.java | 70 ++
external/storm-hive/pom.xml | 2 +-
.../org/apache/storm/hive/bolt/HiveBolt.java | 34 +-
.../storm/hive/bolt/mapper/HiveMapper.java | 4 +-
.../apache/storm/hive/common/HiveWriter.java | 2 +-
.../apache/storm/hive/trident/HiveState.java | 8 +-
.../apache/storm/hive/bolt/HiveTopology.java | 1 -
.../storm/hive/trident/TridentHiveTopology.java | 29 +-
external/storm-kafka-client/README.md | 141 +++-
.../apache/storm/kafka/spout/KafkaSpout.java | 58 +-
.../storm/kafka/spout/KafkaSpoutConfig.java | 27 +-
.../KafkaSpoutRetryExponentialBackoff.java | 6 +-
.../storm/kafka/spout/KafkaSpoutStream.java | 61 +-
.../storm/kafka/spout/KafkaSpoutStreams.java | 133 +--
.../spout/KafkaSpoutStreamsNamedTopics.java | 154 ++++
.../spout/KafkaSpoutStreamsWildcardTopics.java | 61 ++
.../kafka/spout/KafkaSpoutTuplesBuilder.java | 54 +-
.../KafkaSpoutTuplesBuilderNamedTopics.java | 78 ++
.../KafkaSpoutTuplesBuilderWildcardTopics.java | 36 +
.../spout/test/KafkaSpoutTopologyMain.java | 133 ---
.../test/KafkaSpoutTopologyMainNamedTopics.java | 140 ++++
.../KafkaSpoutTopologyMainWildcardTopics.java | 62 ++
external/storm-kafka-monitor/README.md | 41 +
external/storm-kafka-monitor/pom.xml | 127 +++
.../kafka/monitor/KafkaOffsetLagResult.java | 102 +++
.../storm/kafka/monitor/KafkaOffsetLagUtil.java | 403 +++++++++
.../kafka/monitor/KafkaPartitionOffsetLag.java | 71 ++
.../kafka/monitor/NewKafkaSpoutOffsetQuery.java | 76 ++
.../kafka/monitor/OldKafkaSpoutOffsetQuery.java | 124 +++
external/storm-kafka/README.md | 59 +-
external/storm-kafka/pom.xml | 5 +
.../storm/kafka/DynamicBrokersReader.java | 2 +-
.../kafka/DynamicPartitionConnections.java | 2 +-
.../ExponentialBackoffMsgRetryManager.java | 38 +-
.../storm/kafka/FailedMsgRetryManager.java | 60 +-
.../jvm/org/apache/storm/kafka/KafkaConfig.java | 2 +-
.../jvm/org/apache/storm/kafka/KafkaSpout.java | 44 +-
.../jvm/org/apache/storm/kafka/KafkaUtils.java | 2 +-
.../jvm/org/apache/storm/kafka/Partition.java | 3 +-
.../apache/storm/kafka/PartitionManager.java | 57 +-
.../jvm/org/apache/storm/kafka/SpoutConfig.java | 8 +-
.../storm/kafka/StringMultiSchemeWithTopic.java | 3 +-
.../org/apache/storm/kafka/ZkCoordinator.java | 2 +-
.../src/jvm/org/apache/storm/kafka/ZkState.java | 2 +-
.../bolt/selector/FieldIndexTopicSelector.java | 48 ++
.../bolt/selector/FieldNameTopicSelector.java | 49 ++
.../trident/GlobalPartitionInformation.java | 4 +
.../kafka/trident/TridentKafkaEmitter.java | 2 +-
.../storm/kafka/trident/ZkBrokerReader.java | 2 +-
.../ExponentialBackoffMsgRetryManagerTest.java | 99 ++-
.../org/apache/storm/kafka/KafkaTestBroker.java | 15 +-
.../storm/mongodb/bolt/MongoInsertBolt.java | 61 +-
external/storm-opentsdb/README.md | 76 ++
external/storm-opentsdb/pom.xml | 97 +++
.../storm/opentsdb/OpenTsdbMetricDatapoint.java | 119 +++
.../bolt/ITupleOpenTsdbDatapointMapper.java | 39 +
.../storm/opentsdb/bolt/OpenTsdbBolt.java | 179 ++++
.../bolt/TupleOpenTsdbDatapointMapper.java | 120 +++
.../storm/opentsdb/client/ClientResponse.java | 178 ++++
.../storm/opentsdb/client/OpenTsdbClient.java | 153 ++++
.../storm/opentsdb/trident/OpenTsdbState.java | 89 ++
.../opentsdb/trident/OpenTsdbStateFactory.java | 50 ++
.../opentsdb/trident/OpenTsdbStateUpdater.java | 36 +
.../storm/opentsdb/MetricGenBatchSpout.java | 94 +++
.../apache/storm/opentsdb/MetricGenSpout.java | 72 ++
.../opentsdb/SampleOpenTsdbBoltTopology.java | 70 ++
.../opentsdb/SampleOpenTsdbTridentTopology.java | 87 ++
external/storm-redis/README.md | 57 +-
external/storm-redis/pom.xml | 2 +-
.../storm/redis/bolt/RedisFilterBolt.java | 148 ++++
.../storm/redis/bolt/RedisLookupBolt.java | 6 +-
.../apache/storm/redis/bolt/RedisStoreBolt.java | 17 +-
.../redis/common/config/JedisPoolConfig.java | 3 +-
.../common/container/JedisClusterContainer.java | 7 +-
.../common/mapper/RedisDataTypeDescription.java | 7 +-
.../redis/common/mapper/RedisFilterMapper.java | 32 +
.../redis/common/mapper/RedisLookupMapper.java | 2 +-
.../storm/redis/state/RedisKeyValueState.java | 6 +-
.../redis/topology/WhitelistWordCount.java | 155 ++++
.../storm/solr/topology/SolrTopology.java | 6 +-
log4j2/cluster.xml | 14 +
pom.xml | 26 +-
storm-core/pom.xml | 17 +-
.../apache/storm/command/shell_submission.clj | 3 +-
.../clj/org/apache/storm/daemon/executor.clj | 11 +-
.../src/clj/org/apache/storm/daemon/nimbus.clj | 347 ++++----
.../src/clj/org/apache/storm/daemon/worker.clj | 2 +-
.../src/clj/org/apache/storm/metric/testing.clj | 68 --
storm-core/src/clj/org/apache/storm/testing.clj | 2 +-
storm-core/src/clj/org/apache/storm/ui/core.clj | 123 ++-
.../src/clj/org/apache/storm/ui/helpers.clj | 21 +-
storm-core/src/jvm/org/apache/storm/Config.java | 50 +-
.../jvm/org/apache/storm/StormSubmitter.java | 48 +-
.../storm/blobstore/BlobStoreAclHandler.java | 10 +
.../storm/cluster/StormClusterStateImpl.java | 2 +
.../apache/storm/cluster/ZKStateStorage.java | 10 +-
.../src/jvm/org/apache/storm/daemon/Acker.java | 12 +-
.../org/apache/storm/daemon/StormCommon.java | 44 +-
.../daemon/supervisor/SupervisorUtils.java | 39 +
.../daemon/supervisor/SyncProcessEvent.java | 29 +-
.../daemon/supervisor/SyncSupervisorEvent.java | 108 +--
.../workermanager/DefaultWorkerManager.java | 15 +-
.../apache/storm/generated/HBMessageData.java | 6 +-
.../storm/hooks/SubmitterHookException.java | 41 +
.../metric/ClusterMetricsConsumerExecutor.java | 70 ++
.../metric/LoggingClusterMetricsConsumer.java | 86 ++
.../storm/metric/MetricsConsumerBolt.java | 89 +-
.../org/apache/storm/metric/api/DataPoint.java | 43 +
.../metric/api/IClusterMetricsConsumer.java | 63 ++
.../storm/metric/api/IMetricsConsumer.java | 21 +
.../storm/metric/filter/FilterByMetricName.java | 110 +++
.../storm/metric/filter/MetricsFilter.java | 26 +
.../storm/metric/util/DataPointExpander.java | 79 ++
.../apache/storm/pacemaker/PacemakerServer.java | 9 +-
.../storm/pacemaker/codec/ThriftDecoder.java | 21 +-
.../jvm/org/apache/storm/scheduler/Cluster.java | 7 +
.../storm/scheduler/resource/RAS_Node.java | 11 +-
.../DefaultResourceAwareStrategy.java | 228 ++++--
.../apache/storm/security/auth/AuthUtils.java | 139 ++--
.../jvm/org/apache/storm/spout/ShellSpout.java | 56 +-
.../storm/spout/SpoutOutputCollector.java | 20 +-
.../jvm/org/apache/storm/stats/StatsUtil.java | 62 +-
.../jvm/org/apache/storm/task/ShellBolt.java | 20 +-
.../topology/BaseConfigurationDeclarer.java | 14 +-
.../storm/topology/WindowedBoltExecutor.java | 28 +-
.../topology/base/BaseStatefulWindowedBolt.java | 8 +
.../storm/topology/base/BaseWindowedBolt.java | 101 +++
.../apache/storm/trident/TridentTopology.java | 82 +-
.../org/apache/storm/trident/graph/Group.java | 74 +-
.../operation/DefaultResourceDeclarer.java | 14 +-
.../org/apache/storm/trident/planner/Node.java | 9 +-
.../storm/trident/planner/ProcessorNode.java | 5 +
.../spout/PartitionedTridentSpoutExecutor.java | 33 +-
.../topology/TridentTopologyBuilder.java | 29 +-
.../windowing/WindowTridentProcessor.java | 8 +-
.../src/jvm/org/apache/storm/tuple/Fields.java | 9 +
.../jvm/org/apache/storm/utils/BatchHelper.java | 92 +++
.../org/apache/storm/utils/DisruptorQueue.java | 16 +-
.../apache/storm/utils/TopologySpoutLag.java | 185 +++++
.../src/jvm/org/apache/storm/utils/Utils.java | 31 +-
.../storm/validation/ConfigValidation.java | 16 +
.../storm/windowing/CountEvictionPolicy.java | 12 +-
.../storm/windowing/CountTriggerPolicy.java | 2 +-
.../storm/windowing/DefaultEvictionContext.java | 53 ++
.../apache/storm/windowing/EvictionContext.java | 47 ++
.../apache/storm/windowing/EvictionPolicy.java | 3 +-
.../storm/windowing/TimeEvictionPolicy.java | 4 +-
.../storm/windowing/TimeTriggerPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 27 +-
.../windowing/WatermarkCountTriggerPolicy.java | 2 +-
.../windowing/WatermarkTimeTriggerPolicy.java | 3 +-
.../storm/zookeeper/LeaderElectorImp.java | 21 +-
.../org/apache/storm/zookeeper/Zookeeper.java | 48 +-
storm-core/src/py/storm/ttypes.py | 422 +++++-----
storm-core/src/storm.thrift | 26 +-
storm-core/src/ui/public/component.html | 30 +-
.../src/ui/public/deep_search_result.html | 10 +-
storm-core/src/ui/public/index.html | 18 +-
storm-core/src/ui/public/js/script.js | 12 +
storm-core/src/ui/public/js/visualization.js | 4 +-
storm-core/src/ui/public/logviewer_search.html | 6 +-
storm-core/src/ui/public/search_result.html | 6 +-
.../public/templates/index-page-template.html | 10 +-
.../templates/topology-page-template.html | 87 ++
storm-core/src/ui/public/topology.html | 66 +-
.../apache/storm/trident/integration_test.clj | 3 +
.../test/clj/org/apache/storm/fields_test.clj | 59 --
.../apache/storm/messaging/netty_unit_test.clj | 2 +-
.../test/clj/org/apache/storm/metrics_test.clj | 32 +-
.../test/clj/org/apache/storm/nimbus_test.clj | 39 +-
.../storm/security/auth/AuthUtils_test.clj | 75 --
.../storm/security/auth/ReqContext_test.clj | 73 --
.../security/auth/SaslTransportPlugin_test.clj | 43 -
.../storm/security/auth/ThriftClient_test.clj | 61 --
.../storm/security/auth/ThriftServer_test.clj | 32 -
.../BlowfishTupleSerializer_test.clj | 77 --
.../clj/org/apache/storm/supervisor_test.clj | 21 +-
.../apache/storm/metric/FakeMetricConsumer.java | 88 ++
.../metric/filter/FilterByMetricNameTest.java | 95 +++
.../metric/util/DataPointExpanderTest.java | 109 +++
.../apache/storm/nimbus/LocalNimbusTest.java | 132 +++
.../resource/TestResourceAwareScheduler.java | 788 +-----------------
.../TestUtilsForResourceAwareScheduler.java | 68 +-
.../TestDefaultResourceAwareStrategy.java | 333 ++++++++
.../storm/security/auth/AuthUtilsTest.java | 240 ++++++
.../storm/security/auth/AuthUtilsTestMock.java | 82 ++
.../storm/security/auth/ReqContextTest.java | 87 ++
.../security/auth/SaslTransportPluginTest.java | 49 ++
.../storm/security/auth/ThriftClientTest.java | 68 ++
.../storm/security/auth/ThriftServerTest.java | 38 +
.../BlowfishTupleSerializerTest.java | 86 ++
.../topology/WindowedBoltExecutorTest.java | 87 +-
.../jvm/org/apache/storm/tuple/FieldsTest.java | 126 +++
.../apache/storm/utils/DisruptorQueueTest.java | 2 +-
.../utils/ThrowableNestedCauseMatcher.java | 44 +
.../storm/windowing/WindowManagerTest.java | 69 +-
storm-dist/binary/pom.xml | 10 +
storm-dist/binary/src/main/assembly/binary.xml | 41 +-
.../src/main/resources/resources/storm.js | 17 +-
.../src/main/resources/resources/storm.py | 12 +-
268 files changed, 11866 insertions(+), 3861 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/591618a7/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
[07/11] storm git commit: STORM-1837: Minor style cleanup
Posted by bo...@apache.org.
STORM-1837: Minor style cleanup
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/02646984
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/02646984
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/02646984
Branch: refs/heads/master
Commit: 0264698419b7da82eb494c4c23a6f8b185583229
Parents: 3457a03
Author: Stig Rohde D�ssing <sd...@it-minds.dk>
Authored: Mon Aug 29 09:49:42 2016 +0200
Committer: Stig Rohde D�ssing <st...@gmail.com>
Committed: Wed Aug 31 17:45:29 2016 +0200
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/messaging/local/Context.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/02646984/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
index b53b86b..30c7936 100644
--- a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
+++ b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
@@ -17,7 +17,6 @@
*/
package org.apache.storm.messaging.local;
-import java.lang.Thread.UncaughtExceptionHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,6 +110,7 @@ public class Context implements IContext {
flushPending();
} catch (Throwable t) {
LOG.error("Uncaught throwable in pending message flusher thread, messages may be lost", t);
+ throw t;
}
}
}, 5, 5, TimeUnit.SECONDS);
@@ -169,6 +169,11 @@ public class Context implements IContext {
@Override
public void close() {
_pendingFlusher.shutdown();
+ try{
+ _pendingFlusher.awaitTermination(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e){
+ throw new RuntimeException("Interrupted while awaiting flusher shutdown", e);
+ }
}
};
[09/11] storm git commit: STORM-1837: Guard against calls to
registerRecv with null parameter by caching serverCb before using it
Posted by bo...@apache.org.
STORM-1837: Guard against calls to registerRecv with null parameter by caching serverCb before using it
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ee37e57a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ee37e57a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ee37e57a
Branch: refs/heads/master
Commit: ee37e57a4bcdf93d85019314909c04143ef0a96a
Parents: 70ecffc
Author: Stig Rohde D�ssing <st...@gmail.com>
Authored: Mon Sep 19 19:44:57 2016 +0200
Committer: Stig Rohde D�ssing <st...@gmail.com>
Committed: Mon Sep 19 19:44:57 2016 +0200
----------------------------------------------------------------------
.../org/apache/storm/messaging/local/Context.java | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ee37e57a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
index 83a8a99..7300847 100644
--- a/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
+++ b/storm-core/src/jvm/org/apache/storm/messaging/local/Context.java
@@ -122,19 +122,21 @@ public class Context implements IContext {
}
private void flushPending(){
- if (_server._cb != null && !_pendingDueToUnregisteredServer.isEmpty()) {
+ IConnectionCallback serverCb = _server._cb;
+ if (serverCb != null && !_pendingDueToUnregisteredServer.isEmpty()) {
ArrayList<TaskMessage> ret = new ArrayList<>();
_pendingDueToUnregisteredServer.drainTo(ret);
- _server._cb.recv(ret);
+ serverCb.recv(ret);
}
}
@Override
public void send(int taskId, byte[] payload) {
TaskMessage message = new TaskMessage(taskId, payload);
- if (_server._cb != null) {
+ IConnectionCallback serverCb = _server._cb;
+ if (serverCb != null) {
flushPending();
- _server._cb.recv(Arrays.asList(message));
+ serverCb.recv(Arrays.asList(message));
} else {
_pendingDueToUnregisteredServer.add(message);
}
@@ -142,13 +144,14 @@ public class Context implements IContext {
@Override
public void send(Iterator<TaskMessage> msgs) {
- if (_server._cb != null) {
+ IConnectionCallback serverCb = _server._cb;
+ if (serverCb != null) {
flushPending();
ArrayList<TaskMessage> ret = new ArrayList<>();
while (msgs.hasNext()) {
ret.add(msgs.next());
}
- _server._cb.recv(ret);
+ serverCb.recv(ret);
} else {
while(msgs.hasNext()){
_pendingDueToUnregisteredServer.add(msgs.next());
[10/11] storm git commit: Merge branch 'STORM-1837-2' of
https://github.com/srdo/storm into STORM-1837
Posted by bo...@apache.org.
Merge branch 'STORM-1837-2' of https://github.com/srdo/storm into STORM-1837
STORM-1837: Fix complete-topology and prevent message loss
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f8cbfd5b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f8cbfd5b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f8cbfd5b
Branch: refs/heads/master
Commit: f8cbfd5bff20d4a6c87f00f401ff13f83e09a4ae
Parents: 8e6552c ee37e57
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 19 15:12:25 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 19 15:12:25 2016 -0500
----------------------------------------------------------------------
storm-core/src/clj/org/apache/storm/testing.clj | 3 +-
.../apache/storm/messaging/local/Context.java | 70 +++++++++++++++++---
.../org/apache/storm/testing4j_test.clj | 25 ++++---
3 files changed, 79 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[11/11] storm git commit: Added STORM-1837 to Changelog
Posted by bo...@apache.org.
Added STORM-1837 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4ce6f04e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4ce6f04e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4ce6f04e
Branch: refs/heads/master
Commit: 4ce6f04e8352b142b746471a5b96da7ac11c2bf2
Parents: f8cbfd5
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 19 15:41:38 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 19 15:41:38 2016 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4ce6f04e/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0429be2..da834ce 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -148,6 +148,7 @@
* STORM-1769: Added a test to check local nimbus with notifier plugin
## 1.1.0
+ * STORM-1837: Fix complete-topology and prevent message loss
* STORM-2092: optimize TridentKafkaState batch sending
* STORM-1979: Storm Druid Connector implementation.
* STORM-2057: Support JOIN statement in Storm SQL