You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/11/17 16:46:18 UTC
[1/6] storm git commit: STORM-1016: Generate trident bolt ids with
sorted group names
Repository: storm
Updated Branches:
refs/heads/master 5a79ba5f9 -> 1afa5a27c
STORM-1016: Generate trident bolt ids with sorted group names
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f8118e08
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f8118e08
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f8118e08
Branch: refs/heads/master
Commit: f8118e0854e49d70e75707e96ddb6524dff2a930
Parents: 07e0ff2
Author: wangjiasheng <wa...@xiaomi.com>
Authored: Thu Aug 27 14:57:27 2015 +0800
Committer: wangjiasheng <wa...@xiaomi.com>
Committed: Fri Aug 28 17:28:37 2015 +0800
----------------------------------------------------------------------
.../src/jvm/storm/trident/TridentTopology.java | 28 +++++++--
.../jvm/storm/trident/TestTridentTopology.java | 64 ++++++++++++++++++++
2 files changed, 86 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f8118e08/storm-core/src/jvm/storm/trident/TridentTopology.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/storm/trident/TridentTopology.java b/storm-core/src/jvm/storm/trident/TridentTopology.java
index 8164717..519a020 100644
--- a/storm-core/src/jvm/storm/trident/TridentTopology.java
+++ b/storm-core/src/jvm/storm/trident/TridentTopology.java
@@ -33,6 +33,8 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -171,7 +173,7 @@ public class TridentTopology {
}
public Stream multiReduce(Stream s1, Stream s2, MultiReducer function, Fields outputFields) {
- return multiReduce(Arrays.asList(s1, s2), function, outputFields);
+ return multiReduce(Arrays.asList(s1, s2), function, outputFields);
}
public Stream multiReduce(Fields inputFields1, Stream s1, Fields inputFields2, Stream s2, MultiReducer function, Fields outputFields) {
@@ -526,15 +528,29 @@ public class TridentTopology {
private static Map<Group, String> genBoltIds(Collection<Group> groups) {
Map<Group, String> ret = new HashMap();
+ //sort by group name
+ final Map<Group, String> groupNames = new HashMap();
+ for (Group g : groups) {
+ String name = getGroupName(g);
+ groupNames.put(g, name == null ? "" : name);
+ }
+ List<Group> groupList = new ArrayList(groups);
+ Collections.sort(groupList, new Comparator<Group>() {
+ @Override
+ public int compare(Group o1, Group o2) {
+ return groupNames.get(o1).compareTo(groupNames.get(o2));
+ }
+ });
+
int ctr = 0;
- for(Group g: groups) {
- if(!isSpoutGroup(g)) {
+ for (Group g : groupList) {
+ if (!isSpoutGroup(g)) {
List<String> name = new ArrayList();
name.add("b");
name.add("" + ctr);
- String groupName = getGroupName(g);
- if(groupName!=null && !groupName.isEmpty()) {
- name.add(getGroupName(g));
+ String groupName = groupNames.get(g);
+ if (groupName != null && !groupName.isEmpty()) {
+ name.add(groupName);
}
ret.put(g, Utils.join(name, "-"));
ctr++;
http://git-wip-us.apache.org/repos/asf/storm/blob/f8118e08/storm-core/test/jvm/storm/trident/TestTridentTopology.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/storm/trident/TestTridentTopology.java b/storm-core/test/jvm/storm/trident/TestTridentTopology.java
new file mode 100644
index 0000000..2c8910e
--- /dev/null
+++ b/storm-core/test/jvm/storm/trident/TestTridentTopology.java
@@ -0,0 +1,64 @@
+package storm.trident;
+
+import backtype.storm.generated.Bolt;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+import org.junit.Assert;
+import org.junit.Test;
+import storm.trident.operation.builtin.Count;
+import storm.trident.operation.builtin.FilterNull;
+import storm.trident.operation.builtin.Sum;
+import storm.trident.operation.impl.FilterExecutor;
+import storm.trident.testing.FixedBatchSpout;
+import storm.trident.testing.Split;
+import storm.trident.testing.StringLength;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Created by jiasheng on 15-8-25.
+ */
+public class TestTridentTopology {
+
+ private StormTopology buildTopology() {
+ FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
+ new Values("the cow jumped over the moon"),
+ new Values("the man went to the store and bought some candy"),
+ new Values("four score and seven years ago"),
+ new Values("how many apples can you eat"));
+ spout.setCycle(true);
+
+ TridentTopology topology = new TridentTopology();
+ topology.newStream("spout", spout)
+ //no name
+ .each(new Fields("sentence"), new Split(), new Fields("word"))
+ .partitionBy(new Fields("word"))
+ .name("abc")
+ .each(new Fields("word"), new StringLength(), new Fields("length"))
+ .partitionBy(new Fields("length"))
+ .name("def")
+ .aggregate(new Fields("length"), new Count(), new Fields("count"))
+ .partitionBy(new Fields("count"))
+ .name("ghi")
+ .aggregate(new Fields("count"), new Sum(), new Fields("sum"));
+ return topology.build();
+ }
+
+ @Test
+ public void testGenBoltId() {
+ Set<String> pre = null;
+ for (int i = 0; i < 100; i++) {
+ StormTopology topology = buildTopology();
+ Map<String, Bolt> cur = topology.get_bolts();
+ System.out.println(cur.keySet());
+ if (pre != null) {
+ Assert.assertTrue("bold id not consistent with group name", pre.equals(cur.keySet()));
+ }
+ pre = cur.keySet();
+ }
+ }
+
+}
[2/6] storm git commit: replace HashSet with LinkedHashSet in
TridentTopology.java
Posted by bo...@apache.org.
replace HashSet with LinkedHashSet in TridentTopology.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3dbfa10c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3dbfa10c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3dbfa10c
Branch: refs/heads/master
Commit: 3dbfa10c26a71d33f35037edc4e57a47f50a11f8
Parents: f8118e0
Author: wangjiasheng <wa...@xiaomi.com>
Authored: Fri Nov 13 16:02:32 2015 +0800
Committer: wangjiasheng <wa...@xiaomi.com>
Committed: Fri Nov 13 16:02:32 2015 +0800
----------------------------------------------------------------------
.../src/jvm/storm/trident/TridentTopology.java | 45 +++++---------------
.../jvm/storm/trident/graph/GraphGrouper.java | 9 ++--
2 files changed, 14 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3dbfa10c/storm-core/src/jvm/storm/trident/TridentTopology.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/storm/trident/TridentTopology.java b/storm-core/src/jvm/storm/trident/TridentTopology.java
index 519a020..aec6eb8 100644
--- a/storm-core/src/jvm/storm/trident/TridentTopology.java
+++ b/storm-core/src/jvm/storm/trident/TridentTopology.java
@@ -30,17 +30,8 @@ import backtype.storm.tuple.Fields;
import backtype.storm.utils.Utils;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
+import java.util.*;
+
import org.jgrapht.DirectedGraph;
import org.jgrapht.UndirectedGraph;
import org.jgrapht.alg.ConnectivityInspector;
@@ -93,7 +84,7 @@ public class TridentTopology {
// is it too expensive to do a batch per drpc request?
DefaultDirectedGraph<Node, IndexedEdge> _graph;
- Map<String, List<Node>> _colocate = new HashMap();
+ Map<String, List<Node>> _colocate = new LinkedHashMap();
UniqueIdGen _gen;
public TridentTopology() {
@@ -173,7 +164,7 @@ public class TridentTopology {
}
public Stream multiReduce(Stream s1, Stream s2, MultiReducer function, Fields outputFields) {
- return multiReduce(Arrays.asList(s1, s2), function, outputFields);
+ return multiReduce(Arrays.asList(s1, s2), function, outputFields);
}
public Stream multiReduce(Fields inputFields1, Stream s1, Fields inputFields2, Stream s2, MultiReducer function, Fields outputFields) {
@@ -275,7 +266,7 @@ public class TridentTopology {
List<SpoutNode> spoutNodes = new ArrayList<SpoutNode>();
// can be regular nodes (static state) or processor nodes
- Set<Node> boltNodes = new HashSet<Node>();
+ Set<Node> boltNodes = new LinkedHashSet<Node>();
for(Node n: graph.vertexSet()) {
if(n instanceof SpoutNode) {
spoutNodes.add((SpoutNode) n);
@@ -285,7 +276,7 @@ public class TridentTopology {
}
- Set<Group> initialGroups = new HashSet<Group>();
+ Set<Group> initialGroups = new LinkedHashSet<Group>();
for(List<Node> colocate: _colocate.values()) {
Group g = new Group(graph, colocate);
boltNodes.removeAll(colocate);
@@ -528,29 +519,15 @@ public class TridentTopology {
private static Map<Group, String> genBoltIds(Collection<Group> groups) {
Map<Group, String> ret = new HashMap();
- //sort by group name
- final Map<Group, String> groupNames = new HashMap();
- for (Group g : groups) {
- String name = getGroupName(g);
- groupNames.put(g, name == null ? "" : name);
- }
- List<Group> groupList = new ArrayList(groups);
- Collections.sort(groupList, new Comparator<Group>() {
- @Override
- public int compare(Group o1, Group o2) {
- return groupNames.get(o1).compareTo(groupNames.get(o2));
- }
- });
-
int ctr = 0;
- for (Group g : groupList) {
- if (!isSpoutGroup(g)) {
+ for(Group g: groups) {
+ if(!isSpoutGroup(g)) {
List<String> name = new ArrayList();
name.add("b");
name.add("" + ctr);
- String groupName = groupNames.get(g);
- if (groupName != null && !groupName.isEmpty()) {
- name.add(groupName);
+ String groupName = getGroupName(g);
+ if(groupName!=null && !groupName.isEmpty()) {
+ name.add(getGroupName(g));
}
ret.put(g, Utils.join(name, "-"));
ctr++;
http://git-wip-us.apache.org/repos/asf/storm/blob/3dbfa10c/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java b/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
index 94db077..5822ddc 100644
--- a/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
+++ b/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
@@ -17,11 +17,8 @@
*/
package storm.trident.graph;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+
import org.jgrapht.DirectedGraph;
import storm.trident.planner.Node;
import storm.trident.util.IndexedEdge;
@@ -35,7 +32,7 @@ public class GraphGrouper {
public GraphGrouper(DirectedGraph<Node, IndexedEdge> graph, Collection<Group> initialGroups) {
this.graph = graph;
- this.currGroups = new HashSet(initialGroups);
+ this.currGroups = new LinkedHashSet(initialGroups);
reindex();
}
[3/6] storm git commit: Merge branch 'apache' into feature_genBoltId
Posted by bo...@apache.org.
Merge branch 'apache' into feature_genBoltId
Conflicts:
storm-core/src/jvm/storm/trident/TridentTopology.java
storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1496f27a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1496f27a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1496f27a
Branch: refs/heads/master
Commit: 1496f27a9c6f8678f7f24dc3505693f6b3ff57fa
Parents: 3dbfa10 5a79ba5
Author: wangjiasheng <wa...@xiaomi.com>
Authored: Mon Nov 16 17:27:53 2015 +0800
Committer: wangjiasheng <wa...@xiaomi.com>
Committed: Mon Nov 16 17:27:53 2015 +0800
----------------------------------------------------------------------
.gitignore | 3 +
CHANGELOG.md | 125 +-
DEVELOPER.md | 54 +-
DISCLAIMER | 10 -
LICENSE | 21 +
README.markdown | 22 +
SECURITY.md | 244 +-
STORM-UI-REST-API.md | 707 -
bin/flight.bash | 154 +
bin/storm | 20 +-
bin/storm-config.cmd | 10 +-
bin/storm.py | 92 +-
conf/defaults.yaml | 39 +-
conf/storm.yaml.example | 2 +-
dev-tools/storm-merge.py | 2 +-
dev-tools/travis/ratprint.py | 26 +
dev-tools/travis/travis-install.sh | 9 +-
dev-tools/travis/travis-script.sh | 15 +-
docs/about/multi-language.md | 2 +-
docs/documentation/Documentation.md | 4 +
docs/documentation/FAQ.md | 2 +-
docs/documentation/Log-Search.md | 14 +
.../Message-passing-implementation.md | 34 +-
.../documentation/Setting-up-a-Storm-cluster.md | 29 +-
docs/documentation/Tutorial.md | 2 +-
.../documentation/dynamic-log-level-settings.md | 41 +
docs/documentation/dynamic-worker-profiling.md | 29 +
.../images/dynamic_log_level_settings_1.png | Bin 0 -> 93689 bytes
.../images/dynamic_log_level_settings_2.png | Bin 0 -> 78785 bytes
.../images/dynamic_profiling_debugging_1.png | Bin 0 -> 93635 bytes
.../images/dynamic_profiling_debugging_2.png | Bin 0 -> 138120 bytes
.../images/dynamic_profiling_debugging_3.png | Bin 0 -> 96974 bytes
docs/documentation/images/search-a-topology.png | Bin 0 -> 671031 bytes
.../images/search-for-a-single-worker-log.png | Bin 0 -> 736579 bytes
.../storm-metrics-profiling-internal-actions.md | 70 +
docs/documentation/ui-rest-api.md | 984 +
docs/images/viewing_metrics_with_VisualVM.png | Bin 0 -> 225100 bytes
examples/storm-starter/pom.xml | 17 +-
.../storm/starter/FastWordCountTopology.java | 198 +
.../jvm/storm/starter/InOrderDeliveryTest.java | 175 +
.../storm/starter/MultipleLoggerTopology.java | 105 +
.../starter/ResourceAwareExampleTopology.java | 101 +
.../jvm/storm/starter/ThroughputVsLatency.java | 432 +
.../bolt/IntermediateRankingsBoltTest.java | 2 +-
.../starter/bolt/RollingCountBoltTest.java | 2 +-
.../starter/bolt/TotalRankingsBoltTest.java | 2 +-
.../storm/starter/tools/MockTupleHelpers.java | 40 -
external/flux/README.md | 1 +
external/flux/flux-core/pom.xml | 1 -
.../main/java/org/apache/storm/flux/Flux.java | 3 +-
.../java/org/apache/storm/flux/FluxBuilder.java | 13 +
.../org/apache/storm/flux/test/TestBolt.java | 4 +
.../resources/configs/config-methods-test.yaml | 1 +
external/flux/flux-examples/pom.xml | 1 -
external/storm-elasticsearch/README.md | 37 +-
external/storm-elasticsearch/pom.xml | 11 +
.../elasticsearch/bolt/AbstractEsBolt.java | 10 +-
.../elasticsearch/bolt/ElasticSearchClient.java | 57 -
.../storm/elasticsearch/bolt/EsIndexBolt.java | 4 +-
.../storm/elasticsearch/bolt/EsLookupBolt.java | 10 +
.../elasticsearch/bolt/EsPercolateBolt.java | 4 +-
.../elasticsearch/bolt/TransportAddresses.java | 72 -
.../storm/elasticsearch/common/EsConfig.java | 64 +-
.../common/StormElasticSearchClient.java | 48 +
.../common/TransportAddresses.java | 72 +
.../storm/elasticsearch/trident/EsState.java | 32 +-
.../elasticsearch/trident/EsStateFactory.java | 20 +-
.../storm/elasticsearch/trident/EsUpdater.java | 2 +-
.../elasticsearch/bolt/AbstractEsBoltTest.java | 15 +-
.../elasticsearch/bolt/EsIndexBoltTest.java | 5 +
.../elasticsearch/bolt/EsIndexTopology.java | 4 +-
.../bolt/EsLookupBoltIntegrationTest.java | 5 +
.../elasticsearch/bolt/EsLookupBoltTest.java | 5 +
.../elasticsearch/bolt/EsPercolateBoltTest.java | 5 +
.../bolt/TransportAddressesTest.java | 81 -
.../elasticsearch/common/EsConfigTest.java | 71 +
.../common/TransportAddressesTest.java | 81 +
.../trident/EsStateFactoryTest.java | 32 +
.../trident/TridentEsTopology.java | 8 +-
external/storm-eventhubs/pom.xml | 1 -
.../trident/OpaqueTridentEventHubSpout.java | 2 +-
.../TransactionalTridentEventHubSpout.java | 2 +-
external/storm-hbase/README.md | 10 +
.../storm/hbase/bolt/AbstractHBaseBolt.java | 2 +
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 75 +-
.../org/apache/storm/hbase/common/Utils.java | 10 +-
external/storm-hdfs/README.md | 33 +
external/storm-hdfs/pom.xml | 71 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 124 +
.../storm/hdfs/bolt/AvroGenericRecordBolt.java | 145 +
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 51 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 42 +-
.../ha/codedistributor/HDFSCodeDistributor.java | 17 +
.../hdfs/bolt/AvroGenericRecordBoltTest.java | 220 +
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 258 +
.../storm/hdfs/bolt/TestSequenceFileBolt.java | 186 +
.../storm/hdfs/trident/FixedBatchSpout.java | 2 +-
.../storm/hdfs/trident/HdfsStateTest.java | 17 +
external/storm-hive/pom.xml | 7 +
.../org/apache/storm/hive/bolt/HiveBolt.java | 9 +
.../apache/storm/hive/bolt/TestHiveBolt.java | 56 +-
.../storm/hive/trident/TridentHiveTopology.java | 2 +-
external/storm-jdbc/README.md | 6 +-
.../storm/jdbc/bolt/AbstractJdbcBolt.java | 2 +
.../apache/storm/jdbc/bolt/JdbcInsertBolt.java | 9 +
.../apache/storm/jdbc/bolt/JdbcLookupBolt.java | 5 +
.../jdbc/mapper/SimpleJdbcLookupMapper.java | 3 +
.../storm/jdbc/mapper/SimpleJdbcMapper.java | 5 +
.../storm/jdbc/bolt/JdbcInsertBoltTest.java | 71 +
.../storm/jdbc/bolt/JdbcLookupBoltTest.java | 59 +
external/storm-kafka/README.md | 107 +-
external/storm-kafka/pom.xml | 5 +
.../jvm/storm/kafka/DynamicBrokersReader.java | 97 +-
.../kafka/DynamicPartitionConnections.java | 20 +-
.../ExponentialBackoffMsgRetryManager.java | 4 +-
.../src/jvm/storm/kafka/KafkaConfig.java | 3 +-
.../src/jvm/storm/kafka/KafkaSpout.java | 34 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 93 +-
.../jvm/storm/kafka/MessageMetadataScheme.java | 25 +
.../MessageMetadataSchemeAsMultiScheme.java | 40 +
.../src/jvm/storm/kafka/Partition.java | 26 +-
.../src/jvm/storm/kafka/PartitionManager.java | 47 +-
.../src/jvm/storm/kafka/StaticCoordinator.java | 11 +-
.../kafka/StringMessageAndMetadataScheme.java | 42 +
.../storm/kafka/StringMultiSchemeWithTopic.java | 57 +
.../src/jvm/storm/kafka/ZkCoordinator.java | 2 +-
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 17 +-
.../jvm/storm/kafka/trident/Coordinator.java | 7 +-
.../trident/GlobalPartitionInformation.java | 26 +-
.../jvm/storm/kafka/trident/IBrokerReader.java | 7 +-
.../kafka/trident/OpaqueTridentKafkaSpout.java | 9 +-
.../storm/kafka/trident/StaticBrokerReader.java | 23 +-
.../trident/TransactionalTridentKafkaSpout.java | 4 +-
.../kafka/trident/TridentKafkaEmitter.java | 48 +-
.../storm/kafka/trident/TridentKafkaState.java | 25 +-
.../jvm/storm/kafka/trident/ZkBrokerReader.java | 20 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 114 +-
.../src/test/storm/kafka/KafkaUtilsTest.java | 112 +-
.../src/test/storm/kafka/TestUtils.java | 20 +-
.../test/storm/kafka/TridentKafkaTopology.java | 8 +-
.../src/test/storm/kafka/ZkCoordinatorTest.java | 8 +-
.../test/storm/kafka/bolt/KafkaBoltTest.java | 41 +-
external/storm-metrics/pom.xml | 58 +
.../metrics/hdrhistogram/HistogramMetric.java | 79 +
.../apache/storm/metrics/sigar/CPUMetric.java | 60 +
.../resources/libsigar-amd64-freebsd-6.so | Bin 0 -> 210641 bytes
.../resources/resources/libsigar-amd64-linux.so | Bin 0 -> 246605 bytes
.../resources/libsigar-amd64-solaris.so | Bin 0 -> 251360 bytes
.../resources/libsigar-ia64-hpux-11.sl | Bin 0 -> 577452 bytes
.../resources/resources/libsigar-ia64-linux.so | Bin 0 -> 494929 bytes
.../resources/resources/libsigar-pa-hpux-11.sl | Bin 0 -> 516096 bytes
.../resources/resources/libsigar-ppc-aix-5.so | Bin 0 -> 400925 bytes
.../resources/resources/libsigar-ppc-linux.so | Bin 0 -> 258547 bytes
.../resources/resources/libsigar-ppc64-aix-5.so | Bin 0 -> 425077 bytes
.../resources/resources/libsigar-ppc64-linux.so | Bin 0 -> 330767 bytes
.../resources/resources/libsigar-s390x-linux.so | Bin 0 -> 269932 bytes
.../resources/libsigar-sparc-solaris.so | Bin 0 -> 285004 bytes
.../resources/libsigar-sparc64-solaris.so | Bin 0 -> 261896 bytes
.../resources/libsigar-universal-macosx.dylib | Bin 0 -> 377668 bytes
.../resources/libsigar-universal64-macosx.dylib | Bin 0 -> 397440 bytes
.../resources/libsigar-x86-freebsd-5.so | Bin 0 -> 179751 bytes
.../resources/libsigar-x86-freebsd-6.so | Bin 0 -> 179379 bytes
.../resources/resources/libsigar-x86-linux.so | Bin 0 -> 233385 bytes
.../resources/resources/libsigar-x86-solaris.so | Bin 0 -> 242880 bytes
.../resources/resources/sigar-amd64-winnt.dll | Bin 0 -> 402432 bytes
.../resources/resources/sigar-x86-winnt.dll | Bin 0 -> 266240 bytes
.../resources/resources/sigar-x86-winnt.lib | Bin 0 -> 99584 bytes
external/storm-solr/pom.xml | 21 +-
log4j2/cluster.xml | 34 +-
log4j2/worker.xml | 12 +-
pom.xml | 270 +-
storm-core/pom.xml | 323 +-
storm-core/src/clj/backtype/storm/cluster.clj | 208 +-
.../clj/backtype/storm/command/healthcheck.clj | 88 +
.../clj/backtype/storm/command/kill_workers.clj | 33 +
.../backtype/storm/command/set_log_level.clj | 75 +
storm-core/src/clj/backtype/storm/config.clj | 117 +-
storm-core/src/clj/backtype/storm/converter.clj | 94 +-
.../backtype/storm/daemon/builtin_metrics.clj | 99 +-
.../src/clj/backtype/storm/daemon/common.clj | 61 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 46 +-
.../src/clj/backtype/storm/daemon/executor.clj | 369 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 1039 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 670 +-
.../clj/backtype/storm/daemon/supervisor.clj | 286 +-
.../src/clj/backtype/storm/daemon/task.clj | 24 +-
.../src/clj/backtype/storm/daemon/worker.clj | 323 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 72 +-
.../src/clj/backtype/storm/local_state.clj | 44 +-
storm-core/src/clj/backtype/storm/log.clj | 12 +-
.../src/clj/backtype/storm/messaging/loader.clj | 76 +-
.../src/clj/backtype/storm/messaging/local.clj | 56 +-
storm-core/src/clj/backtype/storm/stats.clj | 1521 +-
storm-core/src/clj/backtype/storm/testing.clj | 34 +-
storm-core/src/clj/backtype/storm/timer.clj | 20 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 1347 +-
.../src/clj/backtype/storm/ui/helpers.clj | 77 +-
storm-core/src/clj/backtype/storm/util.clj | 67 +-
.../src/dev/logviewer-search-context-tests.log | 1 +
.../dev/logviewer-search-context-tests.log.gz | Bin 0 -> 72 bytes
storm-core/src/dev/small-worker.log | 1 +
storm-core/src/dev/test-3072.log | 3 +
storm-core/src/dev/test-worker.log | 380 +
storm-core/src/genthrift.sh | 2 +-
storm-core/src/jvm/backtype/storm/Config.java | 797 +-
.../jvm/backtype/storm/ConfigValidation.java | 375 -
.../src/jvm/backtype/storm/LogWriter.java | 2 +-
.../src/jvm/backtype/storm/StormSubmitter.java | 55 +-
.../storm/codedistributor/ICodeDistributor.java | 17 +
.../LocalFileSystemCodeDistributor.java | 17 +
.../storm/coordination/BatchBoltExecutor.java | 4 +-
.../coordination/BatchSubtopologyBuilder.java | 6 +-
.../storm/coordination/CoordinatedBolt.java | 16 +-
.../storm/drpc/DRPCInvocationsClient.java | 5 +-
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 10 +-
.../src/jvm/backtype/storm/drpc/JoinResult.java | 8 +-
.../storm/drpc/LinearDRPCTopologyBuilder.java | 9 +-
.../storm/generated/AlreadyAliveException.java | 4 +-
.../backtype/storm/generated/Assignment.java | 380 +-
.../storm/generated/AuthorizationException.java | 4 +-
.../src/jvm/backtype/storm/generated/Bolt.java | 4 +-
.../storm/generated/BoltAggregateStats.java | 704 +
.../jvm/backtype/storm/generated/BoltStats.java | 444 +-
.../storm/generated/ClusterSummary.java | 221 +-
.../storm/generated/ClusterWorkerHeartbeat.java | 61 +-
.../storm/generated/CommonAggregateStats.java | 902 +
.../generated/ComponentAggregateStats.java | 752 +
.../storm/generated/ComponentCommon.java | 6 +-
.../storm/generated/ComponentObject.java | 2 +-
.../storm/generated/ComponentPageInfo.java | 2194 ++
.../backtype/storm/generated/ComponentType.java | 62 +
.../backtype/storm/generated/Credentials.java | 48 +-
.../storm/generated/DRPCExecutionException.java | 4 +-
.../backtype/storm/generated/DRPCRequest.java | 4 +-
.../backtype/storm/generated/DebugOptions.java | 506 +
.../storm/generated/DistributedRPC.java | 4 +-
.../generated/DistributedRPCInvocations.java | 4 +-
.../jvm/backtype/storm/generated/ErrorInfo.java | 8 +-
.../storm/generated/ExecutorAggregateStats.java | 526 +
.../backtype/storm/generated/ExecutorInfo.java | 8 +-
.../storm/generated/ExecutorSpecificStats.java | 2 +-
.../backtype/storm/generated/ExecutorStats.java | 174 +-
.../storm/generated/ExecutorSummary.java | 8 +-
.../storm/generated/GetInfoOptions.java | 4 +-
.../storm/generated/GlobalStreamId.java | 4 +-
.../jvm/backtype/storm/generated/Grouping.java | 2 +-
.../generated/InvalidTopologyException.java | 4 +-
.../backtype/storm/generated/JavaObject.java | 4 +-
.../backtype/storm/generated/JavaObjectArg.java | 2 +-
.../backtype/storm/generated/KillOptions.java | 6 +-
.../storm/generated/LSApprovedWorkers.java | 48 +-
.../generated/LSSupervisorAssignments.java | 52 +-
.../storm/generated/LSSupervisorId.java | 4 +-
.../backtype/storm/generated/LSTopoHistory.java | 805 +
.../storm/generated/LSTopoHistoryList.java | 460 +
.../storm/generated/LSWorkerHeartbeat.java | 44 +-
.../storm/generated/LocalAssignment.java | 157 +-
.../storm/generated/LocalStateData.java | 52 +-
.../jvm/backtype/storm/generated/LogConfig.java | 475 +
.../jvm/backtype/storm/generated/LogLevel.java | 836 +
.../storm/generated/LogLevelAction.java | 65 +
.../jvm/backtype/storm/generated/Nimbus.java | 18314 ++++++++++++-----
.../backtype/storm/generated/NimbusSummary.java | 10 +-
.../jvm/backtype/storm/generated/NodeInfo.java | 36 +-
.../storm/generated/NotAliveException.java | 4 +-
.../backtype/storm/generated/NullStruct.java | 4 +-
.../storm/generated/NumErrorsChoice.java | 2 +-
.../backtype/storm/generated/ProfileAction.java | 74 +
.../storm/generated/ProfileRequest.java | 631 +
.../storm/generated/RebalanceOptions.java | 52 +-
.../storm/generated/ShellComponent.java | 4 +-
.../storm/generated/SpecificAggregateStats.java | 387 +
.../storm/generated/SpoutAggregateStats.java | 407 +
.../jvm/backtype/storm/generated/SpoutSpec.java | 4 +-
.../backtype/storm/generated/SpoutStats.java | 256 +-
.../storm/generated/StateSpoutSpec.java | 4 +-
.../jvm/backtype/storm/generated/StormBase.java | 231 +-
.../backtype/storm/generated/StormTopology.java | 4 +-
.../backtype/storm/generated/StreamInfo.java | 6 +-
.../backtype/storm/generated/SubmitOptions.java | 4 +-
.../storm/generated/SupervisorInfo.java | 282 +-
.../storm/generated/SupervisorSummary.java | 176 +-
.../storm/generated/ThriftSerializedObject.java | 4 +-
.../storm/generated/TopologyActionOptions.java | 2 +-
.../storm/generated/TopologyHistoryInfo.java | 461 +
.../backtype/storm/generated/TopologyInfo.java | 903 +-
.../storm/generated/TopologyInitialStatus.java | 2 +-
.../storm/generated/TopologyPageInfo.java | 2597 +++
.../backtype/storm/generated/TopologyStats.java | 1094 +
.../storm/generated/TopologyStatus.java | 2 +-
.../storm/generated/TopologySummary.java | 618 +-
.../storm/generated/WorkerResources.java | 605 +
.../src/jvm/backtype/storm/grouping/Load.java | 77 +
.../grouping/LoadAwareCustomStreamGrouping.java | 24 +
.../grouping/LoadAwareShuffleGrouping.java | 76 +
.../backtype/storm/grouping/LoadMapping.java | 64 +
.../storm/grouping/PartialKeyGrouping.java | 5 +-
.../storm/grouping/ShuffleGrouping.java | 65 +
.../storm/logging/ThriftAccessLogger.java | 27 +
.../logging/filters/AccessLoggingFilter.java | 52 +
.../storm/messaging/AddressedTuple.java | 46 +
.../storm/messaging/ConnectionWithStatus.java | 4 +-
.../DeserializingConnectionCallback.java | 60 +
.../backtype/storm/messaging/IConnection.java | 26 +-
.../storm/messaging/IConnectionCallback.java | 31 +
.../jvm/backtype/storm/messaging/IContext.java | 2 +-
.../storm/messaging/TransportFactory.java | 2 +-
.../backtype/storm/messaging/local/Context.java | 164 +
.../backtype/storm/messaging/netty/Client.java | 154 +-
.../backtype/storm/messaging/netty/Context.java | 8 +-
.../storm/messaging/netty/ControlMessage.java | 5 +-
.../storm/messaging/netty/ISaslClient.java | 28 +
.../storm/messaging/netty/ISaslServer.java | 26 +
.../backtype/storm/messaging/netty/IServer.java | 26 +
.../storm/messaging/netty/MessageBatch.java | 14 +-
.../storm/messaging/netty/MessageDecoder.java | 7 +-
.../storm/messaging/netty/SaslMessageToken.java | 10 +-
.../storm/messaging/netty/SaslNettyClient.java | 6 +-
.../storm/messaging/netty/SaslNettyServer.java | 4 -
.../messaging/netty/SaslStormClientHandler.java | 41 +-
.../messaging/netty/SaslStormServerHandler.java | 11 +-
.../storm/messaging/netty/SaslUtils.java | 11 +-
.../backtype/storm/messaging/netty/Server.java | 198 +-
.../messaging/netty/StormClientHandler.java | 51 +-
.../netty/StormClientPipelineFactory.java | 11 +-
.../backtype/storm/metric/EventLoggerBolt.java | 67 +
.../storm/metric/FileBasedEventLogger.java | 132 +
.../metric/HttpForwardingMetricsConsumer.java | 80 +
.../metric/HttpForwardingMetricsServer.java | 118 +
.../jvm/backtype/storm/metric/IEventLogger.java | 70 +
.../storm/metric/LoggingMetricsConsumer.java | 1 -
.../storm/metric/MetricsConsumerBolt.java | 1 -
.../jvm/backtype/storm/metric/SystemBolt.java | 20 +-
.../backtype/storm/metric/api/CountMetric.java | 2 -
.../backtype/storm/metric/api/MeanReducer.java | 4 +-
.../storm/metric/api/MultiCountMetric.java | 2 +-
.../storm/metric/api/MultiReducedMetric.java | 2 +-
.../storm/metric/api/rpc/CountShellMetric.java | 3 +-
.../metric/internal/CountStatAndMetric.java | 211 +
.../metric/internal/LatencyStatAndMetric.java | 259 +
.../storm/metric/internal/MetricStatTimer.java | 27 +
.../internal/MultiCountStatAndMetric.java | 112 +
.../internal/MultiLatencyStatAndMetric.java | 109 +
.../storm/metric/internal/RateTracker.java | 165 +
.../AbstractDNSToSwitchMapping.java | 95 +
.../networktopography/DNSToSwitchMapping.java | 50 +
.../DefaultRackDNSToSwitchMapping.java | 52 +
.../backtype/storm/nimbus/ILeaderElector.java | 23 +-
.../nimbus/ITopologyActionNotifierPlugin.java | 43 +
.../jvm/backtype/storm/nimbus/NimbusInfo.java | 29 +-
.../jvm/backtype/storm/scheduler/Cluster.java | 119 +-
.../scheduler/SchedulerAssignmentImpl.java | 15 +-
.../storm/scheduler/SupervisorDetails.java | 63 +-
.../backtype/storm/scheduler/Topologies.java | 27 +-
.../storm/scheduler/TopologyDetails.java | 377 +-
.../backtype/storm/scheduler/WorkerSlot.java | 25 +
.../scheduler/multitenant/DefaultPool.java | 22 +-
.../storm/scheduler/multitenant/FreePool.java | 6 +-
.../scheduler/multitenant/IsolatedPool.java | 32 +-
.../multitenant/MultitenantScheduler.java | 6 +-
.../storm/scheduler/multitenant/Node.java | 17 +-
.../storm/scheduler/multitenant/NodePool.java | 16 +-
.../storm/scheduler/resource/Component.java | 54 +
.../storm/scheduler/resource/RAS_Node.java | 575 +
.../resource/ResourceAwareScheduler.java | 167 +
.../storm/scheduler/resource/ResourceUtils.java | 133 +
.../resource/strategies/IStrategy.java | 37 +
.../strategies/ResourceAwareStrategy.java | 479 +
.../backtype/storm/security/auth/AuthUtils.java | 27 +-
.../auth/DefaultHttpCredentialsPlugin.java | 6 +-
.../security/auth/DefaultPrincipalToLocal.java | 1 -
.../storm/security/auth/IAuthorizer.java | 4 +-
.../security/auth/ICredentialsRenewer.java | 3 +-
.../security/auth/IHttpCredentialsPlugin.java | 2 -
.../storm/security/auth/IPrincipalToLocal.java | 2 +-
.../storm/security/auth/ITransportPlugin.java | 4 -
.../security/auth/KerberosPrincipalToLocal.java | 2 +-
.../storm/security/auth/ReqContext.java | 18 +-
.../security/auth/SaslTransportPlugin.java | 14 +-
.../security/auth/ShellBasedGroupsMapping.java | 12 +-
.../security/auth/SimpleTransportPlugin.java | 8 +-
.../security/auth/SingleUserPrincipal.java | 5 +-
.../storm/security/auth/TBackoffConnect.java | 1 -
.../storm/security/auth/ThriftClient.java | 10 +-
.../storm/security/auth/ThriftServer.java | 6 +-
.../auth/authorizer/DRPCAuthorizerBase.java | 2 +-
.../authorizer/DRPCSimpleACLAuthorizer.java | 21 +-
.../auth/authorizer/DenyAuthorizer.java | 16 +-
.../authorizer/ImpersonationAuthorizer.java | 19 +-
.../auth/authorizer/NoopAuthorizer.java | 12 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 45 +-
.../authorizer/SimpleWhitelistAuthorizer.java | 16 +-
.../auth/digest/ClientCallbackHandler.java | 2 -
.../auth/digest/DigestSaslTransportPlugin.java | 2 -
.../auth/digest/ServerCallbackHandler.java | 5 +-
.../storm/security/auth/kerberos/AutoTGT.java | 10 +-
.../security/auth/kerberos/NoOpTTrasport.java | 20 +-
.../auth/kerberos/ServerCallbackHandler.java | 2 +
.../serialization/BlowfishTupleSerializer.java | 6 +-
.../GzipThriftSerializationDelegate.java | 1 -
.../storm/serialization/ITupleDeserializer.java | 1 -
.../serialization/KryoTupleDeserializer.java | 3 -
.../serialization/KryoValuesDeserializer.java | 3 +-
.../serialization/SerializationFactory.java | 23 +-
.../jvm/backtype/storm/spout/ShellSpout.java | 4 +-
.../storm/task/GeneralTopologyContext.java | 15 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 42 +-
.../backtype/storm/task/TopologyContext.java | 9 +-
.../AlternateRackDNSToSwitchMapping.java | 65 +
.../storm/testing/MemoryTransactionalSpout.java | 9 +-
.../testing/OpaqueMemoryTransactionalSpout.java | 10 +-
.../storm/testing/TupleCaptureBolt.java | 4 +-
.../topology/BaseConfigurationDeclarer.java | 33 +-
.../storm/topology/BasicBoltExecutor.java | 2 +-
.../ComponentConfigurationDeclarer.java | 5 +-
.../storm/topology/OutputFieldsGetter.java | 2 +-
.../storm/topology/TopologyBuilder.java | 50 +-
.../storm/topology/base/BaseBatchBolt.java | 1 -
.../topology/base/BaseTransactionalSpout.java | 1 -
.../TransactionalSpoutBatchExecutor.java | 4 +-
.../TransactionalSpoutCoordinator.java | 2 +-
.../TransactionalTopologyBuilder.java | 12 +-
...uePartitionedTransactionalSpoutExecutor.java | 13 +-
.../PartitionedTransactionalSpoutExecutor.java | 2 +-
.../backtype/storm/tuple/AddressedTuple.java | 48 +
.../src/jvm/backtype/storm/tuple/Fields.java | 29 +-
.../src/jvm/backtype/storm/tuple/ITuple.java | 126 +-
.../src/jvm/backtype/storm/tuple/MessageId.java | 10 +-
.../src/jvm/backtype/storm/tuple/Tuple.java | 9 +-
.../src/jvm/backtype/storm/tuple/TupleImpl.java | 17 +-
.../jvm/backtype/storm/utils/DRPCClient.java | 1 -
.../utils/DisruptorBackpressureCallback.java | 27 +
.../backtype/storm/utils/DisruptorQueue.java | 592 +-
.../backtype/storm/utils/InprocMessaging.java | 4 +-
.../storm/utils/KeyedRoundRobinQueue.java | 6 +-
.../jvm/backtype/storm/utils/ListDelegate.java | 6 +-
.../jvm/backtype/storm/utils/LocalState.java | 22 +-
.../src/jvm/backtype/storm/utils/Monitor.java | 3 +-
.../jvm/backtype/storm/utils/MutableObject.java | 6 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 10 +-
.../storm/utils/RegisteredGlobalState.java | 6 +-
.../jvm/backtype/storm/utils/RotatingMap.java | 2 +-
.../backtype/storm/utils/ServiceRegistry.java | 2 +-
.../jvm/backtype/storm/utils/ShellProcess.java | 6 +-
.../jvm/backtype/storm/utils/ShellUtils.java | 2 +-
.../StormBoundedExponentialBackoffRetry.java | 3 +-
.../src/jvm/backtype/storm/utils/Time.java | 16 +-
.../backtype/storm/utils/TransferDrainer.java | 17 +-
.../src/jvm/backtype/storm/utils/Utils.java | 372 +-
.../jvm/backtype/storm/utils/VersionInfo.java | 2 +-
.../storm/utils/WorkerBackpressureCallback.java | 26 +
.../storm/utils/WorkerBackpressureThread.java | 59 +
.../storm/validation/ConfigValidation.java | 628 +
.../validation/ConfigValidationAnnotations.java | 214 +
.../storm/validation/ConfigValidationUtils.java | 175 +
.../src/jvm/storm/trident/TridentTopology.java | 89 +-
.../trident/drpc/ReturnResultsReducer.java | 4 +-
.../fluent/ChainedAggregatorDeclarer.java | 8 +-
.../jvm/storm/trident/graph/GraphGrouper.java | 13 +-
.../src/jvm/storm/trident/graph/Group.java | 23 +-
.../trident/operation/builtin/SnapshotGet.java | 4 +-
.../operation/builtin/TupleCollectionGet.java | 6 +-
.../storm/trident/partition/GlobalGrouping.java | 5 +-
.../trident/partition/IdentityGrouping.java | 8 +-
.../src/jvm/storm/trident/planner/Node.java | 5 +-
.../storm/trident/planner/PartitionNode.java | 2 -
.../storm/trident/planner/SubtopologyBolt.java | 21 +-
.../processor/MultiReducerProcessor.java | 2 +-
.../storm/trident/spout/BatchSpoutExecutor.java | 2 +-
.../jvm/storm/trident/spout/IBatchSpout.java | 2 +-
.../spout/IOpaquePartitionedTridentSpout.java | 2 +-
.../trident/spout/IPartitionedTridentSpout.java | 2 +-
.../jvm/storm/trident/spout/ITridentSpout.java | 57 +-
.../OpaquePartitionedTridentSpoutExecutor.java | 30 +-
.../spout/PartitionedTridentSpoutExecutor.java | 26 +-
.../trident/spout/RichSpoutBatchExecutor.java | 8 +-
.../trident/spout/RichSpoutBatchTriggerer.java | 14 +-
.../trident/spout/TridentSpoutCoordinator.java | 6 +-
.../trident/spout/TridentSpoutExecutor.java | 14 +-
.../storm/trident/testing/FeederBatchSpout.java | 4 +-
.../testing/FeederCommitterBatchSpout.java | 2 +-
.../storm/trident/testing/FixedBatchSpout.java | 2 +-
.../trident/topology/TridentBoltExecutor.java | 27 +-
.../topology/TridentTopologyBuilder.java | 45 +-
.../storm/trident/tuple/TridentTupleView.java | 18 +-
.../src/native/worker-launcher/impl/main.c | 10 +
.../worker-launcher/impl/worker-launcher.c | 49 +-
.../worker-launcher/impl/worker-launcher.h | 2 +
storm-core/src/py/storm/DistributedRPC-remote | 2 +-
storm-core/src/py/storm/DistributedRPC.py | 20 +-
.../py/storm/DistributedRPCInvocations-remote | 2 +-
.../src/py/storm/DistributedRPCInvocations.py | 41 +-
storm-core/src/py/storm/Nimbus-remote | 58 +-
storm-core/src/py/storm/Nimbus.py | 2676 ++-
storm-core/src/py/storm/constants.py | 2 +-
storm-core/src/py/storm/ttypes.py | 5120 ++++-
storm-core/src/storm.thrift | 204 +
storm-core/src/ui/public/component.html | 173 +-
storm-core/src/ui/public/css/style.css | 8 +
.../src/ui/public/deep_search_result.html | 155 +
storm-core/src/ui/public/images/search.png | Bin 0 -> 2354 bytes
storm-core/src/ui/public/index.html | 6 +-
storm-core/src/ui/public/js/script.js | 98 +-
.../src/ui/public/js/typeahead.jquery.min.js | 7 +
storm-core/src/ui/public/js/visualization.js | 92 +-
storm-core/src/ui/public/logviewer_search.html | 65 +
storm-core/src/ui/public/search_result.html | 100 +
.../templates/component-page-template.html | 68 +-
.../deep-search-result-page-template.html | 66 +
.../public/templates/index-page-template.html | 20 +-
.../logviewer-search-page-template.html | 44 +
.../templates/search-result-page-template.html | 60 +
.../templates/topology-page-template.html | 201 +-
.../src/ui/public/templates/user-template.html | 17 +-
storm-core/src/ui/public/topology.html | 166 +-
.../test/clj/backtype/storm/cluster_test.clj | 12 +-
.../test/clj/backtype/storm/config_test.clj | 170 -
.../test/clj/backtype/storm/grouping_test.clj | 90 +-
.../clj/backtype/storm/integration_test.clj | 12 +-
.../test/clj/backtype/storm/logviewer_test.clj | 730 +-
.../storm/messaging/netty_integration_test.clj | 3 +-
.../storm/messaging/netty_unit_test.clj | 288 +-
.../test/clj/backtype/storm/messaging_test.clj | 28 +-
.../test/clj/backtype/storm/metrics_test.clj | 2 +-
.../test/clj/backtype/storm/multilang_test.clj | 4 +-
.../test/clj/backtype/storm/nimbus_test.clj | 263 +-
.../scheduler/multitenant_scheduler_test.clj | 34 +-
.../scheduler/resource_aware_scheduler_test.clj | 669 +
.../test/clj/backtype/storm/scheduler_test.clj | 3 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 40 +-
.../clj/backtype/storm/serialization_test.clj | 14 +-
.../test/clj/backtype/storm/supervisor_test.clj | 397 +-
.../test/clj/backtype/storm/testing4j_test.clj | 1 +
.../clj/backtype/storm/transactional_test.clj | 5 +-
.../test/clj/backtype/storm/worker_test.clj | 179 +-
.../jvm/backtype/storm/TestConfigValidate.java | 642 +
.../metric/internal/CountStatAndMetricTest.java | 86 +
.../internal/LatencyStatAndMetricTest.java | 83 +
.../storm/metric/internal/RateTrackerTest.java | 94 +
.../nimbus/InMemoryTopologyActionNotifier.java | 53 +
.../storm/topology/TopologyBuilderTest.java | 48 +
.../utils/DisruptorQueueBackpressureTest.java | 110 +
.../storm/utils/DisruptorQueueTest.java | 106 +-
.../backtype/storm/utils/MockTupleHelpers.java | 40 +
storm-dist/binary/src/main/assembly/binary.xml | 35 -
545 files changed, 62578 insertions(+), 13780 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1496f27a/storm-core/src/jvm/storm/trident/TridentTopology.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/storm/trident/TridentTopology.java
index aec6eb8,58c83e4..1d9c867
--- a/storm-core/src/jvm/storm/trident/TridentTopology.java
+++ b/storm-core/src/jvm/storm/trident/TridentTopology.java
@@@ -80,16 -87,17 +80,17 @@@ import storm.trident.util.TridentUtils
// all operations have finishBatch and can optionally be committers
public class TridentTopology {
- //TODO: add a method for drpc stream, needs to know how to automatically do returnresults, etc
+ //TODO: add a method for drpc stream, needs to know how to automatically do return results, etc
// is it too expensive to do a batch per drpc request?
- DefaultDirectedGraph<Node, IndexedEdge> _graph;
- Map<String, List<Node>> _colocate = new LinkedHashMap();
- UniqueIdGen _gen;
-
+ final DefaultDirectedGraph<Node, IndexedEdge> _graph;
+ final Map<String, List<Node>> _colocate;
+ final UniqueIdGen _gen;
-
++
public TridentTopology() {
- _graph = new DefaultDirectedGraph(new ErrorEdgeFactory());
- _gen = new UniqueIdGen();
+ this(new DefaultDirectedGraph<Node, IndexedEdge>(new ErrorEdgeFactory()),
- new HashMap<String, List<Node>>(),
++ new LinkedHashMap<String, List<Node>>(),
+ new UniqueIdGen());
}
private TridentTopology(DefaultDirectedGraph<Node, IndexedEdge> graph, Map<String, List<Node>> colocate, UniqueIdGen gen) {
@@@ -263,10 -271,10 +264,10 @@@
completeDRPC(graph, _colocate, _gen);
- List<SpoutNode> spoutNodes = new ArrayList<SpoutNode>();
+ List<SpoutNode> spoutNodes = new ArrayList<>();
// can be regular nodes (static state) or processor nodes
- Set<Node> boltNodes = new LinkedHashSet<Node>();
- Set<Node> boltNodes = new HashSet<>();
++ Set<Node> boltNodes = new LinkedHashSet<>();
for(Node n: graph.vertexSet()) {
if(n instanceof SpoutNode) {
spoutNodes.add((SpoutNode) n);
@@@ -276,7 -284,7 +277,7 @@@
}
- Set<Group> initialGroups = new LinkedHashSet<Group>();
- Set<Group> initialGroups = new HashSet<>();
++ Set<Group> initialGroups = new LinkedHashSet<>();
for(List<Node> colocate: _colocate.values()) {
Group g = new Group(graph, colocate);
boltNodes.removeAll(colocate);
http://git-wip-us.apache.org/repos/asf/storm/blob/1496f27a/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
index 5822ddc,6f34e8c..37f00a5
--- a/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
+++ b/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
@@@ -32,8 -34,8 +31,8 @@@ public class GraphGrouper
public GraphGrouper(DirectedGraph<Node, IndexedEdge> graph, Collection<Group> initialGroups) {
this.graph = graph;
- this.currGroups = new HashSet<>(initialGroups);
- reindex();
+ this.currGroups = new LinkedHashSet(initialGroups);
- reindex();
++ reindex();
}
public Collection<Group> getAllGroups() {
[5/6] storm git commit: Merge branch 'STORM-1016' of
https://github.com/victor-wong/storm into STORM-1016
Posted by bo...@apache.org.
Merge branch 'STORM-1016' of https://github.com/victor-wong/storm into STORM-1016
STORM-1016: Generate trident bolt ids with sorted group names
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/696ca47b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/696ca47b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/696ca47b
Branch: refs/heads/master
Commit: 696ca47b59194d8dd6537bf6cb11fd9f6abd2583
Parents: 5a79ba5 532a594
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Nov 17 09:35:43 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Nov 17 09:35:43 2015 -0600
----------------------------------------------------------------------
.../src/jvm/storm/trident/TridentTopology.java | 19 ++---
.../jvm/storm/trident/graph/GraphGrouper.java | 11 ++-
.../jvm/storm/trident/TestTridentTopology.java | 76 ++++++++++++++++++++
3 files changed, 86 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
[6/6] storm git commit: Added STORM-1016 to Changelog
Posted by bo...@apache.org.
Added STORM-1016 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1afa5a27
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1afa5a27
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1afa5a27
Branch: refs/heads/master
Commit: 1afa5a27cb7ab5191905d0cfafdb46f542a7c039
Parents: 696ca47
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Nov 17 09:36:57 2015 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Nov 17 09:36:57 2015 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
README.markdown | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1afa5a27/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c7e7d23..cc4bfe8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-1016: Generate trident bolt ids with sorted group names
* STORM-1190: System Load too high after recent changes
* STORM-1098: Nimbus hook for topology actions.
* STORM-1145: Have IConnection push tuples instead of pull them
http://git-wip-us.apache.org/repos/asf/storm/blob/1afa5a27/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index 683e04f..d0963a8 100644
--- a/README.markdown
+++ b/README.markdown
@@ -241,6 +241,7 @@ under the License.
* Chuck Burgess ([@ashnazg](https://github.com/ashnazg))
* Tom Graves ([@tgravescs](https://github.com/tgravescs))
* Dror Weiss ([@drorweiss](https://github.com/drorweiss))
+* Victor Wong ([@victor-wong](https://github.com/victor-wong))
## Acknowledgements
[4/6] storm git commit: add apache license header
Posted by bo...@apache.org.
add apache license header
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/532a5944
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/532a5944
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/532a5944
Branch: refs/heads/master
Commit: 532a594426c686da38aeab5179c0369cbd2892f1
Parents: 1496f27
Author: wangjiasheng <wa...@xiaomi.com>
Authored: Tue Nov 17 11:35:22 2015 +0800
Committer: wangjiasheng <wa...@xiaomi.com>
Committed: Tue Nov 17 11:35:22 2015 +0800
----------------------------------------------------------------------
.../jvm/storm/trident/graph/GraphGrouper.java | 2 +-
.../jvm/storm/trident/TestTridentTopology.java | 24 +++++++++++++++-----
2 files changed, 19 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/532a5944/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java b/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
index 37f00a5..671498e 100644
--- a/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
+++ b/storm-core/src/jvm/storm/trident/graph/GraphGrouper.java
@@ -31,7 +31,7 @@ public class GraphGrouper {
public GraphGrouper(DirectedGraph<Node, IndexedEdge> graph, Collection<Group> initialGroups) {
this.graph = graph;
- this.currGroups = new LinkedHashSet(initialGroups);
+ this.currGroups = new LinkedHashSet<>(initialGroups);
reindex();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/532a5944/storm-core/test/jvm/storm/trident/TestTridentTopology.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/storm/trident/TestTridentTopology.java b/storm-core/test/jvm/storm/trident/TestTridentTopology.java
index 2c8910e..0b0218a 100644
--- a/storm-core/test/jvm/storm/trident/TestTridentTopology.java
+++ b/storm-core/test/jvm/storm/trident/TestTridentTopology.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package storm.trident;
import backtype.storm.generated.Bolt;
@@ -7,20 +25,14 @@ import backtype.storm.tuple.Values;
import org.junit.Assert;
import org.junit.Test;
import storm.trident.operation.builtin.Count;
-import storm.trident.operation.builtin.FilterNull;
import storm.trident.operation.builtin.Sum;
-import storm.trident.operation.impl.FilterExecutor;
import storm.trident.testing.FixedBatchSpout;
import storm.trident.testing.Split;
import storm.trident.testing.StringLength;
-import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-/**
- * Created by jiasheng on 15-8-25.
- */
public class TestTridentTopology {
private StormTopology buildTopology() {