You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/04/18 14:59:37 UTC
[07/52] [abbrv] metron git commit: METRON-1471: Migrate shuffle
connections to local or shuffle closes apache/metron#949
METRON-1471: Migrate shuffle connections to local or shuffle closes apache/metron#949
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bc7732e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bc7732e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bc7732e
Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 8bc7732e4c904085e4f78469abded84635971632
Parents: abb152b
Author: cstella <ce...@gmail.com>
Authored: Thu Mar 8 17:17:57 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Mar 8 17:17:57 2018 -0500
----------------------------------------------------------------------
.../metron-profiler/src/main/flux/profiler/remote.yaml | 6 +++---
.../metron-indexing/src/main/flux/indexing/batch/remote.yaml | 4 ++--
.../src/main/flux/indexing/random_access/remote.yaml | 4 ++--
.../apache/metron/parsers/topology/ParserTopologyBuilder.java | 4 ++--
4 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
index 752640a..9ec5ba4 100644
--- a/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler/src/main/flux/profiler/remote.yaml
@@ -168,7 +168,7 @@ streams:
from: "kafkaSpout"
to: "splitterBolt"
grouping:
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
- name: "splitter -> builder"
from: "splitterBolt"
@@ -182,11 +182,11 @@ streams:
to: "hbaseBolt"
grouping:
streamId: "hbase"
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
- name: "builder -> kafka"
from: "builderBolt"
to: "kafkaBolt"
grouping:
streamId: "kafka"
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
index 85e3baa..3087dd9 100644
--- a/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/batch/remote.yaml
@@ -158,7 +158,7 @@ streams:
from: "kafkaSpout"
to: "hdfsIndexingBolt"
grouping:
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
- name: "hdfsBolt -> errorIndexingBolt"
@@ -166,4 +166,4 @@ streams:
to: "indexingErrorBolt"
grouping:
streamId: "error"
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
index cadc1ec..429ba45 100644
--- a/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
+++ b/metron-platform/metron-indexing/src/main/flux/indexing/random_access/remote.yaml
@@ -130,11 +130,11 @@ streams:
from: "kafkaSpout"
to: "indexingBolt"
grouping:
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
- name: "indexingBolt -> errorIndexingBolt"
from: "indexingBolt"
to: "indexingErrorBolt"
grouping:
streamId: "error"
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
http://git-wip-us.apache.org/repos/asf/metron/blob/8bc7732e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 1039e56..2865dd6 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -126,14 +126,14 @@ public class ParserTopologyBuilder {
ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig, outputTopic);
builder.setBolt("parserBolt", parserBolt, parserParallelism)
.setNumTasks(parserNumTasks)
- .shuffleGrouping("kafkaSpout");
+ .localOrShuffleGrouping("kafkaSpout");
// create the error bolt, if needed
if (errorWriterNumTasks > 0) {
WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig);
builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism)
.setNumTasks(errorWriterNumTasks)
- .shuffleGrouping("parserBolt", Constants.ERROR_STREAM);
+ .localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
}
return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));