You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/04/18 14:59:35 UTC
[05/52] [abbrv] metron git commit: METRON-1463: Adjust the groupings
and shuffles in enrichment to be more efficient closes apache/metron#944
METRON-1463: Adjust the groupings and shuffles in enrichment to be more efficient closes apache/metron#944
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9fb0d06e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9fb0d06e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9fb0d06e
Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 9fb0d06e644597032da58c5df020a841b93d19c0
Parents: 1d95b83
Author: cstella <ce...@gmail.com>
Authored: Wed Mar 7 09:21:52 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Wed Mar 7 09:21:52 2018 -0500
----------------------------------------------------------------------
.../src/main/flux/enrichment/remote.yaml | 42 +++++++++-----------
.../enrichment/bolt/EnrichmentJoinBolt.java | 4 +-
.../enrichment/bolt/EnrichmentSplitterBolt.java | 4 +-
.../enrichment/bolt/GenericEnrichmentBolt.java | 2 +-
.../enrichment/bolt/ThreatIntelJoinBolt.java | 4 +-
.../bolt/ThreatIntelSplitterBolt.java | 4 +-
6 files changed, 28 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
index 7ee1b2e..fd7ceff 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote.yaml
@@ -411,7 +411,7 @@ streams:
from: "kafkaSpout"
to: "enrichmentSplitBolt"
grouping:
- type: SHUFFLE
+ type: LOCAL_OR_SHUFFLE
#enrichment
- name: "enrichmentSplit -> host"
@@ -420,14 +420,15 @@ streams:
grouping:
streamId: "host"
type: FIELDS
- args: ["key"]
+ args: ["message"]
+
- name: "enrichmentSplit -> geo"
from: "enrichmentSplitBolt"
to: "geoEnrichmentBolt"
grouping:
streamId: "geo"
type: FIELDS
- args: ["key"]
+ args: ["message"]
- name: "enrichmentSplit -> stellar"
from: "enrichmentSplitBolt"
@@ -435,7 +436,7 @@ streams:
grouping:
streamId: "stellar"
type: FIELDS
- args: ["key"]
+ args: ["message"]
- name: "enrichmentSplit -> simpleHBaseEnrichmentBolt"
@@ -444,7 +445,7 @@ streams:
grouping:
streamId: "hbaseEnrichment"
type: FIELDS
- args: ["key"]
+ args: ["message"]
- name: "splitter -> join"
from: "enrichmentSplitBolt"
@@ -453,6 +454,7 @@ streams:
streamId: "message"
type: FIELDS
args: ["key"]
+
- name: "geo -> join"
from: "geoEnrichmentBolt"
to: "enrichmentJoinBolt"
@@ -460,6 +462,7 @@ streams:
streamId: "geo"
type: FIELDS
args: ["key"]
+
- name: "stellar -> join"
from: "stellarEnrichmentBolt"
to: "enrichmentJoinBolt"
@@ -468,8 +471,6 @@ streams:
type: FIELDS
args: ["key"]
-
-
- name: "simpleHBaseEnrichmentBolt -> join"
from: "simpleHBaseEnrichmentBolt"
to: "enrichmentJoinBolt"
@@ -477,6 +478,7 @@ streams:
streamId: "hbaseEnrichment"
type: FIELDS
args: ["key"]
+
- name: "host -> join"
from: "hostEnrichmentBolt"
to: "enrichmentJoinBolt"
@@ -491,32 +493,28 @@ streams:
to: "enrichmentErrorOutputBolt"
grouping:
streamId: "error"
- type: FIELDS
- args: ["message"]
+ type: LOCAL_OR_SHUFFLE
- name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt"
from: "stellarEnrichmentBolt"
to: "enrichmentErrorOutputBolt"
grouping:
streamId: "error"
- type: FIELDS
- args: ["message"]
+ type: LOCAL_OR_SHUFFLE
- name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt"
from: "hostEnrichmentBolt"
to: "enrichmentErrorOutputBolt"
grouping:
streamId: "error"
- type: FIELDS
- args: ["message"]
+ type: LOCAL_OR_SHUFFLE
- name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt"
from: "simpleHBaseEnrichmentBolt"
to: "enrichmentErrorOutputBolt"
grouping:
streamId: "error"
- type: FIELDS
- args: ["message"]
+ type: LOCAL_OR_SHUFFLE
#threat intel
- name: "enrichmentJoin -> threatSplit"
@@ -533,14 +531,15 @@ streams:
grouping:
streamId: "hbaseThreatIntel"
type: FIELDS
- args: ["key"]
+ args: ["message"]
+
- name: "threatSplit -> stellarThreatIntel"
from: "threatIntelSplitBolt"
to: "stellarThreatIntelBolt"
grouping:
streamId: "stellar"
type: FIELDS
- args: ["key"]
+ args: ["message"]
- name: "simpleHBaseThreatIntel -> join"
@@ -572,8 +571,7 @@ streams:
to: "outputBolt"
grouping:
streamId: "message"
- type: FIELDS
- args: ["key"]
+ type: LOCAL_OR_SHUFFLE
# Error output
- name: "simpleHBaseThreatIntelBolt -> threatIntelErrorOutputBolt"
@@ -581,14 +579,12 @@ streams:
to: "threatIntelErrorOutputBolt"
grouping:
streamId: "error"
- type: FIELDS
- args: ["message"]
+ type: LOCAL_OR_SHUFFLE
- name: "stellarThreatIntelBolt -> threatIntelErrorOutputBolt"
from: "stellarThreatIntelBolt"
to: "threatIntelErrorOutputBolt"
grouping:
streamId: "error"
- type: FIELDS
- args: ["message"]
+ type: LOCAL_OR_SHUFFLE
http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
index 3f25479..fd5f874 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentJoinBolt.java
@@ -96,7 +96,7 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
if (config != null) {
return config.getEnrichment().getEnrichmentConfigs();
} else {
- LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType);
+ LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType);
}
} else {
LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -111,7 +111,7 @@ public class EnrichmentJoinBolt extends JoinBolt<JSONObject> {
return config.getEnrichment().getFieldMap();
}
else {
- LOG.info("Unable to retrieve a sensor enrichment config of {}", sourceType);
+ LOG.debug("Unable to retrieve a sensor enrichment config of {}", sourceType);
}
}
else {
http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
index ae75508..3298c76 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
@@ -147,7 +147,7 @@ public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> {
if (config != null) {
return config.getEnrichment().getEnrichmentConfigs();
} else {
- LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType);
+ LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType);
}
} else {
LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -160,7 +160,7 @@ public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> {
if (config != null) {
return config.getEnrichment().getFieldMap();
} else {
- LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType);
+ LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType);
}
} else {
LOG.error("Trying to retrieve a field map with sensor type of null");
http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
index dbbb7b6..7d67d2d 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
@@ -212,7 +212,7 @@ public class GenericEnrichmentBolt extends ConfiguredEnrichmentBolt {
if (value != null) {
SensorEnrichmentConfig config = getConfigurations().getSensorEnrichmentConfig(sourceType);
if(config == null) {
- LOG.error("Unable to find SensorEnrichmentConfig for sourceType: {}", sourceType);
+ LOG.debug("Unable to find SensorEnrichmentConfig for sourceType: {}", sourceType);
MetronError metronError = new MetronError()
.withErrorType(Constants.ErrorType.ENRICHMENT_ERROR)
.withMessage("Unable to find SensorEnrichmentConfig for sourceType: " + sourceType)
http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index 1ce0b16..cfa101d 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -68,7 +68,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
if (config != null) {
return config.getThreatIntel().getEnrichmentConfigs();
} else {
- LOG.info("Unable to retrieve a sensor enrichment config of {}", sensorType);
+ LOG.debug("Unable to retrieve a sensor enrichment config of {}", sensorType);
}
} else {
LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -100,7 +100,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
return config.getThreatIntel().getFieldMap();
}
else {
- LOG.info("Unable to retrieve sensor config: {}", sourceType);
+ LOG.debug("Unable to retrieve sensor config: {}", sourceType);
return null;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/9fb0d06e/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
index bd408e0..76c65c6 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
@@ -37,7 +37,7 @@ public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt {
if (config != null) {
return config.getThreatIntel().getEnrichmentConfigs();
} else {
- LOG.info("Unable to retrieve a sensor config of {}", sensorType);
+ LOG.debug("Unable to retrieve a sensor config of {}", sensorType);
}
} else {
LOG.error("Trying to retrieve a field map with sensor type of null");
@@ -52,7 +52,7 @@ public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt {
if (config != null) {
return config.getThreatIntel().getFieldMap();
} else {
- LOG.info("Unable to retrieve sensor config: {}", sensorType);
+ LOG.debug("Unable to retrieve sensor config: {}", sensorType);
}
} else {
LOG.error("Trying to retrieve a field map with sensor type of null");