You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/04/18 15:00:05 UTC
[35/52] [abbrv] metron git commit: METRON-1504: Enriching missing
values does not match the semantics between the new enrichment topology and
old closes apache/incubator-metron#976
METRON-1504: Enriching missing values does not match the semantics between the new enrichment topology and old closes apache/incubator-metron#976
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3ba9ae25
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3ba9ae25
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3ba9ae25
Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 3ba9ae25126bc393e3b9307f4ffe63ac66a909f4
Parents: 46bc63d
Author: cstella <ce...@gmail.com>
Authored: Mon Apr 9 11:47:51 2018 -0400
Committer: cstella <ce...@gmail.com>
Committed: Mon Apr 9 11:47:51 2018 -0400
----------------------------------------------------------------------
.../enrichment/parallel/ParallelEnricher.java | 3 +
.../integration/EnrichmentIntegrationTest.java | 12 +++-
.../parallel/ParallelEnricherTest.java | 59 +++++++++++++++++++-
3 files changed, 70 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
index 6ddb892..b10c148 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
@@ -170,6 +170,9 @@ public class ParallelEnricher {
for(Object o : m.keySet()) {
String field = (String) o;
Object value = m.get(o);
+ if(value == null) {
+ continue;
+ }
CacheKey cacheKey = new CacheKey(field, value, config);
String prefix = adapter.getOutputPrefix(cacheKey);
Supplier<JSONObject> supplier = () -> {
http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 267ca62..3c55c95 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -95,7 +95,15 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
private static List<byte[]> getInputMessages(String path){
try{
- return TestUtils.readSampleData(path);
+ List<byte[]> ret = TestUtils.readSampleData(path);
+ {
+ //we want one of the fields without a destination IP to ensure that enrichments can function
+ Map<String, Object> sansDestinationIp = JSONUtils.INSTANCE.load(new String(ret.get(ret.size() -1))
+ , JSONUtils.MAP_SUPPLIER);
+ sansDestinationIp.remove(Constants.Fields.DST_ADDR.getName());
+ ret.add(JSONUtils.INSTANCE.toJSONPretty(sansDestinationIp));
+ }
+ return ret;
}catch(IOException ioe){
return null;
}
@@ -262,8 +270,6 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
//ensure we always have a source ip and destination ip
Assert.assertNotNull(jsonDoc.get(SRC_IP));
- Assert.assertNotNull(jsonDoc.get(DST_IP));
-
Assert.assertNotNull(jsonDoc.get("ALL_CAPS"));
Assert.assertNotNull(jsonDoc.get("map.blah"));
Assert.assertNull(jsonDoc.get("map"));
http://git-wip-us.apache.org/repos/asf/metron/blob/3ba9ae25/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
index 4a4573b..d4fcdf4 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
@@ -79,7 +79,39 @@ public class ParallelEnricherTest {
}
}.ofType("ENRICHMENT");
adapter.initializeAdapter(new HashMap<>());
- enrichmentsByType = ImmutableMap.of("stellar", adapter);
+ EnrichmentAdapter<CacheKey> dummy = new EnrichmentAdapter<CacheKey>() {
+ @Override
+ public void logAccess(CacheKey value) {
+
+ }
+
+ @Override
+ public JSONObject enrich(CacheKey value) {
+ return null;
+ }
+
+ @Override
+ public boolean initializeAdapter(Map<String, Object> config) {
+ return false;
+ }
+
+ @Override
+ public void updateAdapter(Map<String, Object> config) {
+
+ }
+
+ @Override
+ public void cleanup() {
+
+ }
+
+ @Override
+ public String getOutputPrefix(CacheKey value) {
+ return null;
+ }
+ };
+
+ enrichmentsByType = ImmutableMap.of("stellar", adapter, "dummy", dummy);
enricher = new ParallelEnricher(enrichmentsByType, infrastructure, false);
}
@@ -115,6 +147,31 @@ public class ParallelEnricherTest {
Assert.assertEquals("TEST", ret.get("ALL_CAPS"));
Assert.assertEquals(0, result.getEnrichmentErrors().size());
}
+/**
+ * {
+ "enrichment": {
+ "fieldMap": {
+ "dummy" : ["notthere"]
+ }
+ ,"fieldToTypeMap": { }
+ },
+ "threatIntel": { }
+}
+ */
+ @Multiline
+ public static String nullConfig;
+
+ @Test
+ public void testNullEnrichment() throws Exception {
+ SensorEnrichmentConfig config = JSONUtils.INSTANCE.load(nullConfig, SensorEnrichmentConfig.class);
+ config.getConfiguration().putIfAbsent("stellarContext", stellarContext);
+ JSONObject message = new JSONObject() {{
+ put(Constants.SENSOR_TYPE, "test");
+ }};
+ ParallelEnricher.EnrichmentResult result = enricher.apply(message, EnrichmentStrategies.ENRICHMENT, config, null);
+ JSONObject ret = result.getResult();
+ Assert.assertEquals("Got the wrong result count: " + ret, 4, ret.size());
+ }
/**
* {