You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/06/19 14:38:06 UTC
metron git commit: METRON-995: Temporary variables in stellar
enrichments which are maps do not function as expected closes
apache/metron#615
Repository: metron
Updated Branches:
refs/heads/master f523c1795 -> de2c871ab
METRON-995: Temporary variables in stellar enrichments which are maps do not function as expected closes apache/metron#615
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/de2c871a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/de2c871a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/de2c871a
Branch: refs/heads/master
Commit: de2c871abf3e259311281743952eb465362cf90c
Parents: f523c17
Author: cstella <ce...@gmail.com>
Authored: Mon Jun 19 10:37:50 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Mon Jun 19 10:37:50 2017 -0400
----------------------------------------------------------------------
.../enrichment/handler/StellarConfig.java | 13 ++---
.../metron/common/stellar/StellarCompiler.java | 2 +
.../StellarEnrichmentConfigTest.java | 37 +++++++++++++
.../configuration/StellarEnrichmentTest.java | 44 +++++++++++++++-
.../metron/common/stellar/StellarTest.java | 5 ++
.../adapters/stellar/StellarAdapter.java | 35 ++++++++++---
.../adapters/stellar/StellarAdapterTest.java | 55 ++++++++++++++++++++
.../integration/EnrichmentIntegrationTest.java | 5 ++
.../main/config/zookeeper/enrichments/test.json | 4 +-
9 files changed, 185 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 7d49fbd..a888b81 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -17,18 +17,18 @@
*/
package org.apache.metron.common.configuration.enrichment.handler;
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
+import com.google.common.base.Joiner;
import org.apache.metron.common.stellar.StellarAssignment;
import org.apache.metron.common.stellar.StellarProcessor;
import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.function.Function;
public class StellarConfig implements Config {
-
+ protected static final Logger _LOG = LoggerFactory.getLogger(StellarConfig.class);
@Override
public List<String> getSubgroups(Iterable<Map.Entry<String, Object>> config) {
boolean includeEmpty = false;
@@ -81,10 +81,10 @@ public class StellarConfig implements Config {
{
StellarProcessor processor = new StellarProcessor();
List<JSONObject> messages = new ArrayList<>();
- Map<String, String> defaultStellarStatementGroup = new HashMap<>();
+ List<String> defaultStellarStatementGroup = new ArrayList<>();
for(Map.Entry<String, Object> kv : config) {
if(kv.getValue() instanceof String) {
- defaultStellarStatementGroup.put(kv.getKey(), (String)kv.getValue());
+ defaultStellarStatementGroup.add((String)kv.getValue());
}
else if(kv.getValue() instanceof Map) {
JSONObject ret = new JSONObject();
@@ -103,6 +103,7 @@ public class StellarConfig implements Config {
ret.put("", getMessage(getFields(processor, defaultStellarStatementGroup), message));
messages.add(ret);
}
+ _LOG.debug("Stellar enrichment split: {}", messages );
return messages;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
index 136728d..f4e101b 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.common.stellar;
+import org.antlr.v4.runtime.ParserRuleContext;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.metron.common.dsl.Context;
import org.apache.metron.common.dsl.Token;
@@ -347,6 +348,7 @@ public class StellarCompiler extends StellarBaseListener {
}, DeferredFunction.class, context));
}
+
@Override
public void exitVariable(StellarParser.VariableContext ctx) {
final FrameContext.Context context = getArgContext();
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
index 0883672..32fb00e 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
@@ -18,7 +18,9 @@
package org.apache.metron.common.configuration;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
+import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
import org.apache.metron.common.configuration.enrichment.handler.Configs;
@@ -33,6 +35,41 @@ import java.util.Map;
public class StellarEnrichmentConfigTest extends StellarEnrichmentTest {
+ /**
+ {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "dga_model_endpoint := MAAS_GET_ENDPOINT('dga')",
+ "dga_result_map := MAAS_MODEL_APPLY( dga_model_endpoint, { 'host' : domain_without_subdomains } )",
+ "dga_result := MAP_GET('is_malicious', dga_result_map)",
+ "is_dga := dga_result != null && dga_result == 'dga'",
+ "dga_model_version := MAP_GET('version', dga_model_endpoint)",
+ "dga_model_endpoint := null",
+ "dga_result_map := null",
+ "dga_result := null"
+ ]
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String conf;
+
+ @Test
+ public void testSplitter_listWithTemporaryVariables() throws IOException {
+ JSONObject message = new JSONObject(ImmutableMap.of("domain_without_subdomains", "yahoo.com"));
+ EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(conf, EnrichmentConfig.class);
+ Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
+ ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
+ List<JSONObject> splits = Configs.STELLAR.splitByFields(message, null, x -> null, handler );
+ Assert.assertEquals(1, splits.size());
+ Map<String, Object> split = (Map<String, Object>)(splits.get(0)).get("");
+ Assert.assertEquals("yahoo.com", split.get("domain_without_subdomains"));
+ Assert.assertTrue(split.containsKey("dga_result"));
+ Assert.assertTrue(split.containsKey("dga_model_endpoint"));
+ Assert.assertTrue(split.containsKey("dga_result_map"));
+ }
@Test
public void testSplitter_default() throws IOException {
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
index 3af15cd..8eb9bbc 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
@@ -59,7 +59,25 @@ public class StellarEnrichmentTest {
*/
@Multiline
public static String defaultStellarConfig_list;
- public static List<String> DEFAULT_CONFIGS = ImmutableList.of(defaultStellarConfig_list, defaultStellarConfig_map);
+
+ /**
+ {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "stmt1 := TO_UPPER(source.type)",
+ "stmt4 := TO_LOWER(string)",
+ "stmt2 := TO_LOWER(stmt1)",
+ "stmt3 := TO_LOWER(string)",
+ "stmt4 := null"
+ ]
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String defaultStellarConfig_listWithTemp;
+ public static List<String> DEFAULT_CONFIGS = ImmutableList.of(defaultStellarConfig_list, defaultStellarConfig_map, defaultStellarConfig_listWithTemp);
/**
{
@@ -100,7 +118,29 @@ public class StellarEnrichmentTest {
*/
@Multiline
public static String groupedStellarConfig_list;
- public static List<String> GROUPED_CONFIGS = ImmutableList.of(groupedStellarConfig_list, groupedStellarConfig_map);
+
+ /**
+ {
+ "fieldMap": {
+ "stellar" : {
+ "config" : {
+ "group1" : [
+ "stmt1 := TO_UPPER(source.type)",
+ "stmt2 := TO_LOWER(stmt1)"
+ ],
+ "group2" : [
+ "stmt3 := TO_LOWER(string)",
+ "stmt4 := TO_LOWER(string)",
+ "stmt4 := null"
+ ]
+ }
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String groupedStellarConfig_listWithTemp;
+ public static List<String> GROUPED_CONFIGS = ImmutableList.of(groupedStellarConfig_listWithTemp, groupedStellarConfig_list, groupedStellarConfig_map);
/**
{
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
index 98abd57..d549e19 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
@@ -183,8 +183,13 @@ public class StellarTest {
Assert.assertEquals(ImmutableSet.of("one", "two")
, processor.variablesUsed("if 1 < 2 then one else two"));
}
+ {
+ Assert.assertEquals(ImmutableSet.of("bar")
+ , processor.variablesUsed("MAP_GET('foo', { 'foo' : bar})"));
+ }
}
+
@Test
public void testFunctionEmptyArgs() {
{
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
index 53ef16f..4b78821 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
@@ -32,6 +32,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.function.Function;
@@ -101,7 +103,9 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
JSONObject ret = new JSONObject();
Iterable<Map.Entry<String, Object>> stellarStatements = getStellarStatements(handler, field);
+ _LOG.debug("message := {}", message);
if(stellarStatements != null) {
+ List<String> mapEntries = new ArrayList<>();
for (Map.Entry<String, Object> kv : stellarStatements) {
if(kv.getKey() != null && kv.getValue() != null) {
if (kv.getValue() instanceof String) {
@@ -114,14 +118,11 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
_PERF_LOG.debug("SLOW LOG: " + stellarStatement + " took" + duration + "ms");
}
}
+ _LOG.debug("{} := {} yields {}", kv.getKey(), stellarStatement , o);
if (o != null && o instanceof Map) {
- for (Map.Entry<Object, Object> valueKv : ((Map<Object, Object>) o).entrySet()) {
- String newKey = ((kv.getKey().length() > 0) ? kv.getKey() + "." : "") + valueKv.getKey();
- message.put(newKey, valueKv.getValue());
- ret.put(newKey, valueKv.getValue());
- }
+ mapEntries.add(kv.getKey());
}
- else if(o == null) {
+ if(o == null) {
message.remove(kv.getKey());
ret.remove(kv.getKey());
}
@@ -132,6 +133,28 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
}
}
}
+ /*
+ We need to handle the map entries separately now.
+ We want to explode them out, so if "var" is
+ {
+ "foo" : "bar"
+ }
+ then we want "var.foo" == "bar"
+ and no "var"
+ */
+ for(String mapEntry : mapEntries) {
+ String key = mapEntry;
+ Map<Object, Object> value = (Map<Object, Object>) ret.get(key);
+ if(value != null) {
+ _LOG.debug("Exploding map: {} == {}", key, value);
+ for (Map.Entry<Object, Object> valueKv : value.entrySet()) {
+ String newKey = ((key.length() > 0) ? key + "." : "") + valueKv.getKey();
+ ret.put(newKey, valueKv.getValue());
+ }
+ //removing the map from downstream
+ ret.remove(key);
+ }
+ }
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
index 5624d4b..724221d 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.enrichment.adapters.stellar;
import com.google.common.collect.ImmutableList;
+import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.StellarEnrichmentTest;
import org.apache.metron.common.configuration.enrichment.EnrichmentConfig;
import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
@@ -30,6 +31,8 @@ import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Test;
+import java.util.List;
+
public class StellarAdapterTest extends StellarEnrichmentTest {
StellarProcessor processor = new StellarProcessor();
@@ -131,4 +134,56 @@ public class StellarAdapterTest extends StellarEnrichmentTest {
Assert.assertEquals(2, enriched.size());
}
}
+
+ /**
+ {
+ "fieldMap": {
+ "stellar" : {
+ "config" : {
+ "group1" : [
+ "stmt1 := TO_UPPER(source.type)",
+ "stmt2 := { 'foo' : source.type }"
+ ]
+ }
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String mapConfig_subgroup;
+ /**
+ {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "stmt1 := TO_UPPER(source.type)",
+ "stmt2 := { 'foo' : source.type }"
+ ]
+ }
+ }
+ }
+ */
+ @Multiline
+ public static String mapConfig_default;
+
+ private void testMapEnrichment(String config, String field) throws Exception {
+ JSONObject message = getMessage();
+ EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(config, EnrichmentConfig.class);
+ Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar"));
+ ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar");
+ JSONObject enriched = enrich(message, field, handler);
+ Assert.assertEquals(2, enriched.size());
+ Assert.assertEquals("stellar_test", enriched.get("stmt2.foo"));
+ Assert.assertEquals("stellar_test".toUpperCase(), enriched.get("stmt1"));
+ }
+
+ @Test
+ public void testMapEnrichment_subgroup() throws Exception {
+ testMapEnrichment(mapConfig_subgroup, "group1");
+ }
+
+ @Test
+ public void testMapEnrichment_default() throws Exception {
+ testMapEnrichment(mapConfig_default, "");
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index 9634805..2149cb9 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -260,6 +260,11 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
Assert.assertNotNull(jsonDoc.get(DST_IP));
Assert.assertNotNull(jsonDoc.get("ALL_CAPS"));
+ Assert.assertNotNull(jsonDoc.get("map.blah"));
+ Assert.assertNull(jsonDoc.get("map"));
+ Assert.assertNotNull(jsonDoc.get("one"));
+ Assert.assertEquals(1, jsonDoc.get("one"));
+ Assert.assertEquals(1, jsonDoc.get("map.blah"));
Assert.assertNotNull(jsonDoc.get("foo"));
Assert.assertEquals("TEST", jsonDoc.get("ALL_CAPS"));
Assert.assertNotNull(jsonDoc.get("bar"));
http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
index 1037b69..bba6679 100644
--- a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
+++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json
@@ -16,7 +16,9 @@
"stellar" : {
"config" : {
"numeric" : {
- "foo": "1 + 1"
+ "map" : "{ 'blah' : 1}"
+ ,"one" : "MAP_GET('blah', map)"
+ ,"foo": "1 + 1"
}
,"ALL_CAPS" : "TO_UPPER(source.type)"
,"src_enrichment" : {