You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/02/27 14:02:18 UTC
incubator-metron git commit: METRON-741: Stellar Field
Transformations should execute all of the transformations,
not just the output closes apache/incubator-metron#465
Repository: incubator-metron
Updated Branches:
refs/heads/master a12faeb0e -> 1f83fdcba
METRON-741: Stellar Field Transformations should execute all of the transformations, not just the output closes apache/incubator-metron#465
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/1f83fdcb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/1f83fdcb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/1f83fdcb
Branch: refs/heads/master
Commit: 1f83fdcba7d04242be23fe7462f72267d476bc7e
Parents: a12faeb
Author: cstella <ce...@gmail.com>
Authored: Mon Feb 27 09:02:11 2017 -0500
Committer: cstella <ce...@gmail.com>
Committed: Mon Feb 27 09:02:11 2017 -0500
----------------------------------------------------------------------
.../common/configuration/FieldTransformer.java | 9 ++--
.../metron/common/dsl/MapVariableResolver.java | 6 ++-
.../transformation/FieldTransformation.java | 3 +-
.../transformation/RemoveTransformation.java | 3 +-
.../SimpleFieldTransformation.java | 3 +-
.../transformation/StellarTransformation.java | 22 ++++++---
.../transformation/FieldTransformationTest.java | 3 +-
.../StellarTransformationTest.java | 51 ++++++++++++++++----
8 files changed, 71 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
index c5ca7a2..46f420e 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
@@ -26,17 +26,14 @@ import org.apache.metron.common.field.transformation.FieldTransformations;
import org.json.simple.JSONObject;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
public class FieldTransformer implements Serializable {
private List<String> input = new ArrayList<>();
private List<String> output;
private FieldTransformation transformation;
private String transformationName;
- private Map<String, Object> config = new HashMap<>();
+ private LinkedHashMap<String, Object> config = new LinkedHashMap<>();
private boolean initialized = false;
public FieldTransformer() {
}
@@ -71,7 +68,7 @@ public class FieldTransformer implements Serializable {
return config;
}
- public void setConfig(Map<String, Object> config) {
+ public void setConfig(LinkedHashMap<String, Object> config) {
this.config = config;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java
index 8d56af2..f180e80 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java
@@ -27,9 +27,11 @@ public class MapVariableResolver implements VariableResolver {
List<Map> variableMappings = new ArrayList<>();
public MapVariableResolver(Map variableMappingOne, Map... variableMapping) {
- variableMappings.add(variableMappingOne);
+ if(variableMappingOne != null) {
+ variableMappings.add(variableMappingOne);
+ }
for(Map m : variableMapping) {
- if(m != null && m.size() > 0) {
+ if(m != null) {
this.variableMappings.add(m);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java
index a19d165..bb33bf7 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java
@@ -21,13 +21,14 @@ package org.apache.metron.common.field.transformation;
import org.apache.metron.common.dsl.Context;
import java.io.Serializable;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public interface FieldTransformation extends Serializable {
Map<String, Object> map( Map<String, Object> input
, List<String> outputField
- , Map<String, Object> fieldMappingConfig
+ , LinkedHashMap<String, Object> fieldMappingConfig
, Map<String, Object> sensorConfig
, Context context
);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java
index 19aab1d..3b5ae5f 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java
@@ -23,6 +23,7 @@ import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
import org.apache.metron.common.stellar.StellarPredicateProcessor;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -66,7 +67,7 @@ public class RemoveTransformation implements FieldTransformation {
@Override
public Map<String, Object> map( Map<String, Object> input
, final List<String> outputFields
- , Map<String, Object> fieldMappingConfig
+ , LinkedHashMap<String, Object> fieldMappingConfig
, Map<String, Object> sensorConfig
, Context context
) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java
index 71b830c..1f8efbd 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java
@@ -21,6 +21,7 @@ package org.apache.metron.common.field.transformation;
import com.google.common.collect.Iterables;
import org.apache.metron.common.dsl.Context;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -28,7 +29,7 @@ public abstract class SimpleFieldTransformation implements FieldTransformation {
@Override
public Map<String, Object> map (Map<String, Object> input
, List<String> outputField
- , Map<String, Object> fieldMappingConfig
+ , LinkedHashMap<String, Object> fieldMappingConfig
, Map<String, Object> sensorConfig
, Context context
)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java
index 2bbbc80..e2ef7c9 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java
@@ -24,29 +24,35 @@ import org.apache.metron.common.dsl.StellarFunctions;
import org.apache.metron.common.dsl.VariableResolver;
import org.apache.metron.common.stellar.StellarProcessor;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
public class StellarTransformation implements FieldTransformation {
@Override
public Map<String, Object> map( Map<String, Object> input
, List<String> outputField
- , Map<String, Object> fieldMappingConfig
+ , LinkedHashMap<String, Object> fieldMappingConfig
, Map<String, Object> sensorConfig
, Context context
)
{
Map<String, Object> ret = new HashMap<>();
- VariableResolver resolver = new MapVariableResolver(ret, input, sensorConfig);
+ Map<String, Object> intermediateVariables = new HashMap<>();
+ Set<String> outputs = new HashSet<>(outputField);
+ VariableResolver resolver = new MapVariableResolver(ret, intermediateVariables, input, sensorConfig);
StellarProcessor processor = new StellarProcessor();
- for(String oField : outputField) {
- Object transformObj = fieldMappingConfig.get(oField);
+ for(Map.Entry<String, Object> kv : fieldMappingConfig.entrySet()) {
+ String oField = kv.getKey();
+ Object transformObj = kv.getValue();
if(transformObj != null) {
try {
Object o = processor.parse(transformObj.toString(), resolver, StellarFunctions.FUNCTION_RESOLVER(), context);
if (o != null) {
- ret.put(oField, o);
+ if(outputs.contains(oField)) {
+ ret.put(oField, o);
+ }
+ else {
+ intermediateVariables.put(oField, o);
+ }
}
}
catch(Exception ex) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
index ba03b70..b4df514 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
@@ -32,6 +32,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -41,7 +42,7 @@ public class FieldTransformationTest {
@Override
public Map<String, Object> map( Map<String, Object> input
, List<String> outputField
- , Map<String, Object> fieldMappingConfig
+ , LinkedHashMap<String, Object> fieldMappingConfig
, Map<String, Object> sensorConfig
, Context context
)
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index a517986..ac9b74c 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -115,16 +115,48 @@ public class StellarTransformationTest {
/**
{
- "fieldTransformations" : [
- {
- "transformation" : "STELLAR"
- ,"output" : ["newStellarField","utc_timestamp"]
- ,"config" : {
- "newStellarField" : "'<<??>>'",
- "utc_timestamp" : "TO_EPOCH_TIMESTAMP(timestamp, 'yyyy-MM-dd HH:mm:ss', 'UTC')"
- }
+ "fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : "final_value"
+ ,"config" : {
+ "value1" : "1"
+ ,"value2" : "value1 + 1"
+ ,"final_value" : "value2 + 1"
+ }
+ }
+ ]
}
- ]
+ */
+ @Multiline
+ public static String intermediateValuesConfig;
+
+ @Test
+ public void testIntermediateValues() throws Exception {
+
+ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(intermediateValuesConfig));
+ FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
+ JSONObject input = new JSONObject(new HashMap<String, Object>() {{
+ }});
+ handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT());
+ int expected = 3;
+ Assert.assertEquals(expected, input.get("final_value"));
+ Assert.assertFalse(input.containsKey("value1"));
+ Assert.assertFalse(input.containsKey("value2"));
+ }
+
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : ["newStellarField","utc_timestamp"]
+ ,"config" : {
+ "newStellarField" : "'<<??>>'",
+ "utc_timestamp" : "TO_EPOCH_TIMESTAMP(timestamp, 'yyyy-MM-dd HH:mm:ss', 'UTC')"
+ }
+ }
+ ]
}
*/
@Multiline
@@ -163,6 +195,7 @@ public class StellarTransformationTest {
Assert.assertEquals(expected, input.get("utc_timestamp"));
Assert.assertTrue(input.containsKey("timestamp"));
}
+
/**
* Ensures that if we try to transform with a field which does not exist, it does not
* 1. throw an exception