You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2018/04/25 15:50:09 UTC
metron git commit: METRON-1539: Specialized RENAME field transformer
closes apache/incubator-metron#1002
Repository: metron
Updated Branches:
refs/heads/master 1c5435ccb -> 2b4f0b840
METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b4f0b84
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b4f0b84
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b4f0b84
Branch: refs/heads/master
Commit: 2b4f0b84062d65f9400421d66ec3b7d6d093bebf
Parents: 1c5435c
Author: cstella <ce...@gmail.com>
Authored: Wed Apr 25 11:49:56 2018 -0400
Committer: cstella <ce...@gmail.com>
Committed: Wed Apr 25 11:49:56 2018 -0400
----------------------------------------------------------------------
.../common/configuration/FieldTransformer.java | 4 +-
.../transformation/FieldTransformations.java | 1 +
.../transformation/RenameTransformation.java | 55 +++++++++++
.../transformation/FieldTransformationTest.java | 17 +---
.../RenameTransformationTest.java | 99 ++++++++++++++++++++
metron-platform/metron-parsers/README.md | 25 ++++-
6 files changed, 183 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
index df80691..43ce9d8 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java
@@ -94,7 +94,9 @@ public class FieldTransformer implements Serializable {
if (output == null || output.isEmpty()) {
if (input == null || input.isEmpty()) {
- throw new IllegalStateException("You must specify an input field if you want to leave the output fields empty");
+ //both are empty, so let's set them both to null
+ output = null;
+ input = null;
} else {
output = input;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
index a905123..95ff390 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
@@ -25,6 +25,7 @@ public enum FieldTransformations {
,REMOVE(new RemoveTransformation())
,STELLAR(new StellarTransformation())
,SELECT(new SelectTransformation())
+ ,RENAME(new RenameTransformation())
;
FieldTransformation mapping;
FieldTransformations(FieldTransformation mapping) {
http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
new file mode 100644
index 0000000..f8b9374
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RenameTransformation.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.field.transformation;
+
+import org.apache.metron.stellar.dsl.Context;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RenameTransformation implements FieldTransformation{
+ @Override
+ public Map<String, Object> map( Map<String, Object> input
+ , List<String> outputField
+ , LinkedHashMap<String, Object> fieldMappingConfig
+ , Context context
+ , Map<String, Object>... sensorConfig
+ )
+ {
+ if(fieldMappingConfig == null || fieldMappingConfig.isEmpty()) {
+ return input;
+ }
+ Map<String, Object> ret = new HashMap<>();
+ for(Map.Entry<String, Object> kv : input.entrySet()) {
+ Object renamed = fieldMappingConfig.get(kv.getKey());
+ if(renamed != null) {
+ //if we're renaming, then we want to copy the field to the new name
+ ret.put(renamed.toString(), kv.getValue());
+ //and remove the old field
+ ret.put(kv.getKey(), null);
+ }
+ else {
+ ret.put(kv.getKey(), kv.getValue());
+ }
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
index 71a0298..b7557e8 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java
@@ -85,18 +85,6 @@ public class FieldTransformationTest {
{
"fieldTransformations" : [
{
- "transformation" : "IP_PROTOCOL"
- }
- ]
- }
- */
- @Multiline
- public static String badConfigMissingInput;
-
- /**
- {
- "fieldTransformations" : [
- {
"input" : "protocol"
}
]
@@ -113,10 +101,7 @@ public class FieldTransformationTest {
Assert.assertEquals(ImmutableList.of("protocol"), c.getFieldTransformations().get(0).getInput());
}
- @Test(expected = IllegalStateException.class)
- public void testInValidSerde_missingInput() throws IOException {
- SensorParserConfig.fromBytes(Bytes.toBytes(badConfigMissingInput));
- }
+
@Test(expected = IllegalStateException.class)
public void testInValidSerde_missingMapping() throws IOException {
http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java
new file mode 100644
index 0000000..cacc818
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RenameTransformationTest.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.field.transformation;
+
+import com.google.common.collect.Iterables;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.FieldTransformer;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class RenameTransformationTest {
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "transformation" : "RENAME",
+ "config" : {
+ "old_field1" : "new_field1",
+ "old_field2" : "new_field2"
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String smoketestConfig;
+
+ @Test
+ public void smokeTest() throws Exception {
+ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(smoketestConfig));
+ FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
+ JSONObject input = new JSONObject(new HashMap<String, Object>() {{
+ for(int i = 1;i <= 10;++i) {
+ put("old_field" + i, "f" + i);
+ }
+ }});
+ handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+ Assert.assertEquals("f1", input.get("new_field1"));
+ Assert.assertEquals("f2", input.get("new_field2"));
+ for(int i = 3;i <= 10;++i) {
+ Assert.assertEquals("f" + i, input.get("old_field" + i));
+ }
+ Assert.assertFalse(input.containsKey("old_field1"));
+ Assert.assertFalse(input.containsKey("old_field2"));
+ Assert.assertEquals(10, input.size());
+ }
+
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "transformation" : "RENAME",
+ "config" : {
+ "old_field1" : "new_field1"
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String renameMissingField;
+ @Test
+ public void renameMissingField() throws Exception {
+ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(renameMissingField));
+ FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
+ JSONObject input = new JSONObject(new HashMap<String, Object>() {{
+ for(int i = 2;i <= 10;++i) {
+ put("old_field" + i, "f" + i);
+ }
+ }});
+ handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+ Assert.assertFalse(input.containsKey("new_field1"));
+ for(int i = 2;i <= 10;++i) {
+ Assert.assertEquals("f" + i, input.get("old_field" + i));
+ }
+ Assert.assertEquals(9, input.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/2b4f0b84/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index 1d2d834..e8b2896 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -313,10 +313,33 @@ into `{ "protocol" : "TCP", "source.type" : "bro", ...}`
* `STELLAR` : This transformation executes a set of transformations
expressed as [Stellar Language](../metron-common) statements.
+* `RENAME` : This transformation allows users to rename a set of fields. Specifically,
+the config is presumed to be the mapping. The keys to the config are the existing field names
+and the values for the config map are the associated new field name.
+
+The following config will rename the fields `old_field` and `different_old_field` to
+`new_field` and `different_new_field` respectively:
+```
+{
+...
+ "fieldTransformations" : [
+ {
+ "transformation" : "RENAME",
+ , "config" : {
+ "old_field" : "new_field",
+ "different_old_field" : "different_new_field"
+ }
+ }
+ ]
+}
+```
+
+
### Assignment to `null`
If, in your field transformation, you assign a field to `null`, the field will be removed.
-You can use this capability to rename variables.
+You can use this capability to rename variables. It is preferred, however, that the `RENAME`
+field transformation is used in this situation as it is less awkward.
Consider this example:
```