You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2018/07/06 15:42:25 UTC
metron git commit: METRON-1643: Create a REGEX_ROUTING field
transformation closes apache/incubator-metron#1083
Repository: metron
Updated Branches:
refs/heads/master 4eb9a8ebc -> f183d987a
METRON-1643: Create a REGEX_ROUTING field transformation closes apache/incubator-metron#1083
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f183d987
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f183d987
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f183d987
Branch: refs/heads/master
Commit: f183d987ab1c9856c585c45b0f9f549aadccb65f
Parents: 4eb9a8e
Author: cstella <ce...@gmail.com>
Authored: Fri Jul 6 11:40:54 2018 -0400
Committer: cstella <ce...@gmail.com>
Committed: Fri Jul 6 11:40:54 2018 -0400
----------------------------------------------------------------------
.../transformation/FieldTransformations.java | 1 +
.../RegexSelectTransformation.java | 97 +++++++++++
.../RegexSelectTransformationTest.java | 169 +++++++++++++++++++
metron-platform/metron-parsers/README.md | 23 +++
4 files changed, 290 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
index 95ff390..d80c1a4 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformations.java
@@ -26,6 +26,7 @@ public enum FieldTransformations {
,STELLAR(new StellarTransformation())
,SELECT(new SelectTransformation())
,RENAME(new RenameTransformation())
+ ,REGEX_SELECT(new RegexSelectTransformation())
;
FieldTransformation mapping;
FieldTransformations(FieldTransformation mapping) {
http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
new file mode 100644
index 0000000..b8d1c07
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RegexSelectTransformation.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.field.transformation;
+
+import com.google.common.collect.Iterables;
+import org.apache.metron.stellar.common.utils.PatternCache;
+import org.apache.metron.stellar.dsl.Context;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+public class RegexSelectTransformation implements FieldTransformation {
+ @Override
+ public Map<String, Object> map( Map<String, Object> input
+ , List<String> outputField
+ , LinkedHashMap<String, Object> fieldMappingConfig
+ , Context context
+ , Map<String, Object>... sensorConfig
+ ) {
+ String outField = null;
+ if(!(outputField == null || outputField.isEmpty())) {
+ outField = outputField.get(0);
+ }
+ String inVal = null;
+ if(!(input == null || input.isEmpty() || input.size() > 1)) {
+ Object inValObj = Iterables.getFirst(input.entrySet(), null).getValue();
+ if(inValObj != null) {
+ inVal = inValObj.toString();
+ }
+ }
+ Map<String, Object> ret = new HashMap<>(1);
+ if(outField == null || inVal == null) {
+ //in the situation where we don't have valid input or output, then we want to do nothing
+ return ret;
+ }
+ for(Map.Entry<String, Object> valToRegex : fieldMappingConfig.entrySet()) {
+ if(isMatch(valToRegex.getValue(), inVal)) {
+ ret.put(outField, valToRegex.getKey());
+ break;
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Return true if there is a regex match or false otherwise.
+ * @param regexes This could be either a list of regexes or a single regex
+ * @param field The field to match
+ * @return
+ */
+ private static boolean isMatch(Object regexes, String field) {
+ if(regexes instanceof String) {
+ return isMatch((String)regexes, field);
+ }
+ else if(regexes instanceof List) {
+ for(Object regex : (List)regexes) {
+ if(isMatch(regex.toString(), field)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isMatch(String regex, String field) {
+ try {
+ Pattern p = PatternCache.INSTANCE.getPattern(regex);
+ if (p == null) {
+ return false;
+ }
+ return p.asPredicate().test(field);
+ }
+ catch(PatternSyntaxException pse) {
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java
new file mode 100644
index 0000000..8d9e935
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/RegexSelectTransformationTest.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.field.transformation;
+
+import com.google.common.collect.Iterables;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metron.common.configuration.FieldTransformer;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class RegexSelectTransformationTest {
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "input" : "in_field"
+ , "output" : "out_field"
+ , "transformation" : "REGEX_SELECT"
+ , "config" : {
+ "option_1" : ".*foo.*",
+ "option_2" : [ ".*metron.*", ".*mortron.*" ]
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String routeSingleInSingleOut;
+
+ private String transform(String in, String config) throws Exception {
+ SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(config));
+ FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null);
+ JSONObject input = new JSONObject(new HashMap<String, Object>() {{
+ put("in_field", in);
+ put("dummy_field", "dummy"); //this is added to ensure that it looks like something approaching a real message
+ }});
+ handler.transformAndUpdate(input, Context.EMPTY_CONTEXT());
+ return (String) input.get("out_field");
+ }
+
+ @Test
+ public void smoketest() throws Exception{
+ Assert.assertEquals("option_1", transform("foo", routeSingleInSingleOut));
+ Assert.assertNull(transform("bar", routeSingleInSingleOut));
+ }
+
+ @Test
+ public void testListOfRegexes() throws Exception {
+ Assert.assertEquals("option_2", transform("I am mortron", routeSingleInSingleOut));
+ Assert.assertEquals("option_2", transform("metron is for smelling", routeSingleInSingleOut));
+ }
+ @Test
+ public void testPrecedence() throws Exception {
+ Assert.assertEquals("option_1", transform("metron is for foorensic cybersecurity", routeSingleInSingleOut));
+ }
+
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "output" : "out_field"
+ , "transformation" : "REGEX_SELECT"
+ , "config" : {
+ "option_1" : ".*foo.*",
+ "option_2" : [ ".*metron.*", ".*mortron.*" ]
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String routeMissingInput;
+
+ @Test
+ public void testMissingInput() throws Exception {
+ Assert.assertNull(transform("metron", routeMissingInput));
+ }
+
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "input" : "in_field"
+ , "transformation" : "REGEX_SELECT"
+ , "config" : {
+ "option_1" : ".*foo.*",
+ "option_2" : [ ".*metron.*", ".*mortron.*" ]
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String routeMissingOutput;
+
+ @Test
+ public void testMissingOutput() throws Exception {
+ Assert.assertNull(transform("metron", routeMissingOutput));
+ }
+
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "input" : "in_field"
+ ,"output" : [ "out_field", "baz_field" ]
+ , "transformation" : "REGEX_SELECT"
+ , "config" : {
+ "option_1" : ".*foo.*",
+ "option_2" : [ ".*metron.*", ".*mortron.*" ]
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String routeMultiOutput;
+
+ @Test
+ public void testMultiOutput() throws Exception{
+ Assert.assertEquals("option_1", transform("foo", routeMultiOutput));
+ Assert.assertNull(transform("bar", routeMultiOutput));
+ }
+
+ /**
+ {
+ "fieldTransformations" : [
+ {
+ "input" : "in_field"
+ ,"output" : "out_field"
+ , "transformation" : "REGEX_SELECT"
+ , "config" : {
+ "option_1" : "[a-z",
+ "option_2" : [ ".*metron.*", ".*mortron.*" ]
+ }
+ }
+ ]
+ }
+ */
+ @Multiline
+ public static String routeBadRegex;
+
+ @Test
+ public void testBadRegex() throws Exception{
+ Assert.assertEquals("option_2", transform("metron", routeBadRegex));
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/f183d987/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index 7ddfdea..c453758 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -338,6 +338,29 @@ The following config will rename the fields `old_field` and `different_old_field
]
}
```
+* `REGEX_SELECT` : This transformation lets users set an output field to one of a set of possibilities based on matching regexes. This transformation is useful when the number or conditions are large enough to make a stellar language match statement unwieldy.
+
+The following config will set the field `logical_source_type` to one of the
+following, dependent upon the value of the `pix_type` field:
+* `cisco-6-302` if `pix_type` starts with either `6-302` or `06-302`
+* `cisco-5-304` if `pix_type` starts with `5-304`
+```
+{
+...
+ "fieldTransformations" : [
+ {
+ "transformation" : "REGEX_ROUTING"
+ ,"input" : "pix_type"
+ ,"output" : "logical_source_type"
+ ,"config" : {
+ "cisco-6-302" : [ "^6-302.*", "^06-302.*"]
+ "cisco-5-304" : "^5-304.*"
+ }
+ }
+ ]
+...
+}
+```
### Assignment to `null`