You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2016/12/07 17:28:28 UTC
incubator-metron git commit: METRON-586 STELLAR should have FILL_LEFT
and FILL_RIGHT functions (ottobackwards) closes apache/incubator-metron#370
Repository: incubator-metron
Updated Branches:
refs/heads/master e15f4ba09 -> ba2722e7a
METRON-586 STELLAR should have FILL_LEFT and FILL_RIGHT functions (ottobackwards) closes apache/incubator-metron#370
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/ba2722e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/ba2722e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/ba2722e7
Branch: refs/heads/master
Commit: ba2722e7a7097fc92db5e741d11423eae8543f0e
Parents: e15f4ba
Author: ottobackwards <ot...@gmail.com>
Authored: Wed Dec 7 11:54:24 2016 -0500
Committer: Otto Fowler <ot...@apache.org>
Committed: Wed Dec 7 11:54:24 2016 -0500
----------------------------------------------------------------------
metron-platform/metron-common/README.md | 18 +++
.../common/dsl/functions/StringFunctions.java | 64 +++++++-
.../dsl/functions/StringFunctionsTest.java | 155 +++++++++++++++++++
.../network/DomainValidationTest.java | 2 +-
.../validation/network/EmailValidationTest.java | 2 +-
.../validation/network/IPValidationTest.java | 2 +-
.../validation/network/URLValidationTest.java | 2 +-
.../primitive/DateValidationTest.java | 2 +-
.../primitive/IntegerValidationTest.java | 2 +-
.../metron/common/stellar/BloomFilterTest.java | 2 +-
.../metron/common/stellar/StellarTest.java | 80 +---------
.../maas/StellarMaaSIntegrationTest.java | 17 +-
.../common/utils/StellarProcessorUtils.java | 72 +++++++++
.../management/ConfigurationFunctionsTest.java | 40 ++---
.../EnrichmentConfigFunctionsTest.java | 1 -
.../management/FileSystemFunctionsTest.java | 5 -
.../metron/management/GrokFunctionsTest.java | 13 +-
.../KafkaFunctionsIntegrationTest.java | 1 +
.../management/ParserConfigFunctionsTest.java | 36 ++---
.../metron/management/ShellFunctionsTest.java | 25 +--
20 files changed, 390 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md
index f2ed82b..e73cd8a 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -58,6 +58,8 @@ Using parens such as: "foo" : "\<ok\>" requires escaping; "foo": "\'\<ok\>\'"
| [ `ENDS_WITH`](#ends_with)|
| [ `ENRICHMENT_EXISTS`](#enrichment_exists)|
| [ `ENRICHMENT_GET`](#enrichment_get)|
+| [ `FILL_LEFT`](#fill_left)|
+| [ `FILL_RIGHT`](#fill_right)|
| [ `GET`](#get)|
| [ `GET_FIRST`](#get_first)|
| [ `GET_LAST`](#get_last)|
@@ -227,6 +229,22 @@ Using parens such as: "foo" : "\<ok\>" requires escaping; "foo": "\'\<ok\>\'"
* input - List
* Returns: Last element of the list
+### `FILL_LEFT`
+ * Description: Fills or pads a given string with a given character, to a given length on the left
+ * Input:
+ * input - string
+ * fill - the fill character
+ * len - the required length
+ * Returns: the filled string
+
+### `FILL_RIGHT`
+ * Description: Fills or pads a given string with a given character, to a given length on the right
+ * Input:
+ * input - string
+ * fill - the fill character string
+ * len - the required length
+ * Returns: Last element of the list
+
### `IN_SUBNET`
* Description: Returns true if an IP is within a subnet range.
* Input:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java
index 7e9cada..1239abb 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/functions/StringFunctions.java
@@ -22,11 +22,12 @@ import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import org.apache.metron.common.dsl.BaseStellarFunction;
+import org.apache.metron.common.dsl.ParseException;
import org.apache.metron.common.dsl.Stellar;
+import org.apache.metron.common.utils.ConversionUtils;
import java.util.ArrayList;
import java.util.List;
-import java.util.function.Function;
public class StringFunctions {
@@ -222,4 +223,65 @@ public class StringFunctions {
return null;
}
}
+
+ private enum FillDirection{
+ LEFT,
+ RIGHT
+ }
+
+ @Stellar(name="FILL_LEFT"
+ , description="Fills or pads a given string with a given character, to a given length on the left"
+ , params = { "input - string", "fill - the fill character", "len - the required length"}
+ , returns = "Filled String"
+ )
+ public static class FillLeft extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ if(args.size() < 3) {
+ throw new IllegalStateException("FILL_LEFT expects three args: [string,char,length] where char is the fill character string and length is the required length of the result");
+ }
+ return fill(FillDirection.LEFT,args.get(0),args.get(1),args.get(2));
+ }
+ }
+
+ @Stellar(name="FILL_RIGHT"
+ , description="Fills or pads a given string with a given character, to a given length on the right"
+ , params = { "input - string", "fill - the fill character", "len - the required length"}
+ , returns = "Filled String"
+ )
+ public static class FillRight extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ if(args.size() < 3) {
+ throw new IllegalStateException("FILL_RIGHT expects three args: [string,char,length] where char is the fill character string and length is the required length of the result");
+ }
+ return fill(FillDirection.RIGHT,args.get(0),args.get(1),args.get(2));
+ }
+ }
+
+ private static Object fill(FillDirection direction, Object inputObject, Object fillObject, Object requiredLengthObject)throws ParseException{
+ if(inputObject == null) {
+ return null;
+ }
+ String input = inputObject.toString();
+
+ if(requiredLengthObject == null || fillObject == null) {
+ throw new IllegalStateException("Required Length and Fill String are both required");
+ }
+
+ String fill = fillObject.toString();
+ if(org.apache.commons.lang.StringUtils.isEmpty(fill)){
+ throw new IllegalStateException("The fill cannot be an empty string");
+ }
+ fill = fill.substring(0,1);
+ Integer requiredLength = ConversionUtils.convert(requiredLengthObject,Integer.class);
+ if(requiredLength == null){
+ throw new IllegalStateException("Required Length not a valid Integer: " + requiredLengthObject.toString());
+ }
+
+ if(direction == FillDirection.LEFT) {
+ return org.apache.commons.lang.StringUtils.leftPad(input,requiredLength,fill);
+ }
+ return org.apache.commons.lang.StringUtils.rightPad(input,requiredLength,fill);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/dsl/functions/StringFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/dsl/functions/StringFunctionsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/dsl/functions/StringFunctionsTest.java
new file mode 100644
index 0000000..bee54b3
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/dsl/functions/StringFunctionsTest.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.dsl.functions;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.collections.map.HashedMap;
+import org.apache.metron.common.dsl.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
+
+public class StringFunctionsTest {
+
+ @Test
+ public void testStringFunctions() throws Exception {
+ final Map<String, String> variableMap = new HashMap<String, String>() {{
+ put("foo", "casey");
+ put("ip", "192.168.0.1");
+ put("empty", "");
+ put("spaced", "metron is great");
+ }};
+ Assert.assertTrue(runPredicate("true and TO_UPPER(foo) == 'CASEY'", v -> variableMap.get(v)));
+ Assert.assertTrue(runPredicate("foo in [ TO_LOWER('CASEY'), 'david' ]", v -> variableMap.get(v)));
+ Assert.assertTrue(runPredicate("TO_UPPER(foo) in [ TO_UPPER('casey'), 'david' ] and IN_SUBNET(ip, '192.168.0.0/24')", v -> variableMap.get(v)));
+ Assert.assertFalse(runPredicate("TO_LOWER(foo) in [ TO_UPPER('casey'), 'david' ]", v -> variableMap.get(v)));
+ }
+
+ @Test
+ public void testStringFunctions_advanced() throws Exception {
+ final Map<String, Object> variableMap = new HashMap<String, Object>() {{
+ put("foo", "casey");
+ put("bar", "bar.casey.grok");
+ put("ip", "192.168.0.1");
+ put("empty", "");
+ put("spaced", "metron is great");
+ put("myList", ImmutableList.of("casey", "apple", "orange"));
+ }};
+ Assert.assertTrue(runPredicate("foo in SPLIT(bar, '.')", v -> variableMap.get(v)));
+ Assert.assertFalse(runPredicate("foo in SPLIT(ip, '.')", v -> variableMap.get(v)));
+ Assert.assertTrue(runPredicate("foo in myList", v -> variableMap.get(v)));
+ Assert.assertFalse(runPredicate("foo not in myList", v -> variableMap.get(v)));
+ }
+
+ @Test
+ public void testLeftRightFills() throws Exception{
+ final Map<String, Object> variableMap = new HashMap<String, Object>() {{
+ put("foo", null);
+ put("bar", null);
+ put("notInt","oh my");
+ }};
+
+ //LEFT
+ Object left = run("FILL_LEFT('123','X', 10)",new HashedMap());
+ Assert.assertNotNull(left);
+ Assert.assertEquals(10,((String)left).length());
+ Assert.assertEquals("XXXXXXX123",(String)left);
+
+ //RIGHT
+ Object right = run("FILL_RIGHT('123','X', 10)", new HashedMap());
+ Assert.assertNotNull(right);
+ Assert.assertEquals(10,((String)right).length());
+ Assert.assertEquals("123XXXXXXX",(String)right);
+
+ //INPUT ALREADY LENGTH
+ Object same = run("FILL_RIGHT('123','X', 3)", new HashedMap());
+ Assert.assertEquals(3,((String)same).length());
+ Assert.assertEquals("123",(String)same);
+
+ //INPUT BIGGER THAN LENGTH
+ Object tooBig = run("FILL_RIGHT('1234567890','X', 3)", new HashedMap());
+ Assert.assertEquals(10,((String)tooBig).length());
+ Assert.assertEquals("1234567890",(String)tooBig);
+
+ //NULL VARIABLES
+ boolean thrown = false;
+ try{
+ run("FILL_RIGHT('123',foo,bar)", variableMap);
+ }catch(ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("are both required"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // NULL LENGTH
+ try{
+ run("FILL_RIGHT('123','X',bar)", variableMap);
+ }catch(ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("are both required"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // NULL FILL
+ try{
+ run("FILL_RIGHT('123',foo, 7)", variableMap);
+ }catch(ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("are both required"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // NON INTEGER LENGTH
+ try {
+ run("FILL_RIGHT('123','X', 'z' )", new HashedMap());
+ }catch(ParseException pe){
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("not a valid Integer"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // EMPTY STRING PAD
+ try {
+ Object returnValue = run("FILL_RIGHT('123','', 10 )", new HashedMap());
+ }catch(ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("cannot be an empty"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ //MISSING LENGTH PARAMETER
+ try {
+ run("FILL_RIGHT('123',foo)", variableMap);
+ }catch(ParseException pe){
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("expects three"));
+ }
+ Assert.assertTrue(thrown);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/DomainValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/DomainValidationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/DomainValidationTest.java
index 88cf37b..779c6fc 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/DomainValidationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/DomainValidationTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
import java.io.IOException;
-import static org.apache.metron.common.stellar.StellarTest.runPredicate;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
public class DomainValidationTest extends BaseValidationTest{
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/EmailValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/EmailValidationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/EmailValidationTest.java
index c82fe0b..ed8896e 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/EmailValidationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/EmailValidationTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
import java.io.IOException;
-import static org.apache.metron.common.stellar.StellarTest.runPredicate;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
public class EmailValidationTest extends BaseValidationTest {
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/IPValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/IPValidationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/IPValidationTest.java
index b088b3d..5c7483f 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/IPValidationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/IPValidationTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
import java.io.IOException;
-import static org.apache.metron.common.stellar.StellarTest.runPredicate;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
public class IPValidationTest extends BaseValidationTest {
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/URLValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/URLValidationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/URLValidationTest.java
index 70a69f8..59e9f47 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/URLValidationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/network/URLValidationTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
import java.io.IOException;
-import static org.apache.metron.common.stellar.StellarTest.runPredicate;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
public class URLValidationTest extends BaseValidationTest {
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/DateValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/DateValidationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/DateValidationTest.java
index cacc997..3aaf899 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/DateValidationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/DateValidationTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
import java.io.IOException;
-import static org.apache.metron.common.stellar.StellarTest.runPredicate;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
public class DateValidationTest extends BaseValidationTest{
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
index c89d786..887918d 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/validation/primitive/IntegerValidationTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
import java.io.IOException;
-import static org.apache.metron.common.stellar.StellarTest.runPredicate;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
public class IntegerValidationTest extends BaseValidationTest{
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/BloomFilterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/BloomFilterTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/BloomFilterTest.java
index 86da4d8..df283c9 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/BloomFilterTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/BloomFilterTest.java
@@ -27,7 +27,7 @@ import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
-import static org.apache.metron.common.stellar.StellarTest.run;
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
public class BloomFilterTest {
private Map<String, Object> variables = new HashMap<String, Object>() {{
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
index f284212..921fb7c 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java
@@ -21,7 +21,11 @@ package org.apache.metron.common.stellar;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import org.apache.commons.collections.map.HashedMap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.Stellar;
+import org.apache.metron.common.dsl.StellarFunction;
import org.apache.metron.common.dsl.Context;
import org.apache.metron.common.dsl.MapVariableResolver;
import org.apache.metron.common.dsl.ParseException;
@@ -45,6 +49,9 @@ import java.util.HashSet;
import java.util.Map;
import static org.apache.metron.common.dsl.functions.resolver.ClasspathFunctionResolver.effectiveClassPathUrls;
+import static java.util.function.Function.identity;
+import static org.apache.metron.common.utils.StellarProcessorUtils.runPredicate;
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
public class StellarTest {
@@ -456,32 +463,6 @@ public class StellarTest {
Assert.assertEquals("google", run("GET(SPLIT(DOMAIN_REMOVE_TLD(foo), '.'), 1)", variables));
}
-
-
- public static Object run(String rule, Map<String, Object> variables) {
- return run(rule, variables, Context.EMPTY_CONTEXT());
- }
-
- /**
- * This ensures the basic contract of a stellar expression is adhered to:
- * 1. Validate works on the expression
- * 2. The output can be serialized and deserialized properly
- *
- * @param rule
- * @param variables
- * @param context
- * @return
- */
- public static Object run(String rule, Map<String, Object> variables, Context context) {
- StellarProcessor processor = new StellarProcessor();
- Assert.assertTrue(rule + " not valid.", processor.validate(rule, context));
- Object ret = processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context);
- byte[] raw = SerDeUtils.toBytes(ret);
- Object actual = SerDeUtils.fromBytes(raw, Object.class);
- Assert.assertEquals(ret, actual);
- return ret;
- }
-
@Test
public void testValidation() throws Exception {
StellarPredicateProcessor processor = new StellarPredicateProcessor();
@@ -501,23 +482,6 @@ public class StellarTest {
}
}
- public static boolean runPredicate(String rule, Map resolver) {
- return runPredicate(rule, resolver, Context.EMPTY_CONTEXT());
- }
-
- public static boolean runPredicate(String rule, Map resolver, Context context) {
- return runPredicate(rule, new MapVariableResolver(resolver), context);
- }
-
- public static boolean runPredicate(String rule, VariableResolver resolver) {
- return runPredicate(rule, resolver, Context.EMPTY_CONTEXT());
- }
-
- public static boolean runPredicate(String rule, VariableResolver resolver, Context context) {
- StellarPredicateProcessor processor = new StellarPredicateProcessor();
- Assert.assertTrue(rule + " not valid.", processor.validate(rule));
- return processor.parse(rule, resolver, StellarFunctions.FUNCTION_RESOLVER(), context);
- }
@Test
public void testSimpleOps() throws Exception {
@@ -586,36 +550,6 @@ public class StellarTest {
}
@Test
- public void testStringFunctions() throws Exception {
- final Map<String, String> variableMap = new HashMap<String, String>() {{
- put("foo", "casey");
- put("ip", "192.168.0.1");
- put("empty", "");
- put("spaced", "metron is great");
- }};
- Assert.assertTrue(runPredicate("true and TO_UPPER(foo) == 'CASEY'", v -> variableMap.get(v)));
- Assert.assertTrue(runPredicate("foo in [ TO_LOWER('CASEY'), 'david' ]", v -> variableMap.get(v)));
- Assert.assertTrue(runPredicate("TO_UPPER(foo) in [ TO_UPPER('casey'), 'david' ] and IN_SUBNET(ip, '192.168.0.0/24')", v -> variableMap.get(v)));
- Assert.assertFalse(runPredicate("TO_LOWER(foo) in [ TO_UPPER('casey'), 'david' ]", v -> variableMap.get(v)));
- }
-
- @Test
- public void testStringFunctions_advanced() throws Exception {
- final Map<String, Object> variableMap = new HashMap<String, Object>() {{
- put("foo", "casey");
- put("bar", "bar.casey.grok");
- put("ip", "192.168.0.1");
- put("empty", "");
- put("spaced", "metron is great");
- put("myList", ImmutableList.of("casey", "apple", "orange"));
- }};
- Assert.assertTrue(runPredicate("foo in SPLIT(bar, '.')", v -> variableMap.get(v)));
- Assert.assertFalse(runPredicate("foo in SPLIT(ip, '.')", v -> variableMap.get(v)));
- Assert.assertTrue(runPredicate("foo in myList", v -> variableMap.get(v)));
- Assert.assertFalse(runPredicate("foo not in myList", v -> variableMap.get(v)));
- }
-
- @Test
public void testMapFunctions_advanced() throws Exception {
final Map<String, Object> variableMap = new HashMap<String, Object>() {{
put("foo", "casey");
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java
index a4591a5..0896d86 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/maas/StellarMaaSIntegrationTest.java
@@ -29,7 +29,6 @@ import org.apache.curator.x.discovery.ServiceInstance;
import org.apache.curator.x.discovery.ServiceInstanceBuilder;
import org.apache.curator.x.discovery.ServiceType;
import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.stellar.StellarTest;
import org.apache.metron.maas.config.Endpoint;
import org.apache.metron.maas.config.MaaSConfig;
import org.apache.metron.maas.config.ModelEndpoint;
@@ -43,6 +42,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
+
public class StellarMaaSIntegrationTest {
private static Context context;
private static TestingServer testZkServer;
@@ -106,7 +107,7 @@ public class StellarMaaSIntegrationTest {
@Test
public void testGetEndpointWithoutVersion() throws Exception {
String stellar = "MAAS_GET_ENDPOINT('dga')";
- Object result = StellarTest.run(stellar, new HashMap<>(), context);
+ Object result = run(stellar, new HashMap<>(), context);
Assert.assertTrue(result instanceof Map);
Map<String, String> resMap = (Map<String, String>)result;
Assert.assertEquals(resMap.get("url"), "http://localhost:8282");
@@ -119,7 +120,7 @@ public class StellarMaaSIntegrationTest {
@Test
public void testGetEndpointWithVersion() throws Exception {
String stellar = "MAAS_GET_ENDPOINT('dga', '1.0')";
- Object result = StellarTest.run(stellar, new HashMap<>(), context);
+ Object result = run(stellar, new HashMap<>(), context);
Assert.assertTrue(result instanceof Map);
Map<String, String> resMap = (Map<String, String>)result;
Assert.assertEquals(resMap.get("url"), "http://localhost:8282");
@@ -131,7 +132,7 @@ public class StellarMaaSIntegrationTest {
@Test
public void testGetEndpointWithWrongVersion() throws Exception {
String stellar = "MAAS_GET_ENDPOINT('dga', '2.0')";
- Object result = StellarTest.run(stellar, new HashMap<>(), context);
+ Object result = run(stellar, new HashMap<>(), context);
Assert.assertNull(result);
}
@@ -139,17 +140,17 @@ public class StellarMaaSIntegrationTest {
public void testModelApply() throws Exception {
{
String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))";
- Object result = StellarTest.run(stellar, ImmutableMap.of("host", "badguy.com"), context);
+ Object result = run(stellar, ImmutableMap.of("host", "badguy.com"), context);
Assert.assertTrue((Boolean) result);
}
{
String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))";
- Object result = StellarTest.run(stellar, ImmutableMap.of("host", "youtube.com"), context);
+ Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context);
Assert.assertFalse((Boolean) result);
}
{
String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), 'apply', {'host': host}))";
- Object result = StellarTest.run(stellar, ImmutableMap.of("host", "youtube.com"), context);
+ Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context);
Assert.assertFalse((Boolean) result);
}
@@ -159,7 +160,7 @@ public class StellarMaaSIntegrationTest {
public void testModelApplyNegative() {
{
String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga', '2.0'), {'host': host}))";
- Object result = StellarTest.run(stellar, ImmutableMap.of("host", "youtube.com"), context);
+ Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context);
Assert.assertNull( result);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/StellarProcessorUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/StellarProcessorUtils.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/StellarProcessorUtils.java
new file mode 100644
index 0000000..e7a58bf
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/StellarProcessorUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.utils;
+
+import org.apache.metron.common.dsl.*;
+import org.apache.metron.common.stellar.StellarPredicateProcessor;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class StellarProcessorUtils {
+
+ /**
+ * This ensures the basic contract of a stellar expression is adhered to:
+ * 1. Validate works on the expression
+ * 2. The output can be serialized and deserialized properly
+ *
+ * @param rule
+ * @param variables
+ * @param context
+ * @return
+ */
+ public static Object run(String rule, Map<String, Object> variables, Context context) {
+ StellarProcessor processor = new StellarProcessor();
+ Assert.assertTrue(rule + " not valid.", processor.validate(rule, context));
+ Object ret = processor.parse(rule, x -> variables.get(x), StellarFunctions.FUNCTION_RESOLVER(), context);
+ byte[] raw = SerDeUtils.toBytes(ret);
+ Object actual = SerDeUtils.fromBytes(raw, Object.class);
+ Assert.assertEquals(ret, actual);
+ return ret;
+ }
+
+ public static Object run(String rule, Map<String, Object> variables) {
+ return run(rule, variables, Context.EMPTY_CONTEXT());
+ }
+
+ public static boolean runPredicate(String rule, Map resolver) {
+ return runPredicate(rule, resolver, Context.EMPTY_CONTEXT());
+ }
+
+ public static boolean runPredicate(String rule, Map resolver, Context context) {
+ return runPredicate(rule, new MapVariableResolver(resolver), context);
+ }
+
+ public static boolean runPredicate(String rule, VariableResolver resolver) {
+ return runPredicate(rule, resolver, Context.EMPTY_CONTEXT());
+ }
+
+ public static boolean runPredicate(String rule, VariableResolver resolver, Context context) {
+ StellarPredicateProcessor processor = new StellarPredicateProcessor();
+ Assert.assertTrue(rule + " not valid.", processor.validate(rule));
+ return processor.parse(rule, resolver, StellarFunctions.FUNCTION_RESOLVER(), context);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
index 2cc9631..8d202c6 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
@@ -27,7 +27,6 @@ import org.apache.metron.common.cli.ConfigurationManager;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.dsl.Context;
import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.stellar.StellarTest;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.parser.JSONParser;
import org.json.simple.JSONObject;
@@ -40,6 +39,7 @@ import java.util.HashMap;
import static org.apache.metron.TestConstants.PARSER_CONFIGS_PATH;
import static org.apache.metron.TestConstants.SAMPLE_CONFIG_PATH;
import static org.apache.metron.management.utils.FileUtils.slurp;
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
public class ConfigurationFunctionsTest {
private TestingServer testZkServer;
@@ -88,7 +88,7 @@ public class ConfigurationFunctionsTest {
@Test
public void testParserGetHappyPath() {
- Object out = StellarTest.run("CONFIG_GET('PARSER', 'bro')", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('PARSER', 'bro')", new HashMap<>(), context);
Assert.assertEquals(goodBroParserConfig, out);
}
@@ -96,7 +96,7 @@ public class ConfigurationFunctionsTest {
public void testParserGetMissWithoutDefault() {
{
- Object out = StellarTest.run("CONFIG_GET('PARSER', 'brop', false)", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('PARSER', 'brop', false)", new HashMap<>(), context);
Assert.assertNull(out);
}
}
@@ -106,12 +106,12 @@ public class ConfigurationFunctionsTest {
JSONObject expected = (JSONObject) new JSONParser().parse(defaultBropParserConfig);
{
- Object out = StellarTest.run("CONFIG_GET('PARSER', 'brop')", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('PARSER', 'brop')", new HashMap<>(), context);
JSONObject actual = (JSONObject) new JSONParser().parse(out.toString().trim());
Assert.assertEquals(expected, actual);
}
{
- Object out = StellarTest.run("CONFIG_GET('PARSER', 'brop', true)", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('PARSER', 'brop', true)", new HashMap<>(), context);
JSONObject actual = (JSONObject) new JSONParser().parse(out.toString().trim());
Assert.assertEquals(expected, actual);
}
@@ -148,7 +148,7 @@ public class ConfigurationFunctionsTest {
@Test
public void testEnrichmentGetHappyPath() {
- Object out = StellarTest.run("CONFIG_GET('ENRICHMENT', 'test')", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('ENRICHMENT', 'test')", new HashMap<>(), context);
Assert.assertEquals(goodTestEnrichmentConfig, out.toString().trim());
}
@@ -156,7 +156,7 @@ public class ConfigurationFunctionsTest {
public void testEnrichmentGetMissWithoutDefault() {
{
- Object out = StellarTest.run("CONFIG_GET('ENRICHMENT', 'brop', false)", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('ENRICHMENT', 'brop', false)", new HashMap<>(), context);
Assert.assertNull(out);
}
}
@@ -166,12 +166,12 @@ public class ConfigurationFunctionsTest {
JSONObject expected = (JSONObject) new JSONParser().parse(defaultBropEnrichmentConfig);
{
- Object out = StellarTest.run("CONFIG_GET('ENRICHMENT', 'brop')", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('ENRICHMENT', 'brop')", new HashMap<>(), context);
JSONObject actual = (JSONObject) new JSONParser().parse(out.toString().trim());
Assert.assertEquals(expected, actual);
}
{
- Object out = StellarTest.run("CONFIG_GET('ENRICHMENT', 'brop', true)", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('ENRICHMENT', 'brop', true)", new HashMap<>(), context);
JSONObject actual = (JSONObject) new JSONParser().parse(out.toString().trim());
Assert.assertEquals(expected, actual);
}
@@ -182,14 +182,14 @@ public class ConfigurationFunctionsTest {
@Test
public void testGlobalGet() {
- Object out = StellarTest.run("CONFIG_GET('GLOBAL')", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('GLOBAL')", new HashMap<>(), context);
Assert.assertEquals(goodGlobalConfig, out.toString().trim());
}
@Test
public void testGlobalPut() {
- Object out = StellarTest.run("CONFIG_GET('GLOBAL')", new HashMap<>(), context);
+ Object out = run("CONFIG_GET('GLOBAL')", new HashMap<>(), context);
Assert.assertEquals(goodGlobalConfig, out.toString().trim());
}
@@ -198,7 +198,7 @@ public class ConfigurationFunctionsTest {
{
UnitTestHelper.setLog4jLevel(ConfigurationFunctions.class, Level.FATAL);
try {
- StellarTest.run("CONFIG_PUT('GLOBAL', 'foo bar')", new HashMap<>(), context);
+ run("CONFIG_PUT('GLOBAL', 'foo bar')", new HashMap<>(), context);
} catch(ParseException e) {
UnitTestHelper.setLog4jLevel(ConfigurationFunctions.class, Level.ERROR);
throw e;
@@ -208,11 +208,11 @@ public class ConfigurationFunctionsTest {
@Test
public void testEnrichmentPut() throws InterruptedException {
- String brop= (String) StellarTest.run("CONFIG_GET('ENRICHMENT', 'testEnrichmentPut')", new HashMap<>(), context);
- StellarTest.run("CONFIG_PUT('ENRICHMENT', config, 'testEnrichmentPut')", ImmutableMap.of("config", brop), context);
+ String brop= (String) run("CONFIG_GET('ENRICHMENT', 'testEnrichmentPut')", new HashMap<>(), context);
+ run("CONFIG_PUT('ENRICHMENT', config, 'testEnrichmentPut')", ImmutableMap.of("config", brop), context);
boolean foundMatch = false;
for(int i = 0;i < 10 && !foundMatch;++i) {
- String bropNew = (String) StellarTest.run("CONFIG_GET('ENRICHMENT', 'testEnrichmentPut', false)", new HashMap<>(), context);
+ String bropNew = (String) run("CONFIG_GET('ENRICHMENT', 'testEnrichmentPut', false)", new HashMap<>(), context);
foundMatch = brop.equals(bropNew);
if(foundMatch) {
break;
@@ -228,7 +228,7 @@ public class ConfigurationFunctionsTest {
{
UnitTestHelper.setLog4jLevel(ConfigurationFunctions.class, Level.FATAL);
try {
- StellarTest.run("CONFIG_PUT('ENRICHMENT', config, 'brop')", ImmutableMap.of("config", "foo bar"), context);
+ run("CONFIG_PUT('ENRICHMENT', config, 'brop')", ImmutableMap.of("config", "foo bar"), context);
} catch(ParseException e) {
UnitTestHelper.setLog4jLevel(ConfigurationFunctions.class, Level.ERROR);
throw e;
@@ -239,11 +239,11 @@ public class ConfigurationFunctionsTest {
@Test
public void testParserPut() throws InterruptedException {
- String brop= (String) StellarTest.run("CONFIG_GET('PARSER', 'testParserPut')", new HashMap<>(), context);
- StellarTest.run("CONFIG_PUT('PARSER', config, 'testParserPut')", ImmutableMap.of("config", brop), context);
+ String brop= (String) run("CONFIG_GET('PARSER', 'testParserPut')", new HashMap<>(), context);
+ run("CONFIG_PUT('PARSER', config, 'testParserPut')", ImmutableMap.of("config", brop), context);
boolean foundMatch = false;
for(int i = 0;i < 10 && !foundMatch;++i) {
- String bropNew = (String) StellarTest.run("CONFIG_GET('PARSER', 'testParserPut', false)", new HashMap<>(), context);
+ String bropNew = (String) run("CONFIG_GET('PARSER', 'testParserPut', false)", new HashMap<>(), context);
foundMatch = brop.equals(bropNew);
if(foundMatch) {
break;
@@ -258,7 +258,7 @@ public class ConfigurationFunctionsTest {
{
UnitTestHelper.setLog4jLevel(ConfigurationFunctions.class, Level.FATAL);
try {
- StellarTest.run("CONFIG_PUT('PARSER', config, 'brop')", ImmutableMap.of("config", "foo bar"), context);
+ run("CONFIG_PUT('PARSER', config, 'brop')", ImmutableMap.of("config", "foo bar"), context);
} catch(ParseException e) {
UnitTestHelper.setLog4jLevel(ConfigurationFunctions.class, Level.ERROR);
throw e;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
index 850991a..21eba0e 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/EnrichmentConfigFunctionsTest.java
@@ -25,7 +25,6 @@ import org.apache.metron.common.dsl.Context;
import org.apache.metron.common.dsl.ParseException;
import org.apache.metron.common.dsl.StellarFunctions;
import org.apache.metron.common.stellar.StellarProcessor;
-import org.apache.metron.common.stellar.StellarTest;
import org.apache.metron.common.stellar.shell.StellarExecutor;
import org.apache.metron.common.utils.JSONUtils;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java
index f165f9c..88eabe0 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/FileSystemFunctionsTest.java
@@ -17,15 +17,10 @@
*/
package org.apache.metron.management;
-import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.StellarFunctions;
-import org.apache.metron.common.stellar.StellarProcessor;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java
index 969798b..6158978 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/GrokFunctionsTest.java
@@ -20,12 +20,11 @@ package org.apache.metron.management;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.stellar.StellarTest;
import org.junit.Assert;
import org.junit.Test;
import java.util.HashMap;
-import java.util.Map;
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
public class GrokFunctionsTest {
private String grokExpr = "%{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?";
@@ -34,7 +33,7 @@ public class GrokFunctionsTest {
@Test
public void testGrokEvalSingleMessage() {
String message = "1474583120.343 142 127.0.0.1 TCP_MISS/301 494 GET http://cnn.com/ - DIRECT/157.166.226.26 text/html";
- String out = (String) StellarTest.run( "GROK_EVAL( grok, messages )"
+ String out = (String) run( "GROK_EVAL( grok, messages )"
, ImmutableMap.of("messages", ImmutableList.of(message), "grok", grokExpr)
, Context.EMPTY_CONTEXT()
);
@@ -47,7 +46,7 @@ public class GrokFunctionsTest {
public void testGrokEvalMultiMessages() {
String message = "1474583120.343 142 127.0.0.1 TCP_MISS/301 494 GET http://cnn.com/ - DIRECT/157.166.226.26 text/html";
String message2 = "1474583120.343 142 127.0.0.1 TCP_MISS/404 494 GET http://google.com/ - DIRECT/157.166.226.26 text/html";
- String out = (String) StellarTest.run( "GROK_EVAL( grok, messages )"
+ String out = (String) run( "GROK_EVAL( grok, messages )"
, ImmutableMap.of("messages", ImmutableList.of(message, message2), "grok", grokExpr)
, Context.EMPTY_CONTEXT()
);
@@ -60,7 +59,7 @@ public class GrokFunctionsTest {
@Test
public void testGrokEvalBadData() {
String message = "1474583120.343 142 foo TCP_MISS/301 494 GET http://cnn.com/ - DIRECT/157.166.226.26 text/html";
- String out = (String) StellarTest.run( "GROK_EVAL( grok, message )"
+ String out = (String) run( "GROK_EVAL( grok, message )"
, ImmutableMap.of("message", message, "grok", grokExpr)
, Context.EMPTY_CONTEXT()
);
@@ -71,7 +70,7 @@ public class GrokFunctionsTest {
public void testGrokEvalBadDataMultiMessages() {
String message = "1474583120.343 142 foo TCP_MISS/301 494 GET http://cnn.com/ - DIRECT/157.166.226.26 text/html";
String message2 = "1474583120.343 142 127.0.0.1 TCP_MISS/404 494 GET http://google.com/ - DIRECT/157.166.226.26 text/html";
- String out = (String) StellarTest.run( "GROK_EVAL( grok, messages )"
+ String out = (String) run( "GROK_EVAL( grok, messages )"
, ImmutableMap.of("messages", ImmutableList.of(message, message2), "grok", grokExpr)
, Context.EMPTY_CONTEXT()
);
@@ -81,7 +80,7 @@ public class GrokFunctionsTest {
@Test
public void testGrokDiscover() {
- String out = (String) StellarTest.run("GROK_PREDICT( '1474583120.343 142 127.0.0.1 TCP_MISS/301')"
+ String out = (String) run("GROK_PREDICT( '1474583120.343 142 127.0.0.1 TCP_MISS/301')"
, new HashMap<>()
, Context.EMPTY_CONTEXT()
);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java
index 040b98d..28d9489 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/KafkaFunctionsIntegrationTest.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
+
import static org.junit.Assert.assertEquals;
/**
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
index c8e5591..f425140 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ParserConfigFunctionsTest.java
@@ -22,7 +22,6 @@ import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.FieldTransformer;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.stellar.StellarTest;
import org.apache.metron.common.stellar.shell.StellarExecutor;
import org.json.simple.JSONObject;
import org.junit.Assert;
@@ -35,6 +34,7 @@ import java.util.Map;
import static org.apache.metron.TestConstants.PARSER_CONFIGS_PATH;
import static org.apache.metron.management.utils.FileUtils.slurp;
import static org.apache.metron.common.configuration.ConfigurationType.PARSER;
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
public class ParserConfigFunctionsTest {
@@ -72,7 +72,7 @@ public class ParserConfigFunctionsTest {
@Test
public void testAddEmpty() {
- String newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ String newConfig = (String)run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
Map<String, Object> transformations = transform(newConfig);
Assert.assertEquals(1, transformations.size());
Assert.assertEquals("FOO", transformations.get("upper") );
@@ -80,7 +80,7 @@ public class ParserConfigFunctionsTest {
@Test
public void testAddHasExisting() {
- String newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))"
+ String newConfig = (String)run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))"
, ImmutableMap.of("config", existingTransformationsConfig )
, context
);
@@ -92,15 +92,15 @@ public class ParserConfigFunctionsTest {
@Test
public void testAddMalformed() {
- String newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('blah'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ String newConfig = (String)run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('blah'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
Map<String, Object> transformations = transform(newConfig);
Assert.assertEquals(0, transformations.size());
}
@Test
public void testAddDuplicate() {
- String newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
- newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", newConfig), context);
+ String newConfig = (String)run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ newConfig = (String)run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", newConfig), context);
Map<String, Object> transformations = transform(newConfig);
Assert.assertEquals(1, transformations.size());
Assert.assertEquals("FOO", transformations.get("upper") );
@@ -108,16 +108,16 @@ public class ParserConfigFunctionsTest {
@Test
public void testRemove() {
- String newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
- newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_REMOVE(config, ['upper'])", ImmutableMap.of("config", newConfig), context);
+ String newConfig = (String)run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ newConfig = (String)run("PARSER_STELLAR_TRANSFORM_REMOVE(config, ['upper'])", ImmutableMap.of("config", newConfig), context);
Map<String, Object> transformations = transform(newConfig);
Assert.assertEquals(0, transformations.size());
}
@Test
public void testRemoveMultiple() {
- String newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper', 'lower'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
- newConfig = (String)StellarTest.run("PARSER_STELLAR_TRANSFORM_REMOVE(config, ['upper', 'lower'])", ImmutableMap.of("config", newConfig), context);
+ String newConfig = (String)run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper', 'lower'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ newConfig = (String)run("PARSER_STELLAR_TRANSFORM_REMOVE(config, ['upper', 'lower'])", ImmutableMap.of("config", newConfig), context);
Map<String, Object> transformations = transform(newConfig);
Assert.assertEquals(0, transformations.size());
}
@@ -125,15 +125,15 @@ public class ParserConfigFunctionsTest {
@Test
public void testRemoveMissing() {
{
- String newConfig = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
- newConfig = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_REMOVE(config, ['lower'])", ImmutableMap.of("config", newConfig), context);
+ String newConfig = (String) run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ newConfig = (String) run("PARSER_STELLAR_TRANSFORM_REMOVE(config, ['lower'])", ImmutableMap.of("config", newConfig), context);
Map<String, Object> transformations = transform(newConfig);
Assert.assertEquals(1, transformations.size());
Assert.assertEquals("FOO", transformations.get("upper"));
}
{
- String newConfig = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
- newConfig = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_REMOVE(config, [''])", ImmutableMap.of("config", newConfig), context);
+ String newConfig = (String) run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ newConfig = (String) run("PARSER_STELLAR_TRANSFORM_REMOVE(config, [''])", ImmutableMap.of("config", newConfig), context);
Map<String, Object> transformations = transform(newConfig);
Assert.assertEquals(1, transformations.size());
Assert.assertEquals("FOO", transformations.get("upper"));
@@ -151,8 +151,8 @@ public class ParserConfigFunctionsTest {
static String testPrintExpected;
@Test
public void testPrint() {
- String newConfig = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
- String out = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_PRINT(config )", ImmutableMap.of("config", newConfig), context);
+ String newConfig = (String) run("PARSER_STELLAR_TRANSFORM_ADD(config, SHELL_VARS2MAP('upper'))", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ String out = (String) run("PARSER_STELLAR_TRANSFORM_PRINT(config )", ImmutableMap.of("config", newConfig), context);
Assert.assertEquals(testPrintExpected, out);
}
/**
@@ -167,14 +167,14 @@ public class ParserConfigFunctionsTest {
@Test
public void testPrintEmpty() {
- String out = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_PRINT(config )", ImmutableMap.of("config", emptyTransformationsConfig), context);
+ String out = (String) run("PARSER_STELLAR_TRANSFORM_PRINT(config )", ImmutableMap.of("config", emptyTransformationsConfig), context);
Assert.assertEquals(testPrintEmptyExpected, out);
}
@Test
public void testPrintNull() {
- String out = (String) StellarTest.run("PARSER_STELLAR_TRANSFORM_PRINT(config )", new HashMap<>(), context);
+ String out = (String) run("PARSER_STELLAR_TRANSFORM_PRINT(config )", new HashMap<>(), context);
Assert.assertNull( out);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/ba2722e7/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
index 87f6642..cf40de8 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
@@ -20,14 +20,17 @@ package org.apache.metron.management;
import com.google.common.collect.ImmutableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.stellar.StellarTest;
import org.apache.metron.common.stellar.shell.StellarExecutor;
import org.junit.Assert;
import org.junit.Test;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import static org.apache.metron.common.utils.StellarProcessorUtils.run;
+
public class ShellFunctionsTest {
Map<String, StellarExecutor.VariableResult> variables = ImmutableMap.of(
@@ -57,7 +60,7 @@ public class ShellFunctionsTest {
Context context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES , () -> variables)
.build();
- Object out = StellarTest.run("SHELL_LIST_VARS()", new HashMap<>(), context);
+ Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
Assert.assertEquals(expectedListWithFoo, out);
}
@@ -76,7 +79,7 @@ public class ShellFunctionsTest {
Context context = new Context.Builder()
.with(StellarExecutor.SHELL_VARIABLES , () -> new HashMap<>())
.build();
- Object out = StellarTest.run("SHELL_LIST_VARS()", new HashMap<>(), context);
+ Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
Assert.assertEquals(expectedEmptyList, out);
}
/**
@@ -95,7 +98,7 @@ public class ShellFunctionsTest {
public void testMap2Table() {
Map<String, Object> variables = ImmutableMap.of("map_field", ImmutableMap.of("field1", "val1", "field2", "val2"));
Context context = Context.EMPTY_CONTEXT();
- Object out = StellarTest.run("SHELL_MAP2TABLE(map_field)", variables, context);
+ Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
Assert.assertEquals(expectedMap2Table, out);
}
/**
@@ -112,7 +115,7 @@ public class ShellFunctionsTest {
public void testMap2TableNullInput() {
Map<String, Object> variables = new HashMap<>();
Context context = Context.EMPTY_CONTEXT();
- Object out = StellarTest.run("SHELL_MAP2TABLE(map_field)", variables, context);
+ Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
Assert.assertEquals(expectedMap2TableNullInput, out);
}
@@ -120,13 +123,13 @@ public class ShellFunctionsTest {
public void testMap2TableInsufficientArgs() {
Map<String, Object> variables = new HashMap<>();
Context context = Context.EMPTY_CONTEXT();
- Object out = StellarTest.run("SHELL_MAP2TABLE()", variables, context);
+ Object out = run("SHELL_MAP2TABLE()", variables, context);
Assert.assertNull(out);
}
@Test
public void testVars2Map() {
- Object out = StellarTest.run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), context);
+ Object out = run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), context);
Assert.assertTrue(out instanceof Map);
Map<String, String> mapOut = (Map<String, String>)out;
//second one is null, so we don't want it there.
@@ -136,14 +139,14 @@ public class ShellFunctionsTest {
@Test
public void testVars2MapEmpty() {
- Object out = StellarTest.run("SHELL_VARS2MAP()", new HashMap<>(), context);
+ Object out = run("SHELL_VARS2MAP()", new HashMap<>(), context);
Map<String, String> mapOut = (Map<String, String>)out;
Assert.assertEquals(0, mapOut.size());
}
@Test
public void testGetExpression() {
- Object out = StellarTest.run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context);
+ Object out = run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context);
Assert.assertTrue(out instanceof String);
String expression = (String)out;
//second one is null, so we don't want it there.
@@ -152,14 +155,14 @@ public class ShellFunctionsTest {
@Test
public void testGetExpressionEmpty() {
- Object out = StellarTest.run("SHELL_GET_EXPRESSION()", new HashMap<>(), context);
+ Object out = run("SHELL_GET_EXPRESSION()", new HashMap<>(), context);
Assert.assertNull(out );
}
@Test
public void testEdit() throws Exception {
System.getProperties().put("EDITOR", "/bin/cat");
- Object out = StellarTest.run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", "foo"), context);
+ Object out = run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", "foo"), context);
Assert.assertEquals("FOO", out);
}