You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2017/12/04 16:14:04 UTC
metron git commit: METRON-1286 Add MIN & MAX Stellar functions
(jasper-k via justinleet) closes apache/metron#823
Repository: metron
Updated Branches:
refs/heads/master 05cb59b4f -> 548617185
METRON-1286 Add MIN & MAX Stellar functions (jasper-k via justinleet) closes apache/metron#823
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/54861718
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/54861718
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/54861718
Branch: refs/heads/master
Commit: 548617185d58224dd74a2345be03d3f9af35b2a1
Parents: 05cb59b
Author: jasper-k <jk...@hortonworks.com>
Authored: Mon Dec 4 11:13:25 2017 -0500
Committer: leet <le...@apache.org>
Committed: Mon Dec 4 11:13:25 2017 -0500
----------------------------------------------------------------------
metron-stellar/stellar-common/README.md | 67 +++---
.../stellar/dsl/functions/OrdinalFunctions.java | 114 ++++++++++
.../dsl/functions/OrdinalFunctionsTest.java | 227 +++++++++++++++++++
3 files changed, 382 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/54861718/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index e6bfc7c..fd8b5d8 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -196,6 +196,9 @@ Where:
| [ `MAAS_MODEL_APPLY`](#maas_model_apply) |
| [ `MAP`](#map) |
| [ `MAP_EXISTS`](#map_exists) |
+| [ `MAP_GET`](#MAP_GET) |
+| [ `MAX`](#MAX) |
+| [ `MIN`](#MIN) |
| [ `MONTH`](#month) |
| [ `MULTISET_ADD`](#multiset_add) |
| [ `MULTISET_INIT`](#multiset_init) |
@@ -544,6 +547,10 @@ Where:
* input - List
* Returns: Last element of the list
+### `GET_SUPPORTED_ENCODINGS`
+ * Description: Returns a list of the encodings that are currently supported.
+ * Returns: A List of String
+
### `HASH`
* Description: Hashes a given value using the given hashing algorithm and returns a hex encoded string.
* Input:
@@ -673,31 +680,6 @@ Where:
* element - Element to add to list
* Returns: Resulting list with the item added at the end.
-### `GET_SUPPORTED_ENCODINGS`
- * Description: Returns a list of the encodings that are currently supported.
- * Returns: A List of String
-
-### `TO_JSON_LIST`
- * Description: Accepts JSON string as an input and returns a List object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
- For e.g. `GET_FIRST( TO_JSON_LIST( '[ "foo", 2]')` would yield `foo`
- * Input:
- * string - The JSON string to be parsed
- * Returns: A parsed List object
-
-### `TO_JSON_MAP`
- * Description: Accepts JSON string as an input and returns a Map object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
- For e.g. `MAP_GET( 'bar', TO_JSON_MAP( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
- * Input:
- * string - The JSON string to be parsed
- * Returns: A parsed Map object
-
-### `TO_JSON_OBJECT`
- * Description: Accepts JSON string as an input and returns a JSON Object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
- For e.g. `MAP_GET( 'bar', TO_JSON_OBJECT( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
- * Input:
- * string - The JSON string to be parsed
- * Returns: A parsed JSON object
-
### `LOG2`
* Description: Returns the log (base `2`) of a number.
* Input:
@@ -753,11 +735,23 @@ Where:
* default - Optionally the default value to return if the key is not in the map.
* Returns: The object associated with the key in the map. If no value is associated with the key and default is specified, then default is returned. If no value is associated with the key or default, then null is returned.
+### `MAX`
+ * Description: Returns the maximum value of a list of input values.
+ * Input:
+ * "list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface). Multi type numeric comparisons are supported: MAX([10,15L,15.3]) would return 15.3, but MAX(['23',25]) will fail and return null as strings and numbers can't be compared.
+ * Returns: The maximum value of the list, or null if the list is empty or the input values were not comparable.
+
+### `MIN`
+ * Description: Returns the minimum value of a list of input values.
+ * Input:
+ * "list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface). Multi type numeric comparisons are supported: MIN([10,15L,15.3]) would return 10, but MIN(['23',25]) will fail and return null as strings and numbers can't be compared.
+ * Returns: The minimum value of the list, or null if the list is empty or the input values were not comparable.
+
### `MONTH`
* Description: The number representing the month. The first month, January, has a value of 0.
* Input:
* dateTime - The datetime as a long representing the milliseconds since unix epoch
- * Returns: The current month (0-based).
+ * Returns: The current month (0-based)
### `MULTISET_ADD`
* Description: Adds to a multiset, which is a map associating objects to their instance counts.
@@ -978,6 +972,27 @@ Where:
* input - Object of string or numeric type
* Returns: Integer version of the first argument
+### `TO_JSON_LIST`
+ * Description: Accepts JSON string as an input and returns a List object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `GET_FIRST( TO_JSON_LIST( '[ "foo", 2]')` would yield `foo`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed List object
+
+### `TO_JSON_MAP`
+ * Description: Accepts JSON string as an input and returns a Map object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `MAP_GET( 'bar', TO_JSON_MAP( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed Map object
+
+### `TO_JSON_OBJECT`
+ * Description: Accepts JSON string as an input and returns a JSON Object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `MAP_GET( 'bar', TO_JSON_OBJECT( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed JSON object
+
### `TO_LONG`
* Description: Transforms the first argument to a long integer
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/54861718/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
new file mode 100644
index 0000000..49e9369
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctions.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.dsl.functions;
+
+import com.google.common.collect.Iterables;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Stellar;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.function.BiFunction;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+public class OrdinalFunctions {
+
+ /**
+ * Stellar Function: MAX
+ * <p>
+ * Return the maximum value of a list of input values in a Stellar list
+ */
+ @Stellar(name = "MAX"
+ , description = "Returns the maximum value of a list of input values"
+ , params = {"list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface)" +
+ " Multi type numeric comparisons are supported: MAX([10,15L,15.3]) would return 15.3, but MAX(['23',25]) will fail and return null as strings and numbers can't be compared."}
+ , returns = "The maximum value in the list, or null if the list is empty or the input values were not comparable.")
+ public static class Max extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> args) {
+ if (args.size() < 1 || args.get(0) == null) {
+ throw new IllegalStateException("MAX function requires at least a Stellar list of values");
+ }
+ Iterable list = (Iterable<Object>) args.get(0);
+ return orderList(list, (ret, val) -> ret.compareTo(val) < 0, "MAX");
+ }
+ }
+
+ /**
+ * Stellar Function: MIN
+ * <p>
+ * Return the minimum value of a list of input values in a Stellar list
+ */
+ @Stellar(name = "MIN"
+ , description = "Returns the minimum value of a list of input values"
+ , params = {"list - List of arguments. The list may only contain objects that are mutually comparable / ordinal (implement java.lang.Comparable interface)" +
+ " Multi type numeric comparisons are supported: MIN([10,15L,15.3]) would return 10, but MIN(['23',25]) will fail and return null as strings and numbers can't be compared."}
+ , returns = "The minimum value in the list, or null if the list is empty or the input values were not comparable.")
+ public static class Min extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> args) {
+ if (args.size() < 1 || args.get(0) == null) {
+ throw new IllegalStateException("MIN function requires at least a Stellar list of values");
+ }
+ Iterable<Comparable> list = (Iterable<Comparable>) args.get(0);
+ return orderList(list, (ret, val) -> ret.compareTo(val) > 0, "MIN");
+
+ }
+ }
+
+ private static Comparable orderList(Iterable<Comparable> list, BiFunction<Comparable, Comparable, Boolean> eval, String funcName) {
+ if (Iterables.isEmpty(list)) {
+ return null;
+ }
+ Object o = Iterables.getFirst(list, null);
+ Comparable ret = null;
+ for(Object valueVal : list) {
+ if(valueVal == null) {
+ continue;
+ }
+ Comparable value = null;
+ if(!(valueVal instanceof Comparable)) {
+ throw new IllegalStateException("Noncomparable object type " + valueVal.getClass().getName()
+ + " submitted to " + funcName);
+ }
+ else {
+ value = (Comparable)valueVal;
+ }
+ try {
+ Comparable convertedRet = ConversionUtils.convert(ret, value.getClass());
+ if(convertedRet == null && ret != null) {
+ throw new IllegalStateException("Incomparable objects were submitted to " + funcName
+ + ": " + ret.getClass() + " is incomparable to " + value.getClass());
+ }
+ if(ret == null || eval.apply(convertedRet, value) ) {
+ ret = value;
+ }
+ }
+ catch(ClassCastException cce) {
+ throw new IllegalStateException("Incomparable objects were submitted to " + funcName
+ + ": " + cce.getMessage(), cce);
+ }
+ }
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/54861718/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java
new file mode 100644
index 0000000..5e06fdd
--- /dev/null
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/OrdinalFunctionsTest.java
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.dsl.functions;
+
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.DefaultVariableResolver;
+import org.apache.metron.stellar.dsl.StellarFunctions;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+import java.util.*;
+
+
+public class OrdinalFunctionsTest {
+
+
+ private static Context context;
+
+ @Before
+ public void setup() throws Exception {
+ context = new Context.Builder().build();
+ }
+ @Test
+ public void testMaxOfMixedNumerical() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(12L);
+ add(56.0);
+ add(56.3);
+ }};
+
+ Object res = run("MAX(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNotNull(res);
+ Assert.assertEquals(56.3, res);
+ }
+
+ @Test
+ public void testMinOfMixedNumerical() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(12L);
+ add(56.0);
+ add(457L);
+ }};
+
+ Object res = run("MIN(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNotNull(res);
+ Assert.assertEquals(res, 12L);
+ }
+ @Test
+ public void testMaxOfStringList() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add("value3");
+ add("value1");
+ add("23");
+ add("value2");
+ }};
+
+ Object res = run("MAX(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNotNull(res);
+ Assert.assertTrue(res.equals("value3"));
+ }
+
+ @Test
+ public void testMaxOfIntegerList() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(12);
+ add(56);
+ }};
+
+ Object res = run("MAX(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNotNull(res);
+ Assert.assertTrue(res.equals(56));
+ }
+
+ @Test
+ public void testMaxWithVarList() throws Exception {
+
+ Object res = run("MAX([string1,string2])", ImmutableMap.of("string1","abc","string2","def"));
+ Assert.assertNotNull(res);
+ Assert.assertTrue(res.equals("def"));
+ }
+
+ @Test
+ public void testMinWithNullInList() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(145);
+ add(null);
+ }};
+
+ Object res = run("MIN(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNotNull(res);
+ Assert.assertTrue(res.equals(145));
+ }
+
+ @Test
+ public void testAllNullList() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(null);
+ add(null);
+ }};
+
+ Object res = run("MAX(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNull(res);
+ }
+
+ @Test
+ public void testMinOfIntegerList() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(56);
+ add(12);
+ add(23);
+ add(null);
+ }};
+
+ Object res = run("MIN(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNotNull(res);
+ Assert.assertTrue(res.equals(12));
+ }
+
+
+ @Test
+ public void testMaxOfLongList() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(12L);
+ add(56L);
+ add(457L);
+ }};
+
+ Object res = run("MAX(input_list)", ImmutableMap.of("input_list", inputList));
+ Assert.assertNotNull(res);
+ Assert.assertTrue(res.equals(457L));
+ }
+
+ @Test
+ public void testMaxOfMixedList() throws Exception {
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(12);
+ add("string");
+ add(457L);
+ }};
+
+ Object res = null;
+
+ try {
+ res = run("MAX(input_list)", ImmutableMap.of("input_list", inputList));
+ } catch(IllegalStateException e) {
+ Assert.assertThat(e.getMessage(), is("Incomparable objects were submitted to MAX: class java.lang.String is incomparable to class java.lang.Long"));
+ Assert.assertNull(res);
+ }
+ }
+
+ @Test
+ public void testSetInput() throws Exception {
+
+ Set<Object> inputSet = new HashSet<Object>(){{
+ add(14L);
+ add(15.3d);
+ add(15);
+ }};
+
+ Object res = run("MAX(input_set)", ImmutableMap.of("input_set", inputSet));
+ Assert.assertNotNull(res);
+ Assert.assertTrue(res.equals(15.3d));
+ }
+
+ @Test
+ public void testNonComparableList() throws Exception {
+
+ class TestObject {
+ private String arg;
+ public TestObject(String arg) {
+ this.arg = arg;
+ }
+ }
+
+ List<Object> inputList = new ArrayList<Object>(){{
+ add(new TestObject("one"));
+ add(new TestObject("two"));
+ add(new TestObject("three"));
+ }};
+
+ Object res = null;
+
+ try {
+ res = run("MIN(input_list)", ImmutableMap.of("input_list", inputList));
+ } catch(IllegalStateException e) {
+ Assert.assertThat(e.getMessage(), is("Noncomparable object type org.apache.metron.stellar.dsl.functions.OrdinalFunctionsTest$1TestObject submitted to MIN"));
+ Assert.assertNull(res);
+ }
+ }
+
+ public Object run(String rule, Map<String, Object> variables) throws Exception {
+ StellarProcessor processor = new StellarProcessor();
+ return processor.parse(rule, new DefaultVariableResolver(x -> variables.get(x), x -> variables.containsKey(x)), StellarFunctions.FUNCTION_RESOLVER(), context);
+ }
+}