You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2017/09/13 02:58:42 UTC
[1/6] metron git commit: METRON-1136 Track Master in Feature Branch
(ottobackwards) closes apache/metron#752
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1136-extensions-parsers 1c63c1eb3 -> 24b668b0b
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java
new file mode 100644
index 0000000..1b4df1e
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/SetFunctions.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+import com.google.common.collect.Iterables;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Stellar;
+
+import java.util.*;
+
+public class SetFunctions {
+ @Stellar(name="INIT"
+ , namespace="SET"
+ , description="Creates a new set"
+ , params = { "input? - An initialization of the set"}
+ , returns = "A Set"
+ )
+ public static class SetInit extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ LinkedHashSet<Object> ret = new LinkedHashSet<>();
+ if(list.size() == 1) {
+ Object o = list.get(0);
+ if(o != null) {
+ if (o instanceof Iterable) {
+ Iterables.addAll(ret, (Iterable) o);
+ }
+ else {
+ throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass());
+ }
+ }
+
+ }
+ return ret;
+ }
+ }
+
+ @Stellar(name="ADD"
+ , namespace="SET"
+ , description="Adds to a set"
+ , params = {"set - The set to add to"
+ ,"o - object to add to set"
+ }
+ , returns = "A Set"
+ )
+ public static class SetAdd extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ if(list.size() < 1) {
+ return null;
+ }
+ LinkedHashSet<Object> ret = (LinkedHashSet<Object>)list.get(0);
+ if(ret == null) {
+ ret = new LinkedHashSet<>();
+ }
+ for(int i = 1;i < list.size();++i) {
+ Object o = list.get(i);
+ if (o != null) {
+ ret.add(o);
+ }
+ }
+ return ret;
+ }
+ }
+
+ @Stellar(name="REMOVE"
+ , namespace="SET"
+ , description="Removes from a set"
+ , params = {"set - The set to add to"
+ ,"o - object to add to set"
+ }
+ , returns = "A Set"
+ )
+ public static class SetRemove extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ if(list.size() < 1) {
+ return null;
+ }
+ LinkedHashSet<Object> ret = (LinkedHashSet<Object>)list.get(0);
+ if(ret == null) {
+ ret = new LinkedHashSet<>();
+ }
+ for(int i = 1;i < list.size();++i) {
+ Object o = list.get(i);
+ if (o != null) {
+ ret.remove(o);
+ }
+ }
+ return ret;
+ }
+ }
+
+ @Stellar(name="MERGE"
+ , namespace="SET"
+ , description="Merges a list of sets"
+ , params = {"sets - A collection of sets to merge"
+ }
+ , returns = "A Set"
+ )
+ public static class SetMerge extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ if(list.size() < 1) {
+ return null;
+ }
+ LinkedHashSet<Object> ret = new LinkedHashSet<>();
+ if(list.size() > 0) {
+ Object o = list.get(0);
+ if(o != null) {
+ if(!(o instanceof Iterable)) {
+ throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass());
+ }
+ Iterable<? extends Iterable> sets = (Iterable<? extends Iterable>) o;
+
+ for (Iterable s : sets) {
+ if (s != null) {
+ if(!(s instanceof Iterable)) {
+ throw new IllegalArgumentException("Expected an Iterable, but " + s + " is of type " + s.getClass());
+ }
+ Iterables.addAll(ret, s);
+ }
+ }
+ }
+ }
+ return ret;
+ }
+ }
+
+ @Stellar(name="INIT"
+ , namespace="MULTISET"
+ , description="Creates an empty multiset, which is a map associating objects to their instance counts."
+ , params = { "input? - An initialization of the multiset"}
+ , returns = "A multiset"
+ )
+ public static class MultiSetInit extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ LinkedHashMap<Object, Integer> ret = new LinkedHashMap<>();
+ if (list.size() >= 1) {
+ Object o = list.get(0);
+ if (o != null) {
+ if (!(o instanceof Iterable)) {
+ throw new IllegalArgumentException("Expected an Iterable, but " + o + " is of type " + o.getClass());
+ }
+ for (Object obj : (Iterable) o) {
+ ret.merge(obj, 1, (k, one) -> k + one);
+ }
+ }
+ }
+ return ret;
+ }
+ }
+
+ @Stellar(name="ADD"
+ , namespace="MULTISET"
+ , description="Adds to a multiset, which is a map associating objects to their instance counts."
+ , params = {"set - The multiset to add to"
+ ,"o - object to add to multiset"
+ }
+ , returns = "A multiset"
+ )
+ public static class MultiSetAdd extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ if(list.size() < 1) {
+ return null;
+ }
+ LinkedHashMap<Object, Integer> ret = (LinkedHashMap<Object, Integer>)list.get(0);
+ if(ret == null) {
+ ret = new LinkedHashMap<>();
+ }
+ for(int i = 1;i < list.size();++i) {
+ Object o = list.get(i);
+ if (o != null) {
+ ret.merge(o, 1, (k, one) -> k + one);
+ }
+ }
+ return ret;
+ }
+ }
+
+ @Stellar(name="REMOVE"
+ , namespace="MULTISET"
+ , description="Removes from a multiset, which is a map associating objects to their instance counts."
+ , params = {"set - The multiset to add to"
+ ,"o - object to remove from multiset"
+ }
+ , returns = "A multiset"
+ )
+ public static class MultiSetRemove extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ if(list.size() < 1) {
+ return null;
+ }
+ LinkedHashMap<Object, Integer> ret = (LinkedHashMap<Object, Integer>)list.get(0);
+ if(ret == null) {
+ ret = new LinkedHashMap<>();
+ }
+ for(int i = 1;i < list.size();++i) {
+ Object o = list.get(i);
+ if (o != null) {
+ Integer cnt = ret.get(o);
+ if(cnt == null) {
+ continue;
+ }
+ if(cnt == 1) {
+ ret.remove(o);
+ }
+ else {
+ ret.put(o, cnt - 1);
+ }
+ }
+ }
+ return ret;
+ }
+ }
+
+ @Stellar(name="MERGE"
+ , namespace="MULTISET"
+ , description="Merges a list of multisets, which is a map associating objects to their instance counts."
+ , params = {"sets - A collection of multisets to merge"
+ }
+ , returns = "A multiset"
+ )
+ public static class MultiSetMerge extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ if(list.size() < 1) {
+ return null;
+ }
+ LinkedHashMap<Object, Integer> ret = new LinkedHashMap<>();
+ if(list.size() > 0) {
+ Iterable<Map<Object, Integer>> maps = (Iterable<Map<Object, Integer>>)list.get(0);
+ for(Map<Object, Integer> s : maps) {
+ if(s != null) {
+ for (Map.Entry<Object, Integer> kv : s.entrySet()) {
+ ret.merge(kv.getKey(), kv.getValue(), (k, cnt) -> k + cnt);
+ }
+ }
+ }
+ }
+ return ret;
+ }
+ }
+
+
+ @Stellar(name="TO_SET"
+ , namespace="MULTISET"
+ , description="Create a set out of a multiset, which is a map associating objects to their instance counts."
+ , params = {"multiset - The multiset to convert."
+ }
+ , returns = "The set of objects in the multiset ignoring multiplicity"
+ )
+ public static class MultiSetToSet extends BaseStellarFunction {
+ @Override
+ public Object apply(List<Object> list) {
+ if(list.size() < 1) {
+ return null;
+ }
+ LinkedHashSet<Object> ret = new LinkedHashSet<>();
+ if(list.size() == 1) {
+ Map<Object, Integer> multiset = (Map<Object, Integer>)list.get(0);
+ if(multiset != null) {
+ ret.addAll(multiset.keySet());
+ }
+ }
+ return ret;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
index 289fa7f..4dc4790 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
@@ -18,15 +18,20 @@
package org.apache.metron.stellar.dsl.functions;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.stellar.common.utils.JSONUtils;
import org.apache.metron.stellar.dsl.BaseStellarFunction;
import org.apache.metron.stellar.dsl.ParseException;
import org.apache.metron.stellar.dsl.Stellar;
import org.apache.metron.stellar.common.utils.ConversionUtils;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -321,6 +326,58 @@ public class StringFunctions {
}
}
+ @Stellar( name="SUBSTRING"
+ , description = "Returns a substring of a string"
+ , params = {
+ "input - The string to take the substring of",
+ "start - The starting position (0-based and inclusive)",
+ "end? - The ending position (0-based and exclusive)"
+ }
+ , returns = "The substring of the input"
+ )
+ public static class Substring extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> strings) {
+
+ if(strings == null || strings.size() < 2 ) {
+ throw new IllegalArgumentException("SUBSTRING requires (at least) 2 arguments: the input and the start position (inclusive)");
+ }
+ Object varObj = strings.get(0);
+ if(varObj != null && !(varObj instanceof String)) {
+ throw new IllegalArgumentException("SUBSTRING input must be a String");
+ }
+ String var = varObj == null?null: (String) varObj;
+ Object startObj = strings.get(1);
+ if(startObj != null && !(startObj instanceof Number)) {
+ throw new IllegalArgumentException("SUBSTRING start must be an Number");
+ }
+ Integer start = startObj == null?null:((Number)startObj).intValue();
+ Integer end = null;
+ if(strings.size() > 2) {
+ Object endObj = strings.get(2);
+ if(endObj != null && !(endObj instanceof Number)) {
+ throw new IllegalArgumentException("SUBSTRING end must be an Number");
+ }
+ end = endObj == null ? null : ((Number) endObj).intValue();
+ }
+ if(var == null || start == null) {
+ return null;
+ }
+ else if(var.length() == 0) {
+ return var;
+ }
+ else {
+ if(end == null) {
+ return var.substring(start);
+ }
+ else {
+ return var.substring(start, end);
+ }
+ }
+ }
+ }
+
@Stellar( name="CHOMP"
, description = "Removes one newline from end of a String if it's there, otherwise leave it alone. A newline is \"\\n\", \"\\r\", or \"\\r\\n\""
, params = { "the String to chomp a newline from, may be null"}
@@ -450,4 +507,114 @@ public class StringFunctions {
}
}
+ @Stellar(name = "TO_JSON_OBJECT"
+ , description = "Returns a JSON object for the specified JSON string"
+ , params = {
+ "str - the JSON String to convert, may be null"
+ }
+ , returns = "an Object containing the parsed JSON string"
+ )
+ public static class ToJsonObject extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> strings) {
+
+ if (strings == null || strings.size() == 0) {
+ throw new IllegalArgumentException("[TO_JSON_OBJECT] incorrect arguments. Usage: TO_JSON_OBJECT <String>");
+ }
+ String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+ if (var == null) {
+ return null;
+ } else if (var.length() == 0) {
+ return var;
+ } else {
+ if (!(strings.get(0) instanceof String)) {
+ throw new ParseException("Valid JSON string not supplied");
+ }
+ // Return JSON Object
+ try {
+ return JSONUtils.INSTANCE.load((String) strings.get(0), Object.class);
+ } catch (JsonProcessingException ex) {
+ throw new ParseException("Valid JSON string not supplied", ex);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return new ParseException("Unable to parse JSON string");
+ }
+ }
+
+ @Stellar(name = "TO_JSON_MAP"
+ , description = "Returns a MAP object for the specified JSON string"
+ , params = {
+ "str - the JSON String to convert, may be null"
+ }
+ , returns = "a MAP object containing the parsed JSON string"
+ )
+ public static class ToJsonMap extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> strings) {
+
+ if (strings == null || strings.size() == 0) {
+ throw new IllegalArgumentException("[TO_JSON_MAP] incorrect arguments. Usage: TO_JSON_MAP <JSON String>");
+ }
+ String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+ if (var == null) {
+ return null;
+ } else if (var.length() == 0) {
+ return var;
+ } else {
+ if (!(strings.get(0) instanceof String)) {
+ throw new ParseException("Valid JSON string not supplied");
+ }
+ // Return parsed JSON Object as a HashMap
+ try {
+ return JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<Map<String, Object>>(){});
+ } catch (JsonProcessingException ex) {
+ throw new ParseException("Valid JSON string not supplied", ex);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return new ParseException("Unable to parse JSON string");
+ }
+ }
+
+ @Stellar(name = "TO_JSON_LIST"
+ , description = "Returns a List object for the specified JSON string"
+ , params = {
+ "str - the JSON String to convert, may be null"
+ }
+ , returns = "a List object containing the parsed JSON string"
+ )
+ public static class ToJsonList extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> strings) {
+
+ if (strings == null || strings.size() == 0) {
+ throw new IllegalArgumentException("[TO_JSON_LIST] incorrect arguments. Usage: TO_JSON_LIST <JSON String>");
+ }
+ String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+ if (var == null) {
+ return null;
+ } else if (var.length() == 0) {
+ return var;
+ } else {
+ if (!(strings.get(0) instanceof String)) {
+ throw new ParseException("Valid JSON string not supplied");
+ }
+ // Return parsed JSON Object as a List
+ try {
+ return (List) JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<List<Object>>(){});
+ } catch (JsonProcessingException ex) {
+ throw new ParseException("Valid JSON string not supplied", ex);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new ParseException("Valid JSON string not supplied", e);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index d6c3713..af86902 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -625,6 +625,18 @@ public class BasicStellarTest {
}
@Test
+
+ public void testShortCircuit_mixedBoolOps() throws Exception {
+ final Map<String, String> variableMap = new HashMap<String, String>();
+ Assert.assertTrue(runPredicate("(false && true) || true"
+ , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertTrue(runPredicate("(false && false) || true"
+ , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertFalse(runPredicate("(true || true) && false"
+ , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ }
+
+ @Test
public void testInString() throws Exception {
final Map<String, String> variableMap = new HashMap<String, String>() {{
put("foo", "casey");
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java
new file mode 100644
index 0000000..1aa8eb4
--- /dev/null
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/SetFunctionsTest.java
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.stellar.dsl.functions;
+
+import org.apache.metron.stellar.common.utils.StellarProcessorUtils;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class SetFunctionsTest {
+
+ @Test(expected=ParseException.class)
+ public void multisetInitTest_wrongType() throws Exception {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT({ 'foo' : 'bar'})", new HashMap<>());
+ }
+
+ @Test
+ public void multisetInitTest() throws Exception {
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT()", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ //int initialization
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT([1,2,3,2])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.containsKey(1));
+ Assert.assertEquals(1,(int)s.get(1));
+ Assert.assertTrue(s.containsKey(2));
+ Assert.assertEquals(2,(int)s.get(2));
+ Assert.assertTrue(s.containsKey(3));
+ Assert.assertEquals(1,(int)s.get(3));
+ }
+ //string initialization
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_INIT(['one','two','three','two'])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.containsKey("one"));
+ Assert.assertEquals(1,(int)s.get("one"));
+ Assert.assertTrue(s.containsKey("two"));
+ Assert.assertEquals(2,(int)s.get("two"));
+ Assert.assertTrue(s.containsKey("three"));
+ Assert.assertEquals(1,(int)s.get("three"));
+
+ }
+ }
+
+ @Test
+ public void multisetAddTest() throws Exception {
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(MULTISET_INIT(), 1)", new HashMap<>());
+ Assert.assertEquals(1, s.size());
+ Assert.assertTrue(s.containsKey(1));
+ Assert.assertEquals(1,(int)s.get(1));
+ }
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(null, 1)", new HashMap<>());
+ Assert.assertEquals(1, s.size());
+ Assert.assertTrue(s.containsKey(1));
+ Assert.assertEquals(1,(int)s.get(1));
+ }
+ //int
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(MULTISET_INIT([1,2,3,4,4]), 4)", new HashMap<>());
+ Assert.assertEquals(4, s.size());
+ Assert.assertTrue(s.containsKey(1));
+ Assert.assertEquals(1,(int)s.get(1));
+ Assert.assertTrue(s.containsKey(2));
+ Assert.assertEquals(1,(int)s.get(2));
+ Assert.assertTrue(s.containsKey(3));
+ Assert.assertEquals(1,(int)s.get(3));
+ Assert.assertTrue(s.containsKey(4));
+ Assert.assertEquals(3,(int)s.get(4));
+ }
+ //string
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_ADD(MULTISET_INIT(['one','two','three', 'four', 'four']), 'four')", new HashMap<>());
+ Assert.assertEquals(4, s.size());
+ Assert.assertTrue(s.containsKey("one"));
+ Assert.assertEquals(1,(int)s.get("one"));
+ Assert.assertTrue(s.containsKey("two"));
+ Assert.assertEquals(1,(int)s.get("two"));
+ Assert.assertTrue(s.containsKey("three"));
+ Assert.assertEquals(1,(int)s.get("three"));
+ Assert.assertTrue(s.containsKey("four"));
+ Assert.assertEquals(3,(int)s.get("four"));
+ }
+ }
+@Test
+ public void multisetRemoveTest() throws Exception {
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(MULTISET_INIT([1]), 1)", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(null, 1)", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ //int
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(MULTISET_INIT([1,2,3,2]), 2)", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.containsKey(1));
+ Assert.assertEquals(1, (int)s.get(1));
+ Assert.assertTrue(s.containsKey(2));
+ Assert.assertEquals(1, (int)s.get(2));
+ Assert.assertTrue(s.containsKey(3));
+ Assert.assertEquals(1, (int)s.get(3));
+ }
+ //string
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_REMOVE(MULTISET_INIT(['one','two','three', 'two']), 'two')", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.containsKey("one"));
+ Assert.assertEquals(1, (int)s.get("one"));
+ Assert.assertTrue(s.containsKey("two"));
+ Assert.assertEquals(1, (int)s.get("two"));
+ Assert.assertTrue(s.containsKey("three"));
+ Assert.assertEquals(1, (int)s.get("three"));
+ }
+ }
+
+ @Test(expected=ParseException.class)
+ public void multisetMergeTest_wrongType() throws Exception {
+
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_MERGE({ 'bar' : 'foo' } )", new HashMap<>());
+ }
+
+ @Test
+ public void multisetMergeTest() throws Exception {
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_MERGE([MULTISET_INIT(), MULTISET_INIT(null), null])", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ //int
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>) StellarProcessorUtils.run("MULTISET_MERGE([MULTISET_INIT([1,2]), MULTISET_INIT([2,3]), null, MULTISET_INIT()])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.containsKey(1));
+ Assert.assertEquals(1, (int)s.get(1));
+ Assert.assertTrue(s.containsKey(2));
+ Assert.assertEquals(2, (int)s.get(2));
+ Assert.assertTrue(s.containsKey(3));
+ Assert.assertEquals(1, (int)s.get(3));
+ }
+ //string
+ {
+ Map<Object, Integer> s = (Map<Object, Integer>)StellarProcessorUtils.run("MULTISET_MERGE([MULTISET_INIT(['one','two']), MULTISET_INIT(['two', 'three'])])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.containsKey("one"));
+ Assert.assertEquals(1, (int)s.get("one"));
+ Assert.assertTrue(s.containsKey("two"));
+ Assert.assertEquals(2, (int)s.get("two"));
+ Assert.assertTrue(s.containsKey("three"));
+ Assert.assertEquals(1, (int)s.get("three"));
+ }
+ }
+
+ @Test(expected=ParseException.class)
+ public void setInitTest_wrongType() throws Exception {
+ Set s = (Set) StellarProcessorUtils.run("SET_INIT({ 'foo' : 2})", new HashMap<>());
+ }
+
+ @Test
+ public void setInitTest() throws Exception {
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_INIT()", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ //int initialization
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_INIT([1,2,3])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.contains(1));
+ Assert.assertTrue(s.contains(2));
+ Assert.assertTrue(s.contains(3));
+ }
+ //string initialization
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_INIT(['one','two','three'])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.contains("one"));
+ Assert.assertTrue(s.contains("two"));
+ Assert.assertTrue(s.contains("three"));
+ }
+ }
+
+ @Test
+ public void multisetToSetTest() throws Exception {
+ {
+ Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(MULTISET_INIT(), 1))", new HashMap<>());
+ Assert.assertEquals(1, s.size());
+ Assert.assertTrue(s.contains(1));
+ }
+ {
+ Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(null, 1))", new HashMap<>());
+ Assert.assertEquals(1, s.size());
+ Assert.assertTrue(s.contains(1));
+ }
+ //int
+ {
+ Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(MULTISET_INIT([1,2,3]), 4))", new HashMap<>());
+ Assert.assertEquals(4, s.size());
+ Assert.assertTrue(s.contains(1));
+ Assert.assertTrue(s.contains(2));
+ Assert.assertTrue(s.contains(3));
+ Assert.assertTrue(s.contains(4));
+ }
+ //string
+ {
+ Set s = (Set) StellarProcessorUtils.run("MULTISET_TO_SET(MULTISET_ADD(MULTISET_INIT(['one','two','three']), 'four'))", new HashMap<>());
+ Assert.assertEquals(4, s.size());
+ Assert.assertTrue(s.contains("one"));
+ Assert.assertTrue(s.contains("two"));
+ Assert.assertTrue(s.contains("three"));
+ Assert.assertTrue(s.contains("four"));
+ }
+ }
+
+ @Test
+ public void setAddTest() throws Exception {
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_ADD(SET_INIT(), 1)", new HashMap<>());
+ Assert.assertEquals(1, s.size());
+ Assert.assertTrue(s.contains(1));
+ }
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_ADD(null, 1)", new HashMap<>());
+ Assert.assertEquals(1, s.size());
+ Assert.assertTrue(s.contains(1));
+ }
+ //int
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_ADD(SET_INIT([1,2,3]), 4)", new HashMap<>());
+ Assert.assertEquals(4, s.size());
+ Assert.assertTrue(s.contains(1));
+ Assert.assertTrue(s.contains(2));
+ Assert.assertTrue(s.contains(3));
+ Assert.assertTrue(s.contains(4));
+ }
+ //string
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_ADD(SET_INIT(['one','two','three']), 'four')", new HashMap<>());
+ Assert.assertEquals(4, s.size());
+ Assert.assertTrue(s.contains("one"));
+ Assert.assertTrue(s.contains("two"));
+ Assert.assertTrue(s.contains("three"));
+ Assert.assertTrue(s.contains("four"));
+ }
+ }
+
+ @Test
+ public void setRemoveTest() throws Exception {
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(SET_INIT([1]), 1)", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(null, 1)", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ //int
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(SET_INIT([1,2,3]), 2)", new HashMap<>());
+ Assert.assertEquals(2, s.size());
+ Assert.assertTrue(s.contains(1));
+ Assert.assertTrue(s.contains(3));
+ }
+ //string
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_REMOVE(SET_INIT(['one','two','three']), 'three')", new HashMap<>());
+ Assert.assertEquals(2, s.size());
+ Assert.assertTrue(s.contains("one"));
+ Assert.assertTrue(s.contains("two"));
+ }
+ }
+
+ @Test(expected=ParseException.class)
+ public void setMergeTest_wrongType() throws Exception {
+ Set s = (Set) StellarProcessorUtils.run("SET_MERGE({ 'foo' : 'bar'} )", new HashMap<>());
+ }
+
+ @Test
+ public void setMergeTest() throws Exception {
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_MERGE([SET_INIT(), SET_INIT(null), null])", new HashMap<>());
+ Assert.assertEquals(0, s.size());
+ }
+ //int
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_MERGE([SET_INIT([1,2]), SET_INIT([3]), null, SET_INIT()])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.contains(1));
+ Assert.assertTrue(s.contains(2));
+ Assert.assertTrue(s.contains(3));
+ }
+ //string
+ {
+ Set s = (Set) StellarProcessorUtils.run("SET_MERGE([SET_INIT(['one','two']), SET_INIT(['three'])])", new HashMap<>());
+ Assert.assertEquals(3, s.size());
+ Assert.assertTrue(s.contains("one"));
+ Assert.assertTrue(s.contains("two"));
+ Assert.assertTrue(s.contains("three"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
index 858a043..418bf2d 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
@@ -20,15 +20,18 @@ package org.apache.metron.stellar.dsl.functions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.metron.stellar.dsl.DefaultVariableResolver;
import org.apache.metron.stellar.dsl.ParseException;
import org.junit.Assert;
import org.junit.Test;
+import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
@@ -408,6 +411,41 @@ public class StringFunctionsTest {
}
+ @Test
+ public void testSubstring() throws Exception {
+ Map<String, Object> variables = ImmutableMap.of("s", "apache metron");
+ Assert.assertEquals("metron", run("SUBSTRING(s, 7)", variables));
+ Assert.assertEquals("me", run("SUBSTRING(s, 7, 9)", variables));
+ Assert.assertNull(run("SUBSTRING(null, 7, 9)", new HashMap<>()));
+ Assert.assertNull(run("SUBSTRING(null, null, 9)", new HashMap<>()));
+ Assert.assertNull(run("SUBSTRING(s, null, 9)", variables));
+ Assert.assertNull(run("SUBSTRING(null, null, null)", new HashMap<>()));
+ Assert.assertEquals("metron", run("SUBSTRING(s, 7, null)", variables));
+ }
+
+ @Test(expected=ParseException.class)
+ public void testSubstring_invalidEmpty() throws Exception {
+ Assert.assertEquals("metron", run("SUBSTRING()", new HashMap<>()));
+ }
+
+ @Test(expected=ParseException.class)
+ public void testSubstring_invalidWrongTypeStart() throws Exception {
+ Map<String, Object> variables = ImmutableMap.of("s", "apache metron");
+ Assert.assertEquals("metron", (String) run("SUBSTRING(s, '7')", variables));
+ }
+
+ @Test(expected=ParseException.class)
+ public void testSubstring_invalidWrongTypeEnd() throws Exception {
+ Map<String, Object> variables = ImmutableMap.of("s", "apache metron");
+ Assert.assertEquals("metron", (String) run("SUBSTRING(s, 7, '9')", variables));
+ }
+
+ @Test(expected=ParseException.class)
+ public void testSubstring_invalidWrongTypeInput() throws Exception {
+ Map<String, Object> variables = ImmutableMap.of("s", 7);
+ Assert.assertEquals("metron", (String) run("SUBSTRING(s, 7, '9')", variables));
+ }
+
/**
* COUNT_MATCHES StringFunction
*/
@@ -449,4 +487,262 @@ public class StringFunctionsTest {
Assert.assertTrue(thrown);
}
+
+ /**
+ * TO_JSON_OBJECT StringFunction
+ */
+
+ // Input strings to be used
+ /**
+ { "foo" : 2 }
+ */
+ @Multiline
+ private String string1;
+
+ /**
+ {
+ "foo" : "abc",
+ "bar" : "def"
+ }
+ */
+ @Multiline
+ private String string2;
+
+ /**
+ [ "foo", 2 ]
+ */
+ @Multiline
+ private String string3;
+
+ /**
+ [ "foo", "bar", "car" ]
+ */
+ @Multiline
+ private String string4;
+
+ /**
+ [
+ {
+ "foo1":"abc",
+ "bar1":"def"
+ },
+ {
+ "foo2":"ghi",
+ "bar2":"jkl"
+ }
+ ]
+ */
+ @Multiline
+ private String string5;
+
+ @Test
+ public void testToJsonObject() throws Exception {
+ //JSON Object
+ Object ret1 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string1));
+ Assert.assertNotNull(ret1);
+ Assert.assertTrue (ret1 instanceof HashMap);
+
+ Object ret2 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string2));
+ Assert.assertNotNull(ret2);
+ Assert.assertTrue (ret2 instanceof HashMap);
+ Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2)));
+
+ //Simple Arrays
+ Object ret3 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string3));
+ Assert.assertNotNull(ret3);
+ Assert.assertTrue (ret3 instanceof ArrayList);
+ List<Object> result3 = (List<Object>) ret3;
+ Assert.assertEquals(2, result3.get(1));
+
+ Object ret4 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string4));
+ Assert.assertNotNull(ret4);
+ Assert.assertTrue (ret4 instanceof ArrayList);
+ List<Object> result4 = (List<Object>) ret4;
+ Assert.assertEquals("car", result4.get(2));
+
+ //JSON Array
+ Object ret5 = run( "TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string5));
+ Assert.assertNotNull(ret5);
+ Assert.assertTrue (ret5 instanceof ArrayList);
+ List<List<Object>> result5 = (List<List<Object>>) ret5;
+ HashMap<String,String> results5Map1 = (HashMap) result5.get(0);
+ Assert.assertEquals("def", results5Map1.get("bar1"));
+ HashMap<String,String> results5Map2 = (HashMap) result5.get(1);
+ Assert.assertEquals("ghi", results5Map2.get("foo2"));
+
+ // No input
+ boolean thrown = false;
+ try {
+ run("TO_JSON_OBJECT()", Collections.emptyMap());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Invalid input
+ try {
+ run("TO_JSON_OBJECT('123, 456')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Malformed JSON String
+ try {
+ run("TO_JSON_OBJECT('{\"foo\" : 2')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+ }
+
+ @Test
+ public void testToJsonMap() throws Exception {
+ //JSON Object
+ Object ret1 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string1));
+ Assert.assertNotNull(ret1);
+ Assert.assertTrue (ret1 instanceof HashMap);
+
+ Object ret2 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string2));
+ Assert.assertNotNull(ret2);
+ Assert.assertTrue (ret2 instanceof HashMap);
+ Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2)));
+
+ //Simple Arrays
+ boolean thrown = false;
+ try {
+ run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string3));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue(thrown);
+
+ thrown = false;
+ try {
+ run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string4));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue (thrown);
+
+ //JSON Array
+ thrown = false;
+ try {
+ run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string5));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue(thrown);
+
+
+ // No input
+ try {
+ run("TO_JSON_MAP()", Collections.emptyMap());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Invalid input
+ try {
+ run("TO_JSON_MAP('123, 456')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Malformed JSON String
+ try {
+ run("TO_JSON_MAP('{\"foo\" : 2')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+ }
+
+ @Test
+ public void testToJsonList() throws Exception {
+ //Simple Arrays
+ Object ret3 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string3));
+ Assert.assertNotNull(ret3);
+ Assert.assertTrue (ret3 instanceof ArrayList);
+ List<Object> result3 = (List<Object>) ret3;
+ Assert.assertEquals(2, result3.get(1));
+
+ Object ret4 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string4));
+ Assert.assertNotNull(ret4);
+ Assert.assertTrue (ret4 instanceof ArrayList);
+ List<Object> result4 = (List<Object>) ret4;
+ Assert.assertEquals("car", result4.get(2));
+
+ //JSON Array
+ Object ret5 = run( "TO_JSON_LIST(msg)", ImmutableMap.of("msg", string5));
+ Assert.assertNotNull(ret5);
+ Assert.assertTrue (ret5 instanceof ArrayList);
+ List<List<Object>> result5 = (List<List<Object>>) ret5;
+ HashMap<String,String> results5Map1 = (HashMap) result5.get(0);
+ Assert.assertEquals("def", results5Map1.get("bar1"));
+ HashMap<String,String> results5Map2 = (HashMap) result5.get(1);
+ Assert.assertEquals("ghi", results5Map2.get("foo2"));
+
+ //JSON Object - throws exception
+ boolean thrown = false;
+ try {
+ run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string1));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue(thrown);
+
+ thrown = false;
+ try {
+ run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string2));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue (thrown);
+
+ // No input
+ thrown = false;
+ try {
+ run("TO_JSON_LIST()", Collections.emptyMap());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+ }
+ Assert.assertTrue(thrown);
+
+ // Invalid input
+ thrown = false;
+ try {
+ run("TO_JSON_LIST('123, 456')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+
+ // Malformed JSON String
+ thrown = false;
+ try {
+ run("TO_JSON_LIST('{\"foo\" : 2')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9a86314..dcadfe8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -344,8 +344,12 @@
<!-- ACE editor assets are covered in the metron-config NOTICE file -->
<exclude>**/src/assets/ace/**</exclude>
<exclude>dist/assets/ace/**</exclude>
- <!-- Generated svg containing Font Awesome fonts are covered in the metron-config README and NOTICE file -->
+ <!-- Generated svg and bundle.css containing Font Awesome fonts are covered in the
+ metron-interface/metron-config README and NOTICE files -->
<exclude>dist/*.svg</exclude>
+ <exclude>dist/styles.a0b6b99c10d9a13dc67e.bundle.css</exclude>
+ <!-- 3rdpartylicenses.txt is an empty file carried along by imported libraries -->
+ <exclude>dist/3rdpartylicenses.txt</exclude>
<exclude>e2e/*.js.map</exclude>
<!-- Checkstyle is LGPL. We derive ours from their base, but don't ship it, so it's fine use.
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/use-cases/README.md
----------------------------------------------------------------------
diff --git a/use-cases/README.md b/use-cases/README.md
new file mode 100644
index 0000000..02be32d
--- /dev/null
+++ b/use-cases/README.md
@@ -0,0 +1,4 @@
+# Worked Examples
+
+The following are worked examples of use-cases that showcase some (or
+many) component(s) of Metron.
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/use-cases/geographic_login_outliers/README.md
----------------------------------------------------------------------
diff --git a/use-cases/geographic_login_outliers/README.md b/use-cases/geographic_login_outliers/README.md
new file mode 100644
index 0000000..99e9a5b
--- /dev/null
+++ b/use-cases/geographic_login_outliers/README.md
@@ -0,0 +1,267 @@
+# Problem Statement
+
+One way to find anomalous behavior in a network is by inspecting user
+login behavior. In particular, if a user is logging in via vastly
+differing geographic locations in a short period of time, this may be
+evidence of malicious behavior.
+
+More formally, we can encode this potentially malicious event in terms
+of how far from the geographic centroid of the user's historic logins
+as compared to all users. For instance, if we track all users and the
+median distance from the central geographic location of all of their
+logins for the last 2 hours is 3 km and the standard deviation is 1 km,
+if we see a user logging in 1700 km from the central geographic location of
+their logins for the last 2 hours, then they MAY be exhibiting a
+deviation that we want to monitor since it would be hard to travel that
+distance in 4 hours. On the other hand, the user may have
+just used a VPN or proxy. Ultimately, this sort of analytic must be
+considered only one piece of evidence in addition to many others before
+we want to indicate an alert.
+
+# Demonstration Design
+For the purposes of demonstration, we will construct synthetic data
+whereby 2 users are logging into a system rather quickly (once per
+second) from various hosts. Each user's locations share the same first
+2 octets, but will choose the last 2 randomly. We will then inject a
+data point indicating `user1` is logging in via a russian IP address.
+
+## Preliminaries
+We assume that the following environment variables are set:
+* `METRON_HOME` - the home directory for metron
+* `ZOOKEEPER` - The zookeeper quorum (comma separated with port specified: e.g. `node1:2181` for full-dev)
+* `BROKERLIST` - The Kafka broker list (comma separated with port specified: e.g. `node1:6667` for full-dev)
+* `ES_HOST` - The elasticsearch master (and port) e.g. `node1:9200` for full-dev.
+
+Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.
+
+Before editing configurations, be sure to pull the configs from zookeeper locally via
+```
+$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f
+```
+
+## Configure the Profiler
+First, we'll configure the profiler to emit a profiler every 1 minute:
+* In Ambari, set the profiler period duration to `1` minute via the Profiler config section.
+* Adjust `$METRON_HOME/config/zookeeper/global.json` to adjust the capture duration:
+```
+ "profiler.client.period.duration" : "1",
+ "profiler.client.period.duration.units" : "MINUTES"
+```
+
+## Create the Data Generator
+We want to create a new sensor for our synthetic data called `auth`. To
+feed it, we need a synthetic data generator. In particular, we want a
+process which will feed authentication events per second for a set of
+users where the IPs are randomly chosen, but each user's login ip
+addresses share the same first 2 octets.
+
+Edit `~/gen_data.py` and paste the following into it:
+```
+#!/usr/bin/python
+
+import random
+import sys
+import time
+
+domains = { 'user1' : '173.90', 'user2' : '156.33' }
+
+def get_ip(base):
+ return base + '.' + str(random.randint(1,255)) + '.' + str(random.randint(1, 255))
+
+def main():
+ freq_s = 1
+ while True:
+ user='user' + str(random.randint(1,len(domains)))
+ epoch_time = int(time.time())
+ ip=get_ip(domains[user])
+ print user + ',' + ip + ',' + str(epoch_time)
+ sys.stdout.flush()
+ time.sleep(freq_s)
+
+if __name__ == '__main__':
+ main()
+```
+
+## Create the `auth` Parser
+
+The message format for our simple synthetic data is a CSV with:
+* username
+* login ip address
+* timestamp
+
+We will need to parse this via our `CSVParser` and add the geohash of the login ip address.
+
+* To create this parser, edit `$METRON_HOME/config/zookeeper/parsers/auth.json` and paste the following:
+```
+{
+ "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+ ,"sensorTopic" : "auth"
+ ,"parserConfig" : {
+ "columns" : {
+ "user" : 0,
+ "ip" : 1,
+ "timestamp" : 2
+ }
+ }
+ ,"fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : [ "hash" ]
+ ,"config" : {
+ "hash" : "GEOHASH_FROM_LOC(GEO_GET(ip))"
+ }
+ }
+ ]
+}
+```
+* Create the kafka topic via:
+```
+/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic auth --partitions 1 --replication-factor 1
+```
+
+## Create the Profiles for Enrichment
+
+We will need to track 2 profiles to accomplish this task:
+* `locations_by_user` - The geohashes of the locations the user has logged in from. This is a multiset of geohashes per user. Note that the multiset in this case is effectively a map of geohashes to occurrance counts.
+* `geo_distribution_from_centroid` - The statistical distribution of the distance between a login location and the geographic centroid of the user's previous logins from the last 2 minutes. Note, in a real installation this would be a larger temporal lookback.
+
+We can represent these in the `$METRON_HOME/config/zookeeper/profiler.json` via the following:
+```
+{
+ "profiles": [
+ {
+ "profile": "geo_distribution_from_centroid",
+ "foreach": "'global'",
+ "onlyif": "exists(geo_distance) && geo_distance != null",
+ "init" : {
+ "s": "STATS_INIT()"
+ },
+ "update": {
+ "s": "STATS_ADD(s, geo_distance)"
+ },
+ "result": "s"
+ },
+ {
+ "profile": "locations_by_user",
+ "foreach": "user",
+ "onlyif": "exists(hash) && hash != null && LENGTH(hash) > 0",
+ "init" : {
+ "s": "MULTISET_INIT()"
+ },
+ "update": {
+ "s": "MULTISET_ADD(s, hash)"
+ },
+ "result": "s"
+ }
+ ]
+}
+```
+
+## Enrich authentication Events
+
+We will need to enrich the authentication records in a couple of ways to use in the threat triage section as well as the profiles:
+* `geo_distance`: representing the distance between the current geohash and the geographic centroid for the last 2 minutes.
+* `geo_centroid`: representing the geographic centroid for the last 2 minutes
+
+Beyond that, we will need to determine if the authentication event is a geographic outlier by computing the following fields:
+* `dist_median` : representing the median distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the median of the `geo_distance` values across all users).
+* `dist_sd` : representing the standard deviation of the distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the standard deviation of the `geo_distance` values across all users).
+* `geo_outlier` : whether `geo_distance` is more than 5 standard deviations from the median across all users.
+
+We also want to set up a triage rule associating a score and setting an alert if `geo_outlier` is true. In reality, this would be more complex as this metric is at best circumstantial and would need supporting evidence, but for simplicity we'll deal with the false positives.
+
+* Edit `$METRON_HOME/config/zookeeper/enrichments/auth.json` and paste the following:
+```
+{
+ "enrichment": {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "geo_locations := MULTISET_MERGE( PROFILE_GET( 'locations_by_user', user, PROFILE_FIXED( 2, 'MINUTES')))",
+ "geo_centroid := GEOHASH_CENTROID(geo_locations)",
+ "geo_distance := TO_INTEGER(GEOHASH_DIST(geo_centroid, hash))",
+ "geo_locations := null"
+ ]
+ }
+ }
+ ,"fieldToTypeMap": { }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "geo_distance_distr:= STATS_MERGE( PROFILE_GET( 'geo_distribution_from_centroid', 'global', PROFILE_FIXED( 2, 'MINUTES')))",
+ "dist_median := STATS_PERCENTILE(geo_distance_distr, 50.0)",
+ "dist_sd := STATS_SD(geo_distance_distr)",
+ "geo_outlier := ABS(dist_median - geo_distance) >= 5*dist_sd",
+ "is_alert := exists(is_alert) && is_alert",
+ "is_alert := is_alert || (geo_outlier != null && geo_outlier == true)",
+ "geo_distance_distr := null"
+ ]
+ }
+
+ },
+ "fieldToTypeMap": { },
+ "triageConfig" : {
+ "riskLevelRules" : [
+ {
+ "name" : "Geographic Outlier",
+ "comment" : "Determine if the user's geographic distance from the centroid of the historic logins is an outlier as compared to all users.",
+ "rule" : "geo_outlier != null && geo_outlier",
+ "score" : 10,
+ "reason" : "FORMAT('user %s has a distance (%d) from the centroid of their last login is 5 std deviations (%f) from the median (%f)', user, geo_distance, dist_sd, dist_median)"
+ }
+ ],
+ "aggregator" : "MAX"
+ }
+ }
+}
+```
+
+## Execute Demonstration
+
+From here, we've set up our configuration and can push the configs:
+* Push the configs to zookeeper via
+```
+$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z node1:2181 -i $METRON_HOME/config/zookeeper/
+```
+* Start the parser via:
+```
+$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s auth
+```
+* Push synthetic data into the `auth` topic via
+```
+python ~/gen_data.py | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic auth
+```
+* Wait for about `5` minutes and kill the previous command
+* Push a synthetic record indicating `user1` has logged in from a russian IP (`109.252.227.173`):
+```
+echo -e "import time\nprint 'user1,109.252.227.173,'+str(int(time.time()))" | python | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic auth
+```
+* Execute the following to search elasticsearch for our geographic login outliers:
+```
+curl -XPOST "http://$ES_HOST/auth*/_search?pretty" -d '
+{
+ "_source" : [ "is_alert", "threat:triage:rules:0:reason", "user", "ip", "geo_distance" ],
+ "query": { "exists" : { "field" : "threat:triage:rules:0:reason" } }
+}
+'
+```
+
+You should see, among a few other false positive results, something like the following:
+```
+{
+ "_index" : "auth_index_2017.09.07.20",
+ "_type" : "auth_doc",
+ "_id" : "f5bdbf76-9d78-48cc-b21d-bc434c96e62e",
+ "_score" : 1.0,
+ "_source" : {
+ "geo_distance" : 7879,
+ "threat:triage:rules:0:reason" : "user user1 has a distance (7879) from the centroid of their last login is 5 std deviations (334.814719) from the median (128.000000)",
+ "ip" : "109.252.227.173",
+ "is_alert" : "true",
+ "user" : "user1"
+ }
+}
+```
+
[5/6] metron git commit: METRON-1136 Track Master in Feature Branch
(ottobackwards) closes apache/metron#752
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index e568bce..823b7b6 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -184,6 +184,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| |
| ---------- |
+| [ `POST /api/v1/alert/escalate`](#get-apiv1alertescalate)|
| [ `POST /api/v1/ext/parsers`](#post-apiv1extparsers)|
| [ `GET /api/v1/ext/parsers`](#get-apiv1extparsers)|
| [ `GET /api/v1/ext/parsers/{name}`](#get-apiv1extparsersname)|
@@ -203,10 +204,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopicname)|
| [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopicname)|
| [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopicnamesample)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
+| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
+| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
| [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadata)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadatacommon)|
+| [ `GET /api/v1/search/column/metadata`](#get-apiv1searchcolumnmetadata)|
+| [ `GET /api/v1/search/column/metadata/common`](#get-apiv1searchcolumnmetadatacommon)|
| [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)|
| [ `GET /api/v1/sensor/enrichment/config/list/available/enrichments`](#get-apiv1sensorenrichmentconfiglistavailableenrichments)|
| [ `GET /api/v1/sensor/enrichment/config/list/available/threat/triage/aggregators`](#get-apiv1sensorenrichmentconfiglistavailablethreattriageaggregators)|
@@ -251,6 +253,13 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `PUT /api/v1/update/replace`](#patch-apiv1updatereplace)|
| [ `GET /api/v1/user`](#get-apiv1user)|
+### `POST /api/v1/alert/escalate`
+ * Description: Escalates a list of alerts by producing it to the Kafka escalate topic
+ * Input:
+ * alerts - The alerts to be escalated
+ * Returns:
+ * 200 - Alerts were escalated
+
### `POST /api/v1/ext/parsers`
* Description: Install a Metron Parser Extension into the system
* Input:
@@ -320,10 +329,13 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - JSON results
### `POST /api/v1/hdfs`
- * Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists.
+ * Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists. Permissions must be set for all three groups if they are to be set. If any are missing, the default permissions will be used, and if any are invalid an exception will be thrown.
* Input:
* path - Path to HDFS file
* contents - File contents
+ * userMode - [optional] symbolic permission string for user portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute
+ * groupMode - [optional] symbolic permission string for group portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute
+ * otherMode - [optional] symbolic permission string for other portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute
* Returns:
* 200 - Contents were written
@@ -386,6 +398,24 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns sample message
* 404 - Either Kafka topic is missing or contains no messages
+### `POST /api/v1/search/search`
+ * Description: Searches the indexing store
+ * Input:
+ * searchRequest - Search request
+ * Returns:
+ * 200 - Search response
+
+### `POST /api/v1/search/group`
+ * Description: Searches the indexing store and returns field groups. Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. The default sorting within groups is by count descending. A groupOrder type of count will sort based on then number of documents in a group while a groupType of term will sort by the groupBy term.
+ * Input:
+ * groupRequest - Group request
+ * indices - list of indices to search
+ * query - lucene query
+ * scoreField - field used to compute a total score for each group
+ * groups - List of groups (field name and sort order)
+ * Returns:
+ * 200 - Group response
+
### `GET /api/v1/search/findOne`
* Description: Returns latest document for a guid and sensor
* Input:
@@ -402,13 +432,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* Returns:
* 200 - Document representing the output
* 404 - Document with UUID and sensor type not found
-
-### `GET /api/v1/search/search`
- * Description: Searches the indexing store
- * Input:
- * searchRequest - Search request
- * Returns:
- * 200 - Search results
### `GET /api/v1/search/column/metadata`
* Description: Get column metadata for each index in the list of indicies
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
index afd4aeb..33e5ac6 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -30,6 +30,8 @@ kafka:
url: ${BROKERLIST}
security:
protocol: ${KAFKA_SECURITY_PROTOCOL}
+ topics:
+ escalation: ${METRON_ESCALATION_TOPIC}
hdfs:
metron:
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index 55aff29..3947e17 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -50,6 +50,7 @@ public class MetronRestConstants {
public static final String CURATOR_MAX_RETRIES = "curator.max.retries";
public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url";
+ public static final String KAFKA_TOPICS_ESCALATION_PROPERTY = "kafka.topics.escalation";
public static final String HDFS_METRON_APPS_ROOT = "hdfs.metron.apps.root";
@@ -58,6 +59,7 @@ public class MetronRestConstants {
public static final String KERBEROS_KEYTAB_SPRING_PROPERTY = "kerberos.keytab";
public static final String SEARCH_MAX_RESULTS = "search.max.results";
+ public static final String SEARCH_MAX_GROUPS = "search.max.groups";
public static final String INDEX_DAO_IMPL = "index.dao.impl";
public static final String INDEX_HBASE_TABLE_PROVIDER_IMPL = "index.hbase.provider";
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
index 6385116..b6ac5e7 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.rest.config;
+import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL;
+
import org.apache.metron.hbase.HTableProvider;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.indexing.dao.AccessConfig;
@@ -28,14 +30,8 @@ import org.apache.metron.rest.service.GlobalConfigService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Profile;
import org.springframework.core.env.Environment;
-import java.lang.reflect.InvocationTargetException;
-
-import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-
@Configuration
public class IndexConfig {
@@ -55,9 +51,11 @@ public class IndexConfig {
try {
String hbaseProviderImpl = environment.getProperty(MetronRestConstants.INDEX_HBASE_TABLE_PROVIDER_IMPL, String.class, null);
String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null);
- int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, -1);
+ int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, 1000);
+ int searchMaxGroups = environment.getProperty(MetronRestConstants.SEARCH_MAX_GROUPS, Integer.class, 1000);
AccessConfig config = new AccessConfig();
config.setMaxSearchResults(searchMaxResults);
+ config.setMaxSearchGroups(searchMaxGroups);
config.setGlobalConfigSupplier(() -> {
try {
return globalConfigService.get();
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
index 247264b..a2abbeb 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
@@ -21,6 +21,7 @@ import kafka.admin.AdminUtils$;
import kafka.utils.ZkUtils;
import org.I0Itec.zkclient.ZkClient;
import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.metron.rest.MetronRestConstants;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
@@ -100,6 +101,21 @@ public class KafkaConfig {
return new DefaultKafkaConsumerFactory<>(consumerProperties());
}
+ @Bean
+ public Map<String, Object> producerProperties() {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY));
+ producerConfig.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ producerConfig.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ producerConfig.put("request.required.acks", 1);
+ return producerConfig;
+ }
+
+ @Bean
+ public KafkaProducer kafkaProducer() {
+ return new KafkaProducer<>(producerProperties());
+ }
+
/**
* Create a bean for {@link AdminUtils$}. This is primarily done to make testing a bit easier.
*
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/AlertsController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/AlertsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/AlertsController.java
new file mode 100644
index 0000000..6f028a1
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/AlertsController.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.AlertService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * The API resource that is used for alert-related operations.
+ */
+@RestController
+@RequestMapping("/api/v1/alert")
+public class AlertsController {
+
+ /**
+ * Service used to interact with alerts.
+ */
+ @Autowired
+ private AlertService alertService;
+
+ @ApiOperation(value = "Escalates a list of alerts by producing it to the Kafka escalate topic")
+ @ApiResponse(message = "Alerts were escalated", code = 200)
+ @RequestMapping(value = "/escalate", method = RequestMethod.POST)
+ ResponseEntity<Void> escalate(final @ApiParam(name = "alerts", value = "The alerts to be escalated", required = true) @RequestBody List<Map<String, Object>> alerts) throws RestException {
+ alertService.escalateAlerts(alerts);
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
index b8957a9..80b5942 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
@@ -67,9 +67,14 @@ public class HdfsController {
@ApiOperation(value = "Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists.")
@ApiResponse(message = "Contents were written", code = 200)
@RequestMapping(method = RequestMethod.POST)
- ResponseEntity<Void> write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path,
- @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException {
- hdfsService.write(new Path(path), contents.getBytes(UTF_8));
+ ResponseEntity<Void> write(
+ @ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path,
+ @ApiParam(name = "contents", value = "File contents", required = true) @RequestBody String contents,
+ @ApiParam(name = "userMode", value = "requested user permissions") @RequestParam(required = false, defaultValue = "") String userMode,
+ @ApiParam(name = "groupMode", value = "requested group permissions") @RequestParam(required = false, defaultValue = "") String groupMode,
+ @ApiParam(name = "otherMode", value = "requested other permissions") @RequestParam(required = false, defaultValue = "") String otherMode
+ ) throws RestException {
+ hdfsService.write(new Path(path), contents.getBytes(UTF_8), userMode, groupMode, otherMode);
return new ResponseEntity<>(HttpStatus.OK);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
index 2787504..d057ac4 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
@@ -21,6 +21,7 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
+import java.util.Set;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.KafkaTopic;
import org.apache.metron.rest.service.KafkaService;
@@ -33,8 +34,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
-import java.util.Set;
-
/**
* The API resource that is use to interact with Kafka.
*/
@@ -109,4 +108,15 @@ public class KafkaController {
return new ResponseEntity<>(HttpStatus.NOT_FOUND);
}
}
+
+ @ApiOperation(value = "Produces a message to a Kafka topic")
+ @ApiResponses(value = {
+ @ApiResponse(message = "Message produced successfully", code = 200)
+ })
+ @RequestMapping(value = "/topic/{name}/produce", method = RequestMethod.POST)
+ ResponseEntity<String> produce(final @ApiParam(name = "name", value = "Kafka topic name", required = true) @PathVariable String name,
+ final @ApiParam(name = "message", value = "Message", required = true) @RequestBody String message) throws RestException {
+ kafkaService.produceMessage(name, message);
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java
index dea628c..e215413 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java
@@ -21,7 +21,8 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.rest.RestException;
import org.apache.metron.indexing.dao.search.SearchRequest;
@@ -38,7 +39,6 @@ import org.springframework.web.bind.annotation.RestController;
import java.util.Map;
import java.util.Optional;
import java.util.List;
-import java.util.Map;
@RestController
@RequestMapping("/api/v1/search")
@@ -54,6 +54,15 @@ public class SearchController {
return new ResponseEntity<>(searchService.search(searchRequest), HttpStatus.OK);
}
+ @ApiOperation(value = "Searches the indexing store and returns field groups. "
+ + "Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. "
+ + "The default sorting within groups is by count descending.")
+ @ApiResponse(message = "Group response", code = 200)
+ @RequestMapping(value = "/group", method = RequestMethod.POST)
+ ResponseEntity<GroupResponse> group(final @ApiParam(name = "groupRequest", value = "Group request", required = true) @RequestBody GroupRequest groupRequest) throws RestException {
+ return new ResponseEntity<>(searchService.group(groupRequest), HttpStatus.OK);
+ }
+
@ApiOperation(value = "Returns latest document for a guid and sensor")
@ApiResponse(message = "Document representing the output", code = 200)
@RequestMapping(value = "/findOne", method = RequestMethod.POST)
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/AlertService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/AlertService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/AlertService.java
new file mode 100644
index 0000000..9668b7c
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/AlertService.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.rest.RestException;
+
+/**
+ * This is a set of operations created to interact with alerts.
+ */
+public interface AlertService {
+
+ void escalateAlerts(List<Map<String, Object>> alerts) throws RestException;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
index 8c3562e..ada5c0e 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
@@ -17,10 +17,10 @@
*/
package org.apache.metron.rest.service;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
-import java.io.File;
import java.util.Map;
public interface GrokService {
@@ -29,8 +29,6 @@ public interface GrokService {
GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException;
- File saveTemporary(String statement, String name) throws RestException;
-
String getStatement(String path) throws RestException;
void saveStatement(String path, byte[] contents) throws RestException;
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
index 846f19b..55f7863 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -17,8 +17,6 @@
*/
package org.apache.metron.rest.service;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
@@ -28,11 +26,13 @@ public interface HdfsService {
String read(Path path) throws RestException;
- void write(Path path, byte[] contents) throws RestException;
+ void write(Path path, byte[] contents,String userMode, String groupMode, String otherMode) throws RestException;
List<String> list(Path path) throws RestException;
boolean delete(Path path, boolean recursive) throws RestException;
+ boolean mkdirs(Path path) throws RestException;
+
boolean ensureDirectory(Path path) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
index bee00f2..da3b226 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
@@ -17,11 +17,10 @@
*/
package org.apache.metron.rest.service;
+import java.util.Set;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.KafkaTopic;
-import java.util.Set;
-
/**
* This is a set of operations created to interact with Kafka.
*/
@@ -67,4 +66,6 @@ public interface KafkaService {
* @return A string representation of the sample message retrieved. If topic doesn't exist null will be returned.
*/
String getSampleMessage(String topic);
+
+ void produceMessage(String topic, String message) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java
index ea0ae81..5899765 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java
@@ -18,7 +18,8 @@
package org.apache.metron.rest.service;
import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.rest.RestException;
import org.apache.metron.indexing.dao.search.SearchRequest;
@@ -27,11 +28,11 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
import java.util.Map;
import java.util.Optional;
import java.util.List;
-import java.util.Map;
public interface SearchService {
SearchResponse search(SearchRequest searchRequest) throws RestException;
+ GroupResponse group(GroupRequest groupRequest) throws RestException;
Optional<Map<String, Object>> getLatest(GetRequest request) throws RestException;
Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException;
Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws RestException;
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertServiceImpl.java
new file mode 100644
index 0000000..46370eb
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertServiceImpl.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.AlertService;
+import org.apache.metron.rest.service.KafkaService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Service;
+
+/**
+ * The default service layer implementation of {@link AlertService}.
+ *
+ * @see AlertService
+ */
+@Service
+public class AlertServiceImpl implements AlertService {
+
+ private Environment environment;
+ private final KafkaService kafkaService;
+
+ @Autowired
+ public AlertServiceImpl(final KafkaService kafkaService,
+ final Environment environment) {
+ this.kafkaService = kafkaService;
+ this.environment = environment;
+ }
+
+ @Override
+ public void escalateAlerts(List<Map<String, Object>> alerts) throws RestException {
+ try {
+ for (Map<String, Object> alert : alerts) {
+ kafkaService.produceMessage(
+ environment.getProperty(MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY),
+ JSONUtils.INSTANCE.toJSON(alert, false));
+ }
+ } catch (JsonProcessingException e) {
+ throw new RestException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/ExtensionServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/ExtensionServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/ExtensionServiceImpl.java
index 3cbed70..971a5a3 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/ExtensionServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/ExtensionServiceImpl.java
@@ -445,7 +445,7 @@ public class ExtensionServiceImpl implements ExtensionService{
BundleProperties props = context.bundleProperties.get();
org.apache.hadoop.fs.Path altPath = new org.apache.hadoop.fs.Path(props.getProperty("bundle.library.directory.alt"));
org.apache.hadoop.fs.Path targetPath = new org.apache.hadoop.fs.Path(altPath, bundlePath.toFile().getName());
- hdfsService.write(targetPath,FileUtils.readFileToByteArray(bundlePath.toFile()));
+ hdfsService.write(targetPath,FileUtils.readFileToByteArray(bundlePath.toFile()),"rwx","r-x","r-x");
}
private void deleteBundleFromHdfs(String bundleName) throws Exception{
@@ -480,7 +480,7 @@ public class ExtensionServiceImpl implements ExtensionService{
paths.add(parserRulePath);
for(Path thisRule : grokRulePaths){
org.apache.hadoop.fs.Path targetPath = new org.apache.hadoop.fs.Path(parserRulePath,thisRule.toFile().getName());
- hdfsService.write(targetPath,FileUtils.readFileToByteArray(thisRule.toFile()));
+ hdfsService.write(targetPath,FileUtils.readFileToByteArray(thisRule.toFile()),"rwx","r-x","r-x");
}
}
context.hdfsPathContext.put(Paths.GROK_HDFS_DIRS,paths);
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index 9d48203..d420955 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -20,6 +20,7 @@ package org.apache.metron.rest.service.impl;
import java.io.InputStream;
import java.nio.file.Paths;
import java.util.HashMap;
+import java.nio.charset.Charset;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import org.apache.commons.io.IOUtils;
@@ -40,9 +41,6 @@ import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Map;
@@ -104,42 +102,13 @@ public class GrokServiceImpl implements GrokService {
}
@Override
- public File saveTemporary(String statement, String name) throws RestException {
- if (statement != null) {
- try {
- File grokDirectory = new File(getTemporaryGrokRootPath());
- if (!grokDirectory.exists()) {
- grokDirectory.mkdirs();
- }
- File path = new File(grokDirectory, name);
- FileWriter fileWriter = new FileWriter(new File(grokDirectory, name));
- fileWriter.write(statement);
- fileWriter.close();
- return path;
- } catch (IOException e) {
- throw new RestException(e);
- }
- } else {
- throw new RestException("A grokStatement must be provided");
- }
- }
-
- @Override
public void saveStatement(String path, byte[] contents) throws RestException {
String root = (String)configurationMap.get("metron.apps.hdfs.dir");
if(!root.endsWith("/") && !path.startsWith("/")) {
root = root + "/";
}
Path rootedPath = new Path(root + path);
- hdfsService.write(rootedPath, contents);
- }
-
- private String getTemporaryGrokRootPath() {
- String javaTmp = System.getProperty("java.io.tmpdir");
- String grokTempPath = Paths
- .get(javaTmp, environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).toString();
- Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
- return new Path(grokTempPath, authentication.getName()).toString();
+ hdfsService.write(rootedPath, contents, null, null, null);
}
public String getStatement(String path) throws RestException {
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
index 82565eb..eb0d705 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -17,10 +17,13 @@
*/
package org.apache.metron.rest.service.impl;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.HdfsService;
@@ -68,22 +71,49 @@ public class HdfsServiceImpl implements HdfsService {
return new String(byteArrayOutputStream.toByteArray(), UTF_8);
}
+ @Override
+ public void write(Path path, byte[] contents, String userMode, String groupMode, String otherMode)
+ throws RestException {
+ FSDataOutputStream fsDataOutputStream;
+ try {
+ FsPermission permission = null;
+ boolean setPermissions = false;
+ if(StringUtils.isNotEmpty(userMode) && StringUtils.isNotEmpty(groupMode) && StringUtils.isNotEmpty(otherMode)) {
+ // invalid actions will return null
+ FsAction userAction = FsAction.getFsAction(userMode);
+ FsAction groupAction = FsAction.getFsAction(groupMode);
+ FsAction otherAction = FsAction.getFsAction(otherMode);
+ if(userAction == null || groupAction == null || otherAction == null){
+ throw new RestException(String.format("Invalid permission set: user[%s] " +
+ "group[%s] other[%s]", userAction, groupAction, otherAction));
+ }
+ permission = new FsPermission(userAction, groupAction, otherAction);
+ setPermissions = true;
+ }
+ fsDataOutputStream = FileSystem.get(configuration).create(path, true);
+ fsDataOutputStream.write(contents);
+ fsDataOutputStream.close();
+ if(setPermissions) {
+ FileSystem.get(configuration).setPermission(path, permission);
+ }
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ }
+
@Override
- public void write(Path path, byte[] contents) throws RestException {
- FSDataOutputStream fsDataOutputStream;
+ public boolean delete(Path path, boolean recursive) throws RestException {
try {
- fsDataOutputStream = FileSystem.get(configuration).create(path, true);
- fsDataOutputStream.write(contents);
- fsDataOutputStream.close();
+ return FileSystem.get(configuration).delete(path, recursive);
} catch (IOException e) {
throw new RestException(e);
}
}
@Override
- public boolean delete(Path path, boolean recursive) throws RestException {
+ public boolean mkdirs(Path path) throws RestException {
try {
- return FileSystem.get(configuration).delete(path, recursive);
+ return FileSystem.get(configuration).mkdirs(path);
} catch (IOException e) {
throw new RestException(e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
index 61e2618..4f232fb 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
@@ -17,6 +17,11 @@
*/
package org.apache.metron.rest.service.impl;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import kafka.admin.AdminOperationException;
import kafka.admin.AdminUtils$;
import kafka.admin.RackAwareMode;
@@ -24,21 +29,18 @@ import kafka.utils.ZkUtils;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.KafkaTopic;
import org.apache.metron.rest.service.KafkaService;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.stereotype.Service;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
/**
* The default service layer implementation of {@link KafkaService}.
*
@@ -53,19 +55,26 @@ public class KafkaServiceImpl implements KafkaService {
private final ZkUtils zkUtils;
private final ConsumerFactory<String, String> kafkaConsumerFactory;
+ private final KafkaProducer<String, String> kafkaProducer;
private final AdminUtils$ adminUtils;
+ @Autowired
+ private Environment environment;
+
/**
* @param zkUtils A utility class used to interact with ZooKeeper.
* @param kafkaConsumerFactory A class used to create {@link KafkaConsumer} in order to interact with Kafka.
+ * @param kafkaProducer A class used to produce messages to Kafka.
* @param adminUtils A utility class used to do administration operations on Kafka.
*/
@Autowired
public KafkaServiceImpl(final ZkUtils zkUtils,
final ConsumerFactory<String, String> kafkaConsumerFactory,
+ final KafkaProducer<String, String> kafkaProducer,
final AdminUtils$ adminUtils) {
this.zkUtils = zkUtils;
this.kafkaConsumerFactory = kafkaConsumerFactory;
+ this.kafkaProducer = kafkaProducer;
this.adminUtils = adminUtils;
}
@@ -141,4 +150,8 @@ public class KafkaServiceImpl implements KafkaService {
return message;
}
+ @Override
+ public void produceMessage(String topic, String message) throws RestException {
+ kafkaProducer.send(new ProducerRecord<>(topic, message));
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
index bdf6037..d865e0e 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
@@ -19,10 +19,11 @@ package org.apache.metron.rest.service.impl;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.SearchService;
@@ -34,7 +35,6 @@ import java.io.IOException;
import java.util.Map;
import java.util.Optional;
import java.util.List;
-import java.util.Map;
@Service
public class SearchServiceImpl implements SearchService {
@@ -58,6 +58,16 @@ public class SearchServiceImpl implements SearchService {
}
@Override
+ public GroupResponse group(GroupRequest groupRequest) throws RestException {
+ try {
+ return dao.group(groupRequest);
+ }
+ catch(InvalidSearchException ise) {
+ throw new RestException(ise.getMessage(), ise);
+ }
+ }
+
+ @Override
public Optional<Map<String, Object>> getLatest(GetRequest request) throws RestException {
try {
return dao.getLatestResult(request);
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 5d6f264..619f2b2 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -17,7 +17,15 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
+
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.io.File;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
@@ -47,15 +55,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
-
@Service
public class SensorParserConfigServiceImpl implements SensorParserConfigService {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -101,7 +100,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
@Override
public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException {
try {
- ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(),
+ objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
} catch (Exception e) {
throw new RestException(e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/resources/application-docker.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml
index 15fa293..426a0da 100644
--- a/metron-interface/metron-rest/src/main/resources/application-docker.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml
@@ -46,7 +46,6 @@ hdfs:
grok:
path:
temp: target/patterns/temp
- default: target/patterns
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
index 47d1750..29ff02d 100644
--- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
@@ -40,8 +40,7 @@ hdfs:
apps.root: /apps/metron
grok:
path:
- temp: ./patterns/temp
- default: /apps/metron/patterns
+ temp: /apps/metron/patterns/tmp
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index 4aff769..3aa5fd9 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -26,8 +26,7 @@ spring:
grok:
path:
- temp: ./
- default: /apps/metron/patterns
+ temp: /apps/metron/patterns/tmp
zookeeper:
client:
@@ -35,6 +34,10 @@ zookeeper:
session: 10000
connection: 10000
+kafka:
+ topics:
+ escalation: escalation
+
curator:
sleep:
time: 1000
@@ -44,8 +47,10 @@ curator:
search:
max:
results: 1000
+ groups: 1000
index:
dao:
# By default, we use the ElasticsearchDao and HBaseDao for backing updates.
- impl: org.apache.metron.elasticsearch.dao.ElasticsearchDao,org.apache.metron.indexing.dao.HBaseDao
\ No newline at end of file
+ impl: org.apache.metron.elasticsearch.dao.ElasticsearchDao,org.apache.metron.indexing.dao.HBaseDao
+
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
index dbf4aa4..7095d11 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
@@ -36,6 +36,7 @@ import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.integration.ComponentRunner;
@@ -142,6 +143,21 @@ public class TestConfig {
}
@Bean
+ public Map<String, Object> producerProperties(KafkaComponent kafkaWithZKComponent) {
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put("bootstrap.servers", kafkaWithZKComponent.getBrokerList());
+ producerConfig.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ producerConfig.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+ producerConfig.put("request.required.acks", 1);
+ return producerConfig;
+ }
+
+ @Bean
+ public KafkaProducer kafkaProducer(KafkaComponent kafkaWithZKComponent) {
+ return new KafkaProducer<>(producerProperties(kafkaWithZKComponent));
+ }
+
+ @Bean
public StormCLIWrapper stormCLIClientWrapper() {
return new MockStormCLIClientWrapper();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/AlertControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/AlertControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/AlertControllerIntegrationTest.java
new file mode 100644
index 0000000..be320fc
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/AlertControllerIntegrationTest.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class AlertControllerIntegrationTest {
+
+ /**
+ [
+ {
+ "is_alert": true,
+ "field": "value1"
+ },
+ {
+ "is_alert": true,
+ "field": "value2"
+ }
+ ]
+ */
+ @Multiline
+ public static String alerts;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String alertUrl = "/api/v1/alert";
+ private String user = "user";
+ private String password = "password";
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ }
+
+ @Test
+ public void testSecurity() throws Exception {
+ this.mockMvc.perform(post(alertUrl + "/escalate").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(alerts))
+ .andExpect(status().isUnauthorized());
+ }
+
+ @Test
+ public void test() throws Exception {
+ this.mockMvc.perform(post(alertUrl + "/escalate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(alerts))
+ .andExpect(status().isOk());
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java
index 5c00cd2..5719a41 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java
@@ -17,15 +17,24 @@
*/
package org.apache.metron.rest.controller;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
import kafka.common.TopicAlreadyMarkedForDeletionException;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.UnableToStartException;
import org.apache.metron.integration.components.KafkaComponent;
-import org.apache.metron.rest.generator.SampleDataGenerator;
import org.apache.metron.rest.service.KafkaService;
import org.hamcrest.Matchers;
-import org.json.simple.parser.ParseException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -41,19 +50,6 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
import org.springframework.web.util.NestedServletException;
-import java.io.IOException;
-
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles(TEST_PROFILE)
@@ -99,35 +95,6 @@ public class KafkaControllerIntegrationTest {
}
}
- class SampleDataRunner implements Runnable {
-
- private boolean stop = false;
- private String path = "../../metron-platform/metron-extensions/metron-parser-extensions/metron-parser-bro-extension/metron-parser-bro/src/test/resources/data/raw/test.raw";
-
- @Override
- public void run() {
- SampleDataGenerator broSampleDataGenerator = new SampleDataGenerator();
- broSampleDataGenerator.setBrokerUrl(kafkaWithZKComponent.getBrokerList());
- broSampleDataGenerator.setNum(1);
- broSampleDataGenerator.setSelectedSensorType("bro");
- broSampleDataGenerator.setDelay(0);
- try {
- while(!stop) {
- broSampleDataGenerator.generateSampleData(path);
- }
- } catch (ParseException|IOException e) {
- throw new IllegalStateException("Caught an error generating sample data", e);
- }
- }
-
- public void stop() {
- stop = true;
- }
- }
-
- private SampleDataRunner sampleDataRunner = new SampleDataRunner();
- private Thread sampleDataThread = new Thread(sampleDataRunner);
-
/**
{
"name": "bro",
@@ -139,6 +106,30 @@ public class KafkaControllerIntegrationTest {
@Multiline
public static String broTopic;
+ /**
+ * {
+ * "type":"message1"
+ * }
+ */
+ @Multiline
+ public static String message1;
+
+ /**
+ * {
+ * "type":"message2"
+ * }
+ */
+ @Multiline
+ public static String message2;
+
+ /**
+ * {
+ * "type":"message3"
+ * }
+ */
+ @Multiline
+ public static String message3;
+
@Autowired
private WebApplicationContext wac;
@@ -179,6 +170,9 @@ public class KafkaControllerIntegrationTest {
this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample"))
.andExpect(status().isUnauthorized());
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro/produce"))
+ .andExpect(status().isUnauthorized());
+
this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(csrf()))
.andExpect(status().isUnauthorized());
}
@@ -200,7 +194,7 @@ public class KafkaControllerIntegrationTest {
.andExpect(jsonPath("$.numPartitions").value(1))
.andExpect(jsonPath("$.replicationFactor").value(1))
);
- sampleDataThread.start();
+
Thread.sleep(1000);
testAndRetry(() ->
this.mockMvc.perform(get(kafkaUrl + "/topic/bro").with(httpBasic(user,password)))
@@ -222,24 +216,42 @@ public class KafkaControllerIntegrationTest {
.andExpect(jsonPath("$", Matchers.hasItem("bro")))
);
- for(int i = 0;i < KAFKA_RETRY;++i) {
- MvcResult result = this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user, password)))
- .andReturn();
- if(result.getResponse().getStatus() == 200) {
- break;
- }
- Thread.sleep(1000);
- }
+ testAndRetry(() ->
+ this.mockMvc.perform(post(kafkaUrl + "/topic/bro/produce").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(message1))
+ .andExpect(status().isOk())
+ );
+ testAndRetry(() ->
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
+ .andExpect(jsonPath("$.type").value("message1"))
+ );
testAndRetry(() ->
- this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user,password)))
+ this.mockMvc.perform(post(kafkaUrl + "/topic/bro/produce").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(message2))
+ .andExpect(status().isOk())
+ );
+ testAndRetry(() ->
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
- .andExpect(jsonPath("$").isNotEmpty())
+ .andExpect(jsonPath("$.type").value("message2"))
+ );
+
+ testAndRetry(() ->
+ this.mockMvc.perform(post(kafkaUrl + "/topic/bro/produce").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(message3))
+ .andExpect(status().isOk())
+ );
+ testAndRetry(() ->
+ this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user,password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")))
+ .andExpect(jsonPath("$.type").value("message3"))
);
this.mockMvc.perform(get(kafkaUrl + "/topic/someTopic/sample").with(httpBasic(user,password)))
.andExpect(status().isNotFound());
+
boolean deleted = false;
for(int i = 0;i < KAFKA_RETRY;++i) {
try {
@@ -270,7 +282,6 @@ public class KafkaControllerIntegrationTest {
@After
public void tearDown() {
- sampleDataRunner.stop();
runner.stop();
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
index e75c356..645e525 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
@@ -24,7 +24,6 @@ import org.apache.metron.rest.service.SearchService;
import org.json.simple.parser.ParseException;
import org.junit.After;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@@ -44,7 +43,6 @@ import static org.hamcrest.Matchers.hasSize;
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
@@ -174,6 +172,22 @@ public class SearchControllerIntegrationTest extends DaoControllerTest {
.andExpect(jsonPath("$.responseCode").value(500))
.andExpect(jsonPath("$.message").value("Search result size must be less than 100"));
+ this.mockMvc.perform(post(searchUrl + "/group").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.groupByQuery))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.*", hasSize(2)))
+ .andExpect(jsonPath("$.groupedBy").value("is_alert"))
+ .andExpect(jsonPath("$.groupResults.*", hasSize(1)))
+ .andExpect(jsonPath("$.groupResults[0].*", hasSize(5)))
+ .andExpect(jsonPath("$.groupResults[0].key").value("is_alert_value"))
+ .andExpect(jsonPath("$.groupResults[0].total").value(10))
+ .andExpect(jsonPath("$.groupResults[0].groupedBy").value("latitude"))
+ .andExpect(jsonPath("$.groupResults[0].groupResults.*", hasSize(1)))
+ .andExpect(jsonPath("$.groupResults[0].groupResults[0].*", hasSize(3)))
+ .andExpect(jsonPath("$.groupResults[0].groupResults[0].key").value("latitude_value"))
+ .andExpect(jsonPath("$.groupResults[0].groupResults[0].total").value(10))
+ .andExpect(jsonPath("$.groupResults[0].groupResults[0].score").value(50));
+
this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\",\"snort\"]"))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
index ffe6b9a..5cb0ba5 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
@@ -21,6 +21,7 @@ import java.io.FileInputStream;
import java.util.HashMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
+import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.bundles.BundleSystem;
import org.apache.metron.bundles.util.BundleProperties;
import org.apache.metron.rest.MetronRestConstants;
@@ -41,6 +42,7 @@ import org.springframework.web.context.WebApplicationContext;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.Method;
import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
import static org.hamcrest.Matchers.hasSize;
@@ -212,11 +214,17 @@ public class SensorParserConfigControllerIntegrationTest {
cleanFileSystem();
this.sensorParserConfigService.delete("jsonTest");
this.sensorParserConfigService.delete("squidTest");
-
+ Method[] method = SensorParserConfig.class.getMethods();
+ int numFields = 0;
+ for(Method m : method) {
+ if(m.getName().startsWith("set")) {
+ numFields++;
+ }
+ }
this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson))
.andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.*", hasSize(10)))
+ .andExpect(jsonPath("$.*", hasSize(numFields)))
.andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.grok.GrokParser"))
.andExpect(jsonPath("$.sensorTopic").value("squidTest"))
.andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
@@ -231,7 +239,7 @@ public class SensorParserConfigControllerIntegrationTest {
this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.*", hasSize(10)))
+ .andExpect(jsonPath("$.*", hasSize(numFields)))
.andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.grok.GrokParser"))
.andExpect(jsonPath("$.sensorTopic").value("squidTest"))
.andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest"))
@@ -260,7 +268,7 @@ public class SensorParserConfigControllerIntegrationTest {
this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(jsonMapJson))
.andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.*", hasSize(10)))
+ .andExpect(jsonPath("$.*", hasSize(numFields)))
.andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.json.JSONMapParser"))
.andExpect(jsonPath("$.sensorTopic").value("jsonTest"))
.andExpect(jsonPath("$.readMetadata").value("true"))
@@ -270,7 +278,7 @@ public class SensorParserConfigControllerIntegrationTest {
this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(jsonMapJson))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.*", hasSize(10)))
+ .andExpect(jsonPath("$.*", hasSize(numFields)))
.andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.json.JSONMapParser"))
.andExpect(jsonPath("$.sensorTopic").value("jsonTest"))
.andExpect(jsonPath("$.readMetadata").value("true"))
@@ -373,11 +381,6 @@ public class SensorParserConfigControllerIntegrationTest {
FileUtils.cleanDirectory(grokTempPath);
FileUtils.deleteDirectory(grokTempPath);
}
- File grokPath = new File(environment.getProperty(MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY));
- if (grokPath.exists()) {
- FileUtils.cleanDirectory(grokPath);
- FileUtils.deleteDirectory(grokPath);
- }
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertServiceImplTest.java
new file mode 100644
index 0000000..c55e0a5
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/AlertServiceImplTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.mock;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.service.AlertService;
+import org.apache.metron.rest.service.KafkaService;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.core.env.Environment;
+
+@SuppressWarnings("unchecked")
+public class AlertServiceImplTest {
+
+ private KafkaService kafkaService;
+ private Environment environment;
+ private AlertService alertService;
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void setUp() throws Exception {
+ kafkaService = mock(KafkaService.class);
+ environment = mock(Environment.class);
+ alertService = new AlertServiceImpl(kafkaService, environment);
+ }
+
+ @Test
+ public void produceMessageShouldProperlyProduceMessage() throws Exception {
+ String escalationTopic = "escalation";
+ final Map<String, Object> message1 = new HashMap<>();
+ message1.put("field", "value1");
+ final Map<String, Object> message2 = new HashMap<>();
+ message2.put("field", "value2");
+ List<Map<String, Object>> messages = Arrays.asList(message1, message2);
+ when(environment.getProperty(MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY)).thenReturn(escalationTopic);
+
+ alertService.escalateAlerts(messages);
+
+ String expectedMessage1 = "{\"field\":\"value1\"}";
+ String expectedMessage2 = "{\"field\":\"value2\"}";
+ verify(kafkaService).produceMessage("escalation", expectedMessage1);
+ verify(kafkaService).produceMessage("escalation", expectedMessage2);
+ verifyZeroInteractions(kafkaService);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
index e6fdbd0..603a0ca 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -17,12 +17,11 @@
*/
package org.apache.metron.rest.service.impl;
-import javax.security.auth.Subject;
+import java.nio.charset.Charset;
import oi.thekraken.grok.api.Grok;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
@@ -32,7 +31,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.core.env.Environment;
@@ -200,43 +198,4 @@ public class GrokServiceImplTest {
grokService.validateGrokStatement(grokValidation);
}
-
- @Test
- public void saveTemporaryShouldProperlySaveFile() throws Exception {
- new File("./target/user1").delete();
- String statement = "grok statement";
-
- Authentication authentication = mock(Authentication.class);
- when(authentication.getName()).thenReturn("user1");
- SecurityContextHolder.getContext().setAuthentication(authentication);
- when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
-
- File testFile = grokService.saveTemporary(statement, "squid");
-
- assertEquals(statement, FileUtils.readFileToString(testFile));
- testFile.delete();
- }
-
- @Test
- public void saveTemporaryShouldWrapExceptionInRestException() throws Exception {
- exception.expect(RestException.class);
-
- String statement = "grok statement";
-
- Authentication authentication = mock(Authentication.class);
- when(authentication.getName()).thenReturn("user1");
- SecurityContextHolder.getContext().setAuthentication(authentication);
- when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
- whenNew(FileWriter.class).withParameterTypes(File.class).withArguments(any()).thenThrow(new IOException());
-
- grokService.saveTemporary(statement, "squid");
- }
-
- @Test
- public void missingGrokStatementShouldThrowRestException() throws Exception {
- exception.expect(RestException.class);
- exception.expectMessage("A grokStatement must be provided");
-
- grokService.saveTemporary(null, "squid");
- }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
index d11f5a4..7282859 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
@@ -85,7 +85,17 @@ public class HdfsServiceImplExceptionTest {
when(FileSystem.get(configuration)).thenReturn(fileSystem);
when(fileSystem.create(new Path(testDir), true)).thenThrow(new IOException());
- hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8));
+ hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8),null, null,null);
+ }
+
+ @Test
+ public void writeShouldThrowIfInvalidPermissions() throws Exception {
+ exception.expect(RestException.class);
+
+ FileSystem fileSystem = mock(FileSystem.class);
+ when(FileSystem.get(configuration)).thenReturn(fileSystem);
+
+ hdfsService.write(new Path(testDir,"test"),"oops".getBytes(UTF_8), "foo", "r-x","r--");
}
@Test
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
index 005a427..280be63 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
@@ -19,7 +19,10 @@ package org.apache.metron.rest.service.impl;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.metron.rest.service.HdfsService;
import org.junit.After;
import org.junit.Before;
@@ -84,12 +87,23 @@ public class HdfsServiceImplTest {
@Test
public void writeShouldProperlyWriteContents() throws Exception {
String contents = "contents";
- hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8));
+ hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8),null,null,null);
assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest.txt")));
}
@Test
+ public void writeShouldProperlyWriteContentsWithPermissions() throws Exception {
+ String contents = "contents";
+ Path thePath = new Path(testDir,"writeTest2.txt");
+ hdfsService.write(thePath, contents.getBytes(UTF_8),"rw-","r-x","r--");
+
+ assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest2.txt")));
+ assertEquals(FileSystem.get(configuration).listStatus(thePath)[0].getPermission().toShort(),
+ new FsPermission(FsAction.READ_WRITE,FsAction.READ_EXECUTE,FsAction.READ).toShort());
+ }
+
+ @Test
public void deleteShouldProperlyDeleteFile() throws Exception {
String contents = "contents";
FileUtils.writeStringToFile(new File(testDir, "deleteTest.txt"), contents);
[6/6] metron git commit: METRON-1136 Track Master in Feature Branch
(ottobackwards) closes apache/metron#752
Posted by ot...@apache.org.
METRON-1136 Track Master in Feature Branch (ottobackwards) closes apache/metron#752
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/24b668b0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/24b668b0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/24b668b0
Branch: refs/heads/feature/METRON-1136-extensions-parsers
Commit: 24b668b0b4041c7fe251a27bc9c322cc4b201d44
Parents: 1c63c1e
Author: ottobackwards <ot...@gmail.com>
Authored: Tue Sep 12 21:47:20 2017 -0400
Committer: otto <ot...@apache.org>
Committed: Tue Sep 12 21:47:20 2017 -0400
----------------------------------------------------------------------
.travis.yml | 3 +-
dependencies_with_url.csv | 2 +
metron-analytics/metron-maas-service/pom.xml | 10 +
metron-analytics/metron-profiler-client/pom.xml | 12 +-
.../client/stellar/ProfilerFunctions.java | 113 +++--
.../client/stellar/ProfilerFunctionsTest.java | 53 ++-
metron-analytics/metron-profiler/pom.xml | 10 +
metron-analytics/metron-statistics/pom.xml | 10 +
.../CURRENT/configuration/metron-rest-env.xml | 10 +-
.../package/scripts/management_ui_master.py | 1 -
.../package/scripts/params/params_linux.py | 6 +-
.../package/scripts/profiler_commands.py | 4 +
.../CURRENT/package/scripts/profiler_master.py | 16 +
.../CURRENT/package/scripts/rest_commands.py | 7 +-
.../CURRENT/package/scripts/rest_master.py | 1 +
.../METRON/CURRENT/package/templates/metron.j2 | 1 +
.../METRON/CURRENT/themes/metron_theme.json | 10 +
.../packaging/docker/rpm-docker/Dockerfile | 4 +
.../docker/rpm-docker/SPECS/metron.spec | 13 +-
.../packaging/docker/rpm-docker/build.sh | 2 +-
.../packaging/docker/rpm-docker/pom.xml | 6 +
.../roles/ambari_config/vars/small_cluster.yml | 1 +
.../metron-alerts/alerts-server-e2e.js | 20 +-
.../metron-alerts/scripts/alerts-server.js | 14 +-
.../alert-details/alert-details.component.ts | 16 +-
.../alerts/alerts-list/alerts-list.component.ts | 26 +-
.../alerts/alerts-list/alerts-list.module.ts | 4 +-
.../configure-table.component.ts | 10 +-
.../metron-alerts/src/app/app.module.ts | 4 +-
.../src/app/model/alerts-search-response.ts | 23 -
.../src/app/model/search-request.ts | 3 +-
.../src/app/model/search-response.ts | 23 +
.../src/app/service/alert.service.ts | 72 ----
.../src/app/service/cluster-metadata.service.ts | 7 +-
.../src/app/service/data-source.ts | 4 +-
.../service/elasticsearch-localstorage-impl.ts | 4 +-
.../src/app/service/rest-api-impl.ts | 46 --
.../src/app/service/search.service.ts | 112 +++++
.../metron-alerts/src/app/utils/constants.ts | 2 +
.../src/app/utils/elasticsearch-utils.ts | 6 +-
metron-interface/metron-config/README.md | 10 +-
metron-interface/metron-rest/README.md | 45 +-
.../src/main/config/rest_application.yml | 2 +
.../apache/metron/rest/MetronRestConstants.java | 2 +
.../apache/metron/rest/config/IndexConfig.java | 12 +-
.../apache/metron/rest/config/KafkaConfig.java | 16 +
.../rest/controller/AlertsController.java | 55 +++
.../metron/rest/controller/HdfsController.java | 11 +-
.../metron/rest/controller/KafkaController.java | 14 +-
.../rest/controller/SearchController.java | 13 +-
.../metron/rest/service/AlertService.java | 30 ++
.../apache/metron/rest/service/GrokService.java | 4 +-
.../apache/metron/rest/service/HdfsService.java | 6 +-
.../metron/rest/service/KafkaService.java | 5 +-
.../metron/rest/service/SearchService.java | 5 +-
.../rest/service/impl/AlertServiceImpl.java | 62 +++
.../rest/service/impl/ExtensionServiceImpl.java | 4 +-
.../rest/service/impl/GrokServiceImpl.java | 35 +-
.../rest/service/impl/HdfsServiceImpl.java | 44 +-
.../rest/service/impl/KafkaServiceImpl.java | 25 +-
.../rest/service/impl/SearchServiceImpl.java | 14 +-
.../impl/SensorParserConfigServiceImpl.java | 20 +-
.../src/main/resources/application-docker.yml | 1 -
.../src/main/resources/application-vagrant.yml | 3 +-
.../src/main/resources/application.yml | 11 +-
.../apache/metron/rest/config/TestConfig.java | 16 +
.../AlertControllerIntegrationTest.java | 86 ++++
.../KafkaControllerIntegrationTest.java | 123 +++---
.../SearchControllerIntegrationTest.java | 18 +-
...orParserConfigControllerIntegrationTest.java | 23 +-
.../rest/service/impl/AlertServiceImplTest.java | 69 +++
.../rest/service/impl/GrokServiceImplTest.java | 43 +-
.../impl/HdfsServiceImplExceptionTest.java | 12 +-
.../rest/service/impl/HdfsServiceImplTest.java | 16 +-
.../rest/service/impl/KafkaServiceImplTest.java | 55 ++-
.../impl/SensorParserConfigServiceImplTest.java | 9 +-
metron-platform/elasticsearch-shaded/pom.xml | 10 +
metron-platform/metron-api/pom.xml | 10 +
metron-platform/metron-common/pom.xml | 10 +
.../configuration/SensorParserConfig.java | 208 ++++++++-
metron-platform/metron-data-management/pom.xml | 12 +-
metron-platform/metron-elasticsearch/pom.xml | 10 +
.../elasticsearch/dao/ElasticsearchDao.java | 252 +++++++----
.../ElasticsearchSearchIntegrationTest.java | 5 +-
metron-platform/metron-enrichment/pom.xml | 27 ++
.../adapters/geo/GeoLiteDatabase.java | 77 +++-
.../adapters/geo/hash/DistanceStrategies.java | 46 ++
.../adapters/geo/hash/DistanceStrategy.java | 24 ++
.../adapters/geo/hash/GeoHashUtil.java | 189 +++++++++
.../enrichment/stellar/GeoHashFunctions.java | 299 +++++++++++++
.../stellar/GeoHashFunctionsTest.java | 337 +++++++++++++++
metron-platform/metron-hbase-client/pom.xml | 10 +
metron-platform/metron-indexing/pom.xml | 10 +
.../metron/indexing/dao/AccessConfig.java | 14 +-
.../apache/metron/indexing/dao/HBaseDao.java | 7 +
.../apache/metron/indexing/dao/IndexDao.java | 5 +-
.../metron/indexing/dao/MultiIndexDao.java | 13 +
.../metron/indexing/dao/search/Group.java | 43 ++
.../metron/indexing/dao/search/GroupOrder.java | 37 ++
.../indexing/dao/search/GroupOrderType.java | 39 ++
.../indexing/dao/search/GroupRequest.java | 58 +++
.../indexing/dao/search/GroupResponse.java | 39 ++
.../metron/indexing/dao/search/GroupResult.java | 73 ++++
.../apache/metron/indexing/dao/InMemoryDao.java | 22 +-
.../indexing/dao/SearchIntegrationTest.java | 247 ++++++++++-
metron-platform/metron-management/pom.xml | 10 +
.../management/ConfigurationFunctionsTest.java | 10 +-
metron-platform/metron-parsers/README.md | 11 +
metron-platform/metron-parsers/pom.xml | 10 +
.../parsers/topology/ParserTopologyBuilder.java | 87 ++--
.../parsers/topology/ParserTopologyCLI.java | 218 +++++++---
.../parsers/topology/config/ValueSupplier.java | 30 ++
.../components/ParserTopologyComponent.java | 30 +-
.../parsers/topology/ParserTopologyCLITest.java | 425 ++++++++++++++++++-
metron-platform/metron-pcap-backend/pom.xml | 10 +
metron-platform/metron-solr/pom.xml | 10 +
metron-platform/metron-writer/pom.xml | 10 +
.../metron/writer/hdfs/SourceHandler.java | 21 +-
.../metron/writer/hdfs/HdfsWriterTest.java | 91 +++-
metron-stellar/stellar-common/README.md | 262 ++++++++++--
metron-stellar/stellar-common/pom.xml | 10 +
.../metron/stellar/common/StellarCompiler.java | 8 +-
.../stellar/common/shell/StellarExecutor.java | 33 +-
.../stellar/common/shell/StellarShell.java | 146 ++++++-
.../dsl/functions/DataStructureFunctions.java | 5 +
.../stellar/dsl/functions/SetFunctions.java | 287 +++++++++++++
.../stellar/dsl/functions/StringFunctions.java | 167 ++++++++
.../stellar/dsl/functions/BasicStellarTest.java | 12 +
.../stellar/dsl/functions/SetFunctionsTest.java | 326 ++++++++++++++
.../dsl/functions/StringFunctionsTest.java | 296 +++++++++++++
pom.xml | 6 +-
use-cases/README.md | 4 +
use-cases/geographic_login_outliers/README.md | 267 ++++++++++++
133 files changed, 5740 insertions(+), 805 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index d85a7ca..8023830 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,14 +31,13 @@ before_install:
- export PATH=$M2_HOME/bin:$PATH
- npm config set cache $HOME/.npm-cache --global
- npm config set prefix $HOME/.npm-prefix --global
- - time build_utils/verify_licenses.sh
install:
- cd bundles-maven-plugin && mvn -q install && cd ..
- time mvn -q -T 2C -DskipTests clean install
script:
- - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config
+ - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh
before_cache:
- rm -rf $HOME/.m2/repository/org/apache/metron
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index d09fed6..9ebb9e4 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -307,3 +307,5 @@ net.byteseek:byteseek:jar:2.0.3:compile,BSD,https://github.com/nishihatapalmer/b
org.springframework.security.kerberos:spring-security-kerberos-client:jar:1.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security-kerberos
org.springframework.security.kerberos:spring-security-kerberos-core:jar:1.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security-kerberos
org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka
+ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java
+org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-maas-service/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index 4eeceae..555e73d 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -252,6 +252,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
index bba881d..69b8c29 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -304,7 +304,17 @@
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.metron.guava</shadedPattern>
</relocation>
- </relocations>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
index 8df5ca8..d6f2c6a 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerFunctions.java
@@ -20,6 +20,7 @@
package org.apache.metron.profiler.client.stellar;
+import org.apache.commons.collections4.ListUtils;
import org.apache.metron.common.configuration.profiler.ProfilerConfig;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.profiler.ProfileMeasurement;
@@ -41,6 +42,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -109,7 +111,7 @@ public class ProfilerFunctions {
name="APPLY",
description="Apply a message to a local profile runner.",
params={
- "message(s)", "The message to apply. A JSON list can be used to apply multiple messages.",
+ "message(s)", "The message to apply; a JSON string or list of JSON strings.",
"profiler", "A local profile runner returned by PROFILER_INIT."
},
returns="The local profile runner."
@@ -131,50 +133,99 @@ public class ProfilerFunctions {
@Override
public Object apply(List<Object> args, Context context) throws ParseException {
- // user must provide the message as a string
- String arg0 = Util.getArg(0, String.class, args);
- if(arg0 == null) {
- throw new IllegalArgumentException(format("expected string, found null"));
+ // the use can pass in one or more messages in a few different forms
+ Object arg0 = Util.getArg(0, Object.class, args);
+ List<JSONObject> messages = getMessages(arg0);
+
+ // user must provide the stand alone profiler
+ StandAloneProfiler profiler = Util.getArg(1, StandAloneProfiler.class, args);
+ try {
+ for (JSONObject message : messages) {
+ profiler.apply(message);
+ }
+
+ } catch (ExecutionException e) {
+ throw new IllegalArgumentException(format("Failed to apply message; error=%s", e.getMessage()), e);
+ }
+
+ return profiler;
+ }
+
+ /**
+ * Gets a message or messages from the function arguments.
+ *
+ * @param arg The function argument containing the message(s).
+ * @return A list of messages
+ */
+ private List<JSONObject> getMessages(Object arg) {
+ List<JSONObject> messages;
+
+ if (arg instanceof String) {
+ messages = getMessagesFromString((String) arg);
+
+ } else if (arg instanceof Iterable) {
+ messages = getMessagesFromIterable((Iterable<String>) arg);
+
+ } else if (arg instanceof JSONObject) {
+ messages = Collections.singletonList((JSONObject) arg);
+
+ } else {
+ throw new IllegalArgumentException(format("invalid message: found '%s', expected String, List, or JSONObject",
+ ClassUtils.getShortClassName(arg, "null")));
}
- // there could be one or more messages
+ return messages;
+ }
+
+ /**
+ * Gets a message or messages from a List
+ *
+ * @param strings The function argument that is a bunch of strings.
+ * @return A list of messages.
+ */
+ private List<JSONObject> getMessagesFromIterable(Iterable<String> strings) {
List<JSONObject> messages = new ArrayList<>();
+
+ // the user pass in a list of strings
+ for (String str : strings) {
+ messages.addAll(getMessagesFromString(str));
+ }
+
+ return messages;
+ }
+
+ /**
+ * Gets a message or messages from a String argument.
+ *
+ * @param arg0 The function argument is just a List.
+ * @return A list of messages.
+ */
+ private List<JSONObject> getMessagesFromString(String arg0) {
+ List<JSONObject> messages = new ArrayList<>();
+
try {
Object parsedArg0 = parser.parse(arg0);
- if(parsedArg0 instanceof JSONObject) {
- // if there is only one message
+ if (parsedArg0 instanceof JSONObject) {
+ // if the string only contains one message
messages.add((JSONObject) parsedArg0);
- } else if(parsedArg0 instanceof JSONArray) {
- // there are multiple messages
+ } else if (parsedArg0 instanceof JSONArray) {
+ // if the string contains multiple messages
JSONArray jsonArray = (JSONArray) parsedArg0;
- for(Object json: jsonArray) {
- if(json instanceof JSONObject) {
- messages.add((JSONObject) json);
-
- } else {
- throw new IllegalArgumentException(format("invalid message: found '%s', expected JSONObject",
- ClassUtils.getShortClassName(json, "null")));
- }
+ for (Object item : jsonArray) {
+ messages.addAll(getMessages(item));
}
- }
- } catch(org.json.simple.parser.ParseException e) {
- throw new IllegalArgumentException("invalid message", e);
- }
-
- // user must provide the stand alone profiler
- StandAloneProfiler profiler = Util.getArg(1, StandAloneProfiler.class, args);
- try {
- for(JSONObject message : messages) {
- profiler.apply(message);
+ } else {
+ throw new IllegalArgumentException(format("invalid message: found '%s', expected JSONObject or JSONArray",
+ ClassUtils.getShortClassName(parsedArg0, "null")));
}
- } catch(ExecutionException e) {
- throw new IllegalArgumentException(format("Failed to apply message; error=%s", e.getMessage()), e);
+ } catch (org.json.simple.parser.ParseException e) {
+ throw new IllegalArgumentException(format("invalid message: '%s'", e.getMessage()), e);
}
- return profiler;
+ return messages;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
index bad3efe..e1c6aa8 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/ProfilerFunctionsTest.java
@@ -21,13 +21,13 @@
package org.apache.metron.profiler.client.stellar;
import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.profiler.ProfileMeasurement;
import org.apache.metron.profiler.StandAloneProfiler;
import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
import org.apache.metron.stellar.common.StellarStatefulExecutor;
-import org.apache.metron.stellar.common.shell.StellarExecutor;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
import org.junit.Before;
import org.junit.Test;
@@ -176,7 +176,7 @@ public class ProfilerFunctionsTest {
}
@Test
- public void testProfilerApply() {
+ public void testProfilerApplyWithString() {
// initialize the profiler
state.put("config", helloWorldProfilerDef);
@@ -195,7 +195,28 @@ public class ProfilerFunctionsTest {
}
@Test
- public void testProfilerApplyWithMultipleMessages() {
+ public void testProfilerApplyWithJSONObject() throws Exception {
+
+ // initialize the profiler
+ state.put("config", helloWorldProfilerDef);
+ StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class);
+ state.put("profiler", profiler);
+
+ // apply a message to the profiler
+ JSONParser parser = new JSONParser();
+ JSONObject jsonObject = (JSONObject) parser.parse(message);
+ state.put("jsonObj", jsonObject);
+ StandAloneProfiler result = run("PROFILER_APPLY(jsonObj, profiler)", StandAloneProfiler.class);
+
+ // validate
+ assertSame(profiler, result);
+ assertEquals(1, profiler.getProfileCount());
+ assertEquals(1, profiler.getMessageCount());
+ assertEquals(1, profiler.getRouteCount());
+ }
+
+ @Test
+ public void testProfilerApplyWithMultipleMessagesInJSONString() {
// initialize the profiler
state.put("config", helloWorldProfilerDef);
@@ -214,6 +235,26 @@ public class ProfilerFunctionsTest {
}
@Test
+ public void testProfilerApplyWithListOfMessages() {
+
+ // initialize the profiler
+ state.put("config", helloWorldProfilerDef);
+ StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class);
+ state.put("profiler", profiler);
+
+ // apply a message to the profiler
+ state.put("msg", message);
+ StandAloneProfiler result = run("PROFILER_APPLY([msg, msg, msg], profiler)", StandAloneProfiler.class);
+
+ // validate
+ assertSame(profiler, result);
+ assertEquals(1, profiler.getProfileCount());
+ assertEquals(3, profiler.getMessageCount());
+ assertEquals(3, profiler.getRouteCount());
+ }
+
+
+ @Test
public void testProfilerApplyWithEmptyList() {
// initialize the profiler
@@ -250,8 +291,8 @@ public class ProfilerFunctionsTest {
StandAloneProfiler profiler = run("PROFILER_INIT(config)", StandAloneProfiler.class);
state.put("profiler", profiler);
- // there is no 'messages' variable
- StandAloneProfiler result = run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class);
+ // there is no 'messages' variable - should throw exception
+ run("PROFILER_APPLY(messages, profiler)", StandAloneProfiler.class);
}
@Test
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-profiler/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml
index 41888a1..e1ee806 100644
--- a/metron-analytics/metron-profiler/pom.xml
+++ b/metron-analytics/metron-profiler/pom.xml
@@ -305,6 +305,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-analytics/metron-statistics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml
index 5fab63e..b4d2ed6 100644
--- a/metron-analytics/metron-statistics/pom.xml
+++ b/metron-analytics/metron-statistics/pom.xml
@@ -74,6 +74,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.tdunning</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index c68f5b2..c25208d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -90,8 +90,8 @@
</property>
<property>
<name>metron_temp_grok_path</name>
- <description>Temporary local file path where grok patterns are written during testing</description>
- <value>./patterns/temp</value>
+ <description>Temporary HDFS file path where grok patterns are written during testing</description>
+ <value>{{metron_apps_hdfs_dir}}/patterns/tmp</value>
<display-name>Metron temp grok path</display-name>
</property>
<property>
@@ -109,4 +109,10 @@
<empty-value-valid>true</empty-value-valid>
</value-attributes>
</property>
+ <property>
+ <name>metron_escalation_topic</name>
+ <description>Escalated alerts will be produced to this topic</description>
+ <value>escalation</value>
+ <display-name>Metron escalation topic</display-name>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py
index e858ed3..54e91aa 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/management_ui_master.py
@@ -38,7 +38,6 @@ class ManagementUIMaster(Script):
from params import params
env.set_params(params)
self.install_packages(env)
- Execute('npm --prefix ' + params.metron_home + '/web/expressjs/ install')
def configure(self, env, upgrade_type=None, config_dir=None):
print 'configure managment_ui'
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index abbddc5..2796674 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -63,9 +63,9 @@ metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_
metron_jdbc_password = config['configurations']['metron-rest-env']['metron_jdbc_password']
metron_jdbc_platform = config['configurations']['metron-rest-env']['metron_jdbc_platform']
metron_jdbc_client_path = config['configurations']['metron-rest-env']['metron_jdbc_client_path']
-metron_temp_grok_path = config['configurations']['metron-rest-env']['metron_temp_grok_path']
metron_default_grok_path = config['configurations']['metron-rest-env']['metron_default_grok_path']
metron_spring_options = config['configurations']['metron-rest-env']['metron_spring_options']
+metron_escalation_topic = config['configurations']['metron-rest-env']['metron_escalation_topic']
metron_config_path = metron_home + '/config'
metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
@@ -139,6 +139,10 @@ if has_kafka_host:
metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
+# well enough until we find a better way via Ambari
+metron_temp_grok_path = format(format(config['configurations']['metron-rest-env']['metron_temp_grok_path']))
+
metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
local_grok_patterns_dir = format("{metron_home}/patterns")
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
index ddd66cb..4dfa371 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
@@ -98,6 +98,10 @@ class ProfilerCommands:
Logger.info("Done creating HBase Tables")
self.set_hbase_configured()
+ def init_kafka_acls(self):
+ Logger.info('Creating Kafka ACls for profiler')
+ metron_service.init_kafka_acls(self.__params, [self.__profiler_topic], ['profiler'])
+
def set_hbase_acls(self):
Logger.info("Setting HBase ACLs")
if self.__params.security_enabled:
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
index 4946ab0..77c32f0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py
@@ -27,6 +27,7 @@ from resource_management.libraries.script import Script
from metron_security import storm_security_setup
import metron_service
+import metron_security
from profiler_commands import ProfilerCommands
@@ -59,6 +60,9 @@ class Profiler(Script):
commands.create_hbase_tables()
if params.security_enabled and not commands.is_hbase_acl_configured():
commands.set_hbase_acls()
+ if params.security_enabled and not commands.is_acl_configured():
+ commands.init_kafka_acls()
+ commands.set_acl_configured()
Logger.info("Calling security setup")
storm_security_setup(params)
@@ -68,6 +72,18 @@ class Profiler(Script):
env.set_params(params)
self.configure(env)
commands = ProfilerCommands(params)
+ if params.security_enabled:
+ metron_security.kinit(params.kinit_path_local,
+ params.metron_keytab_path,
+ params.metron_principal_name,
+ execute_user=params.metron_user)
+
+ if params.security_enabled and not commands.is_hbase_acl_configured():
+ commands.set_hbase_acls()
+ if params.security_enabled and not commands.is_acl_configured():
+ commands.init_kafka_acls()
+ commands.set_acl_configured()
+
commands.start_profiler_topology(env)
def stop(self, env, upgrade_type=None):
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index fe5fa6e..cf29a28 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -45,10 +45,15 @@ class RestCommands:
owner=self.__params.metron_user,
mode=0755)
+ def init_kafka_topics(self):
+ Logger.info('Creating Kafka topics for rest')
+ topics = [self.__params.metron_escalation_topic]
+ metron_service.init_kafka_topics(self.__params, topics)
+
def init_kafka_acls(self):
Logger.info('Creating Kafka ACLs for rest')
# The following topics must be permissioned for the rest application list operation
- topics = [self.__params.ambari_kafka_service_check_topic, self.__params.consumer_offsets_topic]
+ topics = [self.__params.ambari_kafka_service_check_topic, self.__params.consumer_offsets_topic, self.__params.metron_escalation_topic]
metron_service.init_kafka_acls(self.__params, topics, ['metron-rest'])
def start_rest_application(self):
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
index 9331759..2f419df 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
@@ -44,6 +44,7 @@ class RestMaster(Script):
)
commands = RestCommands(params)
+ commands.init_kafka_topics()
if params.security_enabled and not commands.is_acl_configured():
commands.init_kafka_acls()
commands.set_acl_configured()
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
index 7bef3fa..26daa05 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -41,3 +41,4 @@ SECURITY_ENABLED={{security_enabled|lower}}
{% endif %}
KAFKA_SECURITY_PROTOCOL="{{kafka_security_protocol}}"
PARSER_TOPOLOGY_OPTIONS="/home/{{metron_user}}/.storm/storm.config"
+METRON_ESCALATION_TOPIC="{{metron_escalation_topic}}"
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 748feb8..207051d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -617,6 +617,10 @@
"subsection-name": "subsection-rest"
},
{
+ "config": "metron-rest-env/metron_escalation_topic",
+ "subsection-name": "subsection-rest"
+ },
+ {
"config": "metron-management-ui-env/metron_management_ui_port",
"subsection-name": "subsection-management-ui"
}
@@ -1041,6 +1045,12 @@
}
},
{
+ "config": "metron-rest-env/metron_escalation_topic",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
"config": "metron-management-ui-env/metron_management_ui_port",
"widget": {
"type": "text-field"
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/Dockerfile b/metron-deployment/packaging/docker/rpm-docker/Dockerfile
index 2fd2565..a2dae8e 100644
--- a/metron-deployment/packaging/docker/rpm-docker/Dockerfile
+++ b/metron-deployment/packaging/docker/rpm-docker/Dockerfile
@@ -27,3 +27,7 @@ RUN mv apache-maven-3.2.5 /opt/maven
RUN ln -s /opt/maven/bin/mvn /usr/bin/mvn
RUN yum -y install asciidoc rpm-build rpm2cpio tar unzip xmlto zip rpmlint && yum clean all
WORKDIR /root
+
+# install node so that the node dependencies can be packaged into the RPMs
+RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
+RUN yum -y install gcc-c++ make nodejs
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index c617842..3dd9b9b 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -17,7 +17,7 @@
%define timestamp %(date +%Y%m%d%H%M)
%define version %{?_version}%{!?_version:UNKNOWN}
%define full_version %{version}%{?_prerelease}
-%define prerelease_fmt %{?_prerelease:.%{_prerelease}}
+%define prerelease_fmt %{?_prerelease:.%{_prerelease}}
%define vendor_version %{?_vendor_version}%{!?_vendor_version: UNKNOWN}
%define url http://metron.apache.org/
%define base_name metron
@@ -36,6 +36,8 @@
%define metron_extensions_alt_etc %{metron_home}/extension_alt_etc
%define metron_extensions_alt_etc_parsers %{metron_extensions_alt_etc}/parsers
+%define _binaries_in_noarch_packages_terminate_build 0
+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Name: %{base_name}
@@ -158,6 +160,9 @@ mv %{buildroot}%{metron_extensions_etc_parsers}/websphere/lib/*.bundle %{buildro
install %{buildroot}%{metron_home}/bin/metron-rest %{buildroot}/etc/init.d/
install %{buildroot}%{metron_home}/bin/metron-management-ui %{buildroot}/etc/init.d/
+# allows node dependencies to be packaged in the RPMs
+npm install --prefix="%{buildroot}%{metron_home}/web/expressjs" --only=production
+
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%package common
@@ -833,6 +838,8 @@ This package installs the Metron Management UI %{metron_home}
%dir %{metron_home}/bin
%dir %{metron_home}/web
%dir %{metron_home}/web/expressjs
+%dir %{metron_home}/web/expressjs/node_modules
+%dir %{metron_home}/web/expressjs/node_modules/.bin
%dir %{metron_home}/web/management-ui
%dir %{metron_home}/web/management-ui/assets
%dir %{metron_home}/web/management-ui/assets/ace
@@ -843,6 +850,8 @@ This package installs the Metron Management UI %{metron_home}
%dir %{metron_home}/web/management-ui/license
%{metron_home}/bin/metron-management-ui
/etc/init.d/metron-management-ui
+%attr(0755,root,root) %{metron_home}/web/expressjs/node_modules/*
+%attr(0755,root,root) %{metron_home}/web/expressjs/node_modules/.bin/*
%attr(0755,root,root) %{metron_home}/web/expressjs/server.js
%attr(0644,root,root) %{metron_home}/web/expressjs/package.json
%attr(0644,root,root) %{metron_home}/web/management-ui/favicon.ico
@@ -909,7 +918,7 @@ This package install the Metron MaaS Service files %{metron_home}
* Thu Jan 19 2017 Justin Leet <ju...@gmail.com> - 0.3.1
- Replace GeoIP files with new implementation
* Thu Nov 03 2016 David Lyle <dl...@gmail.com> - 0.2.1
-- Add ASA parser/enrichment configuration files
+- Add ASA parser/enrichment configuration files
* Thu Jul 21 2016 Michael Miklavcic <mi...@gmail.com> - 0.2.1
- Remove parser flux files
- Add new enrichment files
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/build.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/build.sh b/metron-deployment/packaging/docker/rpm-docker/build.sh
index 23ed90d..dd5149d 100755
--- a/metron-deployment/packaging/docker/rpm-docker/build.sh
+++ b/metron-deployment/packaging/docker/rpm-docker/build.sh
@@ -36,7 +36,7 @@ if [ $? -ne 0 ] && [ $OWNER_UID -ne 0 ]; then
fi
rm -rf SRPMS/ RPMS/ && \
-rpmbuild -v -ba --define "_topdir $(pwd)" --define "_version ${VERSION}" --define "_prerelease ${PRERELEASE}" SPECS/metron.spec && \
+QA_SKIP_BUILD_ROOT=1 rpmbuild -v -ba --define "_topdir $(pwd)" --define "_version ${VERSION}" --define "_prerelease ${PRERELEASE}" SPECS/metron.spec && \
rpmlint -i SPECS/metron.spec RPMS/*/metron* SRPMS/metron
# Ensure original user permissions are maintained after build
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index db5f288..b111b94 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -258,6 +258,12 @@
<include>*.tar.gz</include>
</includes>
</resource>
+ <resource>
+ <directory>${metron_dir}/metron-analytics/metron-maas-service/target/</directory>
+ <includes>
+ <include>*.tar.gz</include>
+ </includes>
+ </resource>
</resources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
index 45b15f3..51e0455 100644
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
@@ -100,6 +100,7 @@ required_configurations:
storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744"
es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}"
zeppelin_server_url: "{{ groups.zeppelin[0] }}"
+ - metron-rest-env:
metron_jdbc_driver: "org.h2.Driver"
metron_jdbc_url: "jdbc:h2:file:~/metrondb"
metron_jdbc_username: "root"
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/alerts-server-e2e.js
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/alerts-server-e2e.js b/metron-interface/metron-alerts/alerts-server-e2e.js
index 2a5f80b..f1b6410 100644
--- a/metron-interface/metron-alerts/alerts-server-e2e.js
+++ b/metron-interface/metron-alerts/alerts-server-e2e.js
@@ -30,7 +30,7 @@ var favicon = require('serve-favicon');
var proxy = require('http-proxy-middleware');
var argv = require('optimist')
.demand(['p', 'r'])
- .usage('Usage: server.js -p [port]')
+ .usage('Usage: alerts-server-e2e.js -p [port]')
.describe('p', 'Port to run metron alerts ui')
.describe('r', 'Url where metron rest application is available')
.argv;
@@ -40,7 +40,7 @@ var metronUIAddress = '';
var ifaces = os.networkInterfaces();
var restUrl = argv.r || argv.resturl;
var conf = {
- "elastic": {
+ "restapi": {
"target": restUrl,
"secure": false
}
@@ -134,24 +134,28 @@ var clusterState = function(req, res){
app.use(compression());
-app.use(bodyParser.json());
+
app.use(favicon(path.join(__dirname, 'dist/favicon.ico')));
app.use(serveStatic(path.join(__dirname, 'dist'), {
maxAge: '1d',
setHeaders: setCustomCacheControl
}));
-app.use('/api/v1/user', proxy(conf.elastic));
-app.use('/logout', proxy(conf.elastic));
-app.post('/api/v1/search/search', searchResult);
-app.use('/_cluster', clusterState);
+app.use('/logout', proxy(conf.restapi));
+app.use('/api/v1/user', proxy(conf.restapi));
+app.use('/api/v1/search/findOne', proxy(conf.restapi));
+app.use('/api/v1/search/column/metadata', proxy(conf.restapi));
+
app.get('/alerts-list', indexHTML);
app.get('', indexHTML);
+
+app.use(bodyParser.json());
+app.post('/api/v1/search/search', searchResult);
+
app.use(function(req, res, next){
res.status(404).sendStatus(304);
});
-
app.listen(port, function(){
console.log("Metron alerts ui is listening on " + metronUIAddress);
});
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/scripts/alerts-server.js
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/scripts/alerts-server.js b/metron-interface/metron-alerts/scripts/alerts-server.js
index 6fb35c9..6a999f2 100644
--- a/metron-interface/metron-alerts/scripts/alerts-server.js
+++ b/metron-interface/metron-alerts/scripts/alerts-server.js
@@ -22,15 +22,13 @@
var os = require('os');
var app = require('express')();
var path = require('path');
-var compression = require('compression')
+var compression = require('compression');
var serveStatic = require('serve-static');
var favicon = require('serve-favicon');
var proxy = require('http-proxy-middleware');
var argv = require('optimist')
- .demand(['p', 'r', 'e'])
- .alias('r', 'resturl')
- .alias('e', 'elasticurl')
- .usage('Usage: server.js -p [port] -r [restUrl] -e [elasticURL]')
+ .demand(['p', 'r'])
+ .usage('Usage: alert-server.js -p [port] -r [restUrl]')
.describe('p', 'Port to run metron management ui')
.describe('r', 'Url where metron rest application is available')
.argv;
@@ -39,12 +37,7 @@ var port = argv.p;
var metronUIAddress = '';
var ifaces = os.networkInterfaces();
var restUrl = argv.r || argv.resturl;
-var elasticUrl = argv.e || argv.elasticurl;
var conf = {
- "elastic": {
- "target": elasticUrl,
- "secure": false
- },
"rest": {
"target": restUrl,
"secure": false
@@ -78,7 +71,6 @@ var rewriteSearchProxy = proxy({
app.use(compression());
app.use('/api', proxy(conf.rest));
-app.use('/_cluster', proxy(conf.elastic));
app.use(favicon(path.join(__dirname, '../alerts-ui/favicon.ico')));
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index 915c0c1..2e7884e 100644
--- a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -17,7 +17,7 @@
*/
import { Component, OnInit } from '@angular/core';
import {Router, ActivatedRoute} from '@angular/router';
-import {AlertService} from '../../service/alert.service';
+import {SearchService} from '../../service/search.service';
import {Alert} from '../../model/alert';
import {WorkflowService} from '../../service/workflow.service';
import {AlertSource} from '../../model/alert-source';
@@ -42,7 +42,7 @@ export class AlertDetailsComponent implements OnInit {
constructor(private router: Router,
private activatedRoute: ActivatedRoute,
- private alertsService: AlertService,
+ private searchService: SearchService,
private workflowService: WorkflowService) { }
goBack() {
@@ -51,7 +51,7 @@ export class AlertDetailsComponent implements OnInit {
}
getData() {
- this.alertsService.getAlert(this.alertSourceType, this.alertId).subscribe(alert => {
+ this.searchService.getAlert(this.alertSourceType, this.alertId).subscribe(alert => {
this.alertSource = alert;
this.alertFields = Object.keys(alert).filter(field => !field.includes(':ts') && field !== 'original_string').sort();
});
@@ -70,7 +70,7 @@ export class AlertDetailsComponent implements OnInit {
tAlert.source = this.alertSource;
this.selectedAlertState = AlertState.OPEN;
- this.alertsService.updateAlertState([tAlert], 'OPEN', '').subscribe(results => {
+ this.searchService.updateAlertState([tAlert], 'OPEN', '').subscribe(results => {
this.getData();
});
}
@@ -80,7 +80,7 @@ export class AlertDetailsComponent implements OnInit {
tAlert.source = this.alertSource;
this.selectedAlertState = AlertState.NEW;
- this.alertsService.updateAlertState([tAlert], 'NEW', '').subscribe(results => {
+ this.searchService.updateAlertState([tAlert], 'NEW', '').subscribe(results => {
this.getData();
});
}
@@ -91,7 +91,7 @@ export class AlertDetailsComponent implements OnInit {
this.selectedAlertState = AlertState.ESCALATE;
this.workflowService.start([tAlert]).subscribe(workflowId => {
- this.alertsService.updateAlertState([tAlert], 'ESCALATE', workflowId).subscribe(results => {
+ this.searchService.updateAlertState([tAlert], 'ESCALATE', workflowId).subscribe(results => {
this.getData();
});
});
@@ -102,7 +102,7 @@ export class AlertDetailsComponent implements OnInit {
tAlert.source = this.alertSource;
this.selectedAlertState = AlertState.DISMISS;
- this.alertsService.updateAlertState([tAlert], 'DISMISS', '').subscribe(results => {
+ this.searchService.updateAlertState([tAlert], 'DISMISS', '').subscribe(results => {
this.getData();
});
}
@@ -112,7 +112,7 @@ export class AlertDetailsComponent implements OnInit {
tAlert.source = this.alertSource;
this.selectedAlertState = AlertState.RESOLVE;
- this.alertsService.updateAlertState([tAlert], 'RESOLVE', '').subscribe(results => {
+ this.searchService.updateAlertState([tAlert], 'RESOLVE', '').subscribe(results => {
this.getData();
});
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index 35cbeff..72046fc 100644
--- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@ -20,7 +20,7 @@ import {Router, NavigationStart} from '@angular/router';
import {Observable, Subscription} from 'rxjs/Rx';
import {Alert} from '../../model/alert';
-import {AlertService} from '../../service/alert.service';
+import {SearchService} from '../../service/search.service';
import {QueryBuilder} from './query-builder';
import {ConfigureTableService} from '../../service/configure-table.service';
import {WorkflowService} from '../../service/workflow.service';
@@ -35,7 +35,7 @@ import {SaveSearch} from '../../model/save-search';
import {TableMetadata} from '../../model/table-metadata';
import {MetronDialogBox, DialogType} from '../../shared/metron-dialog-box';
import {AlertSearchDirective} from '../../shared/directives/alert-search.directive';
-import {AlertsSearchResponse} from '../../model/alerts-search-response';
+import {SearchResponse} from '../../model/search-response';
import {ElasticsearchUtils} from '../../utils/elasticsearch-utils';
@Component({
@@ -65,7 +65,7 @@ export class AlertsListComponent implements OnInit, OnDestroy {
queryBuilder: QueryBuilder = new QueryBuilder();
constructor(private router: Router,
- private alertsService: AlertService,
+ private searchService: SearchService,
private configureTableService: ConfigureTableService,
private workflowService: WorkflowService,
private clusterMetaDataService: ClusterMetaDataService,
@@ -204,7 +204,7 @@ export class AlertsListComponent implements OnInit, OnDestroy {
}
onConfigRowsChange() {
- this.alertsService.interval = this.refreshInterval;
+ this.searchService.interval = this.refreshInterval;
this.search();
}
@@ -253,26 +253,26 @@ export class AlertsListComponent implements OnInit, OnDestroy {
processEscalate() {
this.workflowService.start(this.selectedAlerts).subscribe(workflowId => {
- this.alertsService.updateAlertState(this.selectedAlerts, 'ESCALATE', workflowId).subscribe(results => {
+ this.searchService.updateAlertState(this.selectedAlerts, 'ESCALATE', workflowId).subscribe(results => {
this.updateSelectedAlertStatus('ESCALATE');
});
});
}
processDismiss() {
- this.alertsService.updateAlertState(this.selectedAlerts, 'DISMISS', '').subscribe(results => {
+ this.searchService.updateAlertState(this.selectedAlerts, 'DISMISS', '').subscribe(results => {
this.updateSelectedAlertStatus('DISMISS');
});
}
processOpen() {
- this.alertsService.updateAlertState(this.selectedAlerts, 'OPEN', '').subscribe(results => {
+ this.searchService.updateAlertState(this.selectedAlerts, 'OPEN', '').subscribe(results => {
this.updateSelectedAlertStatus('OPEN');
});
}
processResolve() {
- this.alertsService.updateAlertState(this.selectedAlerts, 'RESOLVE', '').subscribe(results => {
+ this.searchService.updateAlertState(this.selectedAlerts, 'RESOLVE', '').subscribe(results => {
this.updateSelectedAlertStatus('RESOLVE');
});
}
@@ -313,10 +313,10 @@ export class AlertsListComponent implements OnInit, OnDestroy {
this.saveSearchService.saveAsRecentSearches(savedSearch).subscribe(() => {});
}
- this.alertsService.search(this.queryBuilder.searchRequest).subscribe(results => {
+ this.searchService.search(this.queryBuilder.searchRequest).subscribe(results => {
this.setData(results);
}, error => {
- this.setData(new AlertsSearchResponse());
+ this.setData(new SearchResponse());
this.metronDialogBox.showConfirmationMessage(ElasticsearchUtils.extractESErrorMessage(error), DialogType.Error);
});
@@ -331,7 +331,7 @@ export class AlertsListComponent implements OnInit, OnDestroy {
}
}
- setData(results: AlertsSearchResponse) {
+ setData(results: SearchResponse) {
this.alerts = results.results;
this.pagingData.total = results.total;
}
@@ -370,7 +370,7 @@ export class AlertsListComponent implements OnInit, OnDestroy {
tryStartPolling() {
if (!this.pauseRefresh) {
this.tryStopPolling();
- this.refreshTimer = this.alertsService.pollSearch(this.queryBuilder.searchRequest).subscribe(results => {
+ this.refreshTimer = this.searchService.pollSearch(this.queryBuilder.searchRequest).subscribe(results => {
this.setData(results);
});
}
@@ -383,7 +383,7 @@ export class AlertsListComponent implements OnInit, OnDestroy {
}
updateConfigRowsSettings() {
- this.alertsService.interval = this.refreshInterval;
+ this.searchService.interval = this.refreshInterval;
this.queryBuilder.setFromAndSize(this.pagingData.from, this.pagingData.size);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts
index adc8cbb..e6adae3 100644
--- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.module.ts
@@ -20,7 +20,7 @@ import {NgModule} from '@angular/core';
import {AlertsListComponent} from './alerts-list.component';
import {routing} from './alerts-list.routing';
import {SharedModule} from '../../shared/shared.module';
-import {AlertService} from '../../service/alert.service';
+import {SearchService} from '../../service/search.service';
import {MetronSorterModule} from '../../shared/metron-table/metron-sorter/metron-sorter.module';
import {ListGroupModule} from '../../shared/list-group/list-grup.module';
import {CollapseModule} from '../../shared/collapse/collapse.module';
@@ -32,7 +32,7 @@ import {ConfigureRowsModule} from '../configure-rows/configure-rows.module';
ListGroupModule, CollapseModule],
exports: [AlertsListComponent],
declarations: [AlertsListComponent],
- providers: [AlertService],
+ providers: [SearchService],
})
export class AlertsListModule {
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts b/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts
index 941343e..0c0117b 100644
--- a/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/configure-table/configure-table.component.ts
@@ -24,6 +24,7 @@ import {ClusterMetaDataService} from '../../service/cluster-metadata.service';
import {ColumnMetadata} from '../../model/column-metadata';
import {ColumnNamesService} from '../../service/column-names.service';
import {ColumnNames} from '../../model/column-names';
+import {SearchService} from '../../service/search.service';
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
@@ -51,8 +52,11 @@ export class ConfigureTableComponent implements OnInit {
allColumns: ColumnMetadataWrapper[] = [];
- constructor(private router: Router, private activatedRoute: ActivatedRoute, private configureTableService: ConfigureTableService,
- private clusterMetaDataService: ClusterMetaDataService, private columnNamesService: ColumnNamesService) { }
+ constructor(private router: Router, private activatedRoute: ActivatedRoute,
+ private configureTableService: ConfigureTableService,
+ private clusterMetaDataService: ClusterMetaDataService,
+ private columnNamesService: ColumnNamesService,
+ private searchService: SearchService) { }
goBack() {
this.router.navigateByUrl('/alerts-list');
@@ -80,7 +84,7 @@ export class ConfigureTableComponent implements OnInit {
ngOnInit() {
Observable.forkJoin(
this.clusterMetaDataService.getDefaultColumns(),
- this.clusterMetaDataService.getColumnMetaData(),
+ this.searchService.getColumnMetaData(),
this.configureTableService.getTableMetadata()
).subscribe((response: any) => {
this.prepareData(response[0], response[1], response[2].tableColumns);
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/app.module.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/app.module.ts b/metron-interface/metron-alerts/src/app/app.module.ts
index 295c748..f16b2d1 100644
--- a/metron-interface/metron-alerts/src/app/app.module.ts
+++ b/metron-interface/metron-alerts/src/app/app.module.ts
@@ -36,7 +36,7 @@ import {ConfigureRowsModule} from './alerts/configure-rows/configure-rows.module
import {SwitchModule} from './shared/switch/switch.module';
import {ColumnNamesService} from './service/column-names.service';
import {DataSource} from './service/data-source';
-import {RestApiImpl} from './service/rest-api-impl';
+import {ElasticSearchLocalstorageImpl} from './service/elasticsearch-localstorage-impl';
import {LoginModule} from './login/login.module';
import {AuthGuard} from './shared/auth-guard';
import {AuthenticationService} from './service/authentication.service';
@@ -67,7 +67,7 @@ export function initConfig(config: ColumnNamesService) {
SwitchModule
],
providers: [{ provide: APP_INITIALIZER, useFactory: initConfig, deps: [ColumnNamesService], multi: true },
- { provide: DataSource, useClass: RestApiImpl },
+ { provide: DataSource, useClass: ElasticSearchLocalstorageImpl },
AuthenticationService,
AuthGuard,
LoginGuard,
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts b/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts
deleted file mode 100644
index 265f66b..0000000
--- a/metron-interface/metron-alerts/src/app/model/alerts-search-response.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import {Alert} from './alert';
-
-export class AlertsSearchResponse {
- total = 0;
- results: Alert[] = [];
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/model/search-request.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/model/search-request.ts b/metron-interface/metron-alerts/src/app/model/search-request.ts
index 2150f2b..a37bd8d 100644
--- a/metron-interface/metron-alerts/src/app/model/search-request.ts
+++ b/metron-interface/metron-alerts/src/app/model/search-request.ts
@@ -1,4 +1,5 @@
import {SortField} from './sort-field';
+import {INDEXES} from '../utils/constants';
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -19,7 +20,7 @@ import {SortField} from './sort-field';
export class SearchRequest {
// _source: string[]; //TODO: This needs to be removed
from: number;
- indices: string[] = ['websphere', 'snort', 'asa', 'bro', 'yaf'];
+ indices: string[] = INDEXES;
query: string;
size: number;
sort: SortField[];
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/model/search-response.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/model/search-response.ts b/metron-interface/metron-alerts/src/app/model/search-response.ts
new file mode 100644
index 0000000..c3fea27
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/model/search-response.ts
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Alert} from './alert';
+
+export class SearchResponse {
+ total = 0;
+ results: Alert[] = [];
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/alert.service.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/alert.service.ts b/metron-interface/metron-alerts/src/app/service/alert.service.ts
deleted file mode 100644
index 7dabc4f..0000000
--- a/metron-interface/metron-alerts/src/app/service/alert.service.ts
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import {Injectable, NgZone} from '@angular/core';
-import {Observable} from 'rxjs/Rx';
-import 'rxjs/add/observable/interval';
-import 'rxjs/add/operator/switchMap';
-import 'rxjs/add/operator/onErrorResumeNext';
-
-import {Alert} from '../model/alert';
-import {Http} from '@angular/http';
-import {DataSource} from './data-source';
-import {AlertsSearchResponse} from '../model/alerts-search-response';
-import {SearchRequest} from '../model/search-request';
-import {AlertSource} from '../model/alert-source';
-
-@Injectable()
-export class AlertService {
-
- interval = 80000;
- defaultHeaders = {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'};
-
- constructor(private http: Http,
- private dataSource: DataSource,
- private ngZone: NgZone) { }
-
- public search(searchRequest: SearchRequest): Observable<AlertsSearchResponse> {
- return this.dataSource.getAlerts(searchRequest);
- }
-
- public pollSearch(searchRequest: SearchRequest): Observable<AlertsSearchResponse> {
- return this.ngZone.runOutsideAngular(() => {
- return this.ngZone.run(() => {
- return Observable.interval(this.interval * 1000).switchMap(() => {
- return this.dataSource.getAlerts(searchRequest);
- });
- });
- });
- }
-
- public getAlert(sourceType: string, alertId: string): Observable<AlertSource> {
- return this.dataSource.getAlert(sourceType, alertId);
- }
-
- public updateAlertState(alerts: Alert[], state: string, workflowId: string) {
- let request = '';
- for (let alert of alerts) {
- request += '{ "update" : { "sensorType" : "' + alert.source['source:type'] + '", "guid" : "' + alert.source.guid + '" } }\n' +
- '{ "doc": { "alert_status": "' + state + '"';
- if (workflowId) {
- request += ', "workflow_id": "' + workflowId + '"';
- }
- request += ' }}\n';
- }
-
- return this.dataSource.updateAlertState(request);
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts b/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts
index 4077f30..ffd4ec1 100644
--- a/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts
+++ b/metron-interface/metron-alerts/src/app/service/cluster-metadata.service.ts
@@ -18,11 +18,14 @@
import {Injectable} from '@angular/core';
import {Observable} from 'rxjs/Rx';
import {Http} from '@angular/http';
+
+
import {ColumnMetadata} from '../model/column-metadata';
import {DataSource} from './data-source';
@Injectable()
export class ClusterMetaDataService {
+ defaultHeaders: {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'};
constructor(private http: Http,
private dataSource: DataSource) {
@@ -31,8 +34,4 @@ export class ClusterMetaDataService {
getDefaultColumns(): Observable<ColumnMetadata[]> {
return this.dataSource.getDefaultAlertTableColumnNames();
}
-
- getColumnMetaData(): Observable<ColumnMetadata[]> {
- return this.dataSource.getAllFieldNames();
- }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/data-source.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/data-source.ts b/metron-interface/metron-alerts/src/app/service/data-source.ts
index 28ee384..f4f90ed 100644
--- a/metron-interface/metron-alerts/src/app/service/data-source.ts
+++ b/metron-interface/metron-alerts/src/app/service/data-source.ts
@@ -22,7 +22,7 @@ import {ColumnMetadata} from '../model/column-metadata';
import {ColumnNames} from '../model/column-names';
import {TableMetadata} from '../model/table-metadata';
import {SaveSearch} from '../model/save-search';
-import {AlertsSearchResponse} from '../model/alerts-search-response';
+import {SearchResponse} from '../model/search-response';
import {SearchRequest} from '../model/search-request';
import {AlertSource} from '../model/alert-source';
@@ -33,7 +33,7 @@ export abstract class DataSource {
constructor(protected http: Http) {}
// Calls to fetch alerts
- abstract getAlerts(searchRequest: SearchRequest): Observable<AlertsSearchResponse>
+ abstract getAlerts(searchRequest: SearchRequest): Observable<SearchResponse>
abstract getAlert(sourceType: string, alertId: string): Observable<AlertSource>
abstract updateAlertState(request: any): Observable<{}>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts b/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts
index f4e43d5..7982102 100644
--- a/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts
+++ b/metron-interface/metron-alerts/src/app/service/elasticsearch-localstorage-impl.ts
@@ -30,7 +30,7 @@ import {ColumnNames} from '../model/column-names';
import {ColumnNamesService} from './column-names.service';
import {TableMetadata} from '../model/table-metadata';
import {SaveSearch} from '../model/save-search';
-import {AlertsSearchResponse} from '../model/alerts-search-response';
+import {SearchResponse} from '../model/search-response';
import {SearchRequest} from '../model/search-request';
import {AlertSource} from '../model/alert-source';
@@ -47,7 +47,7 @@ export class ElasticSearchLocalstorageImpl extends DataSource {
new ColumnMetadata('alert_status', 'string')
];
- getAlerts(searchRequest: SearchRequest): Observable<AlertsSearchResponse> {
+ getAlerts(searchRequest: SearchRequest): Observable<SearchResponse> {
let url = '/search/*' + ElasticsearchUtils.excludeIndexName + '/_search';
let request: any = JSON.parse(JSON.stringify(searchRequest));
request.query = { query_string: { query: searchRequest.query } };
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts b/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts
deleted file mode 100644
index 061708d..0000000
--- a/metron-interface/metron-alerts/src/app/service/rest-api-impl.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import {Observable} from 'rxjs/Rx';
-import {Headers, RequestOptions} from '@angular/http';
-
-import {HttpUtil} from '../utils/httpUtil';
-import {AlertsSearchResponse} from '../model/alerts-search-response';
-import {SearchRequest} from '../model/search-request';
-import {ElasticSearchLocalstorageImpl} from './elasticsearch-localstorage-impl';
-import {AlertSource} from '../model/alert-source';
-
-export class RestApiImpl extends ElasticSearchLocalstorageImpl {
-
- getAlerts(searchRequest: SearchRequest): Observable<AlertsSearchResponse> {
- let url = '/api/v1/search/search';
- return this.http.post(url, searchRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
- .map(HttpUtil.extractData)
- .catch(HttpUtil.handleError)
- .onErrorResumeNext();
- }
-
- getAlert(sourceType: string, alertId: string): Observable<AlertSource> {
- let url = '/api/v1/search/findOne';
- let requestSchema = { guid: alertId, sensorType: sourceType};
-
- return this.http.post(url, requestSchema, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
- .map(HttpUtil.extractData)
- .catch(HttpUtil.handleError)
- .onErrorResumeNext();
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/service/search.service.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/search.service.ts b/metron-interface/metron-alerts/src/app/service/search.service.ts
new file mode 100644
index 0000000..be3b1f6
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/service/search.service.ts
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Injectable, NgZone} from '@angular/core';
+import {Headers, RequestOptions} from '@angular/http';
+import {Observable} from 'rxjs/Rx';
+import 'rxjs/add/observable/interval';
+import 'rxjs/add/operator/switchMap';
+import 'rxjs/add/operator/onErrorResumeNext';
+
+import {HttpUtil} from '../utils/httpUtil';
+import {Alert} from '../model/alert';
+import {Http} from '@angular/http';
+import {DataSource} from './data-source';
+import {SearchResponse} from '../model/search-response';
+import {SearchRequest} from '../model/search-request';
+import {AlertSource} from '../model/alert-source';
+import {INDEXES} from '../utils/constants';
+import {ColumnMetadata} from '../model/column-metadata';
+
+@Injectable()
+export class SearchService {
+
+ interval = 80000;
+ defaultHeaders = {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'};
+
+ private static extractColumnNameDataFromRestApi(res: Response): ColumnMetadata[] {
+ let response: any = res || {};
+ let processedKeys: string[] = [];
+ let columnMetadatas: ColumnMetadata[] = [];
+
+ for (let index of Object.keys(response)) {
+ let indexMetaData = response[index];
+ for (let key of Object.keys(indexMetaData)) {
+ if (processedKeys.indexOf(key) === -1) {
+ processedKeys.push(key);
+ columnMetadatas.push(new ColumnMetadata(key, indexMetaData[key]));
+ }
+ }
+ }
+
+ return columnMetadatas;
+ }
+
+ constructor(private http: Http,
+ private dataSource: DataSource,
+ private ngZone: NgZone) { }
+
+ public getAlert(sourceType: string, alertId: string): Observable<AlertSource> {
+ let url = '/api/v1/search/findOne';
+ let requestSchema = { guid: alertId, sensorType: sourceType};
+
+ return this.http.post(url, requestSchema, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ .map(HttpUtil.extractData)
+ .catch(HttpUtil.handleError)
+ .onErrorResumeNext();
+ }
+
+ public getColumnMetaData(): Observable<ColumnMetadata[]> {
+ let url = '/api/v1/search/column/metadata';
+ return this.http.post(url, INDEXES, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ .map(HttpUtil.extractData)
+ .map(SearchService.extractColumnNameDataFromRestApi)
+ .catch(HttpUtil.handleError);
+ }
+
+ public pollSearch(searchRequest: SearchRequest): Observable<SearchResponse> {
+ return this.ngZone.runOutsideAngular(() => {
+ return this.ngZone.run(() => {
+ return Observable.interval(this.interval * 1000).switchMap(() => {
+ return this.search(searchRequest);
+ });
+ });
+ });
+ }
+
+ public search(searchRequest: SearchRequest): Observable<SearchResponse> {
+ let url = '/api/v1/search/search';
+ return this.http.post(url, searchRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ .map(HttpUtil.extractData)
+ .catch(HttpUtil.handleError)
+ .onErrorResumeNext();
+ }
+
+ public updateAlertState(alerts: Alert[], state: string, workflowId: string) {
+ let request = '';
+ for (let alert of alerts) {
+ request += '{ "update" : { "sensorType" : "' + alert.source['source:type'] + '", "guid" : "' + alert.source.guid + '" } }\n' +
+ '{ "doc": { "alert_status": "' + state + '"';
+ if (workflowId) {
+ request += ', "workflow_id": "' + workflowId + '"';
+ }
+ request += ' }}\n';
+ }
+
+ return this.dataSource.updateAlertState(request);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/utils/constants.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/utils/constants.ts b/metron-interface/metron-alerts/src/app/utils/constants.ts
index da6d50c..a738a1d 100644
--- a/metron-interface/metron-alerts/src/app/utils/constants.ts
+++ b/metron-interface/metron-alerts/src/app/utils/constants.ts
@@ -20,3 +20,5 @@ export const ALERTS_RECENT_SEARCH = 'metron-alerts-recent-saved-search';
export const ALERTS_SAVED_SEARCH = 'metron-alerts-saved-search';
export const ALERTS_TABLE_METADATA = 'metron-alerts-table-metadata';
export const ALERTS_COLUMN_NAMES = 'metron-alerts-column-names';
+
+export let INDEXES = ['websphere', 'snort', 'asa', 'bro', 'yaf'];
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts b/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts
index a86907b..0896f32 100644
--- a/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts
+++ b/metron-interface/metron-alerts/src/app/utils/elasticsearch-utils.ts
@@ -16,7 +16,7 @@
* limitations under the License.
*/
import {ColumnMetadata} from '../model/column-metadata';
-import {AlertsSearchResponse} from '../model/alerts-search-response';
+import {SearchResponse} from '../model/search-response';
export class ElasticsearchUtils {
@@ -54,9 +54,9 @@ export class ElasticsearchUtils {
return columnMetadata;
}
- public static extractAlertsData(res: Response): AlertsSearchResponse {
+ public static extractAlertsData(res: Response): SearchResponse {
let response: any = res || {};
- let alertsSearchResponse: AlertsSearchResponse = new AlertsSearchResponse();
+ let alertsSearchResponse: SearchResponse = new SearchResponse();
alertsSearchResponse.total = response['hits']['total'];
alertsSearchResponse.results = response['hits']['hits'];
return alertsSearchResponse;
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-config/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/README.md b/metron-interface/metron-config/README.md
index 257faba..f0a69e3 100644
--- a/metron-interface/metron-config/README.md
+++ b/metron-interface/metron-config/README.md
@@ -49,12 +49,6 @@ This module provides a user interface for management functions in Metron.
rpm -ih metron-config-$METRON_VERSION-*.noarch.rpm
```
-1. Install the [Express](https://expressjs.com/) web framework from the `package.json` file in `$METRON_HOME/web/expressjs`:
-
- ```
- npm --prefix $METRON_HOME/web/expressjs/ install
- ```
-
## Configuration
The Managment UI is configured in the `$METRON_HOME/config/management_ui.yml` file. Create this file and set the values to match your environment:
@@ -87,7 +81,7 @@ The Management UI can also be started in development mode. This allows changes
cd metron-interface/metron-config
npm install
```
-
+
1. Start the application:
```
@@ -113,4 +107,4 @@ The application will be available at http://localhost:4200/. The REST applicati
## License
-This projects bundles Font Awesome which is available under the SIL Open Font License. See http://fontawesome.io/license/ for more details.
\ No newline at end of file
+This projects bundles Font Awesome which is available under the SIL Open Font License. See http://fontawesome.io/license/ for more details.
[3/6] metron git commit: METRON-1136 Track Master in Feature Branch
(ottobackwards) closes apache/metron#752
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
new file mode 100644
index 0000000..f1a0ec4
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.stellar;
+
+import ch.hsr.geohash.WGS84Point;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metron.stellar.common.utils.StellarProcessorUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.*;
+
+public class GeoHashFunctionsTest {
+ public static WGS84Point empireStatePoint = new WGS84Point(40.748570, -73.985752);
+ public static WGS84Point mosconeCenterPoint = new WGS84Point(37.782891, -122.404166);
+ public static WGS84Point jutlandPoint = new WGS84Point(57.64911, 10.40740);
+ public static String explicitJutlandHash = "u4pruydqmvpb";
+ String empireStateHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", empireStatePoint.getLatitude()
+ ,"long",empireStatePoint.getLongitude()
+ )
+ );
+ String mosconeCenterHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", mosconeCenterPoint.getLatitude()
+ ,"long",mosconeCenterPoint.getLongitude()
+ )
+ );
+ String jutlandHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ );
+
+ @Test
+ public void testToLatLong_happypath() throws Exception {
+ Map<String, Object> latLong = (Map<String, Object>)StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", explicitJutlandHash ) );
+ Assert.assertEquals(jutlandPoint.getLatitude(), (double)latLong.get("latitude"), 1e-3);
+ Assert.assertEquals(jutlandPoint.getLongitude(), (double)latLong.get("longitude"), 1e-3);
+ }
+
+ @Test
+ public void testToLatLong_degenerate() throws Exception {
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", "u"));
+ Assert.assertFalse(Double.isNaN((double) latLong.get("latitude")));
+ Assert.assertFalse(Double.isNaN((double) latLong.get("longitude")));
+ }
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", ""));
+ Assert.assertEquals(0d, (double)latLong.get("latitude"), 1e-3);
+ Assert.assertEquals(0d, (double)latLong.get("longitude"), 1e-3);
+ }
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(null)"
+ , new HashMap<>());
+ Assert.assertNull(latLong);
+ }
+ }
+
+ @Test
+ public void testHash_fromlatlong() throws Exception {
+ Assert.assertEquals("u4pruydqmv", StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 10)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmvpb", StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ Assert.assertEquals("u4pruydqmv".substring(0, 6), StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 6)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 10)"
+ , ImmutableMap.of("lat", "blah"
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testHash_fromLocation() throws Exception {
+ Map<String, String> loc = ImmutableMap.of( "latitude", "" + jutlandPoint.getLatitude()
+ , "longitude","" + jutlandPoint.getLongitude()
+ );
+ Assert.assertEquals("u4pruydqmv", StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 10)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmv".substring(0, 6), StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 6)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmvpb", StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc)"
+ , ImmutableMap.of("loc", ImmutableMap.of( "latitude", "57.64911" ))
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 10)"
+ , ImmutableMap.of("loc", ImmutableMap.of( "latitude", "blah"
+ , "longitude","10.40740"
+ )
+ )
+
+ )
+ );
+ }
+
+ @Test
+ public void testDistanceHaversine() throws Exception {
+ testDistance(Optional.empty());
+ testDistance(Optional.of("HAVERSINE"));
+ }
+
+ @Test
+ public void testDistanceLawOfCosines() throws Exception {
+ testDistance(Optional.of("LAW_OF_COSINES"));
+ }
+
+ @Test
+ public void testDistanceLawOfVicenty() throws Exception {
+ testDistance(Optional.of("VICENTY"));
+ }
+
+ @Test
+ public void testMaxDistance_happyPath() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([empireState, mosconeCenter, jutland])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+
+ @Test
+ public void testMaxDistance_differentOrder() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, mosconeCenter, empireState])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+
+ @Test
+ public void testMaxDistance_withNulls() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, mosconeCenter, empireState, null])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+ @Test
+ public void testMaxDistance_allSame() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, jutland, jutland])"
+ , ImmutableMap.of( "jutland", jutlandHash )
+ );
+ Assert.assertEquals(0, maxDistance, 1e-6d);
+ }
+
+ @Test
+ public void testMaxDistance_emptyList() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([])" , new HashMap<>() );
+ Assert.assertTrue(Double.isNaN(maxDistance));
+ }
+
+ @Test
+ public void testMaxDistance_nullList() throws Exception {
+ Double maxDistance = (Double) StellarProcessorUtils.run("GEOHASH_MAX_DIST(null)" , new HashMap<>() );
+ Assert.assertNull(maxDistance);
+ }
+
+ @Test
+ public void testMaxDistance_invalidList() throws Exception {
+ Double maxDistance = (Double) StellarProcessorUtils.run("GEOHASH_MAX_DIST()" , new HashMap<>() );
+ Assert.assertNull(maxDistance);
+ }
+
+ public void testDistance(Optional<String> method) throws Exception {
+ double expectedDistance = 4128; //in kilometers
+ Map<String, Object> vars = ImmutableMap.of("empireState", empireStateHash, "mosconeCenter", mosconeCenterHash);
+ //ensure that d(x, y) == d(y, x) and that both are the same as the expected (up to 1 km accuracy)
+ {
+ String stellarStatement = getDistStellarStatement(ImmutableList.of("mosconeCenter", "empireState"), method);
+ Assert.assertEquals(expectedDistance, (double) StellarProcessorUtils.run(stellarStatement , vars ), 1D );
+ }
+ {
+ String stellarStatement = getDistStellarStatement(ImmutableList.of("empireState", "mosconeCenter"), method);
+ Assert.assertEquals(expectedDistance, (double) StellarProcessorUtils.run(stellarStatement , vars ), 1D );
+ }
+ }
+
+ private static String getDistStellarStatement(List<String> hashVariables, Optional<String> method) {
+ if(method.isPresent()) {
+ List<String> vars = new ArrayList<>();
+ vars.addAll(hashVariables);
+ vars.add("\'" + method.get() + "\'");
+ return "GEOHASH_DIST(" + Joiner.on(",").skipNulls().join(vars) + ")";
+ }
+ else {
+ return "GEOHASH_DIST(" + Joiner.on(",").skipNulls().join(hashVariables) + ")";
+ }
+ }
+
+ @Test
+ public void testCentroid_List() throws Exception {
+ //happy path
+ {
+ double expectedLong = -98.740087 //calculated via http://www.geomidpoint.com/ using the center of gravity or geographic midpoint.
+ , expectedLat = 41.86921
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState, mosconeCenter]))"
+ , ImmutableMap.of("empireState", empireStateHash, "mosconeCenter", mosconeCenterHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //same point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState, empireState]))"
+ , ImmutableMap.of("empireState", empireStateHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //one point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState]))"
+ , ImmutableMap.of("empireState", empireStateHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //no points
+ {
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([]))"
+ , new HashMap<>()
+ );
+ Assert.assertNull(centroid);
+ }
+ }
+
+ @Test
+ public void testCentroid_weighted() throws Exception {
+ //happy path
+ {
+ double expectedLong = -98.740087 //calculated via http://www.geomidpoint.com/ using the center of gravity or geographic midpoint.
+ , expectedLat = 41.86921
+ ;
+ for(int weight = 1;weight < 10;++weight) {
+ Map<Object, Integer> weightedPoints = ImmutableMap.of(empireStateHash, weight, mosconeCenterHash, weight);
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ }
+ //same point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ for(int weight = 1;weight < 10;++weight) {
+ Map<Object, Integer> weightedPoints = ImmutableMap.of(empireStateHash, weight);
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ }
+ //no points
+ {
+ Map<Object, Integer> weightedPoints = new HashMap<>();
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertNull(centroid);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase-client/pom.xml b/metron-platform/metron-hbase-client/pom.xml
index 5dd6127..1237be7 100644
--- a/metron-platform/metron-hbase-client/pom.xml
+++ b/metron-platform/metron-hbase-client/pom.xml
@@ -80,6 +80,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>org.apache.commons.logging</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index c64c374..7d07665 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -222,6 +222,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java
index ddb88e5..4f47a65 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.indexing.dao;
-import org.apache.metron.hbase.HTableProvider;
import org.apache.metron.hbase.TableProvider;
import java.util.HashMap;
@@ -26,6 +25,7 @@ import java.util.function.Supplier;
public class AccessConfig {
private Integer maxSearchResults;
+ private Integer maxSearchGroups;
private Supplier<Map<String, Object>> globalConfigSupplier;
private Map<String, String> optionalSettings = new HashMap<>();
private TableProvider tableProvider = null;
@@ -55,6 +55,18 @@ public class AccessConfig {
}
/**
+ * The maximum search groups.
+ * @return
+ */
+ public Integer getMaxSearchGroups() {
+ return maxSearchGroups;
+ }
+
+ public void setMaxSearchGroups(Integer maxSearchGroups) {
+ this.maxSearchGroups = maxSearchGroups;
+ }
+
+ /**
* Get optional settings for initializing indices.
* @return
*/
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
index a1cf398..c890544 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
@@ -65,6 +67,11 @@ public class HBaseDao implements IndexDao {
}
@Override
+ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+ return null;
+ }
+
+ @Override
public synchronized void init(AccessConfig config) {
if(this.tableInterface == null) {
this.config = config;
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index 350e402..745dccd 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -22,6 +22,8 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.flipkart.zjsonpatch.JsonPatch;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
@@ -33,7 +35,6 @@ import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import java.io.IOException;
import org.apache.metron.indexing.dao.search.FieldType;
-import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -49,6 +50,8 @@ public interface IndexDao {
*/
SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException;
+ GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException;
+
/**
* Initialize the DAO with the AccessConfig object.
* @param config
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
index e9a4a9a..61c6231 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
@@ -22,6 +22,8 @@ import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
@@ -118,6 +120,17 @@ public class MultiIndexDao implements IndexDao {
}
@Override
+ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+ for(IndexDao dao : indices) {
+ GroupResponse s = dao.group(groupRequest);
+ if(s != null) {
+ return s;
+ }
+ }
+ return null;
+ }
+
+ @Override
public void init(AccessConfig config) {
for(IndexDao dao : indices) {
dao.init(config);
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java
new file mode 100644
index 0000000..be02026
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+public class Group {
+
+ private GroupOrder order;
+ private String field;
+
+ public Group() {
+ order = new GroupOrder();
+ order.setGroupOrderType(GroupOrderType.TERM.toString());
+ order.setSortOrder(SortOrder.DESC.toString());
+ }
+
+ public GroupOrder getOrder() {
+ return order;
+ }
+
+ public void setOrder(GroupOrder order) {
+ this.order = order;
+ }
+
+ public String getField() {
+ return field;
+ }
+
+ public void setField(String field) {
+ this.field = field;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java
new file mode 100644
index 0000000..b90c438
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+public class GroupOrder {
+
+ private SortOrder sortOrder;
+ private GroupOrderType groupOrderType;
+
+ public SortOrder getSortOrder() {
+ return sortOrder;
+ }
+
+ public void setSortOrder(String sortOrder) {
+ this.sortOrder = SortOrder.fromString(sortOrder);
+ }
+
+ public GroupOrderType getGroupOrderType() {
+ return groupOrderType;
+ }
+
+ public void setGroupOrderType(String groupOrderType) {
+ this.groupOrderType = GroupOrderType.fromString(groupOrderType);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java
new file mode 100644
index 0000000..8444e50
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public enum GroupOrderType {
+
+ @JsonProperty("count")
+ COUNT("count"),
+ @JsonProperty("term")
+ TERM("term");
+
+ private String groupOrderType;
+
+ GroupOrderType(String groupOrderType) {
+ this.groupOrderType = groupOrderType;
+ }
+
+ public String getGroupOrderType() {
+ return groupOrderType;
+ }
+
+ public static GroupOrderType fromString(String groupOrderType) {
+ return GroupOrderType.valueOf(groupOrderType.toUpperCase());
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java
new file mode 100644
index 0000000..121da10
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+import java.util.List;
+import java.util.Optional;
+
+public class GroupRequest {
+
+ private List<String> indices;
+ private String query;
+ private String scoreField;
+ private List<Group> groups;
+
+ public List<String> getIndices() {
+ return indices;
+ }
+
+ public void setIndices(List<String> indices) {
+ this.indices = indices;
+ }
+
+ public String getQuery() {
+ return query;
+ }
+
+ public void setQuery(String query) {
+ this.query = query;
+ }
+
+ public Optional<String> getScoreField() {
+ return scoreField == null ? Optional.empty() : Optional.of(scoreField);
+ }
+
+ public void setScoreField(String scoreField) {
+ this.scoreField = scoreField;
+ }
+
+ public List<Group> getGroups() {
+ return groups;
+ }
+
+ public void setGroups(List<Group> groups) {
+ this.groups = groups;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java
new file mode 100644
index 0000000..1b42609
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.metron.indexing.dao.search;
+
+import java.util.List;
+
+public class GroupResponse {
+
+ private String groupedBy;
+ private List<GroupResult> groupResults;
+
+ public String getGroupedBy() {
+ return groupedBy;
+ }
+
+ public void setGroupedBy(String groupedBy) {
+ this.groupedBy = groupedBy;
+ }
+
+ public List<GroupResult> getGroupResults() {
+ return groupResults;
+ }
+
+ public void setGroupResults(List<GroupResult> groupResults) {
+ this.groupResults = groupResults;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java
new file mode 100644
index 0000000..d40f146
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.search;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.List;
+
+public class GroupResult {
+
+ private String key;
+ private long total;
+ private Double score;
+ private String groupedBy;
+ private List<GroupResult> groupResults;
+
+ public String getKey() {
+ return key;
+ }
+
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ public long getTotal() {
+ return total;
+ }
+
+ public void setTotal(long total) {
+ this.total = total;
+ }
+
+ public Double getScore() {
+ return score;
+ }
+
+ public void setScore(Double score) {
+ this.score = score;
+ }
+
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public String getGroupedBy() {
+ return groupedBy;
+ }
+
+ public void setGroupedBy(String groupedBy) {
+ this.groupedBy = groupedBy;
+ }
+
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public List<GroupResult> getGroupResults() {
+ return groupResults;
+ }
+
+ public void setGroupResults(List<GroupResult> groups) {
+ this.groupResults = groups;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
index 2d146e0..6e48b58 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
@@ -22,7 +22,6 @@ import com.google.common.base.Splitter;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.Iterables;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.*;
import org.apache.metron.indexing.dao.update.Document;
@@ -77,6 +76,27 @@ public class InMemoryDao implements IndexDao {
return ret;
}
+ @Override
+ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+ GroupResponse groupResponse = new GroupResponse();
+ groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField());
+ groupResponse.setGroupResults(getGroupResults(groupRequest.getGroups(), 0));
+ return groupResponse;
+ }
+
+ private List<GroupResult> getGroupResults(List<Group> groups, int index) {
+ Group group = groups.get(index);
+ GroupResult groupResult = new GroupResult();
+ groupResult.setKey(group.getField() + "_value");
+ if (index < groups.size() - 1) {
+ groupResult.setGroupedBy(groups.get(index + 1).getField());
+ groupResult.setGroupResults(getGroupResults(groups, index + 1));
+ } else {
+ groupResult.setScore(50.0);
+ }
+ groupResult.setTotal(10);
+ return Collections.singletonList(groupResult);
+ }
private static class ComparableComparator implements Comparator<Comparable> {
SortOrder order = null;
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 2645df2..0db8e37 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -20,10 +20,13 @@ package org.apache.metron.indexing.dao;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.search.GroupResult;
import org.apache.metron.integration.InMemoryComponent;
import org.junit.After;
import org.junit.Assert;
@@ -40,11 +43,11 @@ import java.util.Map;
public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "double_field": 1.00001, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "double_field": 1.00002, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "double_field": 1.00002, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "double_field": 1.00002, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "double_field": 1.00001, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5"}
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5"}
* ]
*/
@Multiline
@@ -52,11 +55,11 @@ public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "double_field": 1.00001, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "double_field": 1.00002, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "double_field": 1.00001, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "double_field": 1.00002, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "double_field": 1.00001, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5}
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5}
* ]
*/
@Multiline
@@ -149,7 +152,7 @@ public abstract class SearchIntegrationTest {
/**
* {
- * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "double_field", "is_alert"],
+ * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"],
* "indices": ["bro", "snort"],
* "query": "*",
* "from": 0,
@@ -253,6 +256,63 @@ public abstract class SearchIntegrationTest {
@Multiline
public static String noResultsFieldsQuery;
+ /**
+ * {
+ * "groups": [
+ * {
+ * "field":"is_alert"
+ * },
+ * {
+ * "field":"latitude"
+ * }
+ * ],
+ * "scoreField":"score",
+ * "indices": ["bro", "snort"],
+ * "query": "*"
+ * }
+ */
+ @Multiline
+ public static String groupByQuery;
+
+ /**
+ * {
+ * "groups": [
+ * {
+ * "field":"is_alert",
+ * "order": {
+ * "groupOrderType": "count",
+ * "sortOrder": "ASC"
+ * }
+ * },
+ * {
+ * "field":"ip_src_addr",
+ * "order": {
+ * "groupOrderType": "term",
+ * "sortOrder": "DESC"
+ * }
+ * }
+ * ],
+ * "indices": ["bro", "snort"],
+ * "query": "*"
+ * }
+ */
+ @Multiline
+ public static String sortedGroupByQuery;
+
+ /**
+ * {
+ * "groups": [
+ * {
+ * "field":"location_point"
+ * }
+ * ],
+ * "indices": ["bro", "snort"],
+ * "query": "*"
+ * }
+ */
+ @Multiline
+ public static String badGroupQuery;
+
protected static IndexDao dao;
protected static InMemoryComponent indexComponent;
@@ -387,14 +447,18 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001);
Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0)));
Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1)));
- Map<String, Long> doubleFieldCounts = facetCounts.get("double_field");
- Assert.assertEquals(2, doubleFieldCounts.size());
- List<String> doubleFieldKeys = new ArrayList<>(doubleFieldCounts.keySet());
- Collections.sort(doubleFieldKeys);
- Assert.assertEquals(1.00001, Double.parseDouble(doubleFieldKeys.get(0)), 0.00001);
- Assert.assertEquals(1.00002, Double.parseDouble(doubleFieldKeys.get(1)), 0.00001);
- Assert.assertEquals(new Long(5), doubleFieldCounts.get(doubleFieldKeys.get(0)));
- Assert.assertEquals(new Long(5), doubleFieldCounts.get(doubleFieldKeys.get(1)));
+ Map<String, Long> scoreFieldCounts = facetCounts.get("score");
+ Assert.assertEquals(4, scoreFieldCounts.size());
+ List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet());
+ Collections.sort(scoreFieldKeys);
+ Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001);
+ Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001);
+ Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001);
+ Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001);
+ Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0)));
+ Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1)));
+ Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2)));
+ Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3)));
Map<String, Long> isAlertCounts = facetCounts.get("is_alert");
Assert.assertEquals(2, isAlertCounts.size());
Assert.assertEquals(new Long(6), isAlertCounts.get("true"));
@@ -440,7 +504,7 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(FieldType.LONG, broTypes.get("long_field"));
Assert.assertEquals(FieldType.DATE, broTypes.get("timestamp"));
Assert.assertEquals(FieldType.FLOAT, broTypes.get("latitude"));
- Assert.assertEquals(FieldType.DOUBLE, broTypes.get("double_field"));
+ Assert.assertEquals(FieldType.DOUBLE, broTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, broTypes.get("is_alert"));
Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point"));
Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field"));
@@ -453,7 +517,7 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(FieldType.LONG, snortTypes.get("long_field"));
Assert.assertEquals(FieldType.DATE, snortTypes.get("timestamp"));
Assert.assertEquals(FieldType.FLOAT, snortTypes.get("latitude"));
- Assert.assertEquals(FieldType.DOUBLE, snortTypes.get("double_field"));
+ Assert.assertEquals(FieldType.DOUBLE, snortTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, snortTypes.get("is_alert"));
Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point"));
Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field"));
@@ -486,7 +550,7 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp"));
Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
- Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("double_field"));
+ Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
}
@@ -527,6 +591,145 @@ public abstract class SearchIntegrationTest {
SearchResponse response = dao.search(request);
Assert.assertEquals(0, response.getTotal());
}
+ // Group by test case, default order is count descending
+ {
+ GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class);
+ GroupResponse response = dao.group(request);
+ Assert.assertEquals("is_alert", response.getGroupedBy());
+ List<GroupResult> isAlertGroups = response.getGroupResults();
+ Assert.assertEquals(2, isAlertGroups.size());
+
+ // isAlert == true group
+ GroupResult trueGroup = isAlertGroups.get(0);
+ Assert.assertEquals("true", trueGroup.getKey());
+ Assert.assertEquals(6, trueGroup.getTotal());
+ Assert.assertEquals("latitude", trueGroup.getGroupedBy());
+ Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001);
+ List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults();
+ Assert.assertEquals(2, trueLatitudeGroups.size());
+
+
+ // isAlert == true && latitude == 48.5839 group
+ GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0);
+ Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001);
+ Assert.assertEquals(5, trueLatitudeGroup2.getTotal());
+ Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001);
+
+ // isAlert == true && latitude == 48.0001 group
+ GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1);
+ Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001);
+ Assert.assertEquals(1, trueLatitudeGroup1.getTotal());
+ Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001);
+
+ // isAlert == false group
+ GroupResult falseGroup = isAlertGroups.get(1);
+ Assert.assertEquals("false", falseGroup.getKey());
+ Assert.assertEquals("latitude", falseGroup.getGroupedBy());
+ Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001);
+ List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults();
+ Assert.assertEquals(2, falseLatitudeGroups.size());
+
+ // isAlert == false && latitude == 48.5839 group
+ GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0);
+ Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001);
+ Assert.assertEquals(3, falseLatitudeGroup2.getTotal());
+ Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001);
+
+ // isAlert == false && latitude == 48.0001 group
+ GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1);
+ Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001);
+ Assert.assertEquals(1, falseLatitudeGroup1.getTotal());
+ Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001);
+ }
+ // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending
+ {
+ GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class);
+ GroupResponse response = dao.group(request);
+ Assert.assertEquals("is_alert", response.getGroupedBy());
+ List<GroupResult> isAlertGroups = response.getGroupResults();
+ Assert.assertEquals(2, isAlertGroups.size());
+
+ // isAlert == false group
+ GroupResult falseGroup = isAlertGroups.get(0);
+ Assert.assertEquals(4, falseGroup.getTotal());
+ Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy());
+ List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults();
+ Assert.assertEquals(4, falseIpSrcAddrGroups.size());
+
+ // isAlert == false && ip_src_addr == 192.168.1.8 group
+ GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0);
+ Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.7 group
+ GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1);
+ Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.6 group
+ GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2);
+ Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.2 group
+ GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3);
+ Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults());
+
+ // isAlert == false group
+ GroupResult trueGroup = isAlertGroups.get(1);
+ Assert.assertEquals(6, trueGroup.getTotal());
+ Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy());
+ List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults();
+ Assert.assertEquals(4, trueIpSrcAddrGroups.size());
+
+ // isAlert == false && ip_src_addr == 192.168.1.5 group
+ GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0);
+ Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey());
+ Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.4 group
+ GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1);
+ Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey());
+ Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.3 group
+ GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2);
+ Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey());
+ Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.1 group
+ GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3);
+ Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey());
+ Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults());
+ }
+ //Bad group query
+ {
+ GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class);
+ try {
+ dao.group(request);
+ Assert.fail("Exception expected, but did not come.");
+ }
+ catch(InvalidSearchException ise) {
+ Assert.assertEquals("Could not execute search", ise.getMessage());
+ }
+ }
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index 4117d69..a5cae38 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -205,6 +205,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
index 7574418..bf3342c 100644
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
+++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java
@@ -82,7 +82,15 @@ public class ConfigurationFunctionsTest {
"parserConfig" : { },
"fieldTransformations" : [ ],
"readMetadata":false,
- "mergeMetadata":false
+ "mergeMetadata":false,
+ "parserParallelism" : 1,
+ "errorWriterParallelism" : 1,
+ "spoutNumTasks" : 1,
+ "stormConfig" : {},
+ "errorWriterNumTasks":1,
+ "spoutConfig":{},
+ "parserNumTasks":1,
+ "spoutParallelism":1
}
*/
@Multiline
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index ea4f1dd..d4a984d 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -103,6 +103,17 @@ then it is assumed to be a regex and will match any topic matching the pattern (
* `mergeMetadata` : Boolean indicating whether to merge metadata with the message or not (`false` by default). See below for a discussion about metadata.
* `parserConfig` : A JSON Map representing the parser implementation specific configuration.
* `fieldTransformations` : An array of complex objects representing the transformations to be done on the message generated from the parser before writing out to the kafka topic.
+* `spoutParallelism` : The kafka spout parallelism (default to `1`). This can be overridden on the command line.
+* `spoutNumTasks` : The number of tasks for the spout (default to `1`). This can be overridden on the command line.
+* `parserParallelism` : The parser bolt parallelism (default to `1`). This can be overridden on the command line.
+* `parserNumTasks` : The number of tasks for the parser bolt (default to `1`). This can be overridden on the command line.
+* `errorWriterParallelism` : The error writer bolt parallelism (default to `1`). This can be overridden on the command line.
+* `errorWriterNumTasks` : The number of tasks for the error writer bolt (default to `1`). This can be overridden on the command line.
+* `numWorkers` : The number of workers to use in the topology (default is the storm default of `1`).
+* `numAckers` : The number of acker executors to use in the topology (default is the storm default of `1`).
+* `spoutConfig` : A map representing a custom spout config (this is a map). This can be overridden on the command line.
+* `securityProtocol` : The security protocol to use for reading from kafka (this is a string). This can be overridden on the command line and also specified in the spout config via the `security.protocol` key. If both are specified, then they are merged and the CLI will take precedence.
+* `stormConfig` : The storm config to use (this is a map). This can be overridden on the command line. If both are specified, they are merged with CLI properties taking precedence.
The `fieldTransformations` is a complex object which defines a
transformation which can be done to a message. This transformation can
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index b3b5657..b80343a 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -276,6 +276,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 53d3d99..6c0dc28 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -18,9 +18,11 @@
package org.apache.metron.parsers.topology;
import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.metron.parsers.topology.config.ValueSupplier;
import org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder;
import org.apache.metron.storm.kafka.flux.SpoutConfiguration;
import org.apache.metron.storm.kafka.flux.StormKafkaSpout;
+import org.apache.storm.Config;
import org.apache.storm.kafka.spout.KafkaSpout;
import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.topology.TopologyBuilder;
@@ -47,34 +49,57 @@ import java.util.*;
*/
public class ParserTopologyBuilder {
+ public static class ParserTopology {
+ private TopologyBuilder builder;
+ private Config topologyConfig;
+
+ private ParserTopology(TopologyBuilder builder, Config topologyConfig) {
+ this.builder = builder;
+ this.topologyConfig = topologyConfig;
+ }
+
+
+ public TopologyBuilder getBuilder() {
+ return builder;
+ }
+
+ public Config getTopologyConfig() {
+ return topologyConfig;
+ }
+ }
+
/**
* Builds a Storm topology that parses telemetry data received from an external sensor.
*
* @param zookeeperUrl Zookeeper URL
* @param brokerUrl Kafka Broker URL
* @param sensorType Type of sensor
- * @param spoutParallelism Parallelism hint for the spout
- * @param spoutNumTasks Number of tasks for the spout
- * @param parserParallelism Parallelism hint for the parser bolt
- * @param parserNumTasks Number of tasks for the parser bolt
- * @param errorWriterParallelism Parallelism hint for the bolt that handles errors
- * @param errorWriterNumTasks Number of tasks for the bolt that handles errors
- * @param kafkaSpoutConfig Configuration options for the kafka spout
+ * @param spoutParallelismSupplier Supplier for the parallelism hint for the spout
+ * @param spoutNumTasksSupplier Supplier for the number of tasks for the spout
+ * @param parserParallelismSupplier Supplier for the parallelism hint for the parser bolt
+ * @param parserNumTasksSupplier Supplier for the number of tasks for the parser bolt
+ * @param errorWriterParallelismSupplier Supplier for the parallelism hint for the bolt that handles errors
+ * @param errorWriterNumTasksSupplier Supplier for the number of tasks for the bolt that handles errors
+ * @param kafkaSpoutConfigSupplier Supplier for the configuration options for the kafka spout
+ * @param securityProtocolSupplier Supplier for the security protocol
+ * @param outputTopic The output kafka topic
+ * @param stormConfigSupplier Supplier for the storm config
* @return A Storm topology that parses telemetry data received from an external sensor
* @throws Exception
*/
- public static TopologyBuilder build(String zookeeperUrl,
+ public static ParserTopology build(String zookeeperUrl,
Optional<String> brokerUrl,
String sensorType,
- int spoutParallelism,
- int spoutNumTasks,
- int parserParallelism,
- int parserNumTasks,
- int errorWriterParallelism,
- int errorWriterNumTasks,
- Map<String, Object> kafkaSpoutConfig,
- Optional<String> securityProtocol,
- Optional<String> outputTopic
+ ValueSupplier<Integer> spoutParallelismSupplier,
+ ValueSupplier<Integer> spoutNumTasksSupplier,
+ ValueSupplier<Integer> parserParallelismSupplier,
+ ValueSupplier<Integer> parserNumTasksSupplier,
+ ValueSupplier<Integer> errorWriterParallelismSupplier,
+ ValueSupplier<Integer> errorWriterNumTasksSupplier,
+ ValueSupplier<Map> kafkaSpoutConfigSupplier,
+ ValueSupplier<String> securityProtocolSupplier,
+ Optional<String> outputTopic,
+ ValueSupplier<Config> stormConfigSupplier
) throws Exception {
@@ -82,6 +107,14 @@ public class ParserTopologyBuilder {
// fetch configuration from zookeeper
ParserConfigurations configs = new ParserConfigurations();
SensorParserConfig parserConfig = getSensorParserConfig(zookeeperUrl, sensorType, configs);
+ int spoutParallelism = spoutParallelismSupplier.get(parserConfig, Integer.class);
+ int spoutNumTasks = spoutNumTasksSupplier.get(parserConfig, Integer.class);
+ int parserParallelism = parserParallelismSupplier.get(parserConfig, Integer.class);
+ int parserNumTasks = parserNumTasksSupplier.get(parserConfig, Integer.class);
+ int errorWriterParallelism = errorWriterParallelismSupplier.get(parserConfig, Integer.class);
+ int errorWriterNumTasks = errorWriterNumTasksSupplier.get(parserConfig, Integer.class);
+ Map<String, Object> kafkaSpoutConfig = kafkaSpoutConfigSupplier.get(parserConfig, Map.class);
+ Optional<String> securityProtocol = Optional.ofNullable(securityProtocolSupplier.get(parserConfig, String.class));
// create the spout
TopologyBuilder builder = new TopologyBuilder();
@@ -103,7 +136,7 @@ public class ParserTopologyBuilder {
.shuffleGrouping("parserBolt", Constants.ERROR_STREAM);
}
- return builder;
+ return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));
}
/**
@@ -243,16 +276,16 @@ public class ParserTopologyBuilder {
* @throws Exception
*/
private static SensorParserConfig getSensorParserConfig(String zookeeperUrl, String sensorType, ParserConfigurations configs) throws Exception {
- CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl);
- client.start();
- ConfigurationsUtils.updateParserConfigsFromZookeeper(configs, client);
- SensorParserConfig parserConfig = configs.getSensorParserConfig(sensorType);
- if (parserConfig == null) {
- throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." +
- " Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command.");
+ try(CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl)) {
+ client.start();
+ ConfigurationsUtils.updateParserConfigsFromZookeeper(configs, client);
+ SensorParserConfig parserConfig = configs.getSensorParserConfig(sensorType);
+ if (parserConfig == null) {
+ throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." +
+ " Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command.");
+ }
+ return parserConfig;
}
- client.close();
- return parserConfig;
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
index 8ff4f93..b5ee628 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
@@ -18,10 +18,14 @@
package org.apache.metron.parsers.topology;
import org.apache.metron.common.Constants;
+import org.apache.metron.parsers.topology.config.ValueSupplier;
import org.apache.metron.storm.kafka.flux.SpoutConfiguration;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
+import org.apache.storm.generated.AlreadyAliveException;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -235,12 +239,19 @@ public class ParserTopologyCLI {
return has(cli)?cli.getOptionValue(shortCode):def;
}
- public static Config getConfig(CommandLine cli) {
- Config config = new Config();
+ public static Optional<Config> getConfig(CommandLine cli) {
+ return getConfig(cli, new Config());
+ }
+
+ public static Optional<Config> getConfig(CommandLine cli, Config config) {
+ if(EXTRA_OPTIONS.has(cli)) {
+ Map<String, Object> extraOptions = readJSONMapFromFile(new File(EXTRA_OPTIONS.get(cli)));
+ config.putAll(extraOptions);
+ }
for(ParserOptions option : ParserOptions.values()) {
config = option.configHandler.apply(new Arg(config, option.get(cli)));
}
- return config;
+ return config.isEmpty()?Optional.empty():Optional.of(config);
}
public static CommandLine parse(CommandLineParser parser, String[] args) throws ParseException {
@@ -273,65 +284,172 @@ public class ParserTopologyCLI {
}
}
+ private static CommandLine parse(Options options, String[] args) {
+ CommandLineParser parser = new PosixParser();
+ try {
+ return ParserOptions.parse(parser, args);
+ } catch (ParseException pe) {
+ pe.printStackTrace();
+ final HelpFormatter usageFormatter = new HelpFormatter();
+ usageFormatter.printHelp("ParserTopologyCLI", null, options, null, true);
+ System.exit(-1);
+ return null;
+ }
+ }
+
+ public ParserTopologyBuilder.ParserTopology createParserTopology(final CommandLine cmd) throws Exception {
+ String zookeeperUrl = ParserOptions.ZK_QUORUM.get(cmd);
+ Optional<String> brokerUrl = ParserOptions.BROKER_URL.has(cmd)?Optional.of(ParserOptions.BROKER_URL.get(cmd)):Optional.empty();
+ String sensorType= ParserOptions.SENSOR_TYPE.get(cmd);
+
+ /*
+ It bears mentioning why we're creating this ValueSupplier indirection here.
+ As a separation of responsibilities, the CLI class defines the order of precedence
+ for the various topological and structural properties for creating a parser. This is
+ desirable because there are now (i.e. integration tests)
+ and may be in the future (i.e. a REST service to start parsers without using the CLI)
+ other mechanisms to construct parser topologies. It's sensible to split those concerns..
+
+ Unfortunately, determining the structural parameters for a parser requires interacting with
+ external services (e.g. zookeeper) that are set up well within the ParserTopology class.
+ Rather than pulling the infrastructure to interact with those services out and moving it into the
+ CLI class and breaking that separation of concerns, we've created a supplier
+ indirection where are providing the logic as to how to create precedence in the CLI class
+ without owning the responsibility of constructing the infrastructure where the values are
+ necessarily supplied.
+
+ */
+ ValueSupplier<Integer> spoutParallelism = (parserConfig, clazz) -> {
+ if(ParserOptions.SPOUT_PARALLELISM.has(cmd)) {
+ return Integer.parseInt(ParserOptions.SPOUT_PARALLELISM.get(cmd, "1"));
+ }
+ return Optional.ofNullable(parserConfig.getSpoutParallelism()).orElse(1);
+ };
+ ValueSupplier<Integer> spoutNumTasks = (parserConfig, clazz) -> {
+ if(ParserOptions.SPOUT_NUM_TASKS.has(cmd)) {
+ return Integer.parseInt(ParserOptions.SPOUT_NUM_TASKS.get(cmd, "1"));
+ }
+ return Optional.ofNullable(parserConfig.getSpoutNumTasks()).orElse(1);
+ };
+ ValueSupplier<Integer> parserParallelism = (parserConfig, clazz) -> {
+ if(ParserOptions.PARSER_PARALLELISM.has(cmd)) {
+ return Integer.parseInt(ParserOptions.PARSER_PARALLELISM.get(cmd, "1"));
+ }
+ return Optional.ofNullable(parserConfig.getParserParallelism()).orElse(1);
+ };
+
+ ValueSupplier<Integer> parserNumTasks = (parserConfig, clazz) -> {
+ if(ParserOptions.PARSER_NUM_TASKS.has(cmd)) {
+ return Integer.parseInt(ParserOptions.PARSER_NUM_TASKS.get(cmd, "1"));
+ }
+ return Optional.ofNullable(parserConfig.getParserNumTasks()).orElse(1);
+ };
+
+ ValueSupplier<Integer> errorParallelism = (parserConfig, clazz) -> {
+ if(ParserOptions.ERROR_WRITER_PARALLELISM.has(cmd)) {
+ return Integer.parseInt(ParserOptions.ERROR_WRITER_PARALLELISM.get(cmd, "1"));
+ }
+ return Optional.ofNullable(parserConfig.getErrorWriterParallelism()).orElse(1);
+ };
+
+ ValueSupplier<Integer> errorNumTasks = (parserConfig, clazz) -> {
+ if(ParserOptions.ERROR_WRITER_NUM_TASKS.has(cmd)) {
+ return Integer.parseInt(ParserOptions.ERROR_WRITER_NUM_TASKS.get(cmd, "1"));
+ }
+ return Optional.ofNullable(parserConfig.getErrorWriterNumTasks()).orElse(1);
+ };
+
+ ValueSupplier<Map> spoutConfig = (parserConfig, clazz) -> {
+ if(ParserOptions.SPOUT_CONFIG.has(cmd)) {
+ return readJSONMapFromFile(new File(ParserOptions.SPOUT_CONFIG.get(cmd)));
+ }
+ return Optional.ofNullable(parserConfig.getSpoutConfig()).orElse(new HashMap<>());
+ };
+
+ ValueSupplier<String> securityProtocol = (parserConfig, clazz) -> {
+ Optional<String> sp = Optional.empty();
+ if (ParserOptions.SECURITY_PROTOCOL.has(cmd)) {
+ sp = Optional.of(ParserOptions.SECURITY_PROTOCOL.get(cmd));
+ }
+ if (!sp.isPresent()) {
+ sp = getSecurityProtocol(sp, spoutConfig.get(parserConfig, Map.class));
+ }
+ return sp.orElse(Optional.ofNullable(parserConfig.getSecurityProtocol()).orElse(null));
+ };
+
+ ValueSupplier<Config> stormConf = (parserConfig, clazz) -> {
+ Map<String, Object> c = parserConfig.getStormConfig();
+ Config finalConfig = new Config();
+ if(c != null && !c.isEmpty()) {
+ finalConfig.putAll(c);
+ }
+ if(parserConfig.getNumAckers() != null) {
+ Config.setNumAckers(finalConfig, parserConfig.getNumAckers());
+ }
+ if(parserConfig.getNumWorkers() != null) {
+ Config.setNumWorkers(finalConfig, parserConfig.getNumWorkers());
+ }
+ return ParserOptions.getConfig(cmd, finalConfig).orElse(finalConfig);
+ };
+
+ Optional<String> outputTopic = ParserOptions.OUTPUT_TOPIC.has(cmd)?Optional.of(ParserOptions.OUTPUT_TOPIC.get(cmd)):Optional.empty();
+
+ return getParserTopology(zookeeperUrl, brokerUrl, sensorType, spoutParallelism, spoutNumTasks, parserParallelism, parserNumTasks, errorParallelism, errorNumTasks, spoutConfig, securityProtocol, stormConf, outputTopic);
+ }
+
+ protected ParserTopologyBuilder.ParserTopology getParserTopology( String zookeeperUrl
+ , Optional<String> brokerUrl
+ , String sensorType
+ , ValueSupplier<Integer> spoutParallelism
+ , ValueSupplier<Integer> spoutNumTasks
+ , ValueSupplier<Integer> parserParallelism
+ , ValueSupplier<Integer> parserNumTasks
+ , ValueSupplier<Integer> errorParallelism
+ , ValueSupplier<Integer> errorNumTasks
+ , ValueSupplier<Map> spoutConfig
+ , ValueSupplier<String> securityProtocol
+ , ValueSupplier<Config> stormConf
+ , Optional<String> outputTopic
+ ) throws Exception
+ {
+ return ParserTopologyBuilder.build(zookeeperUrl,
+ brokerUrl,
+ sensorType,
+ spoutParallelism,
+ spoutNumTasks,
+ parserParallelism,
+ parserNumTasks,
+ errorParallelism,
+ errorNumTasks,
+ spoutConfig,
+ securityProtocol,
+ outputTopic,
+ stormConf
+ );
+ }
+
+
public static void main(String[] args) {
- Options options = new Options();
try {
- CommandLineParser parser = new PosixParser();
- CommandLine cmd = null;
- try {
- cmd = ParserOptions.parse(parser, args);
- } catch (ParseException pe) {
- pe.printStackTrace();
- final HelpFormatter usageFormatter = new HelpFormatter();
- usageFormatter.printHelp("ParserTopologyCLI", null, options, null, true);
- System.exit(-1);
- }
+ Options options = new Options();
+ final CommandLine cmd = parse(options, args);
if (cmd.hasOption("h")) {
final HelpFormatter usageFormatter = new HelpFormatter();
usageFormatter.printHelp("ParserTopologyCLI", null, options, null, true);
System.exit(0);
}
- String zookeeperUrl = ParserOptions.ZK_QUORUM.get(cmd);;
- Optional<String> brokerUrl = ParserOptions.BROKER_URL.has(cmd)?Optional.of(ParserOptions.BROKER_URL.get(cmd)):Optional.empty();
+ ParserTopologyCLI cli = new ParserTopologyCLI();
+ ParserTopologyBuilder.ParserTopology topology = cli.createParserTopology(cmd);
String sensorType= ParserOptions.SENSOR_TYPE.get(cmd);
- int spoutParallelism = Integer.parseInt(ParserOptions.SPOUT_PARALLELISM.get(cmd, "1"));
- int spoutNumTasks = Integer.parseInt(ParserOptions.SPOUT_NUM_TASKS.get(cmd, "1"));
- int parserParallelism = Integer.parseInt(ParserOptions.PARSER_PARALLELISM.get(cmd, "1"));
- int parserNumTasks= Integer.parseInt(ParserOptions.PARSER_NUM_TASKS.get(cmd, "1"));
- int errorParallelism = Integer.parseInt(ParserOptions.ERROR_WRITER_PARALLELISM.get(cmd, "1"));
- int errorNumTasks= Integer.parseInt(ParserOptions.ERROR_WRITER_NUM_TASKS.get(cmd, "1"));
- int invalidParallelism = Integer.parseInt(ParserOptions.INVALID_WRITER_PARALLELISM.get(cmd, "1"));
- int invalidNumTasks= Integer.parseInt(ParserOptions.INVALID_WRITER_NUM_TASKS.get(cmd, "1"));
- Map<String, Object> spoutConfig = new HashMap<>();
- if(ParserOptions.SPOUT_CONFIG.has(cmd)) {
- spoutConfig = readSpoutConfig(new File(ParserOptions.SPOUT_CONFIG.get(cmd)));
- }
- Optional<String> outputTopic = ParserOptions.OUTPUT_TOPIC.has(cmd)?Optional.of(ParserOptions.OUTPUT_TOPIC.get(cmd)):Optional.empty();
- Optional<String> securityProtocol = ParserOptions.SECURITY_PROTOCOL.has(cmd)?Optional.of(ParserOptions.SECURITY_PROTOCOL.get(cmd)):Optional.empty();
- securityProtocol = getSecurityProtocol(securityProtocol, spoutConfig);
- TopologyBuilder builder = ParserTopologyBuilder.build(zookeeperUrl,
- brokerUrl,
- sensorType,
- spoutParallelism,
- spoutNumTasks,
- parserParallelism,
- parserNumTasks,
- errorParallelism,
- errorNumTasks,
- spoutConfig,
- securityProtocol,
- outputTopic
- );
- Config stormConf = ParserOptions.getConfig(cmd);
if (ParserOptions.TEST.has(cmd)) {
- stormConf.put(Config.TOPOLOGY_DEBUG, true);
+ topology.getTopologyConfig().put(Config.TOPOLOGY_DEBUG, true);
LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(sensorType, stormConf, builder.createTopology());
+ cluster.submitTopology(sensorType, topology.getTopologyConfig(), topology.getBuilder().createTopology());
Utils.sleep(300000);
cluster.shutdown();
} else {
- StormSubmitter.submitTopology(sensorType, stormConf, builder.createTopology());
+ StormSubmitter.submitTopology(sensorType, topology.getTopologyConfig(), topology.getBuilder().createTopology());
}
} catch (Exception e) {
e.printStackTrace();
@@ -347,13 +465,13 @@ public class ParserTopologyCLI {
if(!ret.isPresent()) {
ret = Optional.ofNullable((String) spoutConfig.get("security.protocol"));
}
- if(ret.isPresent() && protocol.get().equalsIgnoreCase("PLAINTEXT")) {
+ if(ret.isPresent() && ret.get().equalsIgnoreCase("PLAINTEXT")) {
ret = Optional.empty();
}
return ret;
}
- private static Map<String, Object> readSpoutConfig(File inputFile) {
+ private static Map<String, Object> readJSONMapFromFile(File inputFile) {
String json = null;
if (inputFile.exists()) {
try {
[4/6] metron git commit: METRON-1136 Track Master in Feature Branch
(ottobackwards) closes apache/metron#752
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
index c92feab..1f300ea 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
@@ -17,9 +17,27 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.mock;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
import kafka.admin.AdminOperationException;
import kafka.admin.AdminUtils$;
import kafka.admin.RackAwareMode;
@@ -27,6 +45,8 @@ import kafka.utils.ZkUtils;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
@@ -43,25 +63,6 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.kafka.core.ConsumerFactory;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.verifyZeroInteractions;
-import static org.mockito.Mockito.when;
-import static org.powermock.api.mockito.PowerMockito.mock;
-
@SuppressWarnings("unchecked")
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.management.*") // resolve classloader conflict
@@ -72,6 +73,7 @@ public class KafkaServiceImplTest {
private ZkUtils zkUtils;
private KafkaConsumer<String, String> kafkaConsumer;
+ private KafkaProducer<String, String> kafkaProducer;
private ConsumerFactory<String, String> kafkaConsumerFactory;
private AdminUtils$ adminUtils;
@@ -90,11 +92,12 @@ public class KafkaServiceImplTest {
zkUtils = mock(ZkUtils.class);
kafkaConsumerFactory = mock(ConsumerFactory.class);
kafkaConsumer = mock(KafkaConsumer.class);
+ kafkaProducer = mock(KafkaProducer.class);
adminUtils = mock(AdminUtils$.class);
when(kafkaConsumerFactory.createConsumer()).thenReturn(kafkaConsumer);
- kafkaService = new KafkaServiceImpl(zkUtils, kafkaConsumerFactory, adminUtils);
+ kafkaService = new KafkaServiceImpl(zkUtils, kafkaConsumerFactory, kafkaProducer, adminUtils);
}
@Test
@@ -304,4 +307,16 @@ public class KafkaServiceImplTest {
verifyZeroInteractions(zkUtils, adminUtils);
}
+
+ @Test
+ public void produceMessageShouldProperlyProduceMessage() throws Exception {
+ final String topicName = "t";
+ final String message = "{\"field\":\"value\"}";
+
+ kafkaService.produceMessage(topicName, message);
+
+ String expectedMessage = "{\"field\":\"value\"}";
+ verify(kafkaProducer).send(new ProducerRecord<>(topicName, expectedMessage));
+ verifyZeroInteractions(kafkaProducer);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
index 1ded45e..cfc24f4 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
@@ -53,6 +53,8 @@ import java.io.FileWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -99,6 +101,8 @@ public class SensorParserConfigServiceImplTest {
@Multiline
public static String jsonMapJson;
+ private String user = "user1";
+
@Before
public void setUp() throws Exception {
BundleSystem.reset();
@@ -106,7 +110,10 @@ public class SensorParserConfigServiceImplTest {
curatorFramework = mock(CuratorFramework.class);
grokService = mock(GrokService.class);
environment = mock(Environment.class);
+ Authentication authentication = mock(Authentication.class);
+ when(authentication.getName()).thenReturn(user);
when(environment.getProperty(MetronRestConstants.HDFS_METRON_APPS_ROOT)).thenReturn("./target");
+ SecurityContextHolder.getContext().setAuthentication(authentication);
try(FileInputStream fis = new FileInputStream(new File("src/test/resources/zookeeper/bundle.properties"))) {
BundleProperties properties = BundleProperties.createBasicBundleProperties(fis, new HashMap<>());
properties.setProperty(BundleProperties.BUNDLE_LIBRARY_DIRECTORY,"./target");
@@ -307,8 +314,6 @@ public class SensorParserConfigServiceImplTest {
writer.write(grokStatement);
writer.close();
- when(grokService.saveTemporary(grokStatement, "squid")).thenReturn(patternFile);
-
assertEquals(new HashMap() {{
put("elapsed", 161);
put("code", 200);
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index bf02510..bbf96a0 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -89,6 +89,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-api/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml
index 912859d..8a15251 100644
--- a/metron-platform/metron-api/pom.xml
+++ b/metron-platform/metron-api/pom.xml
@@ -221,6 +221,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 390ec23..9356e13 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -403,6 +403,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
index f08e9c4..2d0ccd8 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java
@@ -37,7 +37,155 @@ public class SensorParserConfig implements Serializable {
private String invalidWriterClassName;
private Boolean readMetadata = false;
private Boolean mergeMetadata = false;
+ private Integer numWorkers = null;
+ private Integer numAckers= null;
+ private Integer spoutParallelism = 1;
+ private Integer spoutNumTasks = 1;
+ private Integer parserParallelism = 1;
+ private Integer parserNumTasks = 1;
+ private Integer errorWriterParallelism = 1;
+ private Integer errorWriterNumTasks = 1;
+ private Map<String, Object> spoutConfig = new HashMap<>();
+ private String securityProtocol = null;
+ private Map<String, Object> stormConfig = new HashMap<>();
+
+ /**
+ * Return the number of workers for the topology. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getNumWorkers() {
+ return numWorkers;
+ }
+
+ public void setNumWorkers(Integer numWorkers) {
+ this.numWorkers = numWorkers;
+ }
+
+ /**
+ * Return the number of ackers for the topology. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getNumAckers() {
+ return numAckers;
+ }
+
+ public void setNumAckers(Integer numAckers) {
+ this.numAckers = numAckers;
+ }
+
+ /**
+ * Return the spout parallelism. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getSpoutParallelism() {
+ return spoutParallelism;
+ }
+
+ public void setSpoutParallelism(Integer spoutParallelism) {
+ this.spoutParallelism = spoutParallelism;
+ }
+
+ /**
+ * Return the spout num tasks. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getSpoutNumTasks() {
+ return spoutNumTasks;
+ }
+
+ public void setSpoutNumTasks(Integer spoutNumTasks) {
+ this.spoutNumTasks = spoutNumTasks;
+ }
+
+ /**
+ * Return the parser parallelism. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getParserParallelism() {
+ return parserParallelism;
+ }
+
+ public void setParserParallelism(Integer parserParallelism) {
+ this.parserParallelism = parserParallelism;
+ }
+
+ /**
+ * Return the parser number of tasks. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getParserNumTasks() {
+ return parserNumTasks;
+ }
+
+ public void setParserNumTasks(Integer parserNumTasks) {
+ this.parserNumTasks = parserNumTasks;
+ }
+
+ /**
+ * Return the error writer bolt parallelism. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getErrorWriterParallelism() {
+ return errorWriterParallelism;
+ }
+
+ public void setErrorWriterParallelism(Integer errorWriterParallelism) {
+ this.errorWriterParallelism = errorWriterParallelism;
+ }
+
+ /**
+ * Return the error writer bolt number of tasks. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Integer getErrorWriterNumTasks() {
+ return errorWriterNumTasks;
+ }
+
+ public void setErrorWriterNumTasks(Integer errorWriterNumTasks) {
+ this.errorWriterNumTasks = errorWriterNumTasks;
+ }
+
+ /**
+ * Return the spout config. This includes kafka properties. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Map<String, Object> getSpoutConfig() {
+ return spoutConfig;
+ }
+ public void setSpoutConfig(Map<String, Object> spoutConfig) {
+ this.spoutConfig = spoutConfig;
+ }
+
+ /**
+ * Return security protocol to use. This property will be used for the parser unless overridden on the CLI.
+ * The order of precedence is CLI > spout config > config in the sensor parser config.
+ * @return
+ */
+ public String getSecurityProtocol() {
+ return securityProtocol;
+ }
+
+ public void setSecurityProtocol(String securityProtocol) {
+ this.securityProtocol = securityProtocol;
+ }
+
+ /**
+ * Return Storm topologyconfig. This property will be used for the parser unless overridden on the CLI.
+ * @return
+ */
+ public Map<String, Object> getStormConfig() {
+ return stormConfig;
+ }
+
+ public void setStormConfig(Map<String, Object> stormConfig) {
+ this.stormConfig = stormConfig;
+ }
+
+ /**
+ * Return whether or not to merge metadata sent into the message. If true, then metadata become proper fields.
+ * @return
+ */
public Boolean getMergeMetadata() {
return mergeMetadata;
}
@@ -46,6 +194,10 @@ public class SensorParserConfig implements Serializable {
this.mergeMetadata = mergeMetadata;
}
+ /**
+ * Return whether or not to read metadata at all.
+ * @return
+ */
public Boolean getReadMetadata() {
return readMetadata;
}
@@ -145,10 +297,21 @@ public class SensorParserConfig implements Serializable {
", writerClassName='" + writerClassName + '\'' +
", errorWriterClassName='" + errorWriterClassName + '\'' +
", invalidWriterClassName='" + invalidWriterClassName + '\'' +
- ", parserConfig=" + parserConfig +
- ", fieldTransformations=" + fieldTransformations +
", readMetadata=" + readMetadata +
", mergeMetadata=" + mergeMetadata +
+ ", numWorkers=" + numWorkers +
+ ", numAckers=" + numAckers +
+ ", spoutParallelism=" + spoutParallelism +
+ ", spoutNumTasks=" + spoutNumTasks +
+ ", parserParallelism=" + parserParallelism +
+ ", parserNumTasks=" + parserNumTasks +
+ ", errorWriterParallelism=" + errorWriterParallelism +
+ ", errorWriterNumTasks=" + errorWriterNumTasks +
+ ", spoutConfig=" + spoutConfig +
+ ", securityProtocol='" + securityProtocol + '\'' +
+ ", stormConfig=" + stormConfig +
+ ", parserConfig=" + parserConfig +
+ ", fieldTransformations=" + fieldTransformations +
'}';
}
@@ -171,12 +334,34 @@ public class SensorParserConfig implements Serializable {
return false;
if (getInvalidWriterClassName() != null ? !getInvalidWriterClassName().equals(that.getInvalidWriterClassName()) : that.getInvalidWriterClassName() != null)
return false;
- if (getParserConfig() != null ? !getParserConfig().equals(that.getParserConfig()) : that.getParserConfig() != null)
- return false;
if (getReadMetadata() != null ? !getReadMetadata().equals(that.getReadMetadata()) : that.getReadMetadata() != null)
return false;
if (getMergeMetadata() != null ? !getMergeMetadata().equals(that.getMergeMetadata()) : that.getMergeMetadata() != null)
return false;
+ if (getNumWorkers() != null ? !getNumWorkers().equals(that.getNumWorkers()) : that.getNumWorkers() != null)
+ return false;
+ if (getNumAckers() != null ? !getNumAckers().equals(that.getNumAckers()) : that.getNumAckers() != null)
+ return false;
+ if (getSpoutParallelism() != null ? !getSpoutParallelism().equals(that.getSpoutParallelism()) : that.getSpoutParallelism() != null)
+ return false;
+ if (getSpoutNumTasks() != null ? !getSpoutNumTasks().equals(that.getSpoutNumTasks()) : that.getSpoutNumTasks() != null)
+ return false;
+ if (getParserParallelism() != null ? !getParserParallelism().equals(that.getParserParallelism()) : that.getParserParallelism() != null)
+ return false;
+ if (getParserNumTasks() != null ? !getParserNumTasks().equals(that.getParserNumTasks()) : that.getParserNumTasks() != null)
+ return false;
+ if (getErrorWriterParallelism() != null ? !getErrorWriterParallelism().equals(that.getErrorWriterParallelism()) : that.getErrorWriterParallelism() != null)
+ return false;
+ if (getErrorWriterNumTasks() != null ? !getErrorWriterNumTasks().equals(that.getErrorWriterNumTasks()) : that.getErrorWriterNumTasks() != null)
+ return false;
+ if (getSpoutConfig() != null ? !getSpoutConfig().equals(that.getSpoutConfig()) : that.getSpoutConfig() != null)
+ return false;
+ if (getSecurityProtocol() != null ? !getSecurityProtocol().equals(that.getSecurityProtocol()) : that.getSecurityProtocol() != null)
+ return false;
+ if (getStormConfig() != null ? !getStormConfig().equals(that.getStormConfig()) : that.getStormConfig() != null)
+ return false;
+ if (getParserConfig() != null ? !getParserConfig().equals(that.getParserConfig()) : that.getParserConfig() != null)
+ return false;
return getFieldTransformations() != null ? getFieldTransformations().equals(that.getFieldTransformations()) : that.getFieldTransformations() == null;
}
@@ -189,10 +374,21 @@ public class SensorParserConfig implements Serializable {
result = 31 * result + (getWriterClassName() != null ? getWriterClassName().hashCode() : 0);
result = 31 * result + (getErrorWriterClassName() != null ? getErrorWriterClassName().hashCode() : 0);
result = 31 * result + (getInvalidWriterClassName() != null ? getInvalidWriterClassName().hashCode() : 0);
- result = 31 * result + (getParserConfig() != null ? getParserConfig().hashCode() : 0);
- result = 31 * result + (getFieldTransformations() != null ? getFieldTransformations().hashCode() : 0);
result = 31 * result + (getReadMetadata() != null ? getReadMetadata().hashCode() : 0);
result = 31 * result + (getMergeMetadata() != null ? getMergeMetadata().hashCode() : 0);
+ result = 31 * result + (getNumWorkers() != null ? getNumWorkers().hashCode() : 0);
+ result = 31 * result + (getNumAckers() != null ? getNumAckers().hashCode() : 0);
+ result = 31 * result + (getSpoutParallelism() != null ? getSpoutParallelism().hashCode() : 0);
+ result = 31 * result + (getSpoutNumTasks() != null ? getSpoutNumTasks().hashCode() : 0);
+ result = 31 * result + (getParserParallelism() != null ? getParserParallelism().hashCode() : 0);
+ result = 31 * result + (getParserNumTasks() != null ? getParserNumTasks().hashCode() : 0);
+ result = 31 * result + (getErrorWriterParallelism() != null ? getErrorWriterParallelism().hashCode() : 0);
+ result = 31 * result + (getErrorWriterNumTasks() != null ? getErrorWriterNumTasks().hashCode() : 0);
+ result = 31 * result + (getSpoutConfig() != null ? getSpoutConfig().hashCode() : 0);
+ result = 31 * result + (getSecurityProtocol() != null ? getSecurityProtocol().hashCode() : 0);
+ result = 31 * result + (getStormConfig() != null ? getStormConfig().hashCode() : 0);
+ result = 31 * result + (getParserConfig() != null ? getParserConfig().hashCode() : 0);
+ result = 31 * result + (getFieldTransformations() != null ? getFieldTransformations().hashCode() : 0);
return result;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 90c2c52..3fccc0a 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -384,7 +384,17 @@
<goal>shade</goal>
</goals>
<configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 40989c6..0005484 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -242,6 +242,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 01c113c..0d7a76c 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -19,59 +19,59 @@ package org.apache.metron.elasticsearch.dao;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
-import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
-import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.IndexDao;
-import org.apache.metron.indexing.dao.search.*;
+import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.metron.indexing.dao.search.Group;
+import org.apache.metron.indexing.dao.search.GroupOrder;
+import org.apache.metron.indexing.dao.search.GroupOrderType;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
+import org.apache.metron.indexing.dao.search.GroupResult;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.elasticsearch.action.get.GetRequestBuilder;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.*;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.action.update.UpdateResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.search.SortOrder;
+import org.apache.metron.indexing.dao.update.Document;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.index.mapper.ip.IpFieldMapper;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
+import org.elasticsearch.search.aggregations.metrics.sum.Sum;
+import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.sort.*;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Date;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Optional;
-import java.util.function.Function;
-import java.util.stream.Collectors;
public class ElasticsearchDao implements IndexDao {
private transient TransportClient client;
@@ -115,26 +115,17 @@ public class ElasticsearchDao implements IndexDao {
.size(searchRequest.getSize())
.from(searchRequest.getFrom())
.query(new QueryStringQueryBuilder(searchRequest.getQuery()))
-
.trackScores(true);
+ searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder())));
Optional<List<String>> fields = searchRequest.getFields();
if (fields.isPresent()) {
searchSourceBuilder.fields(fields.get());
} else {
searchSourceBuilder.fetchSource(true);
}
- for (SortField sortField : searchRequest.getSort()) {
- FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(sortField.getField());
- if (sortField.getSortOrder() == org.apache.metron.indexing.dao.search.SortOrder.DESC) {
- fieldSortBuilder.order(org.elasticsearch.search.sort.SortOrder.DESC);
- } else {
- fieldSortBuilder.order(org.elasticsearch.search.sort.SortOrder.ASC);
- }
- searchSourceBuilder = searchSourceBuilder.sort(fieldSortBuilder);
- }
Optional<List<String>> facetFields = searchRequest.getFacetFields();
if (facetFields.isPresent()) {
- addFacetFields(searchSourceBuilder, facetFields.get());
+ facetFields.get().forEach(field -> searchSourceBuilder.aggregation(new TermsBuilder(getFacentAggregationName(field)).field(field)));
}
String[] wildcardIndices = searchRequest.getIndices().stream().map(index -> String.format("%s*", index)).toArray(value -> new String[searchRequest.getIndices().size()]);
org.elasticsearch.action.search.SearchResponse elasticsearchResponse;
@@ -146,23 +137,8 @@ public class ElasticsearchDao implements IndexDao {
}
SearchResponse searchResponse = new SearchResponse();
searchResponse.setTotal(elasticsearchResponse.getHits().getTotalHits());
- searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> {
- SearchResult searchResult = new SearchResult();
- searchResult.setId(searchHit.getId());
- Map<String, Object> source;
- if (fields.isPresent()) {
- source = new HashMap<>();
- searchHit.getFields().forEach((key, value) -> {
- source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues());
- });
- } else {
- source = searchHit.getSource();
- }
- searchResult.setSource(source);
- searchResult.setScore(searchHit.getScore());
- searchResult.setIndex(searchHit.getIndex());
- return searchResult;
- }).collect(Collectors.toList()));
+ searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit ->
+ getSearchResult(searchHit, fields.isPresent())).collect(Collectors.toList()));
if (facetFields.isPresent()) {
Map<String, FieldType> commonColumnMetadata;
try {
@@ -176,6 +152,37 @@ public class ElasticsearchDao implements IndexDao {
}
@Override
+ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+ if(client == null) {
+ throw new InvalidSearchException("Uninitialized Dao! You must call init() prior to use.");
+ }
+ if (groupRequest.getGroups() == null || groupRequest.getGroups().size() == 0) {
+ throw new InvalidSearchException("At least 1 group must be provided.");
+ }
+ final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(new QueryStringQueryBuilder(groupRequest.getQuery()));
+ searchSourceBuilder.aggregation(getGroupsTermBuilder(groupRequest, 0));
+ String[] wildcardIndices = groupRequest.getIndices().stream().map(index -> String.format("%s*", index)).toArray(value -> new String[groupRequest.getIndices().size()]);
+ org.elasticsearch.action.search.SearchResponse elasticsearchResponse;
+ try {
+ elasticsearchResponse = client.search(new org.elasticsearch.action.search.SearchRequest(wildcardIndices)
+ .source(searchSourceBuilder)).actionGet();
+ } catch (SearchPhaseExecutionException e) {
+ throw new InvalidSearchException("Could not execute search", e);
+ }
+ Map<String, FieldType> commonColumnMetadata;
+ try {
+ commonColumnMetadata = getCommonColumnMetadata(groupRequest.getIndices());
+ } catch (IOException e) {
+ throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(groupRequest.getIndices().toArray())));
+ }
+ GroupResponse groupResponse = new GroupResponse();
+ groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField());
+ groupResponse.setGroupResults(getGroupResults(groupRequest, 0, elasticsearchResponse.getAggregations(), commonColumnMetadata));
+ return groupResponse;
+ }
+
+ @Override
public synchronized void init(AccessConfig config) {
if(this.client == null) {
this.client = ElasticsearchUtils.getClient(config.getGlobalConfigSupplier().get(), config.getOptionalSettings());
@@ -330,9 +337,17 @@ public class ElasticsearchDao implements IndexDao {
return latestIndices.values().toArray(new String[latestIndices.size()]);
}
- public void addFacetFields(SearchSourceBuilder searchSourceBuilder, List<String> fields) {
- for(String field: fields) {
- searchSourceBuilder = searchSourceBuilder.aggregation(new TermsBuilder(getAggregationName(field)).field(field));
+ private org.elasticsearch.search.sort.SortOrder getElasticsearchSortOrder(
+ org.apache.metron.indexing.dao.search.SortOrder sortOrder) {
+ return sortOrder == org.apache.metron.indexing.dao.search.SortOrder.DESC ?
+ org.elasticsearch.search.sort.SortOrder.DESC : org.elasticsearch.search.sort.SortOrder.ASC;
+ }
+
+ private Order getElasticsearchGroupOrder(GroupOrder groupOrder) {
+ if (groupOrder.getGroupOrderType() == GroupOrderType.TERM) {
+ return groupOrder.getSortOrder() == SortOrder.ASC ? Order.term(true) : Order.term(false);
+ } else {
+ return groupOrder.getSortOrder() == SortOrder.ASC ? Order.count(true) : Order.count(false);
}
}
@@ -340,33 +355,94 @@ public class ElasticsearchDao implements IndexDao {
Map<String, Map<String, Long>> fieldCounts = new HashMap<>();
for (String field: fields) {
Map<String, Long> valueCounts = new HashMap<>();
- Aggregation aggregation = aggregations.get(getAggregationName(field));
- if (aggregation instanceof LongTerms) {
- LongTerms longTerms = (LongTerms) aggregation;
- FieldType type = commonColumnMetadata.get(field);
- if (FieldType.IP.equals(type)) {
- longTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(IpFieldMapper.longToIp((Long) bucket.getKey()), bucket.getDocCount()));
- } else if (FieldType.BOOLEAN.equals(type)) {
- longTerms.getBuckets().stream().forEach(bucket -> {
- String key = (Long) bucket.getKey() == 1 ? "true" : "false";
- valueCounts.put(key, bucket.getDocCount());
- });
- } else {
- longTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(bucket.getKeyAsString(), bucket.getDocCount()));
- }
- } else if (aggregation instanceof DoubleTerms) {
- DoubleTerms doubleTerms = (DoubleTerms) aggregation;
- doubleTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(bucket.getKeyAsString(), bucket.getDocCount()));
- } else if (aggregation instanceof StringTerms) {
- StringTerms stringTerms = (StringTerms) aggregation;
- stringTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(bucket.getKeyAsString(), bucket.getDocCount()));
+ Aggregation aggregation = aggregations.get(getFacentAggregationName(field));
+ if (aggregation instanceof Terms) {
+ Terms terms = (Terms) aggregation;
+ terms.getBuckets().stream().forEach(bucket -> valueCounts.put(formatKey(bucket.getKey(), commonColumnMetadata.get(field)), bucket.getDocCount()));
}
fieldCounts.put(field, valueCounts);
}
return fieldCounts;
}
- private String getAggregationName(String field) {
+ private String formatKey(Object key, FieldType type) {
+ if (FieldType.IP.equals(type)) {
+ return IpFieldMapper.longToIp((Long) key);
+ } else if (FieldType.BOOLEAN.equals(type)) {
+ return (Long) key == 1 ? "true" : "false";
+ } else {
+ return key.toString();
+ }
+ }
+
+ private TermsBuilder getGroupsTermBuilder(GroupRequest groupRequest, int index) {
+ List<Group> groups = groupRequest.getGroups();
+ Group group = groups.get(index);
+ String aggregationName = getGroupByAggregationName(group.getField());
+ TermsBuilder termsBuilder = new TermsBuilder(aggregationName)
+ .field(group.getField())
+ .size(accessConfig.getMaxSearchGroups())
+ .order(getElasticsearchGroupOrder(group.getOrder()));
+ if (index < groups.size() - 1) {
+ termsBuilder.subAggregation(getGroupsTermBuilder(groupRequest, index + 1));
+ }
+ Optional<String> scoreField = groupRequest.getScoreField();
+ if (scoreField.isPresent()) {
+ termsBuilder.subAggregation(new SumBuilder(getSumAggregationName(scoreField.get())).field(scoreField.get()).missing(0));
+ }
+ return termsBuilder;
+ }
+
+ private List<GroupResult> getGroupResults(GroupRequest groupRequest, int index, Aggregations aggregations, Map<String, FieldType> commonColumnMetadata) {
+ List<Group> groups = groupRequest.getGroups();
+ String field = groups.get(index).getField();
+ Terms terms = aggregations.get(getGroupByAggregationName(field));
+ List<GroupResult> searchResultGroups = new ArrayList<>();
+ for(Bucket bucket: terms.getBuckets()) {
+ GroupResult groupResult = new GroupResult();
+ groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field)));
+ groupResult.setTotal(bucket.getDocCount());
+ Optional<String> scoreField = groupRequest.getScoreField();
+ if (scoreField.isPresent()) {
+ Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
+ groupResult.setScore(score.getValue());
+ }
+ if (index < groups.size() - 1) {
+ groupResult.setGroupedBy(groups.get(index + 1).getField());
+ groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata));
+ }
+ searchResultGroups.add(groupResult);
+ }
+ return searchResultGroups;
+ }
+
+ private SearchResult getSearchResult(SearchHit searchHit, boolean fieldsPresent) {
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(searchHit.getId());
+ Map<String, Object> source;
+ if (fieldsPresent) {
+ source = new HashMap<>();
+ searchHit.getFields().forEach((key, value) -> {
+ source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues());
+ });
+ } else {
+ source = searchHit.getSource();
+ }
+ searchResult.setSource(source);
+ searchResult.setScore(searchHit.getScore());
+ searchResult.setIndex(searchHit.getIndex());
+ return searchResult;
+ }
+
+ private String getFacentAggregationName(String field) {
return String.format("%s_count", field);
}
+
+ private String getGroupByAggregationName(String field) {
+ return String.format("%s_group", field);
+ }
+
+ private String getSumAggregationName(String field) {
+ return String.format("%s_score", field);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index d794ac9..5de9fd2 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -50,7 +50,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
* "long_field": { "type": "long" },
* "timestamp" : { "type": "date" },
* "latitude" : { "type": "float" },
- * "double_field": { "type": "double" },
+ * "score": { "type": "double" },
* "is_alert": { "type": "boolean" },
* "location_point": { "type": "geo_point" },
* "bro_field": { "type": "string" },
@@ -72,7 +72,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
* "long_field": { "type": "long" },
* "timestamp" : { "type": "date" },
* "latitude" : { "type": "float" },
- * "double_field": { "type": "double" },
+ * "score": { "type": "double" },
* "is_alert": { "type": "boolean" },
* "location_point": { "type": "geo_point" },
* "snort_field": { "type": "integer" },
@@ -91,6 +91,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
ret.init(
new AccessConfig() {{
setMaxSearchResults(100);
+ setMaxSearchGroups(100);
setGlobalConfigSupplier( () ->
new HashMap<String, Object>() {{
put("es.clustername", "metron");
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 37cb49f..dd3998b 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -94,6 +94,23 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>ch.hsr</groupId>
+ <artifactId>geohash</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vividsolutions</groupId>
+ <artifactId>jts-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>com.maxmind.geoip2</groupId>
<artifactId>geoip2</artifactId>
<version>${geoip.version}</version>
@@ -313,6 +330,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
index 0f9bf37..f5d20f7 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.enrichment.adapters.geo;
+import ch.hsr.geohash.WGS84Point;
import com.maxmind.db.CHMCache;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.AddressNotFoundException;
@@ -35,11 +36,16 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.InetAddressValidator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +63,42 @@ public enum GeoLiteDatabase {
private static volatile String hdfsLoc = GEO_HDFS_FILE_DEFAULT;
private static DatabaseReader reader = null;
+ public enum GeoProps {
+ LOC_ID("locID"),
+ COUNTRY("country"),
+ CITY("city"),
+ POSTAL_CODE("postalCode"),
+ DMA_CODE("dmaCode"),
+ LATITUDE("latitude"),
+ LONGITUDE("longitude"),
+ LOCATION_POINT("location_point"),
+ ;
+ Function<Map<String, String>, String> getter;
+ String simpleName;
+
+ GeoProps(String simpleName) {
+ this(simpleName, m -> m.get(simpleName));
+ }
+
+ GeoProps(String simpleName,
+ Function<Map<String, String>, String> getter
+ ) {
+ this.simpleName = simpleName;
+ this.getter = getter;
+ }
+ public String getSimpleName() {
+ return simpleName;
+ }
+
+ public String get(Map<String, String> map) {
+ return getter.apply(map);
+ }
+
+ public void set(Map<String, String> map, String val) {
+ map.put(simpleName, val);
+ }
+ }
+
public synchronized void updateIfNecessary(Map<String, Object> globalConfig) {
// Reload database if necessary (file changes on HDFS)
LOG.trace("[Metron] Determining if GeoIpDatabase update required");
@@ -143,24 +185,24 @@ public enum GeoLiteDatabase {
Postal postal = cityResponse.getPostal();
Location location = cityResponse.getLocation();
- geoInfo.put("locID", convertNullToEmptyString(city.getGeoNameId()));
- geoInfo.put("country", convertNullToEmptyString(country.getIsoCode()));
- geoInfo.put("city", convertNullToEmptyString(city.getName()));
- geoInfo.put("postalCode", convertNullToEmptyString(postal.getCode()));
- geoInfo.put("dmaCode", convertNullToEmptyString(location.getMetroCode()));
+ GeoProps.LOC_ID.set(geoInfo, convertNullToEmptyString(city.getGeoNameId()));
+ GeoProps.COUNTRY.set(geoInfo, convertNullToEmptyString(country.getIsoCode()));
+ GeoProps.CITY.set(geoInfo, convertNullToEmptyString(city.getName()));
+ GeoProps.POSTAL_CODE.set(geoInfo, convertNullToEmptyString(postal.getCode()));
+ GeoProps.DMA_CODE.set(geoInfo, convertNullToEmptyString(location.getMetroCode()));
Double latitudeRaw = location.getLatitude();
String latitude = convertNullToEmptyString(latitudeRaw);
- geoInfo.put("latitude", latitude);
+ GeoProps.LATITUDE.set(geoInfo, latitude);
Double longitudeRaw = location.getLongitude();
String longitude = convertNullToEmptyString(longitudeRaw);
- geoInfo.put("longitude", longitude);
+ GeoProps.LONGITUDE.set(geoInfo, longitude);
if (latitudeRaw == null || longitudeRaw == null) {
- geoInfo.put("location_point", "");
+ GeoProps.LOCATION_POINT.set(geoInfo, "");
} else {
- geoInfo.put("location_point", latitude + "," + longitude);
+ GeoProps.LOCATION_POINT.set(geoInfo, latitude + "," + longitude);
}
return Optional.of(geoInfo);
@@ -174,6 +216,23 @@ public enum GeoLiteDatabase {
return Optional.empty();
}
+ public Optional<WGS84Point> toPoint(Map<String, String> geoInfo) {
+ String latitude = GeoProps.LATITUDE.get(geoInfo);
+ String longitude = GeoProps.LONGITUDE.get(geoInfo);
+ if(latitude == null || longitude == null) {
+ return Optional.empty();
+ }
+
+ try {
+ double latD = Double.parseDouble(latitude.toString());
+ double longD = Double.parseDouble(longitude.toString());
+ return Optional.of(new WGS84Point(latD, longD));
+ } catch (NumberFormatException nfe) {
+ LOG.warn(String.format("Invalid lat/long: %s/%s: %s", latitude, longitude, nfe.getMessage()), nfe);
+ return Optional.empty();
+ }
+ }
+
protected String convertNullToEmptyString(Object raw) {
return raw == null ? "" : String.valueOf(raw);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
new file mode 100644
index 0000000..6af214e
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.WGS84Point;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+
+public enum DistanceStrategies implements DistanceStrategy {
+ HAVERSINE((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distHaversineRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ ),
+ LAW_OF_COSINES((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distLawOfCosinesRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ ),
+ VICENTY((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distVincentyRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ )
+ ;
+ DistanceStrategy strat;
+ DistanceStrategies(DistanceStrategy strat) {
+ this.strat = strat;
+ }
+
+ @Override
+ public double distance(WGS84Point point1, WGS84Point point2) {
+ return strat.distance(point1, point2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
new file mode 100644
index 0000000..0303986
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.WGS84Point;
+
+public interface DistanceStrategy {
+ public double distance(WGS84Point point1, WGS84Point point2);
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
new file mode 100644
index 0000000..902eea3
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.GeoHash;
+import ch.hsr.geohash.WGS84Point;
+import com.google.common.collect.Iterables;
+import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
+
+import java.util.AbstractMap;
+import java.util.Map;
+import java.util.Optional;
+
+public enum GeoHashUtil {
+ INSTANCE;
+
+ public Optional<String> computeHash(Double latitude, Double longitude, int precision) {
+ if(latitude == null || longitude == null) {
+ return Optional.empty();
+ }
+ return computeHash(new WGS84Point(latitude, longitude), precision);
+ }
+
+ public Optional<String> computeHash(WGS84Point point, int precision) {
+ GeoHash hash = GeoHash.withCharacterPrecision(point.getLatitude(), point.getLongitude(), precision);
+ return Optional.of(hash.toBase32());
+ }
+
+ public Optional<String> computeHash(Map<String, String> geoLoc, int precision) {
+ Optional<WGS84Point> point = GeoLiteDatabase.INSTANCE.toPoint(geoLoc);
+ if(point.isPresent()) {
+ return computeHash(point.get(), precision);
+ }
+ else {
+ return Optional.empty();
+ }
+ }
+
+ public Optional<WGS84Point> toPoint(String hash) {
+ if(hash == null) {
+ return Optional.empty();
+ }
+ GeoHash h = GeoHash.fromGeohashString(hash);
+ return Optional.ofNullable(h == null?null:h.getPoint());
+ }
+
+ public double distance(WGS84Point point1, WGS84Point point2, DistanceStrategy strategy) {
+ return strategy.distance(point1, point2);
+ }
+
+ public WGS84Point centroidOfHashes(Iterable<String> hashes) {
+ Iterable<WGS84Point> points = Iterables.transform(hashes, h -> toPoint(h).orElse(null));
+ return centroidOfPoints(points);
+ }
+
+ public WGS84Point centroidOfPoints(Iterable<WGS84Point> points) {
+ Iterable<WGS84Point> nonNullPoints = Iterables.filter(points, p -> p != null);
+ return centroid(Iterables.transform(nonNullPoints
+ , p -> new AbstractMap.SimpleImmutableEntry<>(p, 1)
+ )
+ );
+ }
+
+ public WGS84Point centroidOfWeightedPoints(Map<String, Number> points) {
+
+ Iterable<Map.Entry<WGS84Point, Number>> weightedPoints = Iterables.transform(points.entrySet()
+ , kv -> {
+ WGS84Point pt = toPoint(kv.getKey()).orElse(null);
+ return new AbstractMap.SimpleImmutableEntry<>(pt, kv.getValue());
+ });
+ return centroid(Iterables.filter(weightedPoints, kv -> kv.getKey() != null));
+ }
+
+ /**
+ * Find the equilibrium point of a weighted set of lat/long geo points.
+ * @param points The points and their weights (e.g. multiplicity)
+ * @return
+ */
+ private WGS84Point centroid(Iterable<Map.Entry<WGS84Point, Number>> points) {
+ double x = 0d
+ , y = 0d
+ , z = 0d
+ , totalWeight = 0d
+ ;
+ int n = 0;
+ /**
+ * So, it's first important to realize that long/lat are not cartesian, so simple weighted averaging
+ * is insufficient here as it denies the fact that we're not living on a flat square, but rather the surface of
+ * an ellipsoid. A crow, for instance, does not fly a straight line to an observer outside of Earth, but
+ * rather flies across the arc tracing the surface of earth, or a "great-earth arc". When computing the centroid
+ * you want to find the centroid of the points with distance defined as the great-earth arc.
+ *
+ * The general strategy is to:
+ * 1. Change coordinate systems from degrees on a WGS84 projection (e.g. lat/long)
+ * to a 3 dimensional cartesian surface atop a sphere approximating the earth.
+ * 2. Compute a weighted average of the cartesian coordinates
+ * 3. Change coordinate systems of the resulting centroid in cartesian space back to lat/long
+ *
+ * This is generally detailed at http://www.geomidpoint.com/example.html
+ */
+ for(Map.Entry<WGS84Point, Number> weightedPoint : points) {
+ WGS84Point pt = weightedPoint.getKey();
+ if(pt == null) {
+ continue;
+ }
+ double latRad = Math.toRadians(pt.getLatitude());
+ double longRad = Math.toRadians(pt.getLongitude());
+ double cosLat = Math.cos(latRad);
+ /*
+ Convert from lat/long coordinates to cartesian coordinates. The cartesian coordinate system is a right-hand,
+ rectangular, three-dimensional, earth-fixed coordinate system
+ with an origin at (0, 0, 0). The Z-axis, is parrallel to the axis of rotation of the earth. The Z-coordinate
+ is positive toward the North pole. The X-Y plane lies in the equatorial plane. The X-axis lies along the
+ intersection of the plane containing the prime meridian and the equatorial plane. The X-coordinate is positive
+ toward the intersection of the prime meridian and equator.
+
+ Please see https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
+ for more information about this coordinate transformation.
+ */
+ double ptX = cosLat * Math.cos(longRad);
+ double ptY = cosLat * Math.sin(longRad);
+ double ptZ = Math.sin(latRad);
+ double weight = weightedPoint.getValue().doubleValue();
+ x += ptX*weight;
+ y += ptY*weight;
+ z += ptZ*weight;
+ n++;
+ totalWeight += weight;
+ }
+ if(n == 0) {
+ return null;
+ }
+ //average the vector representation in cartesian space, forming the center of gravity in cartesian space
+ x /= totalWeight;
+ y /= totalWeight;
+ z /= totalWeight;
+
+ //convert the cartesian representation back to radians
+ double longitude = Math.atan2(y, x);
+ double hypotenuse = Math.sqrt(x*x + y*y);
+ double latitude = Math.atan2(z, hypotenuse);
+
+ //convert the radians back to degrees latitude and longitude.
+ return new WGS84Point(Math.toDegrees(latitude), Math.toDegrees(longitude));
+ }
+
+ public double maxDistanceHashes(Iterable<String> hashes, DistanceStrategy strategy) {
+ Iterable<WGS84Point> points = Iterables.transform(hashes, s -> toPoint(s).orElse(null));
+ return maxDistancePoints(Iterables.filter(points, p -> p != null), strategy);
+ }
+
+ public double maxDistancePoints(Iterable<WGS84Point> points, DistanceStrategy strategy) {
+ //Note: because distance is commutative, we only need search the upper triangle
+ int i = 0;
+ double max = Double.NaN;
+ for(WGS84Point pt1 : points) {
+ int j = 0;
+ for(WGS84Point pt2 : points) {
+ if(j <= i) {
+ double d = strategy.distance(pt1, pt2);
+ if(Double.isNaN(max)|| d > max) {
+ max = d;
+ }
+ j++;
+ }
+ else {
+ break;
+ }
+ }
+ i++;
+ }
+ return max;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
new file mode 100644
index 0000000..a1e64c5
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.stellar;
+
+import ch.hsr.geohash.WGS84Point;
+import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
+import org.apache.metron.enrichment.adapters.geo.hash.DistanceStrategies;
+import org.apache.metron.enrichment.adapters.geo.hash.DistanceStrategy;
+import org.apache.metron.enrichment.adapters.geo.hash.GeoHashUtil;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class GeoHashFunctions {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Stellar(name="TO_LATLONG"
+ ,namespace="GEOHASH"
+ ,description="Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)"
+ }
+ ,returns = "A map containing the latitude and longitude of the hash (keys \"latitude\" and \"longitude\")"
+ )
+ public static class ToLatLong implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ String hash = (String)args.get(0);
+ if(hash == null) {
+ return null;
+ }
+
+ Optional<WGS84Point> point = GeoHashUtil.INSTANCE.toPoint(hash);
+ if(point.isPresent()) {
+ Map<String, Object> ret = new HashMap<>();
+ ret.put(GeoLiteDatabase.GeoProps.LONGITUDE.getSimpleName(), point.get().getLongitude());
+ ret.put(GeoLiteDatabase.GeoProps.LATITUDE.getSimpleName(), point.get().getLatitude());
+ return ret;
+ }
+ return null;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="FROM_LATLONG"
+ ,namespace="GEOHASH"
+ ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long"
+ ,params = {
+ "latitude - The latitude",
+ "longitude - The longitude",
+ "character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long"
+ )
+ public static class FromLatLong implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 2) {
+ return null;
+ }
+ Object latObj = args.get(0);
+ Object longObj = args.get(1);
+ if(latObj == null || longObj == null) {
+ return null;
+ }
+ Double latitude = ConversionUtils.convert(latObj, Double.class);
+ Double longitude = ConversionUtils.convert(longObj, Double.class);
+ int charPrecision = 12;
+ if(args.size() > 2) {
+ charPrecision = ConversionUtils.convert(args.get(2), Integer.class);
+ }
+ Optional<String> ret = GeoHashUtil.INSTANCE.computeHash(latitude, longitude, charPrecision);
+ return ret.orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="FROM_LOC"
+ ,namespace="GEOHASH"
+ ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location"
+ ,params = {
+ "map - the latitude and logitude in a map (the output of GEO_GET)",
+ "character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location"
+ )
+ public static class FromLoc implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Map<String, String> map = (Map<String, String>) args.get(0);
+ if(map == null) {
+ return null;
+ }
+ int charPrecision = 12;
+ if(args.size() > 1) {
+ charPrecision = ConversionUtils.convert(args.get(1), Integer.class);
+ }
+ Optional<String> ret = GeoHashUtil.INSTANCE.computeHash(map, charPrecision);
+ return ret.orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+
+ @Stellar(name="DIST"
+ ,namespace="GEOHASH"
+ ,description="Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hash1 - The first location as a geohash",
+ "hash2 - The second location as a geohash",
+ "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default."
+ }
+ ,returns = "The distance in kilometers between the hashes"
+ )
+ public static class Dist implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 2) {
+ return null;
+ }
+ String hash1 = (String)args.get(0);
+ if(hash1 == null) {
+ return null;
+ }
+ Optional<WGS84Point> pt1 = GeoHashUtil.INSTANCE.toPoint(hash1);
+ String hash2 = (String)args.get(1);
+ if(hash2 == null) {
+ return null;
+ }
+ Optional<WGS84Point> pt2 = GeoHashUtil.INSTANCE.toPoint(hash2);
+ DistanceStrategy strat = DistanceStrategies.HAVERSINE;
+ if(args.size() > 2) {
+ strat = DistanceStrategies.valueOf((String) args.get(2));
+ }
+ if(pt1.isPresent() && pt2.isPresent()) {
+ return GeoHashUtil.INSTANCE.distance(pt1.get(), pt2.get(), strat);
+ }
+ return Double.NaN;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="MAX_DIST"
+ ,namespace="GEOHASH"
+ ,description="Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash)",
+ "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default."
+ }
+ ,returns = "The maximum distance in kilometers between any two locations"
+ )
+ public static class MaxDist implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Iterable<String> hashes = (Iterable<String>)args.get(0);
+ if(hashes == null) {
+ return null;
+ }
+ DistanceStrategy strat = DistanceStrategies.HAVERSINE;
+ if(args.size() > 1) {
+ strat = DistanceStrategies.valueOf((String) args.get(1));
+ }
+ return GeoHashUtil.INSTANCE.maxDistanceHashes(hashes, strat);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="CENTROID"
+ ,namespace="GEOHASH"
+ ,description="Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights"
+ ,"character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "The geohash of the centroid"
+ )
+ public static class Centroid implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Object o1 = args.get(0);
+ if(o1 == null) {
+ return null;
+ }
+ WGS84Point centroid = null;
+ if(o1 instanceof Map) {
+ centroid = GeoHashUtil.INSTANCE.centroidOfWeightedPoints((Map<String, Number>)o1);
+ }
+ else if(o1 instanceof Iterable) {
+ centroid = GeoHashUtil.INSTANCE.centroidOfHashes((Iterable<String>)o1);
+ }
+ if(centroid == null) {
+ return null;
+ }
+ Integer precision = 12;
+ if(args.size() > 1) {
+ precision = (Integer)args.get(1);
+ }
+ return GeoHashUtil.INSTANCE.computeHash(centroid, precision).orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+}
[2/6] metron git commit: METRON-1136 Track Master in Feature Branch
(ottobackwards) closes apache/metron#752
Posted by ot...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java
new file mode 100644
index 0000000..0ede0f8
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ValueSupplier.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.topology.config;
+
+import org.apache.metron.common.configuration.SensorParserConfig;
+
+
+/**
+ * Supplies a value given a sensor config.
+ * @param <T>
+ */
+public interface ValueSupplier<T> {
+ T get(SensorParserConfig config, Class<T> clazz);
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java
index 3e8e2db..63d9e52 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/components/ParserTopologyComponent.java
@@ -87,23 +87,29 @@ public class ParserTopologyComponent implements InMemoryComponent {
@Override
public void start() throws UnableToStartException {
try {
- TopologyBuilder topologyBuilder = ParserTopologyBuilder.build(topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY)
+ final Map<String, Object> stormConf = new HashMap<>();
+ stormConf.put(Config.TOPOLOGY_DEBUG, true);
+ ParserTopologyBuilder.ParserTopology topologyBuilder = ParserTopologyBuilder.build(topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY)
, Optional.ofNullable(brokerUrl)
, sensorType
- , 1
- , 1
- , 1
- , 1
- , 1
- , 1
- , null
- , Optional.empty()
+ , (x,y) -> 1
+ , (x,y) -> 1
+ , (x,y) -> 1
+ , (x,y) -> 1
+ , (x,y) -> 1
+ , (x,y) -> 1
+ , (x,y) -> new HashMap<>()
+ , (x,y) -> null
, Optional.ofNullable(outputTopic)
+ , (x,y) -> {
+ Config c = new Config();
+ c.putAll(stormConf);
+ return c;
+ }
);
- Map<String, Object> stormConf = new HashMap<>();
- stormConf.put(Config.TOPOLOGY_DEBUG, true);
+
stormCluster = new LocalCluster();
- stormCluster.submitTopology(sensorType, stormConf, topologyBuilder.createTopology());
+ stormCluster.submitTopology(sensorType, stormConf, topologyBuilder.getBuilder().createTopology());
} catch (Exception e) {
throw new UnableToStartException("Unable to start parser topology for sensorType: " + sensorType, e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java
index 5f536a5..97dac5a 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/topology/ParserTopologyCLITest.java
@@ -18,8 +18,12 @@
package org.apache.metron.parsers.topology;
+import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.commons.cli.Parser;
import org.apache.log4j.Level;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.parsers.topology.config.ValueSupplier;
import org.apache.metron.test.utils.UnitTestHelper;
import org.apache.storm.Config;
import com.google.common.collect.ImmutableMap;
@@ -34,7 +38,10 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.lang.ref.Reference;
import java.util.*;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
public class ParserTopologyCLITest {
@@ -142,7 +149,8 @@ public class ParserTopologyCLITest {
.with(ParserTopologyCLI.ParserOptions.NUM_MAX_TASK_PARALLELISM, "3")
.with(ParserTopologyCLI.ParserOptions.MESSAGE_TIMEOUT, "4")
.build(longOpt);
- Config config = ParserTopologyCLI.ParserOptions.getConfig(cli);
+ Optional<Config> configOptional = ParserTopologyCLI.ParserOptions.getConfig(cli);
+ Config config = configOptional.get();
Assert.assertEquals(1, config.get(Config.TOPOLOGY_WORKERS));
Assert.assertEquals(2, config.get(Config.TOPOLOGY_ACKER_EXECUTORS));
Assert.assertEquals(3, config.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM));
@@ -189,7 +197,8 @@ public class ParserTopologyCLITest {
.with(ParserTopologyCLI.ParserOptions.MESSAGE_TIMEOUT, "4")
.with(ParserTopologyCLI.ParserOptions.EXTRA_OPTIONS, extraFile.getAbsolutePath())
.build(longOpt);
- Config config = ParserTopologyCLI.ParserOptions.getConfig(cli);
+ Optional<Config> configOptional = ParserTopologyCLI.ParserOptions.getConfig(cli);
+ Config config = configOptional.get();
Assert.assertEquals(4, config.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
Assert.assertEquals("foo", config.get("string"));
Assert.assertEquals(1, config.get("integer"));
@@ -197,4 +206,416 @@ public class ParserTopologyCLITest {
extraFile.deleteOnExit();
}
}
+
+ private static class ParserInput {
+ private Integer spoutParallelism;
+ private Integer spoutNumTasks;
+ private Integer parserParallelism;
+ private Integer parserNumTasks;
+ private Integer errorParallelism;
+ private Integer errorNumTasks;
+ private Map<String, Object> spoutConfig;
+ private String securityProtocol;
+ private Config stormConf;
+
+ public ParserInput( ValueSupplier<Integer> spoutParallelism
+ , ValueSupplier<Integer> spoutNumTasks
+ , ValueSupplier<Integer> parserParallelism
+ , ValueSupplier<Integer> parserNumTasks
+ , ValueSupplier<Integer> errorParallelism
+ , ValueSupplier<Integer> errorNumTasks
+ , ValueSupplier<Map> spoutConfig
+ , ValueSupplier<String> securityProtocol
+ , ValueSupplier<Config> stormConf
+ , SensorParserConfig config
+ )
+ {
+ this.spoutParallelism = spoutParallelism.get(config, Integer.class);
+ this.spoutNumTasks = spoutNumTasks.get(config, Integer.class);
+ this.parserParallelism = parserParallelism.get(config, Integer.class);
+ this.parserNumTasks = parserNumTasks.get(config, Integer.class);
+ this.errorParallelism = errorParallelism.get(config, Integer.class);
+ this.errorNumTasks = errorNumTasks.get(config, Integer.class);
+ this.spoutConfig = spoutConfig.get(config, Map.class);
+ this.securityProtocol = securityProtocol.get(config, String.class);
+ this.stormConf = stormConf.get(config, Config.class);
+ }
+
+ public Integer getSpoutParallelism() {
+ return spoutParallelism;
+ }
+
+ public Integer getSpoutNumTasks() {
+ return spoutNumTasks;
+ }
+
+ public Integer getParserParallelism() {
+ return parserParallelism;
+ }
+
+ public Integer getParserNumTasks() {
+ return parserNumTasks;
+ }
+
+ public Integer getErrorParallelism() {
+ return errorParallelism;
+ }
+
+ public Integer getErrorNumTasks() {
+ return errorNumTasks;
+ }
+
+ public Map<String, Object> getSpoutConfig() {
+ return spoutConfig;
+ }
+
+ public String getSecurityProtocol() {
+ return securityProtocol;
+ }
+
+ public Config getStormConf() {
+ return stormConf;
+ }
+ }
+ /**
+{
+ "parserClassName": "org.apache.metron.parsers.GrokParser",
+ "sensorTopic": "squid",
+ "parserConfig": {
+ "grokPath": "/patterns/squid",
+ "patternLabel": "SQUID_DELIMITED",
+ "timestampField": "timestamp"
+ },
+ "fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : [ "full_hostname", "domain_without_subdomains" ]
+ ,"config" : {
+ "full_hostname" : "URL_TO_HOST(url)"
+ ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
+ }
+ }
+ ]
+}
+ */
+ @Multiline
+ public static String baseConfig;
+ private static SensorParserConfig getBaseConfig() {
+ try {
+ return JSONUtils.INSTANCE.load(baseConfig, SensorParserConfig.class);
+ } catch (IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ @Test
+ public void testSpoutParallelism() throws Exception {
+ testConfigOption(ParserTopologyCLI.ParserOptions.SPOUT_PARALLELISM
+ , "10"
+ , input -> input.getSpoutParallelism().equals(10)
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setSpoutParallelism(20);
+ return config;
+ }
+ , input -> input.getSpoutParallelism().equals(20)
+ );
+ }
+
+ @Test
+ public void testSpoutNumTasks() throws Exception {
+ testConfigOption(ParserTopologyCLI.ParserOptions.SPOUT_NUM_TASKS
+ , "10"
+ , input -> input.getSpoutNumTasks().equals(10)
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setSpoutNumTasks(20);
+ return config;
+ }
+ , input -> input.getSpoutNumTasks().equals(20)
+ );
+ }
+
+ @Test
+ public void testParserParallelism() throws Exception {
+ testConfigOption(ParserTopologyCLI.ParserOptions.PARSER_PARALLELISM
+ , "10"
+ , input -> input.getParserParallelism().equals(10)
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setParserParallelism(20);
+ return config;
+ }
+ , input -> input.getParserParallelism().equals(20)
+ );
+ }
+
+ @Test
+ public void testParserNumTasks() throws Exception {
+ testConfigOption(ParserTopologyCLI.ParserOptions.PARSER_NUM_TASKS
+ , "10"
+ , input -> input.getParserNumTasks().equals(10)
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setParserNumTasks(20);
+ return config;
+ }
+ , input -> input.getParserNumTasks().equals(20)
+ );
+ }
+
+ @Test
+ public void testErrorParallelism() throws Exception {
+ testConfigOption(ParserTopologyCLI.ParserOptions.ERROR_WRITER_PARALLELISM
+ , "10"
+ , input -> input.getErrorParallelism().equals(10)
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setErrorWriterParallelism(20);
+ return config;
+ }
+ , input -> input.getErrorParallelism().equals(20)
+ );
+ }
+
+ @Test
+ public void testErrorNumTasks() throws Exception {
+ testConfigOption(ParserTopologyCLI.ParserOptions.ERROR_WRITER_NUM_TASKS
+ , "10"
+ , input -> input.getErrorNumTasks().equals(10)
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setErrorWriterNumTasks(20);
+ return config;
+ }
+ , input -> input.getErrorNumTasks().equals(20)
+ );
+ }
+
+ @Test
+ public void testSecurityProtocol_fromCLI() throws Exception {
+ testConfigOption(ParserTopologyCLI.ParserOptions.SECURITY_PROTOCOL
+ , "PLAINTEXT"
+ , input -> input.getSecurityProtocol().equals("PLAINTEXT")
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setSecurityProtocol("KERBEROS");
+ return config;
+ }
+ , input -> input.getSecurityProtocol().equals("KERBEROS")
+ );
+ }
+
+ @Test
+ public void testSecurityProtocol_fromSpout() throws Exception {
+ //Ultimately the order of precedence is CLI > spout config > parser config
+ File extraConfig = File.createTempFile("spoutConfig", "json");
+ extraConfig.deleteOnExit();
+ writeMap(extraConfig, new HashMap<String, Object>() {{
+ put("security.protocol", "PLAINTEXTSASL");
+ }});
+ {
+ //Ensure that the CLI spout config takes precedence
+
+ testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {{
+ put(ParserTopologyCLI.ParserOptions.SPOUT_CONFIG, extraConfig.getAbsolutePath());
+ put(ParserTopologyCLI.ParserOptions.SECURITY_PROTOCOL, "PLAINTEXT");
+ }}
+ , input -> input.getSecurityProtocol().equals("PLAINTEXT")
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setSecurityProtocol("PLAINTEXTSASL_FROM_ZK");
+ return config;
+ }
+ , input -> input.getSecurityProtocol().equals("PLAINTEXTSASL_FROM_ZK")
+ );
+ }
+ {
+ //Ensure that the spout config takes precedence
+ testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {{
+ put(ParserTopologyCLI.ParserOptions.SPOUT_CONFIG, extraConfig.getAbsolutePath());
+ }}
+ , input -> input.getSecurityProtocol().equals("PLAINTEXTSASL")
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setSecurityProtocol("PLAINTEXTSASL_FROM_ZK");
+ return config;
+ }
+ , input -> input.getSecurityProtocol().equals("PLAINTEXTSASL_FROM_ZK")
+ );
+ }
+ }
+
+ @Test
+ public void testTopologyConfig_fromConfigExplicitly() throws Exception {
+ testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class)
+ {{
+ put(ParserTopologyCLI.ParserOptions.NUM_WORKERS, "10");
+ put(ParserTopologyCLI.ParserOptions.NUM_ACKERS, "20");
+ }}
+ , input -> {
+ Config c = input.getStormConf();
+ return (int)c.get(Config.TOPOLOGY_WORKERS) == 10
+ && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 20;
+ }
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setNumWorkers(100);
+ config.setNumAckers(200);
+ return config;
+ }
+ , input -> {
+ Config c = input.getStormConf();
+ return (int)c.get(Config.TOPOLOGY_WORKERS) == 100
+ && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 200
+ ;
+ }
+ );
+ }
+
+ @Test
+ public void testTopologyConfig() throws Exception {
+ File extraConfig = File.createTempFile("topologyConfig", "json");
+ extraConfig.deleteOnExit();
+ writeMap(extraConfig, new HashMap<String, Object>() {{
+ put(Config.TOPOLOGY_DEBUG, true);
+ }});
+ testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class)
+ {{
+ put(ParserTopologyCLI.ParserOptions.NUM_WORKERS, "10");
+ put(ParserTopologyCLI.ParserOptions.NUM_ACKERS, "20");
+ put(ParserTopologyCLI.ParserOptions.EXTRA_OPTIONS, extraConfig.getAbsolutePath());
+ }}
+ , input -> {
+ Config c = input.getStormConf();
+ return (int)c.get(Config.TOPOLOGY_WORKERS) == 10
+ && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 20
+ && (boolean)c.get(Config.TOPOLOGY_DEBUG);
+ }
+ , () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setStormConfig(
+ new HashMap<String, Object>() {{
+ put(Config.TOPOLOGY_WORKERS, 100);
+ put(Config.TOPOLOGY_ACKER_EXECUTORS, 200);
+ }}
+ );
+ return config;
+ }
+ , input -> {
+ Config c = input.getStormConf();
+ return (int)c.get(Config.TOPOLOGY_WORKERS) == 100
+ && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 200
+ && !c.containsKey(Config.TOPOLOGY_DEBUG);
+ }
+ );
+ }
+
+ @Test
+ public void testSpoutConfig() throws Exception {
+ File extraConfig = File.createTempFile("spoutConfig", "json");
+ extraConfig.deleteOnExit();
+ writeMap(extraConfig, new HashMap<String, Object>() {{
+ put("extra_config", "from_file");
+ }});
+ EnumMap<ParserTopologyCLI.ParserOptions, String> cliOptions = new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class)
+ {{
+ put(ParserTopologyCLI.ParserOptions.SPOUT_CONFIG, extraConfig.getAbsolutePath());
+ }};
+ Predicate<ParserInput> cliOverrideExpected = input -> {
+ return input.getSpoutConfig().get("extra_config").equals("from_file");
+ };
+
+ Predicate<ParserInput> configOverrideExpected = input -> {
+ return input.getSpoutConfig().get("extra_config").equals("from_zk")
+ ;
+ };
+
+ Supplier<SensorParserConfig> configSupplier = () -> {
+ SensorParserConfig config = getBaseConfig();
+ config.setSpoutConfig(
+ new HashMap<String, Object>() {{
+ put("extra_config", "from_zk");
+ }}
+ );
+ return config;
+ };
+ testConfigOption( cliOptions
+ , cliOverrideExpected
+ , configSupplier
+ , configOverrideExpected
+ );
+ }
+
+ private void writeMap(File outFile, Map<String, Object> config) throws IOException {
+ FileUtils.write(outFile, JSONUtils.INSTANCE.toJSON(config, true));
+ }
+
+ private void testConfigOption( ParserTopologyCLI.ParserOptions option
+ , String cliOverride
+ , Predicate<ParserInput> cliOverrideCondition
+ , Supplier<SensorParserConfig> configSupplier
+ , Predicate<ParserInput> configOverrideCondition
+ ) throws Exception {
+ testConfigOption(
+ new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {{
+ put(option, cliOverride);
+ }},
+ cliOverrideCondition,
+ configSupplier,
+ configOverrideCondition
+ );
+ }
+
+ private void testConfigOption( EnumMap<ParserTopologyCLI.ParserOptions, String> options
+ , Predicate<ParserInput> cliOverrideCondition
+ , Supplier<SensorParserConfig> configSupplier
+ , Predicate<ParserInput> configOverrideCondition
+ ) throws Exception {
+ //CLI Override
+ SensorParserConfig config = configSupplier.get();
+ {
+ CLIBuilder builder = new CLIBuilder().with(ParserTopologyCLI.ParserOptions.BROKER_URL, "mybroker")
+ .with(ParserTopologyCLI.ParserOptions.ZK_QUORUM, "myzk")
+ .with(ParserTopologyCLI.ParserOptions.SENSOR_TYPE, "mysensor");
+ for(Map.Entry<ParserTopologyCLI.ParserOptions, String> entry : options.entrySet()) {
+ builder.with(entry.getKey(), entry.getValue());
+ }
+ CommandLine cmd = builder.build(true);
+ ParserInput input = getInput(cmd, config);
+ Assert.assertTrue(cliOverrideCondition.test(input));
+ }
+ // Config Override
+ {
+ CLIBuilder builder = new CLIBuilder().with(ParserTopologyCLI.ParserOptions.BROKER_URL, "mybroker")
+ .with(ParserTopologyCLI.ParserOptions.ZK_QUORUM, "myzk")
+ .with(ParserTopologyCLI.ParserOptions.SENSOR_TYPE, "mysensor");
+ CommandLine cmd = builder.build(true);
+ ParserInput input = getInput(cmd, config);
+ Assert.assertTrue(configOverrideCondition.test(input));
+ }
+ }
+
+ private static ParserInput getInput(CommandLine cmd, SensorParserConfig config ) throws Exception {
+ final ParserInput[] parserInput = new ParserInput[]{null};
+ new ParserTopologyCLI() {
+ @Override
+ protected ParserTopologyBuilder.ParserTopology getParserTopology(String zookeeperUrl, Optional<String> brokerUrl, String sensorType, ValueSupplier<Integer> spoutParallelism, ValueSupplier<Integer> spoutNumTasks, ValueSupplier<Integer> parserParallelism, ValueSupplier<Integer> parserNumTasks, ValueSupplier<Integer> errorParallelism, ValueSupplier<Integer> errorNumTasks, ValueSupplier<Map> spoutConfig, ValueSupplier<String> securityProtocol, ValueSupplier<Config> stormConf, Optional<String> outputTopic) throws Exception {
+ parserInput[0] = new ParserInput( spoutParallelism
+ , spoutNumTasks
+ , parserParallelism
+ , parserNumTasks
+ , errorParallelism
+ , errorNumTasks
+ , spoutConfig
+ , securityProtocol
+ , stormConf
+ , config
+ );
+ return null;
+ }
+ }.createParserTopology(cmd);
+ return parserInput[0];
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-pcap-backend/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml
index 388b1e0..5878873 100644
--- a/metron-platform/metron-pcap-backend/pom.xml
+++ b/metron-platform/metron-pcap-backend/pom.xml
@@ -221,6 +221,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index be1fe33..97132c4 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -232,6 +232,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index 7d3152c..de6b3b8 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -238,6 +238,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
index fbb5561..b841249 100644
--- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/SourceHandler.java
@@ -64,6 +64,7 @@ public class SourceHandler {
this.rotationPolicy = rotationPolicy;
this.syncPolicy = syncPolicy;
this.fileNameFormat = fileNameFormat;
+ this.cleanupCallback = cleanupCallback;
initialize();
}
@@ -71,12 +72,26 @@ public class SourceHandler {
protected void handle(JSONObject message, String sensor, WriterConfiguration config, SyncPolicyCreator syncPolicyCreator) throws IOException {
byte[] bytes = (message.toJSONString() + "\n").getBytes();
synchronized (this.writeLock) {
- out.write(bytes);
+ try {
+ out.write(bytes);
+ } catch (IOException writeException) {
+ LOG.warn("IOException while writing output", writeException);
+ // If the stream is closed, attempt to rotate the file and try again, hoping it's transient
+ if (writeException.getMessage().contains("Stream Closed")) {
+ LOG.warn("Output Stream was closed. Attempting to rotate file and continue");
+ rotateOutputFile();
+ // If this write fails, the exception will be allowed to bubble up.
+ out.write(bytes);
+ } else {
+ throw writeException;
+ }
+ }
this.offset += bytes.length;
if (this.syncPolicy.mark(null, this.offset)) {
if (this.out instanceof HdfsDataOutputStream) {
- ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+ ((HdfsDataOutputStream) this.out)
+ .hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
} else {
this.out.hsync();
}
@@ -146,7 +161,7 @@ public class SourceHandler {
return path;
}
- private void closeOutputFile() throws IOException {
+ protected void closeOutputFile() throws IOException {
this.out.close();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
index 0118a15..832f8bf 100644
--- a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
+++ b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hdfs/HdfsWriterTest.java
@@ -18,11 +18,21 @@
package org.apache.metron.writer.hdfs;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
+import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.json.simple.JSONObject;
@@ -32,11 +42,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.util.*;
-
// Suppress ConstantConditions to avoid NPE warnings that only would occur on test failure anyway
@SuppressWarnings("ConstantConditions")
public class HdfsWriterTest {
@@ -410,6 +415,82 @@ public class HdfsWriterTest {
}
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSingleFileIfNoStreamClosed() throws Exception {
+ String function = "FORMAT('test-%s/%s', test.key, test.key)";
+ WriterConfiguration config = buildWriterConfiguration(function);
+ HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
+ writer.init(new HashMap<String, String>(), createTopologyContext(), config);
+
+ JSONObject message = new JSONObject();
+ message.put("test.key", "test.value");
+ ArrayList<JSONObject> messages = new ArrayList<>();
+ messages.add(message);
+ ArrayList<Tuple> tuples = new ArrayList<>();
+
+ CountSyncPolicy basePolicy = new CountSyncPolicy(5);
+ ClonedSyncPolicyCreator creator = new ClonedSyncPolicyCreator(basePolicy);
+
+ writer.write(SENSOR_NAME, config, tuples, messages);
+ writer.write(SENSOR_NAME, config, tuples, messages);
+ writer.close();
+
+ File outputFolder = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
+
+ // The message should show up twice, once in each file
+ ArrayList<String> expected = new ArrayList<>();
+ expected.add(message.toJSONString());
+ expected.add(message.toJSONString());
+
+ // Assert both messages are in the same file, because the stream stayed open
+ Assert.assertEquals(1, outputFolder.listFiles().length);
+ for (File file : outputFolder.listFiles()) {
+ List<String> lines = Files.readAllLines(file.toPath());
+ // One line per file
+ Assert.assertEquals(2, lines.size());
+ Assert.assertEquals(expected, lines);
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testHandleAttemptsRotateIfStreamClosed() throws Exception {
+ String function = "FORMAT('test-%s/%s', test.key, test.key)";
+ WriterConfiguration config = buildWriterConfiguration(function);
+ HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
+ writer.init(new HashMap<String, String>(), createTopologyContext(), config);
+
+ JSONObject message = new JSONObject();
+ message.put("test.key", "test.value");
+ ArrayList<JSONObject> messages = new ArrayList<>();
+ messages.add(message);
+ ArrayList<Tuple> tuples = new ArrayList<>();
+
+ CountSyncPolicy basePolicy = new CountSyncPolicy(5);
+ ClonedSyncPolicyCreator creator = new ClonedSyncPolicyCreator(basePolicy);
+
+ writer.write(SENSOR_NAME, config, tuples, messages);
+ writer.getSourceHandler(SENSOR_NAME, "test-test.value/test.value", config).closeOutputFile();
+ writer.getSourceHandler(SENSOR_NAME, "test-test.value/test.value", config).handle(message, SENSOR_NAME, config, creator);
+ writer.close();
+
+ File outputFolder = new File(folder.getAbsolutePath() + "/test-test.value/test.value/");
+
+ // The message should show up twice, once in each file
+ ArrayList<String> expected = new ArrayList<>();
+ expected.add(message.toJSONString());
+
+ // Assert this went into a new file because it actually rotated
+ Assert.assertEquals(2, outputFolder.listFiles().length);
+ for (File file : outputFolder.listFiles()) {
+ List<String> lines = Files.readAllLines(file.toPath());
+ // One line per file
+ Assert.assertEquals(1, lines.size());
+ Assert.assertEquals(expected, lines);
+ }
+ }
+
protected WriterConfiguration buildWriterConfiguration(String function) {
IndexingConfigurations indexingConfig = new IndexingConfigurations();
Map<String, Object> sensorIndexingConfig = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 926132e..d464d8c 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -1,24 +1,24 @@
+# Stellar Language
-# Contents
+For a variety of components (threat intelligence triage and field transformations) we have the need to do simple computation and transformation using the data from messages as variables. For those purposes, there exists a simple, scaled down DSL created to do simple computation and transformation.
-* [Stellar Language](#stellar-language)
- * [Stellar Language Keywords](#stellar-language-keywords)
- * [Stellar Core Functions](#stellar-core-functions)
- * [Stellar Benchmarks](#stellar-benchmarks)
- * [Stellar Shell](#stellar-shell)
-* [Stellar Configuration](#stellar-configuration)
+* [Introduction](#introduction)
+* [Stellar Core Functions](#stellar-core-functions)
+* [Stellar Benchmarks](#stellar-benchmarks)
+* [Stellar Shell](#stellar-shell)
+ * [Getting Started](#getting-started)
+ * [Command Line Options](#command-line-options)
+ * [Variable Assignment](#variable-assignment)
+ * [Magic Commands](#magic-commands)
+ * [Advanced Usage](#advanced-usage)
+* [Stellar Configuration](#stellar-configuration)
-# Stellar Language
-For a variety of components (threat intelligence triage and field
-transformations) we have the need to do simple computation and
-transformation using the data from messages as variables.
-For those purposes, there exists a simple, scaled down DSL
-created to do simple computation and transformation.
+## Introduction
-The query language supports the following:
+The Stellar language supports the following:
* Referencing fields in the enriched JSON
* String literals are quoted with either `'` or `"`
* String literals support escaping for `'`, `"`, `\t`, `\r`, `\n`, and backslash
@@ -35,7 +35,7 @@ The query language supports the following:
* The ability to have parenthesis to make order of operations explicit
* User defined functions, including Lambda expressions
-## Stellar Language Keywords
+### Stellar Language Keywords
The following keywords need to be single quote escaped in order to be used in Stellar expressions:
| | | | | |
@@ -47,13 +47,13 @@ The following keywords need to be single quote escaped in order to be used in St
Using parens such as: "foo" : "\<ok\>" requires escaping; "foo": "\'\<ok\>\'"
-## Stellar Language Inclusion Checks (`in` and `not in`)
+### Stellar Language Inclusion Checks (`in` and `not in`)
1. `in` supports string contains. e.g. `'foo' in 'foobar' == true`
2. `in` supports collection contains. e.g. `'foo' in [ 'foo', 'bar' ] == true`
3. `in` supports map key contains. e.g. `'foo' in { 'foo' : 5} == true`
4. `not in` is the negation of the in expression. e.g. `'grok' not in 'foobar' == true`
-## Stellar Language Comparisons (`<`, `<=`, `>`, `>=`)
+### Stellar Language Comparisons (`<`, `<=`, `>`, `>=`)
1. If either side of the comparison is null then return false.
2. If both values being compared implement number then the following:
@@ -64,7 +64,7 @@ Using parens such as: "foo" : "\<ok\>" requires escaping; "foo": "\'\<ok\>\'"
3. If both sides are of the same type and are comparable then use the compareTo method to compare values.
4. If none of the above are met then an exception is thrown.
-## Stellar Language Equality Check (`==`, `!=`)
+### Stellar Language Equality Check (`==`, `!=`)
Below is how the `==` operator is expected to work:
@@ -78,7 +78,7 @@ Below is how the `==` operator is expected to work:
The `!=` operator is the negation of the above.
-## Stellar Language Lambda Expressions
+### Stellar Language Lambda Expressions
Stellar provides the capability to pass lambda expressions to functions
which wish to support that layer of indirection. The syntax is:
@@ -110,8 +110,8 @@ In the core language functions, we support basic functional programming primitiv
| [ `BLOOM_EXISTS`](#bloom_exists) |
| [ `BLOOM_INIT`](#bloom_init) |
| [ `BLOOM_MERGE`](#bloom_merge) |
-| [ `CEILING`](#ceiling) |
-| [ `COS`](#cos) |
+| [ `CEILING`](#ceiling) |
+| [ `COS`](#cos) |
| [ `CHOP`](#chop) |
| [ `CHOMP`](#chomp) |
| [ `COUNT_MATCHES`](#count_matches) |
@@ -126,20 +126,26 @@ In the core language functions, we support basic functional programming primitiv
| [ `ENDS_WITH`](#ends_with) |
| [ `ENRICHMENT_EXISTS`](#enrichment_exists) |
| [ `ENRICHMENT_GET`](#enrichment_get) |
-| [ `EXP`](#exp) |
+| [ `EXP`](#exp) |
| [ `FILL_LEFT`](#fill_left) |
| [ `FILL_RIGHT`](#fill_right) |
| [ `FILTER`](#filter) |
-| [ `FLOOR`](#floor) |
-| [ `FUZZY_LANGS`](#fuzzy_langs) |
-| [ `FUZZY_SCORE`](#fuzzy_score) |
+| [ `FLOOR`](#floor) |
+| [ `FUZZY_LANGS`](#fuzzy_langs) |
+| [ `FUZZY_SCORE`](#fuzzy_score) |
| [ `FORMAT`](#format) |
| [ `GEO_GET`](#geo_get) |
+| [ `GEOHASH_CENTROID`](#geohash_centroid) |
+| [ `GEOHASH_DIST`](#geohash_dist) |
+| [ `GEOHASH_FROM_LATLONG`](#geohash_from_latlong) |
+| [ `GEOHASH_FROM_LOC`](#geohash_from_loc) |
+| [ `GEOHASH_MAX_DIST`](#geohash_max_dist) |
+| [ `GEOHASH_TO_LATLONG`](#geohash_to_latlong) |
| [ `GET`](#get) |
| [ `GET_FIRST`](#get_first) |
| [ `GET_LAST`](#get_last) |
-| [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings) |
-| [ `HASH`](#hash) |
+| [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings) |
+| [ `HASH`](#hash) |
| [ `HLLP_CARDINALITY`](../../metron-analytics/metron-statistics#hllp_cardinality) |
| [ `HLLP_INIT`](../../metron-analytics/metron-statistics#hllp_init) |
| [ `HLLP_MERGE`](../../metron-analytics/metron-statistics#hllp_merge) |
@@ -160,14 +166,19 @@ In the core language functions, we support basic functional programming primitiv
| [ `KAFKA_TAIL`](#kafka_tail) |
| [ `LENGTH`](#length) |
| [ `LIST_ADD`](#list_add) |
-| [ `LOG2`](#log2) |
-| [ `LOG10`](#log10) |
-| [ `LN`](#ln) |
+| [ `LOG2`](#log2) |
+| [ `LOG10`](#log10) |
+| [ `LN`](#ln) |
| [ `MAAS_GET_ENDPOINT`](#maas_get_endpoint) |
| [ `MAAS_MODEL_APPLY`](#maas_model_apply) |
| [ `MAP`](#map) |
| [ `MAP_EXISTS`](#map_exists) |
| [ `MONTH`](#month) |
+| [ `MULTISET_ADD`](#multiset_add) |
+| [ `MULTISET_INIT`](#multiset_init) |
+| [ `MULTISET_MERGE`](#multiset_merge) |
+| [ `MULTISET_REMOVE`](#multiset_remove) |
+| [ `MULTISET_TO_SET`](#multiset_to_set) |
| [ `PREPEND_IF_MISSING`](#prepend_if_missing) |
| [ `PROFILE_GET`](#profile_get) |
| [ `PROFILE_FIXED`](#profile_fixed) |
@@ -175,11 +186,15 @@ In the core language functions, we support basic functional programming primitiv
| [ `PROTOCOL_TO_NAME`](#protocol_to_name) |
| [ `REDUCE`](#reduce) |
| [ `REGEXP_MATCH`](#regexp_match) |
-| [ `REGEXP_GROUP_VAL`](#regexp_group_val) |
-| [ `ROUND`](#round) |
+| [ `REGEXP_GROUP_VAL`](#regexp_group_val) |
+| [ `ROUND`](#round) |
+| [ `SET_ADD`](#set_add) |
+| [ `SET_INIT`](#set_init) |
+| [ `SET_MERGE`](#set_merge) |
+| [ `SET_REMOVE`](#set_remove) |
| [ `SPLIT`](#split) |
-| [ `SIN`](#sin) |
-| [ `SQRT`](#sqrt) |
+| [ `SIN`](#sin) |
+| [ `SQRT`](#sqrt) |
| [ `STARTS_WITH`](#starts_with) |
| [ `STATS_ADD`](../../metron-analytics/metron-statistics#stats_add) |
| [ `STATS_BIN`](../../metron-analytics/metron-statistics#stats_bin) |
@@ -201,13 +216,17 @@ In the core language functions, we support basic functional programming primitiv
| [ `STATS_SUM_SQUARES`](../../metron-analytics/metron-statistics#stats_sum_squares) |
| [ `STATS_VARIANCE`](../../metron-analytics/metron-statistics#stats_variance) |
| [ `STRING_ENTROPY`](#string_entropy) |
+| [ `SUBSTRING`](#substring) |
| [ `SYSTEM_ENV_GET`](#system_env_get) |
| [ `SYSTEM_PROPERTY_GET`](#system_property_get) |
-| [ `TAN`](#tan) |
+| [ `TAN`](#tan) |
| [ `TO_DOUBLE`](#to_double) |
| [ `TO_EPOCH_TIMESTAMP`](#to_epoch_timestamp) |
| [ `TO_FLOAT`](#to_float) |
| [ `TO_INTEGER`](#to_integer) |
+| [ `TO_JSON_LIST`](#to_json_List) |
+| [ `TO_JSON_MAP`](#to_json_map) |
+| [ `TO_JSON_OBJECT`](#to_json_object) |
| [ `TO_LONG`](#to_long) |
| [ `TO_LOWER`](#to_lower) |
| [ `TO_STRING`](#to_string) |
@@ -220,8 +239,8 @@ In the core language functions, we support basic functional programming primitiv
| [ `WEEK_OF_MONTH`](#week_of_month) |
| [ `WEEK_OF_YEAR`](#week_of_year) |
| [ `YEAR`](#year) |
-| [ `ZIP`](#zip) |
-| [ `ZIP_JAGGED`](#zip_jagged) |
+| [ `ZIP`](#zip) |
+| [ `ZIP_JAGGED`](#zip_jagged) |
### `APPEND_IF_MISSING`
* Description: Appends the suffix to the end of the string if the string does not already end with any of the suffixes.
@@ -434,6 +453,50 @@ In the core language functions, we support basic functional programming primitiv
* fields - Optional list of GeoIP fields to grab. Options are locID, country, city postalCode, dmaCode, latitude, longitude, location_point
* Returns: If a Single field is requested a string of the field, If multiple fields a map of string of the fields, and null otherwise
+### `GEOHASH_CENTROID`
+ * Description: Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights
+ * character_precision? - The number of characters to use in the hash. Default is 12
+ * Returns: The geohash of the centroid
+
+### `GEOHASH_DIST`
+ * Description: Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hash1 - The first point as a geohash
+ * hash2 - The second point as a geohash
+ * strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default.
+ * Returns: The distance in kilometers between the hashes.
+
+### `GEOHASH_FROM_LATLONG`
+ * Description: Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long
+ * Input:
+ * latitude - The latitude
+ * longitude - The longitude
+ * character_precision? - The number of characters to use in the hash. Default is 12
+ * Returns: A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long
+
+### `GEOHASH_FROM_LOC`
+ * Description: Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location
+ * Input:
+ * map - the latitude and logitude in a map (the output of [GEO_GET](#geo_get) )
+ * longitude - The longitude
+ * character_precision? - The number of characters to use in the hash. Default is `12`
+ * Returns: A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location
+
+### `GEOHASH_MAX_DIST`
+ * Description: Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hashes - A set of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default.
+ * Returns: The maximum distance in kilometers between any two locations
+
+### `GEOHASH_TO_LATLONG`
+ * Description: Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)
+ * Returns: A map containing the latitude and longitude of the hash (keys "latitude" and "longitude")
+
### `GET`
* Description: Returns the i'th element of the list
* Input:
@@ -502,7 +565,7 @@ In the core language functions, we support basic functional programming primitiv
* Description: Returns true if the passed string is encoded in one of the supported encodings and false if otherwise.
* Input:
* string - The string to test
- * encoding - The name of the encoding as string. See [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings).
+ * encoding - The name of the encoding as string. See [ `GET_SUPPORTED_ENCODINGS`](#get_supported_encodings).
* Returns: True if the passed string is encoded in one of the supported encodings and false if otherwise.
### `IS_INTEGER`
@@ -578,6 +641,27 @@ In the core language functions, we support basic functional programming primitiv
* Description: Returns a list of the encodings that are currently supported.
* Returns: A List of String
+### `TO_JSON_LIST`
+ * Description: Accepts JSON string as an input and returns a List object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `GET_FIRST( TO_JSON_LIST( '[ "foo", 2]')` would yield `foo`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed List object
+
+### `TO_JSON_MAP`
+ * Description: Accepts JSON string as an input and returns a Map object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `MAP_GET( 'bar', TO_JSON_MAP( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed Map object
+
+### `TO_JSON_OBJECT`
+ * Description: Accepts JSON string as an input and returns a JSON Object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `MAP_GET( 'bar', TO_JSON_OBJECT( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed JSON object
+
### `LOG2`
* Description: Returns the log (base `2`) of a number.
* Input:
@@ -638,6 +722,38 @@ In the core language functions, we support basic functional programming primitiv
* Input:
* dateTime - The datetime as a long representing the milliseconds since unix epoch
* Returns: The current month (0-based).
+
+### `MULTISET_ADD`
+ * Description: Adds to a multiset, which is a map associating objects to their instance counts.
+ * Input:
+ * set - The multiset to add to
+ * o - object to add to multiset
+ * Returns: A multiset
+
+### `MULTISET_INIT`
+ * Description: Creates an empty multiset, which is a map associating objects to their instance counts.
+ * Input:
+ * input? - An initialization of the multiset
+ * Returns: A multiset
+
+### `MULTISET_MERGE`
+ * Description: Merges a list of multisets, which is a map associating objects to their instance counts.
+ * Input:
+ * sets - A collection of multisets to merge
+ * Returns: A multiset
+
+### `MULTISET_REMOVE`
+ * Description: Removes from a multiset, which is a map associating objects to their instance counts.
+ * Input:
+ * set - The multiset to add to
+ * o - object to remove from multiset
+ * Returns: A multiset
+
+### `MULTISET_TO_SET`
+ * Description: Create a set out of a multiset, which is a map associating objects to their instance counts.
+ * Input:
+ * multiset - The multiset to convert.
+ * Returns: The set of objects in the multiset ignoring multiplicity
### `PREPEND_IF_MISSING`
* Description: Prepends the prefix to the start of the string if the string does not already start with any of the prefixes.
@@ -708,6 +824,32 @@ In the core language functions, we support basic functional programming primitiv
* number - The number to round
* Returns: The nearest integer (based on half-up rounding).
+### `SET_ADD`
+ * Description: Adds to a set
+ * Input:
+ * set - The set to add to
+ * o - object to add to set
+ * Returns: A Set
+
+### `SET_INIT`
+ * Description: Creates an new set
+ * Input:
+ * input? - An initialization of the set
+ * Returns: A Set
+
+### `SET_MERGE`
+ * Description: Merges a list of sets
+ * Input:
+ * sets - A collection of sets to merge
+ * Returns: A Set
+
+### `SET_REMOVE`
+ * Description: Removes from a set
+ * Input:
+ * set - The set to add to
+ * o - object to add to set
+ * Returns: A Set
+
### `SIN`
* Description: Returns the sine of a number.
* Input:
@@ -752,6 +894,14 @@ In the core language functions, we support basic functional programming primitiv
* key - Property to get the value for
* Returns: String
+### `SUBSTRING`
+ * Description: Returns the substring of a string
+ * Input:
+ * input - The string to take the substring of
+ * start - The starting position (`0`-based and inclusive)
+ * end? - The ending position (`0`-based and exclusive)
+ * Returns: The substring of the input
+
### `TAN`
* Description: Returns the tangent of a number.
* Input:
@@ -1080,8 +1230,6 @@ IN_SUBNET
Lists all variables in the Stellar environment.
```
-Stellar, Go!
-{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH}
[Stellar]>>> %vars
[Stellar]>>> foo := 2 + 2
4.0
@@ -1089,6 +1237,38 @@ Stellar, Go!
foo = 4.0
```
+#### `%globals`
+
+Lists all values that are defined in the global configuration.
+
+Most of Metron's functional components have access to what is called the global configuration. This is a key/value configuration store that can be used to customize Metron. Many Stellar functions accept configuration values from the global configuration. The Stellar Shell also leverages the global configuration for customizing the behavior of many Stellar functions.
+
+```
+[Stellar]>>> %globals
+{es.clustername=metron, es.ip=node1:9300, es.date.format=yyyy.MM.dd.HH, parser.error.topic=indexing, update.hbase.table=metron_update, update.hbase.cf=t}
+```
+
+#### `%define`
+
+Defines a global configuration value in the current shell session. This value will be forgotten once the session is ended.
+
+```
+[Stellar]>>> %define bootstrap.servers := "node1:6667"
+node1:6667
+[Stellar]>>> %globals
+{bootstrap.servers=node1:6667}
+```
+
+#### `%undefine`
+
+Undefine a global configuration value in the current shell session. This will not modify the persisted global configuration.
+
+```
+[Stellar]>>> %undefine bootstrap.servers
+[Stellar]>>> %globals
+{}
+```
+
#### `?<function>`
Returns formatted documentation of the Stellar function. Provides the description of the function along with the expected arguments.
@@ -1138,7 +1318,7 @@ Please note that functions are loading lazily in the background and will be unav
ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, CHOMP, CHOP, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, FILL_LEFT, FILL_RIGHT, FILTER, FORMAT, GET, GET_FIRST, GET_LAST, HLLP_ADD, HLLP_CARDINALITY, HLLP_INIT, HLLP_MERGE, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, JOIN, LENGTH, LIST_ADD, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP, MAP_EXISTS, MAP_GET, MONTH, OUTLIER_MAD_ADD, OUTLIER_MAD_SCORE, OUTLIER_MAD_STATE_MERGE, PREPEND_IF_MISSING, PROFILE_FIXED, PROFILE_GET, PROFILE_WINDOW, PROTOCOL_TO_NAME, REDUCE, REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_BIN, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, STRING_ENTROPY, SYS
TEM_ENV_GET, SYSTEM_PROPERTY_GET, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LONG, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR
```
-# Stellar Configuration
+## Stellar Configuration
Stellar can be configured in a variety of ways from the [Global Configuration](../../metron-platform/metron-common/README.md#global-configuration).
In particular, there are three main configuration parameters around configuring Stellar:
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index 5945bbd..9ec29b8 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -257,6 +257,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
index a8bc773..b669bc7 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
@@ -121,15 +121,15 @@ public class StellarCompiler extends StellarBaseListener {
//if we have a boolean as the current value and the next non-contextual token is a short circuit op
//then we need to short circuit possibly
if(token.getUnderlyingType() == BooleanArg.class) {
- if (curr.getMultiArgContext() != null
- && curr.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_OR
+ if (token.getMultiArgContext() != null
+ && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_OR
&& (Boolean) (curr.getValue())
) {
//short circuit the or
FrameContext.Context context = curr.getMultiArgContext();
shortCircuit(it, context);
- } else if (curr.getMultiArgContext() != null
- && curr.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_AND
+ } else if (token.getMultiArgContext() != null
+ && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_AND
&& !(Boolean) (curr.getValue())
) {
//short circuit the and
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java
index 1181c6f..febde40 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarExecutor.java
@@ -92,9 +92,10 @@ public class StellarExecutor {
}
/**
- * prefix tree index of autocompletes
+ * Prefix tree index of auto-completes.
*/
private PatriciaTrie<AutoCompleteType> autocompleteIndex;
+
/**
* The variables known by Stellar.
*/
@@ -118,7 +119,9 @@ public class StellarExecutor {
private Console console;
public enum OperationType {
- DOC,MAGIC,NORMAL;
+ DOC
+ , MAGIC
+ , NORMAL
}
public interface AutoCompleteTransformation {
@@ -138,6 +141,7 @@ public class StellarExecutor {
, VARIABLE((type, key) -> key )
, TOKEN((type, key) -> key)
;
+
AutoCompleteTransformation transform;
AutoCompleteType(AutoCompleteTransformation transform) {
this.transform = transform;
@@ -147,7 +151,6 @@ public class StellarExecutor {
public String transform(OperationType type, String key) {
return transform.transform(type, key);
}
-
}
/**
@@ -178,18 +181,17 @@ public class StellarExecutor {
// asynchronously update the index with function names found from a classpath scan.
new Thread( () -> {
- Iterable<StellarFunctionInfo> functions = functionResolver.getFunctionInfo();
- indexLock.writeLock().lock();
- try {
- for(StellarFunctionInfo info: functions) {
- String functionName = info.getName();
- autocompleteIndex.put(functionName, AutoCompleteType.FUNCTION);
- }
+ Iterable<StellarFunctionInfo> functions = functionResolver.getFunctionInfo();
+ indexLock.writeLock().lock();
+ try {
+ for(StellarFunctionInfo info: functions) {
+ String functionName = info.getName();
+ autocompleteIndex.put(functionName, AutoCompleteType.FUNCTION);
}
- finally {
- System.out.println("Functions loaded, you may refer to functions now...");
- indexLock.writeLock().unlock();
- }
+ } finally {
+ System.out.println("Functions loaded, you may refer to functions now...");
+ indexLock.writeLock().unlock();
+ }
}).start();
}
@@ -203,6 +205,9 @@ public class StellarExecutor {
index.put("quit", AutoCompleteType.TOKEN);
index.put(StellarShell.MAGIC_FUNCTIONS, AutoCompleteType.FUNCTION);
index.put(StellarShell.MAGIC_VARS, AutoCompleteType.FUNCTION);
+ index.put(StellarShell.MAGIC_GLOBALS, AutoCompleteType.FUNCTION);
+ index.put(StellarShell.MAGIC_DEFINE, AutoCompleteType.FUNCTION);
+ index.put(StellarShell.MAGIC_UNDEFINE, AutoCompleteType.FUNCTION);
return new PatriciaTrie<>(index);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
index a860778..0002c5a 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/StellarShell.java
@@ -30,10 +30,9 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.lang3.StringUtils;
-import org.apache.metron.stellar.dsl.Context;
-import org.apache.metron.stellar.dsl.StellarFunctionInfo;
import org.apache.metron.stellar.common.StellarAssignment;
import org.apache.metron.stellar.common.utils.JSONUtils;
+import org.apache.metron.stellar.dsl.StellarFunctionInfo;
import org.jboss.aesh.complete.CompleteOperation;
import org.jboss.aesh.complete.Completion;
import org.jboss.aesh.console.AeshConsoleCallback;
@@ -51,13 +50,18 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
+import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG;
+
/**
* A REPL environment for Stellar.
*
@@ -90,6 +94,9 @@ public class StellarShell extends AeshConsoleCallback implements Completion {
public static final String MAGIC_VARS = MAGIC_PREFIX + "vars";
public static final String DOC_PREFIX = "?";
public static final String STELLAR_PROPERTIES_FILENAME = "stellar.properties";
+ public static final String MAGIC_GLOBALS = MAGIC_PREFIX + "globals";
+ public static final String MAGIC_DEFINE = MAGIC_PREFIX + "define";
+ public static final String MAGIC_UNDEFINE = MAGIC_PREFIX + "undefine";
private StellarExecutor executor;
@@ -241,7 +248,7 @@ public class StellarShell extends AeshConsoleCallback implements Completion {
// welcome message and print globals
writeLine(WELCOME);
executor.getContext()
- .getCapability(Context.Capabilities.GLOBAL_CONFIG, false)
+ .getCapability(GLOBAL_CONFIG, false)
.ifPresent(conf -> writeLine(conf.toString()));
console.start();
@@ -287,32 +294,25 @@ public class StellarShell extends AeshConsoleCallback implements Completion {
* Executes a magic expression.
* @param rawExpression The expression to execute.
*/
- private void handleMagic( String rawExpression) {
- String[] expression = rawExpression.trim().split(" ");
+ private void handleMagic(String rawExpression) {
+ String[] expression = rawExpression.trim().split("\\s+");
String command = expression[0];
- if(MAGIC_FUNCTIONS.equals(command)) {
- // if '%functions FOO' then show only functions that contain 'FOO'
- Predicate<String> nameFilter = (name -> true);
- if(expression.length > 1) {
- nameFilter = (name -> name.contains(expression[1]));
- }
+ if (MAGIC_FUNCTIONS.equals(command)) {
+ handleMagicFunctions(expression);
+
+ } else if (MAGIC_VARS.equals(command)) {
+ handleMagicVars();
- // list available functions
- String functions = StreamSupport
- .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false)
- .map(info -> String.format("%s", info.getName()))
- .filter(nameFilter)
- .sorted()
- .collect(Collectors.joining(", "));
- writeLine(functions);
+ } else if (MAGIC_GLOBALS.equals(command)) {
+ handleMagicGlobals();
- } else if(MAGIC_VARS.equals(command)) {
+ } else if (MAGIC_DEFINE.equals(command)) {
+ handleMagicDefine(rawExpression);
- // list all variables
- executor.getVariables()
- .forEach((k,v) -> writeLine(String.format("%s = %s", k, v)));
+ } else if(MAGIC_UNDEFINE.equals(command)) {
+ handleMagicUndefine(expression);
} else {
writeLine(ERROR_PROMPT + "undefined magic command: " + rawExpression);
@@ -320,6 +320,106 @@ public class StellarShell extends AeshConsoleCallback implements Completion {
}
/**
+ * Handle a magic '%functions'. Lists all of the variables in-scope.
+ * @param expression
+ */
+ private void handleMagicFunctions(String[] expression) {
+
+ // if '%functions FOO' then show only functions that contain 'FOO'
+ Predicate<String> nameFilter = (name -> true);
+ if (expression.length > 1) {
+ nameFilter = (name -> name.contains(expression[1]));
+ }
+
+ // '%functions' -> list all functions in scope
+ String functions = StreamSupport
+ .stream(executor.getFunctionResolver().getFunctionInfo().spliterator(), false)
+ .map(info -> String.format("%s", info.getName()))
+ .filter(nameFilter)
+ .sorted()
+ .collect(Collectors.joining(", "));
+ writeLine(functions);
+ }
+
+ /**
+ * Handle a magic '%vars'. Lists all of the variables in-scope.
+ */
+ private void handleMagicVars() {
+ executor.getVariables()
+ .forEach((k, v) -> writeLine(String.format("%s = %s", k, v)));
+ }
+
+ /**
+ * Handle a magic '%globals'. List all of the global configuration values.
+ */
+ private void handleMagicGlobals() {
+ Map<String, Object> globals = getOrCreateGlobalConfig(executor);
+ writeLine(globals.toString());
+ }
+
+ /**
+ * Handle a magic '%define var=value'. Alter the global configuration.
+ * @param expression The expression passed to %define
+ */
+ public void handleMagicDefine(String expression) {
+
+ // grab the expression in '%define <assign-expression>'
+ String assignExpr = StringUtils.trimToEmpty(expression.substring(MAGIC_DEFINE.length()));
+ if (assignExpr.length() > 0) {
+
+ // the expression must be an assignment
+ if(StellarAssignment.isAssignment(assignExpr)) {
+ StellarAssignment expr = StellarAssignment.from(assignExpr);
+
+ // execute the expression
+ Object result = executor.execute(expr.getStatement());
+ if (result != null) {
+ writeLine(result.toString());
+
+ // alter the global configuration
+ getOrCreateGlobalConfig(executor).put(expr.getVariable(), result);
+ }
+
+ } else {
+ // the expression is not an assignment. boo!
+ writeLine(ERROR_PROMPT + MAGIC_DEFINE + " expected assignment expression");
+ }
+ }
+ }
+
+ /**
+ * Handle a magic '%undefine var'. Removes a variable from the global configuration.
+ * @param expression
+ */
+ private void handleMagicUndefine(String[] expression) {
+ if(expression.length > 1) {
+ Map<String, Object> globals = getOrCreateGlobalConfig(executor);
+ globals.remove(expression[1]);
+ }
+ }
+
+ /**
+ * Retrieves the GLOBAL_CONFIG, if it exists. If it does not, it creates the GLOBAL_CONFIG
+ * and adds it to the Stellar execution context.
+ * @param executor The Stellar executor.
+ * @return The global configuration.
+ */
+ private Map<String, Object> getOrCreateGlobalConfig(StellarExecutor executor) {
+ Map<String, Object> globals;
+ Optional<Object> capability = executor.getContext().getCapability(GLOBAL_CONFIG, false);
+ if (capability.isPresent()) {
+ globals = (Map<String, Object>) capability.get();
+
+ } else {
+ // if it does not exist, create it. this creates the global config for the current stellar executor
+ // session only. this does not change the global config maintained externally in zookeeper
+ globals = new HashMap<>();
+ executor.getContext().addCapability(GLOBAL_CONFIG, () -> globals);
+ }
+ return globals;
+ }
+
+ /**
* Executes a doc expression.
* @param expression The doc expression to execute.
*/
http://git-wip-us.apache.org/repos/asf/metron/blob/24b668b0/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
index f3d995f..a92406b 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.stellar.dsl.functions;
+import com.google.common.collect.Iterables;
import org.apache.metron.stellar.dsl.BaseStellarFunction;
import org.apache.metron.stellar.dsl.Stellar;
import org.apache.metron.stellar.common.utils.BloomFilter;
@@ -24,7 +25,9 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.stellar.common.utils.SerDeUtils;
import java.util.Collection;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
public class DataStructureFunctions {
@@ -217,4 +220,6 @@ public class DataStructureFunctions {
}
}
}
+
+
}