You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/09/08 20:25:35 UTC
[1/5] metron git commit: METRON-1160 Blueprint configuration
validation failed: Missing required properties (nickwallen via justinleet)
closes apache/metron#735
Repository: metron
Updated Branches:
refs/heads/Metron_0.4.1 7a3de6737 -> f0ae85fb7
METRON-1160 Blueprint configuration validation failed: Missing required properties (nickwallen via justinleet) closes apache/metron#735
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0bbc51d6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0bbc51d6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0bbc51d6
Branch: refs/heads/Metron_0.4.1
Commit: 0bbc51d6858c67015ec7dde3105069f4a678d518
Parents: 7a3de67
Author: nickwallen <ni...@nickallen.org>
Authored: Fri Sep 8 06:38:49 2017 -0400
Committer: leet <le...@apache.org>
Committed: Fri Sep 8 06:38:49 2017 -0400
----------------------------------------------------------------------
metron-deployment/roles/ambari_config/vars/small_cluster.yml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0bbc51d6/metron-deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
index 45b15f3..51e0455 100644
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
@@ -100,6 +100,7 @@ required_configurations:
storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744"
es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}"
zeppelin_server_url: "{{ groups.zeppelin[0] }}"
+ - metron-rest-env:
metron_jdbc_driver: "org.h2.Driver"
metron_jdbc_url: "jdbc:h2:file:~/metrondb"
metron_jdbc_username: "root"
[5/5] metron git commit: METRON-1166: Stellar short circuiting fails
when a complex condition using a boolean op is followed by the opposite
boolean op this closes apache/incubator-metron#738
Posted by ma...@apache.org.
METRON-1166: Stellar short circuiting fails when a complex condition using a boolean op is followed by the opposite boolean op this closes apache/incubator-metron#738
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f0ae85fb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f0ae85fb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f0ae85fb
Branch: refs/heads/Metron_0.4.1
Commit: f0ae85fb71bf1d102d9b72f6b413a89abe8b4bd4
Parents: 224d3d5
Author: cstella <ce...@gmail.com>
Authored: Fri Sep 8 15:04:44 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Fri Sep 8 15:04:44 2017 -0400
----------------------------------------------------------------------
.../apache/metron/stellar/common/StellarCompiler.java | 8 ++++----
.../metron/stellar/dsl/functions/BasicStellarTest.java | 12 ++++++++++++
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f0ae85fb/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
index a8bc773..b669bc7 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
@@ -121,15 +121,15 @@ public class StellarCompiler extends StellarBaseListener {
//if we have a boolean as the current value and the next non-contextual token is a short circuit op
//then we need to short circuit possibly
if(token.getUnderlyingType() == BooleanArg.class) {
- if (curr.getMultiArgContext() != null
- && curr.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_OR
+ if (token.getMultiArgContext() != null
+ && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_OR
&& (Boolean) (curr.getValue())
) {
//short circuit the or
FrameContext.Context context = curr.getMultiArgContext();
shortCircuit(it, context);
- } else if (curr.getMultiArgContext() != null
- && curr.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_AND
+ } else if (token.getMultiArgContext() != null
+ && token.getMultiArgContext().getVariety() == FrameContext.BOOLEAN_AND
&& !(Boolean) (curr.getValue())
) {
//short circuit the and
http://git-wip-us.apache.org/repos/asf/metron/blob/f0ae85fb/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index d6c3713..af86902 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -625,6 +625,18 @@ public class BasicStellarTest {
}
@Test
+
+ public void testShortCircuit_mixedBoolOps() throws Exception {
+ final Map<String, String> variableMap = new HashMap<String, String>();
+ Assert.assertTrue(runPredicate("(false && true) || true"
+ , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertTrue(runPredicate("(false && false) || true"
+ , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertFalse(runPredicate("(true || true) && false"
+ , new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ }
+
+ @Test
public void testInString() throws Exception {
final Map<String, String> variableMap = new HashMap<String, String>() {{
put("foo", "casey");
[3/5] metron git commit: METRON-1151: Grok patterns in HDFS should be
owned by metron this closes apache/incubator-metron#730
Posted by ma...@apache.org.
METRON-1151: Grok patterns in HDFS should be owned by metron this closes apache/incubator-metron#730
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f79214be
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f79214be
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f79214be
Branch: refs/heads/Metron_0.4.1
Commit: f79214be9bcabca3b8b9411e17915c2de5236ea8
Parents: dd71181
Author: merrimanr <me...@gmail.com>
Authored: Fri Sep 8 09:19:39 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Fri Sep 8 09:19:39 2017 -0400
----------------------------------------------------------------------
.../METRON/CURRENT/package/scripts/parser_commands.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f79214be/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index a487298..334dc55 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -77,7 +77,8 @@ class ParserCommands:
action="create_on_execute",
owner=self.__params.metron_user,
mode=0755,
- source=self.__params.local_grok_patterns_dir)
+ source=self.__params.local_grok_patterns_dir,
+ recursive_chown = True)
Logger.info("Done initializing parser configuration")
[4/5] metron git commit: METRON-1150 REST parseMessage endpoint fails
with Unable to load < grok file>
from either classpath or HDFS (merrimanr) closes apache/metron#729
Posted by ma...@apache.org.
METRON-1150 REST parseMessage endpoint fails with Unable to load <grok file> from either classpath or HDFS (merrimanr) closes apache/metron#729
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/224d3d5e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/224d3d5e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/224d3d5e
Branch: refs/heads/Metron_0.4.1
Commit: 224d3d5e2b8c17ca4414ad7d7846dafee0b06d3b
Parents: f79214b
Author: merrimanr <me...@gmail.com>
Authored: Fri Sep 8 09:43:29 2017 -0500
Committer: merrimanr <me...@apache.org>
Committed: Fri Sep 8 09:43:29 2017 -0500
----------------------------------------------------------------------
.../CURRENT/configuration/metron-rest-env.xml | 10 +---
.../package/scripts/params/params_linux.py | 6 ++-
.../METRON/CURRENT/package/templates/metron.j2 | 1 -
.../src/main/config/rest_application.yml | 1 -
.../apache/metron/rest/MetronRestConstants.java | 1 -
.../apache/metron/rest/service/GrokService.java | 6 ++-
.../apache/metron/rest/service/HdfsService.java | 2 +
.../rest/service/impl/GrokServiceImpl.java | 37 ++++++-------
.../rest/service/impl/HdfsServiceImpl.java | 9 ++++
.../impl/SensorParserConfigServiceImpl.java | 41 +++++++-------
.../src/main/resources/application-docker.yml | 1 -
.../src/main/resources/application-test.yml | 1 -
.../src/main/resources/application-vagrant.yml | 3 +-
.../src/main/resources/application.yml | 3 +-
...orParserConfigControllerIntegrationTest.java | 6 ---
.../rest/service/impl/GrokServiceImplTest.java | 56 +++++++-------------
.../impl/SensorParserConfigServiceImplTest.java | 20 +++++--
17 files changed, 97 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 9c11123..0549510 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -90,17 +90,11 @@
</property>
<property>
<name>metron_temp_grok_path</name>
- <description>Temporary local file path where grok patterns are written during testing</description>
- <value>./patterns/temp</value>
+ <description>Temporary HDFS file path where grok patterns are written during testing</description>
+ <value>{{metron_apps_hdfs_dir}}/patterns/tmp</value>
<display-name>Metron temp grok path</display-name>
</property>
<property>
- <name>metron_default_grok_path</name>
- <description>Default HDFS directory path used when storing Grok patterns</description>
- <value>/apps/metron/patterns</value>
- <display-name>Metron default grok path</display-name>
- </property>
- <property>
<name>metron_spring_options</name>
<description>Additional Spring options not included in the rest_application.yml file</description>
<display-name>Metron Spring options</display-name>
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 78d253c..a9d00dd 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -55,8 +55,6 @@ metron_jdbc_username = config['configurations']['metron-rest-env']['metron_jdbc_
metron_jdbc_password = config['configurations']['metron-rest-env']['metron_jdbc_password']
metron_jdbc_platform = config['configurations']['metron-rest-env']['metron_jdbc_platform']
metron_jdbc_client_path = config['configurations']['metron-rest-env']['metron_jdbc_client_path']
-metron_temp_grok_path = config['configurations']['metron-rest-env']['metron_temp_grok_path']
-metron_default_grok_path = config['configurations']['metron-rest-env']['metron_default_grok_path']
metron_spring_options = config['configurations']['metron-rest-env']['metron_spring_options']
metron_escalation_topic = config['configurations']['metron-rest-env']['metron_escalation_topic']
metron_config_path = metron_home + '/config'
@@ -132,6 +130,10 @@ if has_kafka_host:
metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+# the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
+# well enough until we find a better way via Ambari
+metron_temp_grok_path = format(format(config['configurations']['metron-rest-env']['metron_temp_grok_path']))
+
metron_topic_retention = config['configurations']['metron-env']['metron_topic_retention']
local_grok_patterns_dir = format("{metron_home}/patterns")
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
index dd37946..049b6ee 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -27,7 +27,6 @@ METRON_JDBC_USERNAME="{{metron_jdbc_username}}"
METRON_JDBC_PLATFORM="{{metron_jdbc_platform}}"
METRON_JDBC_CLIENT_PATH="{{metron_jdbc_client_path}}"
METRON_TEMP_GROK_PATH="{{metron_temp_grok_path}}"
-METRON_DEFAULT_GROK_PATH="{{metron_default_grok_path}}"
METRON_SPRING_OPTIONS="{{metron_spring_options}}"
ZOOKEEPER="{{zookeeper_quorum}}"
BROKERLIST="{{kafka_brokers}}"
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 0c17580..6e4fb66 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -36,7 +36,6 @@ kafka:
grok:
path:
temp: ${METRON_TEMP_GROK_PATH}
- default: ${METRON_DEFAULT_GROK_PATH}
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index 7c9cdac..7f8110c 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -26,7 +26,6 @@ public class MetronRestConstants {
public static final String DOCKER_PROFILE = "docker";
public static final String CSRF_ENABLE_PROFILE = "csrf-enable";
- public static final String GROK_DEFAULT_PATH_SPRING_PROPERTY = "grok.path.default";
public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp";
public static final String GROK_CLASS_NAME = GrokParser.class.getName();
public static final String GROK_PATH_KEY = "grokPath";
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
index adeb1ed..807a330 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java
@@ -17,10 +17,10 @@
*/
package org.apache.metron.rest.service;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
-import java.io.File;
import java.util.Map;
public interface GrokService {
@@ -29,7 +29,9 @@ public interface GrokService {
GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException;
- File saveTemporary(String statement, String name) throws RestException;
+ Path saveTemporary(String statement, String name) throws RestException;
+
+ void deleteTemporary() throws RestException;
String getStatementFromClasspath(String path) throws RestException;
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
index d5932c7..58dbf9b 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -31,4 +31,6 @@ public interface HdfsService {
List<String> list(Path path) throws RestException;
boolean delete(Path path, boolean recursive) throws RestException;
+
+ boolean mkdirs(Path path) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index edae13b..e185b53 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.service.impl;
+import java.nio.charset.Charset;
import oi.thekraken.grok.api.Grok;
import oi.thekraken.grok.api.Match;
import org.apache.commons.io.IOUtils;
@@ -25,15 +26,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
+import org.apache.metron.rest.service.HdfsService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Map;
@@ -47,10 +46,13 @@ public class GrokServiceImpl implements GrokService {
private Grok commonGrok;
+ private HdfsService hdfsService;
+
@Autowired
- public GrokServiceImpl(Environment environment, Grok commonGrok) {
+ public GrokServiceImpl(Environment environment, Grok commonGrok, HdfsService hdfsService) {
this.environment = environment;
this.commonGrok = commonGrok;
+ this.hdfsService = hdfsService;
}
@Override
@@ -85,30 +87,25 @@ public class GrokServiceImpl implements GrokService {
}
@Override
- public File saveTemporary(String statement, String name) throws RestException {
+ public Path saveTemporary(String statement, String name) throws RestException {
if (statement != null) {
- try {
- File grokDirectory = new File(getTemporaryGrokRootPath());
- if (!grokDirectory.exists()) {
- grokDirectory.mkdirs();
- }
- File path = new File(grokDirectory, name);
- FileWriter fileWriter = new FileWriter(new File(grokDirectory, name));
- fileWriter.write(statement);
- fileWriter.close();
- return path;
- } catch (IOException e) {
- throw new RestException(e);
- }
+ Path path = getTemporaryGrokRootPath();
+ hdfsService.mkdirs(path);
+ hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8")));
+ return path;
} else {
throw new RestException("A grokStatement must be provided");
}
}
- private String getTemporaryGrokRootPath() {
+ public void deleteTemporary() throws RestException {
+ hdfsService.delete(getTemporaryGrokRootPath(), true);
+ }
+
+ private Path getTemporaryGrokRootPath() {
String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY);
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
- return new Path(grokTempPath, authentication.getName()).toString();
+ return new Path(grokTempPath, authentication.getName());
}
public String getStatementFromClasspath(String path) throws RestException {
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
index 789c421..a9ae8eb 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -88,4 +88,13 @@ public class HdfsServiceImpl implements HdfsService {
throw new RestException(e);
}
}
+
+ @Override
+ public boolean mkdirs(Path path) throws RestException {
+ try {
+ return FileSystem.get(configuration).mkdirs(path);
+ } catch (IOException e) {
+ throw new RestException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 37d59d0..f99b41c 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -17,8 +17,16 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
+
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.SensorParserConfig;
@@ -34,15 +42,6 @@ import org.reflections.Reflections;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
-
@Service
public class SensorParserConfigServiceImpl implements SensorParserConfigService {
@@ -53,7 +52,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
private GrokService grokService;
@Autowired
- public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) {
+ public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client,
+ GrokService grokService) {
this.objectMapper = objectMapper;
this.client = client;
this.grokService = grokService;
@@ -64,7 +64,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
@Override
public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException {
try {
- ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
+ ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(),
+ objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client);
} catch (Exception e) {
throw new RestException(e);
}
@@ -126,7 +127,8 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
Set<Class<? extends MessageParser>> parserClasses = getParserClasses();
parserClasses.forEach(parserClass -> {
if (!"BasicParser".equals(parserClass.getSimpleName())) {
- availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""), parserClass.getName());
+ availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""),
+ parserClass.getName());
}
});
}
@@ -154,20 +156,23 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService
} else {
MessageParser<JSONObject> parser;
try {
- parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName()).newInstance();
+ parser = (MessageParser<JSONObject>) Class.forName(sensorParserConfig.getParserClassName())
+ .newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
- File temporaryGrokFile = null;
+ Path temporaryGrokPath = null;
if (isGrokConfig(sensorParserConfig)) {
- temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic());
- sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString());
+ String name = parseMessageRequest.getSensorParserConfig().getSensorTopic();
+ temporaryGrokPath = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), name);
+ sensorParserConfig.getParserConfig()
+ .put(MetronRestConstants.GROK_PATH_KEY, new Path(temporaryGrokPath, name).toString());
}
parser.configure(sensorParserConfig.getParserConfig());
parser.init();
JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0);
- if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) {
- temporaryGrokFile.delete();
+ if (isGrokConfig(sensorParserConfig) && temporaryGrokPath != null) {
+ grokService.deleteTemporary();
}
return results;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-docker.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml
index 15fa293..426a0da 100644
--- a/metron-interface/metron-rest/src/main/resources/application-docker.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml
@@ -46,7 +46,6 @@ hdfs:
grok:
path:
temp: target/patterns/temp
- default: target/patterns
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-test.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml
index 9793840..b5e65a7 100644
--- a/metron-interface/metron-rest/src/main/resources/application-test.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-test.yml
@@ -29,7 +29,6 @@ spring:
grok:
path:
temp: target/patterns/temp
- default: target/patterns
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
index 31b5784..cf2c170 100644
--- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
@@ -39,8 +39,7 @@ hdfs:
grok:
path:
- temp: ./patterns/temp
- default: /apps/metron/patterns
+ temp: /apps/metron/patterns/tmp
storm:
ui:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index cf9218b..d96439c 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -26,8 +26,7 @@ spring:
grok:
path:
- temp: ./
- default: /apps/metron/patterns
+ temp: /apps/metron/patterns/tmp
zookeeper:
client:
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
index 66771eb..6e2d788 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java
@@ -21,7 +21,6 @@ import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.rest.MetronRestConstants;
-import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.junit.Before;
import org.junit.Test;
@@ -365,11 +364,6 @@ public class SensorParserConfigControllerIntegrationTest {
FileUtils.cleanDirectory(grokTempPath);
FileUtils.deleteDirectory(grokTempPath);
}
- File grokPath = new File(environment.getProperty(MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY));
- if (grokPath.exists()) {
- FileUtils.cleanDirectory(grokPath);
- FileUtils.deleteDirectory(grokPath);
- }
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
index 1935269..8f7f40c 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java
@@ -17,51 +17,46 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
import oi.thekraken.grok.api.Grok;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.GrokValidation;
import org.apache.metron.rest.service.GrokService;
+import org.apache.metron.rest.service.HdfsService;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.core.env.Environment;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.when;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({GrokServiceImpl.class, FileWriter.class})
public class GrokServiceImplTest {
@Rule
public final ExpectedException exception = ExpectedException.none();
private Environment environment;
private Grok grok;
+ private HdfsService hdfsService;
private GrokService grokService;
@Before
public void setUp() throws Exception {
environment = mock(Environment.class);
grok = mock(Grok.class);
- grokService = new GrokServiceImpl(environment, grok);
+ hdfsService = new HdfsServiceImpl(new Configuration());
+ grokService = new GrokServiceImpl(environment, grok, hdfsService);
}
@Test
@@ -205,24 +200,9 @@ public class GrokServiceImplTest {
grokService.saveTemporary(statement, "squid");
- File testFile = new File("./target/user1/squid");
- assertEquals(statement, FileUtils.readFileToString(testFile));
- testFile.delete();
- }
-
- @Test
- public void saveTemporaryShouldWrapExceptionInRestException() throws Exception {
- exception.expect(RestException.class);
-
- String statement = "grok statement";
-
- Authentication authentication = mock(Authentication.class);
- when(authentication.getName()).thenReturn("user1");
- SecurityContextHolder.getContext().setAuthentication(authentication);
- when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
- whenNew(FileWriter.class).withParameterTypes(File.class).withArguments(any()).thenThrow(new IOException());
-
- grokService.saveTemporary(statement, "squid");
+ File testRoot = new File("./target/user1");
+ assertEquals(statement, FileUtils.readFileToString(new File(testRoot, "squid"), Charset.forName("utf-8")));
+ testRoot.delete();
}
@Test
http://git-wip-us.apache.org/repos/asf/metron/blob/224d3d5e/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
index d35a48c..c96a796 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java
@@ -18,12 +18,14 @@
package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
+import oi.thekraken.grok.api.Grok;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.DeleteBuilder;
import org.apache.curator.framework.api.GetChildrenBuilder;
import org.apache.curator.framework.api.GetDataBuilder;
import org.apache.curator.framework.api.SetDataBuilder;
+import org.apache.hadoop.conf.Configuration;
import org.apache.metron.common.configuration.ConfigurationType;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.rest.RestException;
@@ -43,7 +45,10 @@ import java.io.FileWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -88,11 +93,18 @@ public class SensorParserConfigServiceImplTest {
@Multiline
public static String broJson;
+ private String user = "user1";
+
@Before
public void setUp() throws Exception {
objectMapper = mock(ObjectMapper.class);
curatorFramework = mock(CuratorFramework.class);
- grokService = mock(GrokService.class);
+ Environment environment = mock(Environment.class);
+ Authentication authentication = mock(Authentication.class);
+ when(authentication.getName()).thenReturn(user);
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+ when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target");
+ grokService = new GrokServiceImpl(environment, mock(Grok.class), new HdfsServiceImpl(new Configuration()));
sensorParserConfigService = new SensorParserConfigServiceImpl(objectMapper, curatorFramework, grokService);
}
@@ -269,13 +281,13 @@ public class SensorParserConfigServiceImplTest {
parseMessageRequest.setGrokStatement(grokStatement);
parseMessageRequest.setSampleData(sampleData);
- File patternFile = new File("./target/squidTest");
+ File grokRoot = new File("./target", user);
+ grokRoot.mkdir();
+ File patternFile = new File(grokRoot, "squid");
FileWriter writer = new FileWriter(patternFile);
writer.write(grokStatement);
writer.close();
- when(grokService.saveTemporary(grokStatement, "squid")).thenReturn(patternFile);
-
assertEquals(new HashMap() {{
put("elapsed", 161);
put("code", 200);
[2/5] metron git commit: METRON-1142: Add Geo Hashing functions to
stellar closes apache/incubator-metron#724
Posted by ma...@apache.org.
METRON-1142: Add Geo Hashing functions to stellar closes apache/incubator-metron#724
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/dd711819
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/dd711819
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/dd711819
Branch: refs/heads/Metron_0.4.1
Commit: dd7118197ea09550192d82161cda4e0614a62b35
Parents: 0bbc51d
Author: cstella <ce...@gmail.com>
Authored: Fri Sep 8 09:17:58 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Fri Sep 8 09:17:58 2017 -0400
----------------------------------------------------------------------
metron-analytics/metron-maas-service/pom.xml | 10 +
metron-analytics/metron-profiler-client/pom.xml | 12 +-
metron-analytics/metron-profiler/pom.xml | 10 +
metron-analytics/metron-statistics/pom.xml | 10 +
metron-platform/elasticsearch-shaded/pom.xml | 10 +
metron-platform/metron-api/pom.xml | 10 +
metron-platform/metron-common/pom.xml | 10 +
metron-platform/metron-data-management/pom.xml | 12 +-
metron-platform/metron-elasticsearch/pom.xml | 10 +
metron-platform/metron-enrichment/pom.xml | 27 ++
.../adapters/geo/GeoLiteDatabase.java | 77 ++++-
.../adapters/geo/hash/DistanceStrategies.java | 46 +++
.../adapters/geo/hash/DistanceStrategy.java | 24 ++
.../adapters/geo/hash/GeoHashUtil.java | 189 +++++++++++
.../enrichment/stellar/GeoHashFunctions.java | 299 ++++++++++++++++
.../stellar/GeoHashFunctionsTest.java | 337 +++++++++++++++++++
metron-platform/metron-hbase-client/pom.xml | 10 +
metron-platform/metron-indexing/pom.xml | 10 +
metron-platform/metron-management/pom.xml | 10 +
metron-platform/metron-parsers/pom.xml | 10 +
metron-platform/metron-pcap-backend/pom.xml | 10 +
metron-platform/metron-solr/pom.xml | 10 +
metron-platform/metron-writer/pom.xml | 10 +
metron-stellar/stellar-common/README.md | 50 +++
metron-stellar/stellar-common/pom.xml | 10 +
use-cases/README.md | 4 +
use-cases/geographic_login_outliers/README.md | 267 +++++++++++++++
27 files changed, 1483 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-maas-service/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index 4eeceae..555e73d 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -252,6 +252,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-profiler-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
index bba881d..69b8c29 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -304,7 +304,17 @@
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.metron.guava</shadedPattern>
</relocation>
- </relocations>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-profiler/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml
index 41888a1..e1ee806 100644
--- a/metron-analytics/metron-profiler/pom.xml
+++ b/metron-analytics/metron-profiler/pom.xml
@@ -305,6 +305,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-statistics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml
index 5fab63e..b4d2ed6 100644
--- a/metron-analytics/metron-statistics/pom.xml
+++ b/metron-analytics/metron-statistics/pom.xml
@@ -74,6 +74,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.tdunning</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index bf02510..bbf96a0 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -89,6 +89,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-api/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml
index 912859d..8a15251 100644
--- a/metron-platform/metron-api/pom.xml
+++ b/metron-platform/metron-api/pom.xml
@@ -221,6 +221,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 390ec23..9356e13 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -403,6 +403,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 90c2c52..3fccc0a 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -384,7 +384,17 @@
<goal>shade</goal>
</goals>
<configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 40989c6..0005484 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -242,6 +242,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 37cb49f..dd3998b 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -94,6 +94,23 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>ch.hsr</groupId>
+ <artifactId>geohash</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vividsolutions</groupId>
+ <artifactId>jts-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>com.maxmind.geoip2</groupId>
<artifactId>geoip2</artifactId>
<version>${geoip.version}</version>
@@ -313,6 +330,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
index 0f9bf37..f5d20f7 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.enrichment.adapters.geo;
+import ch.hsr.geohash.WGS84Point;
import com.maxmind.db.CHMCache;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.AddressNotFoundException;
@@ -35,11 +36,16 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.InetAddressValidator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +63,42 @@ public enum GeoLiteDatabase {
private static volatile String hdfsLoc = GEO_HDFS_FILE_DEFAULT;
private static DatabaseReader reader = null;
+ public enum GeoProps {
+ LOC_ID("locID"),
+ COUNTRY("country"),
+ CITY("city"),
+ POSTAL_CODE("postalCode"),
+ DMA_CODE("dmaCode"),
+ LATITUDE("latitude"),
+ LONGITUDE("longitude"),
+ LOCATION_POINT("location_point"),
+ ;
+ Function<Map<String, String>, String> getter;
+ String simpleName;
+
+ GeoProps(String simpleName) {
+ this(simpleName, m -> m.get(simpleName));
+ }
+
+ GeoProps(String simpleName,
+ Function<Map<String, String>, String> getter
+ ) {
+ this.simpleName = simpleName;
+ this.getter = getter;
+ }
+ public String getSimpleName() {
+ return simpleName;
+ }
+
+ public String get(Map<String, String> map) {
+ return getter.apply(map);
+ }
+
+ public void set(Map<String, String> map, String val) {
+ map.put(simpleName, val);
+ }
+ }
+
public synchronized void updateIfNecessary(Map<String, Object> globalConfig) {
// Reload database if necessary (file changes on HDFS)
LOG.trace("[Metron] Determining if GeoIpDatabase update required");
@@ -143,24 +185,24 @@ public enum GeoLiteDatabase {
Postal postal = cityResponse.getPostal();
Location location = cityResponse.getLocation();
- geoInfo.put("locID", convertNullToEmptyString(city.getGeoNameId()));
- geoInfo.put("country", convertNullToEmptyString(country.getIsoCode()));
- geoInfo.put("city", convertNullToEmptyString(city.getName()));
- geoInfo.put("postalCode", convertNullToEmptyString(postal.getCode()));
- geoInfo.put("dmaCode", convertNullToEmptyString(location.getMetroCode()));
+ GeoProps.LOC_ID.set(geoInfo, convertNullToEmptyString(city.getGeoNameId()));
+ GeoProps.COUNTRY.set(geoInfo, convertNullToEmptyString(country.getIsoCode()));
+ GeoProps.CITY.set(geoInfo, convertNullToEmptyString(city.getName()));
+ GeoProps.POSTAL_CODE.set(geoInfo, convertNullToEmptyString(postal.getCode()));
+ GeoProps.DMA_CODE.set(geoInfo, convertNullToEmptyString(location.getMetroCode()));
Double latitudeRaw = location.getLatitude();
String latitude = convertNullToEmptyString(latitudeRaw);
- geoInfo.put("latitude", latitude);
+ GeoProps.LATITUDE.set(geoInfo, latitude);
Double longitudeRaw = location.getLongitude();
String longitude = convertNullToEmptyString(longitudeRaw);
- geoInfo.put("longitude", longitude);
+ GeoProps.LONGITUDE.set(geoInfo, longitude);
if (latitudeRaw == null || longitudeRaw == null) {
- geoInfo.put("location_point", "");
+ GeoProps.LOCATION_POINT.set(geoInfo, "");
} else {
- geoInfo.put("location_point", latitude + "," + longitude);
+ GeoProps.LOCATION_POINT.set(geoInfo, latitude + "," + longitude);
}
return Optional.of(geoInfo);
@@ -174,6 +216,23 @@ public enum GeoLiteDatabase {
return Optional.empty();
}
+ public Optional<WGS84Point> toPoint(Map<String, String> geoInfo) {
+ String latitude = GeoProps.LATITUDE.get(geoInfo);
+ String longitude = GeoProps.LONGITUDE.get(geoInfo);
+ if(latitude == null || longitude == null) {
+ return Optional.empty();
+ }
+
+ try {
+ double latD = Double.parseDouble(latitude.toString());
+ double longD = Double.parseDouble(longitude.toString());
+ return Optional.of(new WGS84Point(latD, longD));
+ } catch (NumberFormatException nfe) {
+ LOG.warn(String.format("Invalid lat/long: %s/%s: %s", latitude, longitude, nfe.getMessage()), nfe);
+ return Optional.empty();
+ }
+ }
+
protected String convertNullToEmptyString(Object raw) {
return raw == null ? "" : String.valueOf(raw);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
new file mode 100644
index 0000000..6af214e
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.WGS84Point;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+
+public enum DistanceStrategies implements DistanceStrategy {
+ HAVERSINE((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distHaversineRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ ),
+ LAW_OF_COSINES((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distLawOfCosinesRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ ),
+ VICENTY((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distVincentyRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ )
+ ;
+ DistanceStrategy strat;
+ DistanceStrategies(DistanceStrategy strat) {
+ this.strat = strat;
+ }
+
+ @Override
+ public double distance(WGS84Point point1, WGS84Point point2) {
+ return strat.distance(point1, point2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
new file mode 100644
index 0000000..0303986
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.WGS84Point;
+
+public interface DistanceStrategy {
+ public double distance(WGS84Point point1, WGS84Point point2);
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
new file mode 100644
index 0000000..902eea3
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.GeoHash;
+import ch.hsr.geohash.WGS84Point;
+import com.google.common.collect.Iterables;
+import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
+
+import java.util.AbstractMap;
+import java.util.Map;
+import java.util.Optional;
+
+public enum GeoHashUtil {
+ INSTANCE;
+
+ public Optional<String> computeHash(Double latitude, Double longitude, int precision) {
+ if(latitude == null || longitude == null) {
+ return Optional.empty();
+ }
+ return computeHash(new WGS84Point(latitude, longitude), precision);
+ }
+
+ public Optional<String> computeHash(WGS84Point point, int precision) {
+ GeoHash hash = GeoHash.withCharacterPrecision(point.getLatitude(), point.getLongitude(), precision);
+ return Optional.of(hash.toBase32());
+ }
+
+ public Optional<String> computeHash(Map<String, String> geoLoc, int precision) {
+ Optional<WGS84Point> point = GeoLiteDatabase.INSTANCE.toPoint(geoLoc);
+ if(point.isPresent()) {
+ return computeHash(point.get(), precision);
+ }
+ else {
+ return Optional.empty();
+ }
+ }
+
+ public Optional<WGS84Point> toPoint(String hash) {
+ if(hash == null) {
+ return Optional.empty();
+ }
+ GeoHash h = GeoHash.fromGeohashString(hash);
+ return Optional.ofNullable(h == null?null:h.getPoint());
+ }
+
+ public double distance(WGS84Point point1, WGS84Point point2, DistanceStrategy strategy) {
+ return strategy.distance(point1, point2);
+ }
+
+ public WGS84Point centroidOfHashes(Iterable<String> hashes) {
+ Iterable<WGS84Point> points = Iterables.transform(hashes, h -> toPoint(h).orElse(null));
+ return centroidOfPoints(points);
+ }
+
+ public WGS84Point centroidOfPoints(Iterable<WGS84Point> points) {
+ Iterable<WGS84Point> nonNullPoints = Iterables.filter(points, p -> p != null);
+ return centroid(Iterables.transform(nonNullPoints
+ , p -> new AbstractMap.SimpleImmutableEntry<>(p, 1)
+ )
+ );
+ }
+
+ public WGS84Point centroidOfWeightedPoints(Map<String, Number> points) {
+
+ Iterable<Map.Entry<WGS84Point, Number>> weightedPoints = Iterables.transform(points.entrySet()
+ , kv -> {
+ WGS84Point pt = toPoint(kv.getKey()).orElse(null);
+ return new AbstractMap.SimpleImmutableEntry<>(pt, kv.getValue());
+ });
+ return centroid(Iterables.filter(weightedPoints, kv -> kv.getKey() != null));
+ }
+
+ /**
+ * Find the equilibrium point of a weighted set of lat/long geo points.
+ * @param points The points and their weights (e.g. multiplicity)
+ * @return
+ */
+ private WGS84Point centroid(Iterable<Map.Entry<WGS84Point, Number>> points) {
+ double x = 0d
+ , y = 0d
+ , z = 0d
+ , totalWeight = 0d
+ ;
+ int n = 0;
+ /**
+ * So, it's first important to realize that long/lat are not cartesian, so simple weighted averaging
+ * is insufficient here as it denies the fact that we're not living on a flat square, but rather the surface of
+ * an ellipsoid. A crow, for instance, does not fly a straight line to an observer outside of Earth, but
+ * rather flies across the arc tracing the surface of earth, or a "great-earth arc". When computing the centroid
+ * you want to find the centroid of the points with distance defined as the great-earth arc.
+ *
+ * The general strategy is to:
+ * 1. Change coordinate systems from degrees on a WGS84 projection (e.g. lat/long)
+ * to a 3 dimensional cartesian surface atop a sphere approximating the earth.
+ * 2. Compute a weighted average of the cartesian coordinates
+ * 3. Change coordinate systems of the resulting centroid in cartesian space back to lat/long
+ *
+ * This is generally detailed at http://www.geomidpoint.com/example.html
+ */
+ for(Map.Entry<WGS84Point, Number> weightedPoint : points) {
+ WGS84Point pt = weightedPoint.getKey();
+ if(pt == null) {
+ continue;
+ }
+ double latRad = Math.toRadians(pt.getLatitude());
+ double longRad = Math.toRadians(pt.getLongitude());
+ double cosLat = Math.cos(latRad);
+ /*
+ Convert from lat/long coordinates to cartesian coordinates. The cartesian coordinate system is a right-hand,
+ rectangular, three-dimensional, earth-fixed coordinate system
+ with an origin at (0, 0, 0). The Z-axis, is parrallel to the axis of rotation of the earth. The Z-coordinate
+ is positive toward the North pole. The X-Y plane lies in the equatorial plane. The X-axis lies along the
+ intersection of the plane containing the prime meridian and the equatorial plane. The X-coordinate is positive
+ toward the intersection of the prime meridian and equator.
+
+ Please see https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
+ for more information about this coordinate transformation.
+ */
+ double ptX = cosLat * Math.cos(longRad);
+ double ptY = cosLat * Math.sin(longRad);
+ double ptZ = Math.sin(latRad);
+ double weight = weightedPoint.getValue().doubleValue();
+ x += ptX*weight;
+ y += ptY*weight;
+ z += ptZ*weight;
+ n++;
+ totalWeight += weight;
+ }
+ if(n == 0) {
+ return null;
+ }
+ //average the vector representation in cartesian space, forming the center of gravity in cartesian space
+ x /= totalWeight;
+ y /= totalWeight;
+ z /= totalWeight;
+
+ //convert the cartesian representation back to radians
+ double longitude = Math.atan2(y, x);
+ double hypotenuse = Math.sqrt(x*x + y*y);
+ double latitude = Math.atan2(z, hypotenuse);
+
+ //convert the radians back to degrees latitude and longitude.
+ return new WGS84Point(Math.toDegrees(latitude), Math.toDegrees(longitude));
+ }
+
+ public double maxDistanceHashes(Iterable<String> hashes, DistanceStrategy strategy) {
+ Iterable<WGS84Point> points = Iterables.transform(hashes, s -> toPoint(s).orElse(null));
+ return maxDistancePoints(Iterables.filter(points, p -> p != null), strategy);
+ }
+
+ public double maxDistancePoints(Iterable<WGS84Point> points, DistanceStrategy strategy) {
+ //Note: because distance is commutative, we only need search the upper triangle
+ int i = 0;
+ double max = Double.NaN;
+ for(WGS84Point pt1 : points) {
+ int j = 0;
+ for(WGS84Point pt2 : points) {
+ if(j <= i) {
+ double d = strategy.distance(pt1, pt2);
+ if(Double.isNaN(max)|| d > max) {
+ max = d;
+ }
+ j++;
+ }
+ else {
+ break;
+ }
+ }
+ i++;
+ }
+ return max;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
new file mode 100644
index 0000000..a1e64c5
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.stellar;
+
+import ch.hsr.geohash.WGS84Point;
+import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
+import org.apache.metron.enrichment.adapters.geo.hash.DistanceStrategies;
+import org.apache.metron.enrichment.adapters.geo.hash.DistanceStrategy;
+import org.apache.metron.enrichment.adapters.geo.hash.GeoHashUtil;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class GeoHashFunctions {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Stellar(name="TO_LATLONG"
+ ,namespace="GEOHASH"
+ ,description="Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)"
+ }
+ ,returns = "A map containing the latitude and longitude of the hash (keys \"latitude\" and \"longitude\")"
+ )
+ public static class ToLatLong implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ String hash = (String)args.get(0);
+ if(hash == null) {
+ return null;
+ }
+
+ Optional<WGS84Point> point = GeoHashUtil.INSTANCE.toPoint(hash);
+ if(point.isPresent()) {
+ Map<String, Object> ret = new HashMap<>();
+ ret.put(GeoLiteDatabase.GeoProps.LONGITUDE.getSimpleName(), point.get().getLongitude());
+ ret.put(GeoLiteDatabase.GeoProps.LATITUDE.getSimpleName(), point.get().getLatitude());
+ return ret;
+ }
+ return null;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="FROM_LATLONG"
+ ,namespace="GEOHASH"
+ ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long"
+ ,params = {
+ "latitude - The latitude",
+ "longitude - The longitude",
+ "character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long"
+ )
+ public static class FromLatLong implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 2) {
+ return null;
+ }
+ Object latObj = args.get(0);
+ Object longObj = args.get(1);
+ if(latObj == null || longObj == null) {
+ return null;
+ }
+ Double latitude = ConversionUtils.convert(latObj, Double.class);
+ Double longitude = ConversionUtils.convert(longObj, Double.class);
+ int charPrecision = 12;
+ if(args.size() > 2) {
+ charPrecision = ConversionUtils.convert(args.get(2), Integer.class);
+ }
+ Optional<String> ret = GeoHashUtil.INSTANCE.computeHash(latitude, longitude, charPrecision);
+ return ret.orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="FROM_LOC"
+ ,namespace="GEOHASH"
+ ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location"
+ ,params = {
+ "map - the latitude and logitude in a map (the output of GEO_GET)",
+ "character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location"
+ )
+ public static class FromLoc implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Map<String, String> map = (Map<String, String>) args.get(0);
+ if(map == null) {
+ return null;
+ }
+ int charPrecision = 12;
+ if(args.size() > 1) {
+ charPrecision = ConversionUtils.convert(args.get(1), Integer.class);
+ }
+ Optional<String> ret = GeoHashUtil.INSTANCE.computeHash(map, charPrecision);
+ return ret.orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+
+ @Stellar(name="DIST"
+ ,namespace="GEOHASH"
+ ,description="Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hash1 - The first location as a geohash",
+ "hash2 - The second location as a geohash",
+ "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default."
+ }
+ ,returns = "The distance in kilometers between the hashes"
+ )
+ public static class Dist implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 2) {
+ return null;
+ }
+ String hash1 = (String)args.get(0);
+ if(hash1 == null) {
+ return null;
+ }
+ Optional<WGS84Point> pt1 = GeoHashUtil.INSTANCE.toPoint(hash1);
+ String hash2 = (String)args.get(1);
+ if(hash2 == null) {
+ return null;
+ }
+ Optional<WGS84Point> pt2 = GeoHashUtil.INSTANCE.toPoint(hash2);
+ DistanceStrategy strat = DistanceStrategies.HAVERSINE;
+ if(args.size() > 2) {
+ strat = DistanceStrategies.valueOf((String) args.get(2));
+ }
+ if(pt1.isPresent() && pt2.isPresent()) {
+ return GeoHashUtil.INSTANCE.distance(pt1.get(), pt2.get(), strat);
+ }
+ return Double.NaN;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="MAX_DIST"
+ ,namespace="GEOHASH"
+ ,description="Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash)",
+ "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default."
+ }
+ ,returns = "The maximum distance in kilometers between any two locations"
+ )
+ public static class MaxDist implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Iterable<String> hashes = (Iterable<String>)args.get(0);
+ if(hashes == null) {
+ return null;
+ }
+ DistanceStrategy strat = DistanceStrategies.HAVERSINE;
+ if(args.size() > 1) {
+ strat = DistanceStrategies.valueOf((String) args.get(1));
+ }
+ return GeoHashUtil.INSTANCE.maxDistanceHashes(hashes, strat);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="CENTROID"
+ ,namespace="GEOHASH"
+ ,description="Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights"
+ ,"character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "The geohash of the centroid"
+ )
+ public static class Centroid implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Object o1 = args.get(0);
+ if(o1 == null) {
+ return null;
+ }
+ WGS84Point centroid = null;
+ if(o1 instanceof Map) {
+ centroid = GeoHashUtil.INSTANCE.centroidOfWeightedPoints((Map<String, Number>)o1);
+ }
+ else if(o1 instanceof Iterable) {
+ centroid = GeoHashUtil.INSTANCE.centroidOfHashes((Iterable<String>)o1);
+ }
+ if(centroid == null) {
+ return null;
+ }
+ Integer precision = 12;
+ if(args.size() > 1) {
+ precision = (Integer)args.get(1);
+ }
+ return GeoHashUtil.INSTANCE.computeHash(centroid, precision).orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
new file mode 100644
index 0000000..f1a0ec4
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.stellar;
+
+import ch.hsr.geohash.WGS84Point;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metron.stellar.common.utils.StellarProcessorUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.*;
+
+public class GeoHashFunctionsTest {
+ public static WGS84Point empireStatePoint = new WGS84Point(40.748570, -73.985752);
+ public static WGS84Point mosconeCenterPoint = new WGS84Point(37.782891, -122.404166);
+ public static WGS84Point jutlandPoint = new WGS84Point(57.64911, 10.40740);
+ public static String explicitJutlandHash = "u4pruydqmvpb";
+ String empireStateHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", empireStatePoint.getLatitude()
+ ,"long",empireStatePoint.getLongitude()
+ )
+ );
+ String mosconeCenterHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", mosconeCenterPoint.getLatitude()
+ ,"long",mosconeCenterPoint.getLongitude()
+ )
+ );
+ String jutlandHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ );
+
+ @Test
+ public void testToLatLong_happypath() throws Exception {
+ Map<String, Object> latLong = (Map<String, Object>)StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", explicitJutlandHash ) );
+ Assert.assertEquals(jutlandPoint.getLatitude(), (double)latLong.get("latitude"), 1e-3);
+ Assert.assertEquals(jutlandPoint.getLongitude(), (double)latLong.get("longitude"), 1e-3);
+ }
+
+ @Test
+ public void testToLatLong_degenerate() throws Exception {
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", "u"));
+ Assert.assertFalse(Double.isNaN((double) latLong.get("latitude")));
+ Assert.assertFalse(Double.isNaN((double) latLong.get("longitude")));
+ }
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", ""));
+ Assert.assertEquals(0d, (double)latLong.get("latitude"), 1e-3);
+ Assert.assertEquals(0d, (double)latLong.get("longitude"), 1e-3);
+ }
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(null)"
+ , new HashMap<>());
+ Assert.assertNull(latLong);
+ }
+ }
+
+ @Test
+ public void testHash_fromlatlong() throws Exception {
+ Assert.assertEquals("u4pruydqmv", StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 10)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmvpb", StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ Assert.assertEquals("u4pruydqmv".substring(0, 6), StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 6)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 10)"
+ , ImmutableMap.of("lat", "blah"
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testHash_fromLocation() throws Exception {
+ Map<String, String> loc = ImmutableMap.of( "latitude", "" + jutlandPoint.getLatitude()
+ , "longitude","" + jutlandPoint.getLongitude()
+ );
+ Assert.assertEquals("u4pruydqmv", StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 10)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmv".substring(0, 6), StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 6)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmvpb", StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc)"
+ , ImmutableMap.of("loc", ImmutableMap.of( "latitude", "57.64911" ))
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 10)"
+ , ImmutableMap.of("loc", ImmutableMap.of( "latitude", "blah"
+ , "longitude","10.40740"
+ )
+ )
+
+ )
+ );
+ }
+
+ @Test
+ public void testDistanceHaversine() throws Exception {
+ testDistance(Optional.empty());
+ testDistance(Optional.of("HAVERSINE"));
+ }
+
+ @Test
+ public void testDistanceLawOfCosines() throws Exception {
+ testDistance(Optional.of("LAW_OF_COSINES"));
+ }
+
+ @Test
+ public void testDistanceLawOfVicenty() throws Exception {
+ testDistance(Optional.of("VICENTY"));
+ }
+
+ @Test
+ public void testMaxDistance_happyPath() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([empireState, mosconeCenter, jutland])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+
+ @Test
+ public void testMaxDistance_differentOrder() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, mosconeCenter, empireState])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+
+ @Test
+ public void testMaxDistance_withNulls() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, mosconeCenter, empireState, null])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+ @Test
+ public void testMaxDistance_allSame() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, jutland, jutland])"
+ , ImmutableMap.of( "jutland", jutlandHash )
+ );
+ Assert.assertEquals(0, maxDistance, 1e-6d);
+ }
+
+ @Test
+ public void testMaxDistance_emptyList() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([])" , new HashMap<>() );
+ Assert.assertTrue(Double.isNaN(maxDistance));
+ }
+
+ @Test
+ public void testMaxDistance_nullList() throws Exception {
+ Double maxDistance = (Double) StellarProcessorUtils.run("GEOHASH_MAX_DIST(null)" , new HashMap<>() );
+ Assert.assertNull(maxDistance);
+ }
+
+ @Test
+ public void testMaxDistance_invalidList() throws Exception {
+ Double maxDistance = (Double) StellarProcessorUtils.run("GEOHASH_MAX_DIST()" , new HashMap<>() );
+ Assert.assertNull(maxDistance);
+ }
+
+ public void testDistance(Optional<String> method) throws Exception {
+ double expectedDistance = 4128; //in kilometers
+ Map<String, Object> vars = ImmutableMap.of("empireState", empireStateHash, "mosconeCenter", mosconeCenterHash);
+ //ensure that d(x, y) == d(y, x) and that both are the same as the expected (up to 1 km accuracy)
+ {
+ String stellarStatement = getDistStellarStatement(ImmutableList.of("mosconeCenter", "empireState"), method);
+ Assert.assertEquals(expectedDistance, (double) StellarProcessorUtils.run(stellarStatement , vars ), 1D );
+ }
+ {
+ String stellarStatement = getDistStellarStatement(ImmutableList.of("empireState", "mosconeCenter"), method);
+ Assert.assertEquals(expectedDistance, (double) StellarProcessorUtils.run(stellarStatement , vars ), 1D );
+ }
+ }
+
+ private static String getDistStellarStatement(List<String> hashVariables, Optional<String> method) {
+ if(method.isPresent()) {
+ List<String> vars = new ArrayList<>();
+ vars.addAll(hashVariables);
+ vars.add("\'" + method.get() + "\'");
+ return "GEOHASH_DIST(" + Joiner.on(",").skipNulls().join(vars) + ")";
+ }
+ else {
+ return "GEOHASH_DIST(" + Joiner.on(",").skipNulls().join(hashVariables) + ")";
+ }
+ }
+
+ @Test
+ public void testCentroid_List() throws Exception {
+ //happy path
+ {
+ double expectedLong = -98.740087 //calculated via http://www.geomidpoint.com/ using the center of gravity or geographic midpoint.
+ , expectedLat = 41.86921
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState, mosconeCenter]))"
+ , ImmutableMap.of("empireState", empireStateHash, "mosconeCenter", mosconeCenterHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //same point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState, empireState]))"
+ , ImmutableMap.of("empireState", empireStateHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //one point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState]))"
+ , ImmutableMap.of("empireState", empireStateHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //no points
+ {
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([]))"
+ , new HashMap<>()
+ );
+ Assert.assertNull(centroid);
+ }
+ }
+
+ @Test
+ public void testCentroid_weighted() throws Exception {
+ //happy path
+ {
+ double expectedLong = -98.740087 //calculated via http://www.geomidpoint.com/ using the center of gravity or geographic midpoint.
+ , expectedLat = 41.86921
+ ;
+ for(int weight = 1;weight < 10;++weight) {
+ Map<Object, Integer> weightedPoints = ImmutableMap.of(empireStateHash, weight, mosconeCenterHash, weight);
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ }
+ //same point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ for(int weight = 1;weight < 10;++weight) {
+ Map<Object, Integer> weightedPoints = ImmutableMap.of(empireStateHash, weight);
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ }
+ //no points
+ {
+ Map<Object, Integer> weightedPoints = new HashMap<>();
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertNull(centroid);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase-client/pom.xml b/metron-platform/metron-hbase-client/pom.xml
index 5dd6127..1237be7 100644
--- a/metron-platform/metron-hbase-client/pom.xml
+++ b/metron-platform/metron-hbase-client/pom.xml
@@ -80,6 +80,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>org.apache.commons.logging</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index c64c374..7d07665 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -222,6 +222,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index 4117d69..a5cae38 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -205,6 +205,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index b7c21ff..85c6218 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -266,6 +266,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-pcap-backend/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml
index 388b1e0..5878873 100644
--- a/metron-platform/metron-pcap-backend/pom.xml
+++ b/metron-platform/metron-pcap-backend/pom.xml
@@ -221,6 +221,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index be1fe33..97132c4 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -232,6 +232,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index 7d3152c..de6b3b8 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -238,6 +238,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 4dc7d8d..340a7ae 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -135,6 +135,12 @@ In the core language functions, we support basic functional programming primitiv
| [ `FUZZY_SCORE`](#fuzzy_score) |
| [ `FORMAT`](#format) |
| [ `GEO_GET`](#geo_get) |
+| [ `GEOHASH_CENTROID`](#geohash_centroid) |
+| [ `GEOHASH_DIST`](#geohash_dist) |
+| [ `GEOHASH_FROM_LATLONG`](#geohash_from_latlong) |
+| [ `GEOHASH_FROM_LOC`](#geohash_from_loc) |
+| [ `GEOHASH_MAX_DIST`](#geohash_max_dist) |
+| [ `GEOHASH_TO_LATLONG`](#geohash_to_latlong) |
| [ `GET`](#get) |
| [ `GET_FIRST`](#get_first) |
| [ `GET_LAST`](#get_last) |
@@ -443,6 +449,50 @@ In the core language functions, we support basic functional programming primitiv
* fields - Optional list of GeoIP fields to grab. Options are locID, country, city postalCode, dmaCode, latitude, longitude, location_point
* Returns: If a Single field is requested a string of the field, If multiple fields a map of string of the fields, and null otherwise
+### `GEOHASH_CENTROID`
+ * Description: Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights
+ * character_precision? - The number of characters to use in the hash. Default is 12
+ * Returns: The geohash of the centroid
+
+### `GEOHASH_DIST`
+ * Description: Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hash1 - The first point as a geohash
+ * hash2 - The second point as a geohash
+ * strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default.
+ * Returns: The distance in kilometers between the hashes.
+
+### `GEOHASH_FROM_LATLONG`
+ * Description: Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long
+ * Input:
+ * latitude - The latitude
+ * longitude - The longitude
+ * character_precision? - The number of characters to use in the hash. Default is 12
+ * Returns: A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long
+
+### `GEOHASH_FROM_LOC`
+ * Description: Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location
+ * Input:
+ * map - the latitude and logitude in a map (the output of [GEO_GET](#geo_get) )
+ * longitude - The longitude
+ * character_precision? - The number of characters to use in the hash. Default is `12`
+ * Returns: A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location
+
+### `GEOHASH_MAX_DIST`
+ * Description: Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hashes - A set of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default.
+ * Returns: The maximum distance in kilometers between any two locations
+
+### `GEOHASH_TO_LATLONG`
+ * Description: Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)
+ * Returns: A map containing the latitude and longitude of the hash (keys "latitude" and "longitude")
+
### `GET`
* Description: Returns the i'th element of the list
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index 5945bbd..9ec29b8 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -257,6 +257,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/use-cases/README.md
----------------------------------------------------------------------
diff --git a/use-cases/README.md b/use-cases/README.md
new file mode 100644
index 0000000..02be32d
--- /dev/null
+++ b/use-cases/README.md
@@ -0,0 +1,4 @@
+# Worked Examples
+
+The following are worked examples of use-cases that showcase some (or
+many) component(s) of Metron.
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/use-cases/geographic_login_outliers/README.md
----------------------------------------------------------------------
diff --git a/use-cases/geographic_login_outliers/README.md b/use-cases/geographic_login_outliers/README.md
new file mode 100644
index 0000000..99e9a5b
--- /dev/null
+++ b/use-cases/geographic_login_outliers/README.md
@@ -0,0 +1,267 @@
+# Problem Statement
+
+One way to find anomalous behavior in a network is by inspecting user
+login behavior. In particular, if a user is logging in via vastly
+differing geographic locations in a short period of time, this may be
+evidence of malicious behavior.
+
+More formally, we can encode this potentially malicious event in terms
+of how far from the geographic centroid of the user's historic logins
+as compared to all users. For instance, if we track all users and the
+median distance from the central geographic location of all of their
+logins for the last 2 hours is 3 km and the standard deviation is 1 km,
+if we see a user logging in 1700 km from the central geographic location of
+their logins for the last 2 hours, then they MAY be exhibiting a
+deviation that we want to monitor since it would be hard to travel that
+distance in 4 hours. On the other hand, the user may have
+just used a VPN or proxy. Ultimately, this sort of analytic must be
+considered only one piece of evidence in addition to many others before
+we want to indicate an alert.
+
+# Demonstration Design
+For the purposes of demonstration, we will construct synthetic data
+whereby 2 users are logging into a system rather quickly (once per
+second) from various hosts. Each user's locations share the same first
+2 octets, but will choose the last 2 randomly. We will then inject a
+data point indicating `user1` is logging in via a russian IP address.
+
+## Preliminaries
+We assume that the following environment variables are set:
+* `METRON_HOME` - the home directory for metron
+* `ZOOKEEPER` - The zookeeper quorum (comma separated with port specified: e.g. `node1:2181` for full-dev)
+* `BROKERLIST` - The Kafka broker list (comma separated with port specified: e.g. `node1:6667` for full-dev)
+* `ES_HOST` - The elasticsearch master (and port) e.g. `node1:9200` for full-dev.
+
+Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.
+
+Before editing configurations, be sure to pull the configs from zookeeper locally via
+```
+$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f
+```
+
+## Configure the Profiler
+First, we'll configure the profiler to emit a profiler every 1 minute:
+* In Ambari, set the profiler period duration to `1` minute via the Profiler config section.
+* Adjust `$METRON_HOME/config/zookeeper/global.json` to adjust the capture duration:
+```
+ "profiler.client.period.duration" : "1",
+ "profiler.client.period.duration.units" : "MINUTES"
+```
+
+## Create the Data Generator
+We want to create a new sensor for our synthetic data called `auth`. To
+feed it, we need a synthetic data generator. In particular, we want a
+process which will feed authentication events per second for a set of
+users where the IPs are randomly chosen, but each user's login ip
+addresses share the same first 2 octets.
+
+Edit `~/gen_data.py` and paste the following into it:
+```
+#!/usr/bin/python
+
+import random
+import sys
+import time
+
+domains = { 'user1' : '173.90', 'user2' : '156.33' }
+
+def get_ip(base):
+ return base + '.' + str(random.randint(1,255)) + '.' + str(random.randint(1, 255))
+
+def main():
+ freq_s = 1
+ while True:
+ user='user' + str(random.randint(1,len(domains)))
+ epoch_time = int(time.time())
+ ip=get_ip(domains[user])
+ print user + ',' + ip + ',' + str(epoch_time)
+ sys.stdout.flush()
+ time.sleep(freq_s)
+
+if __name__ == '__main__':
+ main()
+```
+
+## Create the `auth` Parser
+
+The message format for our simple synthetic data is a CSV with:
+* username
+* login ip address
+* timestamp
+
+We will need to parse this via our `CSVParser` and add the geohash of the login ip address.
+
+* To create this parser, edit `$METRON_HOME/config/zookeeper/parsers/auth.json` and paste the following:
+```
+{
+ "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+ ,"sensorTopic" : "auth"
+ ,"parserConfig" : {
+ "columns" : {
+ "user" : 0,
+ "ip" : 1,
+ "timestamp" : 2
+ }
+ }
+ ,"fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : [ "hash" ]
+ ,"config" : {
+ "hash" : "GEOHASH_FROM_LOC(GEO_GET(ip))"
+ }
+ }
+ ]
+}
+```
+* Create the kafka topic via:
+```
+/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic auth --partitions 1 --replication-factor 1
+```
+
+## Create the Profiles for Enrichment
+
+We will need to track 2 profiles to accomplish this task:
+* `locations_by_user` - The geohashes of the locations the user has logged in from. This is a multiset of geohashes per user. Note that the multiset in this case is effectively a map of geohashes to occurrance counts.
+* `geo_distribution_from_centroid` - The statistical distribution of the distance between a login location and the geographic centroid of the user's previous logins from the last 2 minutes. Note, in a real installation this would be a larger temporal lookback.
+
+We can represent these in the `$METRON_HOME/config/zookeeper/profiler.json` via the following:
+```
+{
+ "profiles": [
+ {
+ "profile": "geo_distribution_from_centroid",
+ "foreach": "'global'",
+ "onlyif": "exists(geo_distance) && geo_distance != null",
+ "init" : {
+ "s": "STATS_INIT()"
+ },
+ "update": {
+ "s": "STATS_ADD(s, geo_distance)"
+ },
+ "result": "s"
+ },
+ {
+ "profile": "locations_by_user",
+ "foreach": "user",
+ "onlyif": "exists(hash) && hash != null && LENGTH(hash) > 0",
+ "init" : {
+ "s": "MULTISET_INIT()"
+ },
+ "update": {
+ "s": "MULTISET_ADD(s, hash)"
+ },
+ "result": "s"
+ }
+ ]
+}
+```
+
+## Enrich authentication Events
+
+We will need to enrich the authentication records in a couple of ways to use in the threat triage section as well as the profiles:
+* `geo_distance`: representing the distance between the current geohash and the geographic centroid for the last 2 minutes.
+* `geo_centroid`: representing the geographic centroid for the last 2 minutes
+
+Beyond that, we will need to determine if the authentication event is a geographic outlier by computing the following fields:
+* `dist_median` : representing the median distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the median of the `geo_distance` values across all users).
+* `dist_sd` : representing the standard deviation of the distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the standard deviation of the `geo_distance` values across all users).
+* `geo_outlier` : whether `geo_distance` is more than 5 standard deviations from the median across all users.
+
+We also want to set up a triage rule associating a score and setting an alert if `geo_outlier` is true. In reality, this would be more complex as this metric is at best circumstantial and would need supporting evidence, but for simplicity we'll deal with the false positives.
+
+* Edit `$METRON_HOME/config/zookeeper/enrichments/auth.json` and paste the following:
+```
+{
+ "enrichment": {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "geo_locations := MULTISET_MERGE( PROFILE_GET( 'locations_by_user', user, PROFILE_FIXED( 2, 'MINUTES')))",
+ "geo_centroid := GEOHASH_CENTROID(geo_locations)",
+ "geo_distance := TO_INTEGER(GEOHASH_DIST(geo_centroid, hash))",
+ "geo_locations := null"
+ ]
+ }
+ }
+ ,"fieldToTypeMap": { }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "geo_distance_distr:= STATS_MERGE( PROFILE_GET( 'geo_distribution_from_centroid', 'global', PROFILE_FIXED( 2, 'MINUTES')))",
+ "dist_median := STATS_PERCENTILE(geo_distance_distr, 50.0)",
+ "dist_sd := STATS_SD(geo_distance_distr)",
+ "geo_outlier := ABS(dist_median - geo_distance) >= 5*dist_sd",
+ "is_alert := exists(is_alert) && is_alert",
+ "is_alert := is_alert || (geo_outlier != null && geo_outlier == true)",
+ "geo_distance_distr := null"
+ ]
+ }
+
+ },
+ "fieldToTypeMap": { },
+ "triageConfig" : {
+ "riskLevelRules" : [
+ {
+ "name" : "Geographic Outlier",
+ "comment" : "Determine if the user's geographic distance from the centroid of the historic logins is an outlier as compared to all users.",
+ "rule" : "geo_outlier != null && geo_outlier",
+ "score" : 10,
+ "reason" : "FORMAT('user %s has a distance (%d) from the centroid of their last login is 5 std deviations (%f) from the median (%f)', user, geo_distance, dist_sd, dist_median)"
+ }
+ ],
+ "aggregator" : "MAX"
+ }
+ }
+}
+```
+
+## Execute Demonstration
+
+From here, we've set up our configuration and can push the configs:
+* Push the configs to zookeeper via
+```
+$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z node1:2181 -i $METRON_HOME/config/zookeeper/
+```
+* Start the parser via:
+```
+$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s auth
+```
+* Push synthetic data into the `auth` topic via
+```
+python ~/gen_data.py | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic auth
+```
+* Wait for about `5` minutes and kill the previous command
+* Push a synthetic record indicating `user1` has logged in from a russian IP (`109.252.227.173`):
+```
+echo -e "import time\nprint 'user1,109.252.227.173,'+str(int(time.time()))" | python | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic auth
+```
+* Execute the following to search elasticsearch for our geographic login outliers:
+```
+curl -XPOST "http://$ES_HOST/auth*/_search?pretty" -d '
+{
+ "_source" : [ "is_alert", "threat:triage:rules:0:reason", "user", "ip", "geo_distance" ],
+ "query": { "exists" : { "field" : "threat:triage:rules:0:reason" } }
+}
+'
+```
+
+You should see, among a few other false positive results, something like the following:
+```
+{
+ "_index" : "auth_index_2017.09.07.20",
+ "_type" : "auth_doc",
+ "_id" : "f5bdbf76-9d78-48cc-b21d-bc434c96e62e",
+ "_score" : 1.0,
+ "_source" : {
+ "geo_distance" : 7879,
+ "threat:triage:rules:0:reason" : "user user1 has a distance (7879) from the centroid of their last login is 5 std deviations (334.814719) from the median (128.000000)",
+ "ip" : "109.252.227.173",
+ "is_alert" : "true",
+ "user" : "user1"
+ }
+}
+```
+