You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/09/08 20:25:36 UTC
[2/5] metron git commit: METRON-1142: Add Geo Hashing functions to
stellar closes apache/incubator-metron#724
METRON-1142: Add Geo Hashing functions to stellar closes apache/incubator-metron#724
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/dd711819
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/dd711819
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/dd711819
Branch: refs/heads/Metron_0.4.1
Commit: dd7118197ea09550192d82161cda4e0614a62b35
Parents: 0bbc51d
Author: cstella <ce...@gmail.com>
Authored: Fri Sep 8 09:17:58 2017 -0400
Committer: cstella <ce...@gmail.com>
Committed: Fri Sep 8 09:17:58 2017 -0400
----------------------------------------------------------------------
metron-analytics/metron-maas-service/pom.xml | 10 +
metron-analytics/metron-profiler-client/pom.xml | 12 +-
metron-analytics/metron-profiler/pom.xml | 10 +
metron-analytics/metron-statistics/pom.xml | 10 +
metron-platform/elasticsearch-shaded/pom.xml | 10 +
metron-platform/metron-api/pom.xml | 10 +
metron-platform/metron-common/pom.xml | 10 +
metron-platform/metron-data-management/pom.xml | 12 +-
metron-platform/metron-elasticsearch/pom.xml | 10 +
metron-platform/metron-enrichment/pom.xml | 27 ++
.../adapters/geo/GeoLiteDatabase.java | 77 ++++-
.../adapters/geo/hash/DistanceStrategies.java | 46 +++
.../adapters/geo/hash/DistanceStrategy.java | 24 ++
.../adapters/geo/hash/GeoHashUtil.java | 189 +++++++++++
.../enrichment/stellar/GeoHashFunctions.java | 299 ++++++++++++++++
.../stellar/GeoHashFunctionsTest.java | 337 +++++++++++++++++++
metron-platform/metron-hbase-client/pom.xml | 10 +
metron-platform/metron-indexing/pom.xml | 10 +
metron-platform/metron-management/pom.xml | 10 +
metron-platform/metron-parsers/pom.xml | 10 +
metron-platform/metron-pcap-backend/pom.xml | 10 +
metron-platform/metron-solr/pom.xml | 10 +
metron-platform/metron-writer/pom.xml | 10 +
metron-stellar/stellar-common/README.md | 50 +++
metron-stellar/stellar-common/pom.xml | 10 +
use-cases/README.md | 4 +
use-cases/geographic_login_outliers/README.md | 267 +++++++++++++++
27 files changed, 1483 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-maas-service/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index 4eeceae..555e73d 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -252,6 +252,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-profiler-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
index bba881d..69b8c29 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -304,7 +304,17 @@
<pattern>com.google.common</pattern>
<shadedPattern>org.apache.metron.guava</shadedPattern>
</relocation>
- </relocations>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-profiler/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml
index 41888a1..e1ee806 100644
--- a/metron-analytics/metron-profiler/pom.xml
+++ b/metron-analytics/metron-profiler/pom.xml
@@ -305,6 +305,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-analytics/metron-statistics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-statistics/pom.xml b/metron-analytics/metron-statistics/pom.xml
index 5fab63e..b4d2ed6 100644
--- a/metron-analytics/metron-statistics/pom.xml
+++ b/metron-analytics/metron-statistics/pom.xml
@@ -74,6 +74,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.tdunning</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/elasticsearch-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
index bf02510..bbf96a0 100644
--- a/metron-platform/elasticsearch-shaded/pom.xml
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -89,6 +89,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-api/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-api/pom.xml b/metron-platform/metron-api/pom.xml
index 912859d..8a15251 100644
--- a/metron-platform/metron-api/pom.xml
+++ b/metron-platform/metron-api/pom.xml
@@ -221,6 +221,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 390ec23..9356e13 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -403,6 +403,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-data-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 90c2c52..3fccc0a 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -384,7 +384,17 @@
<goal>shade</goal>
</goals>
<configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 40989c6..0005484 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -242,6 +242,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/pom.xml b/metron-platform/metron-enrichment/pom.xml
index 37cb49f..dd3998b 100644
--- a/metron-platform/metron-enrichment/pom.xml
+++ b/metron-platform/metron-enrichment/pom.xml
@@ -94,6 +94,23 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>ch.hsr</groupId>
+ <artifactId>geohash</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.spatial4j</groupId>
+ <artifactId>spatial4j</artifactId>
+ <version>0.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vividsolutions</groupId>
+ <artifactId>jts-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>com.maxmind.geoip2</groupId>
<artifactId>geoip2</artifactId>
<version>${geoip.version}</version>
@@ -313,6 +330,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
index 0f9bf37..f5d20f7 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoLiteDatabase.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.enrichment.adapters.geo;
+import ch.hsr.geohash.WGS84Point;
import com.maxmind.db.CHMCache;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.AddressNotFoundException;
@@ -35,11 +36,16 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.InetAddressValidator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +63,42 @@ public enum GeoLiteDatabase {
private static volatile String hdfsLoc = GEO_HDFS_FILE_DEFAULT;
private static DatabaseReader reader = null;
+ public enum GeoProps {
+ LOC_ID("locID"),
+ COUNTRY("country"),
+ CITY("city"),
+ POSTAL_CODE("postalCode"),
+ DMA_CODE("dmaCode"),
+ LATITUDE("latitude"),
+ LONGITUDE("longitude"),
+ LOCATION_POINT("location_point"),
+ ;
+ Function<Map<String, String>, String> getter;
+ String simpleName;
+
+ GeoProps(String simpleName) {
+ this(simpleName, m -> m.get(simpleName));
+ }
+
+ GeoProps(String simpleName,
+ Function<Map<String, String>, String> getter
+ ) {
+ this.simpleName = simpleName;
+ this.getter = getter;
+ }
+ public String getSimpleName() {
+ return simpleName;
+ }
+
+ public String get(Map<String, String> map) {
+ return getter.apply(map);
+ }
+
+ public void set(Map<String, String> map, String val) {
+ map.put(simpleName, val);
+ }
+ }
+
public synchronized void updateIfNecessary(Map<String, Object> globalConfig) {
// Reload database if necessary (file changes on HDFS)
LOG.trace("[Metron] Determining if GeoIpDatabase update required");
@@ -143,24 +185,24 @@ public enum GeoLiteDatabase {
Postal postal = cityResponse.getPostal();
Location location = cityResponse.getLocation();
- geoInfo.put("locID", convertNullToEmptyString(city.getGeoNameId()));
- geoInfo.put("country", convertNullToEmptyString(country.getIsoCode()));
- geoInfo.put("city", convertNullToEmptyString(city.getName()));
- geoInfo.put("postalCode", convertNullToEmptyString(postal.getCode()));
- geoInfo.put("dmaCode", convertNullToEmptyString(location.getMetroCode()));
+ GeoProps.LOC_ID.set(geoInfo, convertNullToEmptyString(city.getGeoNameId()));
+ GeoProps.COUNTRY.set(geoInfo, convertNullToEmptyString(country.getIsoCode()));
+ GeoProps.CITY.set(geoInfo, convertNullToEmptyString(city.getName()));
+ GeoProps.POSTAL_CODE.set(geoInfo, convertNullToEmptyString(postal.getCode()));
+ GeoProps.DMA_CODE.set(geoInfo, convertNullToEmptyString(location.getMetroCode()));
Double latitudeRaw = location.getLatitude();
String latitude = convertNullToEmptyString(latitudeRaw);
- geoInfo.put("latitude", latitude);
+ GeoProps.LATITUDE.set(geoInfo, latitude);
Double longitudeRaw = location.getLongitude();
String longitude = convertNullToEmptyString(longitudeRaw);
- geoInfo.put("longitude", longitude);
+ GeoProps.LONGITUDE.set(geoInfo, longitude);
if (latitudeRaw == null || longitudeRaw == null) {
- geoInfo.put("location_point", "");
+ GeoProps.LOCATION_POINT.set(geoInfo, "");
} else {
- geoInfo.put("location_point", latitude + "," + longitude);
+ GeoProps.LOCATION_POINT.set(geoInfo, latitude + "," + longitude);
}
return Optional.of(geoInfo);
@@ -174,6 +216,23 @@ public enum GeoLiteDatabase {
return Optional.empty();
}
+ public Optional<WGS84Point> toPoint(Map<String, String> geoInfo) {
+ String latitude = GeoProps.LATITUDE.get(geoInfo);
+ String longitude = GeoProps.LONGITUDE.get(geoInfo);
+ if(latitude == null || longitude == null) {
+ return Optional.empty();
+ }
+
+ try {
+ double latD = Double.parseDouble(latitude.toString());
+ double longD = Double.parseDouble(longitude.toString());
+ return Optional.of(new WGS84Point(latD, longD));
+ } catch (NumberFormatException nfe) {
+ LOG.warn(String.format("Invalid lat/long: %s/%s: %s", latitude, longitude, nfe.getMessage()), nfe);
+ return Optional.empty();
+ }
+ }
+
protected String convertNullToEmptyString(Object raw) {
return raw == null ? "" : String.valueOf(raw);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
new file mode 100644
index 0000000..6af214e
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategies.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.WGS84Point;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+
+public enum DistanceStrategies implements DistanceStrategy {
+ HAVERSINE((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distHaversineRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ ),
+ LAW_OF_COSINES((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distLawOfCosinesRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ ),
+ VICENTY((p1, p2) -> DistanceUtils.EARTH_MEAN_RADIUS_KM*DistanceUtils.distVincentyRAD( Math.toRadians(p1.getLatitude()), Math.toRadians(p1.getLongitude())
+ , Math.toRadians(p2.getLatitude()), Math.toRadians(p2.getLongitude())
+ )
+ )
+ ;
+ DistanceStrategy strat;
+ DistanceStrategies(DistanceStrategy strat) {
+ this.strat = strat;
+ }
+
+ @Override
+ public double distance(WGS84Point point1, WGS84Point point2) {
+ return strat.distance(point1, point2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
new file mode 100644
index 0000000..0303986
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/DistanceStrategy.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.WGS84Point;
+
+public interface DistanceStrategy {
+ public double distance(WGS84Point point1, WGS84Point point2);
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
new file mode 100644
index 0000000..902eea3
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/hash/GeoHashUtil.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.adapters.geo.hash;
+
+import ch.hsr.geohash.GeoHash;
+import ch.hsr.geohash.WGS84Point;
+import com.google.common.collect.Iterables;
+import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
+
+import java.util.AbstractMap;
+import java.util.Map;
+import java.util.Optional;
+
+public enum GeoHashUtil {
+ INSTANCE;
+
+ public Optional<String> computeHash(Double latitude, Double longitude, int precision) {
+ if(latitude == null || longitude == null) {
+ return Optional.empty();
+ }
+ return computeHash(new WGS84Point(latitude, longitude), precision);
+ }
+
+ public Optional<String> computeHash(WGS84Point point, int precision) {
+ GeoHash hash = GeoHash.withCharacterPrecision(point.getLatitude(), point.getLongitude(), precision);
+ return Optional.of(hash.toBase32());
+ }
+
+ public Optional<String> computeHash(Map<String, String> geoLoc, int precision) {
+ Optional<WGS84Point> point = GeoLiteDatabase.INSTANCE.toPoint(geoLoc);
+ if(point.isPresent()) {
+ return computeHash(point.get(), precision);
+ }
+ else {
+ return Optional.empty();
+ }
+ }
+
+ public Optional<WGS84Point> toPoint(String hash) {
+ if(hash == null) {
+ return Optional.empty();
+ }
+ GeoHash h = GeoHash.fromGeohashString(hash);
+ return Optional.ofNullable(h == null?null:h.getPoint());
+ }
+
+ public double distance(WGS84Point point1, WGS84Point point2, DistanceStrategy strategy) {
+ return strategy.distance(point1, point2);
+ }
+
+ public WGS84Point centroidOfHashes(Iterable<String> hashes) {
+ Iterable<WGS84Point> points = Iterables.transform(hashes, h -> toPoint(h).orElse(null));
+ return centroidOfPoints(points);
+ }
+
+ public WGS84Point centroidOfPoints(Iterable<WGS84Point> points) {
+ Iterable<WGS84Point> nonNullPoints = Iterables.filter(points, p -> p != null);
+ return centroid(Iterables.transform(nonNullPoints
+ , p -> new AbstractMap.SimpleImmutableEntry<>(p, 1)
+ )
+ );
+ }
+
+ public WGS84Point centroidOfWeightedPoints(Map<String, Number> points) {
+
+ Iterable<Map.Entry<WGS84Point, Number>> weightedPoints = Iterables.transform(points.entrySet()
+ , kv -> {
+ WGS84Point pt = toPoint(kv.getKey()).orElse(null);
+ return new AbstractMap.SimpleImmutableEntry<>(pt, kv.getValue());
+ });
+ return centroid(Iterables.filter(weightedPoints, kv -> kv.getKey() != null));
+ }
+
+ /**
+ * Find the equilibrium point of a weighted set of lat/long geo points.
+ * @param points The points and their weights (e.g. multiplicity)
+ * @return
+ */
+ private WGS84Point centroid(Iterable<Map.Entry<WGS84Point, Number>> points) {
+ double x = 0d
+ , y = 0d
+ , z = 0d
+ , totalWeight = 0d
+ ;
+ int n = 0;
+ /**
+ * So, it's first important to realize that long/lat are not cartesian, so simple weighted averaging
+ * is insufficient here as it denies the fact that we're not living on a flat square, but rather the surface of
+ * an ellipsoid. A crow, for instance, does not fly a straight line to an observer outside of Earth, but
+ * rather flies across the arc tracing the surface of earth, or a "great-earth arc". When computing the centroid
+ * you want to find the centroid of the points with distance defined as the great-earth arc.
+ *
+ * The general strategy is to:
+ * 1. Change coordinate systems from degrees on a WGS84 projection (e.g. lat/long)
+ * to a 3 dimensional cartesian surface atop a sphere approximating the earth.
+ * 2. Compute a weighted average of the cartesian coordinates
+ * 3. Change coordinate systems of the resulting centroid in cartesian space back to lat/long
+ *
+ * This is generally detailed at http://www.geomidpoint.com/example.html
+ */
+ for(Map.Entry<WGS84Point, Number> weightedPoint : points) {
+ WGS84Point pt = weightedPoint.getKey();
+ if(pt == null) {
+ continue;
+ }
+ double latRad = Math.toRadians(pt.getLatitude());
+ double longRad = Math.toRadians(pt.getLongitude());
+ double cosLat = Math.cos(latRad);
+ /*
+ Convert from lat/long coordinates to cartesian coordinates. The cartesian coordinate system is a right-hand,
+ rectangular, three-dimensional, earth-fixed coordinate system
+ with an origin at (0, 0, 0). The Z-axis, is parrallel to the axis of rotation of the earth. The Z-coordinate
+ is positive toward the North pole. The X-Y plane lies in the equatorial plane. The X-axis lies along the
+ intersection of the plane containing the prime meridian and the equatorial plane. The X-coordinate is positive
+ toward the intersection of the prime meridian and equator.
+
+ Please see https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
+ for more information about this coordinate transformation.
+ */
+ double ptX = cosLat * Math.cos(longRad);
+ double ptY = cosLat * Math.sin(longRad);
+ double ptZ = Math.sin(latRad);
+ double weight = weightedPoint.getValue().doubleValue();
+ x += ptX*weight;
+ y += ptY*weight;
+ z += ptZ*weight;
+ n++;
+ totalWeight += weight;
+ }
+ if(n == 0) {
+ return null;
+ }
+ //average the vector representation in cartesian space, forming the center of gravity in cartesian space
+ x /= totalWeight;
+ y /= totalWeight;
+ z /= totalWeight;
+
+ //convert the cartesian representation back to radians
+ double longitude = Math.atan2(y, x);
+ double hypotenuse = Math.sqrt(x*x + y*y);
+ double latitude = Math.atan2(z, hypotenuse);
+
+ //convert the radians back to degrees latitude and longitude.
+ return new WGS84Point(Math.toDegrees(latitude), Math.toDegrees(longitude));
+ }
+
+ public double maxDistanceHashes(Iterable<String> hashes, DistanceStrategy strategy) {
+ Iterable<WGS84Point> points = Iterables.transform(hashes, s -> toPoint(s).orElse(null));
+ return maxDistancePoints(Iterables.filter(points, p -> p != null), strategy);
+ }
+
+ public double maxDistancePoints(Iterable<WGS84Point> points, DistanceStrategy strategy) {
+ //Note: because distance is commutative, we only need search the upper triangle
+ int i = 0;
+ double max = Double.NaN;
+ for(WGS84Point pt1 : points) {
+ int j = 0;
+ for(WGS84Point pt2 : points) {
+ if(j <= i) {
+ double d = strategy.distance(pt1, pt2);
+ if(Double.isNaN(max)|| d > max) {
+ max = d;
+ }
+ j++;
+ }
+ else {
+ break;
+ }
+ }
+ i++;
+ }
+ return max;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
new file mode 100644
index 0000000..a1e64c5
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/stellar/GeoHashFunctions.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.enrichment.stellar;
+
+import ch.hsr.geohash.WGS84Point;
+import org.apache.metron.enrichment.adapters.geo.GeoLiteDatabase;
+import org.apache.metron.enrichment.adapters.geo.hash.DistanceStrategies;
+import org.apache.metron.enrichment.adapters.geo.hash.DistanceStrategy;
+import org.apache.metron.enrichment.adapters.geo.hash.GeoHashUtil;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class GeoHashFunctions {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Stellar(name="TO_LATLONG"
+ ,namespace="GEOHASH"
+ ,description="Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)"
+ }
+ ,returns = "A map containing the latitude and longitude of the hash (keys \"latitude\" and \"longitude\")"
+ )
+ public static class ToLatLong implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ String hash = (String)args.get(0);
+ if(hash == null) {
+ return null;
+ }
+
+ Optional<WGS84Point> point = GeoHashUtil.INSTANCE.toPoint(hash);
+ if(point.isPresent()) {
+ Map<String, Object> ret = new HashMap<>();
+ ret.put(GeoLiteDatabase.GeoProps.LONGITUDE.getSimpleName(), point.get().getLongitude());
+ ret.put(GeoLiteDatabase.GeoProps.LATITUDE.getSimpleName(), point.get().getLatitude());
+ return ret;
+ }
+ return null;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="FROM_LATLONG"
+ ,namespace="GEOHASH"
+ ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long"
+ ,params = {
+ "latitude - The latitude",
+ "longitude - The longitude",
+ "character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long"
+ )
+ public static class FromLatLong implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 2) {
+ return null;
+ }
+ Object latObj = args.get(0);
+ Object longObj = args.get(1);
+ if(latObj == null || longObj == null) {
+ return null;
+ }
+ Double latitude = ConversionUtils.convert(latObj, Double.class);
+ Double longitude = ConversionUtils.convert(longObj, Double.class);
+ int charPrecision = 12;
+ if(args.size() > 2) {
+ charPrecision = ConversionUtils.convert(args.get(2), Integer.class);
+ }
+ Optional<String> ret = GeoHashUtil.INSTANCE.computeHash(latitude, longitude, charPrecision);
+ return ret.orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="FROM_LOC"
+ ,namespace="GEOHASH"
+ ,description="Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location"
+ ,params = {
+ "map - the latitude and logitude in a map (the output of GEO_GET)",
+ "character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location"
+ )
+ public static class FromLoc implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Map<String, String> map = (Map<String, String>) args.get(0);
+ if(map == null) {
+ return null;
+ }
+ int charPrecision = 12;
+ if(args.size() > 1) {
+ charPrecision = ConversionUtils.convert(args.get(1), Integer.class);
+ }
+ Optional<String> ret = GeoHashUtil.INSTANCE.computeHash(map, charPrecision);
+ return ret.orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+
+ @Stellar(name="DIST"
+ ,namespace="GEOHASH"
+ ,description="Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hash1 - The first location as a geohash",
+ "hash2 - The second location as a geohash",
+ "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default."
+ }
+ ,returns = "The distance in kilometers between the hashes"
+ )
+ public static class Dist implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 2) {
+ return null;
+ }
+ String hash1 = (String)args.get(0);
+ if(hash1 == null) {
+ return null;
+ }
+ Optional<WGS84Point> pt1 = GeoHashUtil.INSTANCE.toPoint(hash1);
+ String hash2 = (String)args.get(1);
+ if(hash2 == null) {
+ return null;
+ }
+ Optional<WGS84Point> pt2 = GeoHashUtil.INSTANCE.toPoint(hash2);
+ DistanceStrategy strat = DistanceStrategies.HAVERSINE;
+ if(args.size() > 2) {
+ strat = DistanceStrategies.valueOf((String) args.get(2));
+ }
+ if(pt1.isPresent() && pt2.isPresent()) {
+ return GeoHashUtil.INSTANCE.distance(pt1.get(), pt2.get(), strat);
+ }
+ return Double.NaN;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="MAX_DIST"
+ ,namespace="GEOHASH"
+ ,description="Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash)",
+ "strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default."
+ }
+ ,returns = "The maximum distance in kilometers between any two locations"
+ )
+ public static class MaxDist implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Iterable<String> hashes = (Iterable<String>)args.get(0);
+ if(hashes == null) {
+ return null;
+ }
+ DistanceStrategy strat = DistanceStrategies.HAVERSINE;
+ if(args.size() > 1) {
+ strat = DistanceStrategies.valueOf((String) args.get(1));
+ }
+ return GeoHashUtil.INSTANCE.maxDistanceHashes(hashes, strat);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(name="CENTROID"
+ ,namespace="GEOHASH"
+ ,description="Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)"
+ ,params = {
+ "hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights"
+ ,"character_precision? - The number of characters to use in the hash. Default is 12"
+ }
+ ,returns = "The geohash of the centroid"
+ )
+ public static class Centroid implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 1) {
+ return null;
+ }
+ Object o1 = args.get(0);
+ if(o1 == null) {
+ return null;
+ }
+ WGS84Point centroid = null;
+ if(o1 instanceof Map) {
+ centroid = GeoHashUtil.INSTANCE.centroidOfWeightedPoints((Map<String, Number>)o1);
+ }
+ else if(o1 instanceof Iterable) {
+ centroid = GeoHashUtil.INSTANCE.centroidOfHashes((Iterable<String>)o1);
+ }
+ if(centroid == null) {
+ return null;
+ }
+ Integer precision = 12;
+ if(args.size() > 1) {
+ precision = (Integer)args.get(1);
+ }
+ return GeoHashUtil.INSTANCE.computeHash(centroid, precision).orElse(null);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
new file mode 100644
index 0000000..f1a0ec4
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/stellar/GeoHashFunctionsTest.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.stellar;
+
+import ch.hsr.geohash.WGS84Point;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metron.stellar.common.utils.StellarProcessorUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.*;
+
+public class GeoHashFunctionsTest {
+ public static WGS84Point empireStatePoint = new WGS84Point(40.748570, -73.985752);
+ public static WGS84Point mosconeCenterPoint = new WGS84Point(37.782891, -122.404166);
+ public static WGS84Point jutlandPoint = new WGS84Point(57.64911, 10.40740);
+ public static String explicitJutlandHash = "u4pruydqmvpb";
+ String empireStateHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", empireStatePoint.getLatitude()
+ ,"long",empireStatePoint.getLongitude()
+ )
+ );
+ String mosconeCenterHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", mosconeCenterPoint.getLatitude()
+ ,"long",mosconeCenterPoint.getLongitude()
+ )
+ );
+ String jutlandHash = (String)StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ );
+
+ @Test
+ public void testToLatLong_happypath() throws Exception {
+ Map<String, Object> latLong = (Map<String, Object>)StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", explicitJutlandHash ) );
+ Assert.assertEquals(jutlandPoint.getLatitude(), (double)latLong.get("latitude"), 1e-3);
+ Assert.assertEquals(jutlandPoint.getLongitude(), (double)latLong.get("longitude"), 1e-3);
+ }
+
+ @Test
+ public void testToLatLong_degenerate() throws Exception {
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", "u"));
+ Assert.assertFalse(Double.isNaN((double) latLong.get("latitude")));
+ Assert.assertFalse(Double.isNaN((double) latLong.get("longitude")));
+ }
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(hash)"
+ , ImmutableMap.of("hash", ""));
+ Assert.assertEquals(0d, (double)latLong.get("latitude"), 1e-3);
+ Assert.assertEquals(0d, (double)latLong.get("longitude"), 1e-3);
+ }
+ {
+ Map<String, Object> latLong = (Map<String, Object>) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(null)"
+ , new HashMap<>());
+ Assert.assertNull(latLong);
+ }
+ }
+
+ @Test
+ public void testHash_fromlatlong() throws Exception {
+ Assert.assertEquals("u4pruydqmv", StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 10)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmvpb", StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ Assert.assertEquals("u4pruydqmv".substring(0, 6), StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 6)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat)"
+ , ImmutableMap.of("lat", jutlandPoint.getLatitude()
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LATLONG(lat, long, 10)"
+ , ImmutableMap.of("lat", "blah"
+ ,"long",jutlandPoint.getLongitude()
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testHash_fromLocation() throws Exception {
+ Map<String, String> loc = ImmutableMap.of( "latitude", "" + jutlandPoint.getLatitude()
+ , "longitude","" + jutlandPoint.getLongitude()
+ );
+ Assert.assertEquals("u4pruydqmv", StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 10)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmv".substring(0, 6), StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 6)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+
+ Assert.assertEquals("u4pruydqmvpb", StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc)"
+ , ImmutableMap.of("loc", loc
+ )
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc)"
+ , ImmutableMap.of("loc", ImmutableMap.of( "latitude", "57.64911" ))
+ )
+ );
+ Assert.assertNull(StellarProcessorUtils.run("GEOHASH_FROM_LOC(loc, 10)"
+ , ImmutableMap.of("loc", ImmutableMap.of( "latitude", "blah"
+ , "longitude","10.40740"
+ )
+ )
+
+ )
+ );
+ }
+
+ @Test
+ public void testDistanceHaversine() throws Exception {
+ testDistance(Optional.empty());
+ testDistance(Optional.of("HAVERSINE"));
+ }
+
+ @Test
+ public void testDistanceLawOfCosines() throws Exception {
+ testDistance(Optional.of("LAW_OF_COSINES"));
+ }
+
+ @Test
+ public void testDistanceLawOfVicenty() throws Exception {
+ testDistance(Optional.of("VICENTY"));
+ }
+
+ @Test
+ public void testMaxDistance_happyPath() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([empireState, mosconeCenter, jutland])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+
+ @Test
+ public void testMaxDistance_differentOrder() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, mosconeCenter, empireState])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+
+ @Test
+ public void testMaxDistance_withNulls() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, mosconeCenter, empireState, null])"
+ , ImmutableMap.of("empireState", empireStateHash
+ , "mosconeCenter", mosconeCenterHash
+ , "jutland", jutlandHash
+ )
+ );
+ double expectedDistance = 8528;
+ Assert.assertEquals(expectedDistance, maxDistance, 1d);
+ }
+ @Test
+ public void testMaxDistance_allSame() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([jutland, jutland, jutland])"
+ , ImmutableMap.of( "jutland", jutlandHash )
+ );
+ Assert.assertEquals(0, maxDistance, 1e-6d);
+ }
+
+ @Test
+ public void testMaxDistance_emptyList() throws Exception {
+ Double maxDistance = (double) StellarProcessorUtils.run("GEOHASH_MAX_DIST([])" , new HashMap<>() );
+ Assert.assertTrue(Double.isNaN(maxDistance));
+ }
+
+ @Test
+ public void testMaxDistance_nullList() throws Exception {
+ Double maxDistance = (Double) StellarProcessorUtils.run("GEOHASH_MAX_DIST(null)" , new HashMap<>() );
+ Assert.assertNull(maxDistance);
+ }
+
+ @Test
+ public void testMaxDistance_invalidList() throws Exception {
+ Double maxDistance = (Double) StellarProcessorUtils.run("GEOHASH_MAX_DIST()" , new HashMap<>() );
+ Assert.assertNull(maxDistance);
+ }
+
+ public void testDistance(Optional<String> method) throws Exception {
+ double expectedDistance = 4128; //in kilometers
+ Map<String, Object> vars = ImmutableMap.of("empireState", empireStateHash, "mosconeCenter", mosconeCenterHash);
+ //ensure that d(x, y) == d(y, x) and that both are the same as the expected (up to 1 km accuracy)
+ {
+ String stellarStatement = getDistStellarStatement(ImmutableList.of("mosconeCenter", "empireState"), method);
+ Assert.assertEquals(expectedDistance, (double) StellarProcessorUtils.run(stellarStatement , vars ), 1D );
+ }
+ {
+ String stellarStatement = getDistStellarStatement(ImmutableList.of("empireState", "mosconeCenter"), method);
+ Assert.assertEquals(expectedDistance, (double) StellarProcessorUtils.run(stellarStatement , vars ), 1D );
+ }
+ }
+
+ private static String getDistStellarStatement(List<String> hashVariables, Optional<String> method) {
+ if(method.isPresent()) {
+ List<String> vars = new ArrayList<>();
+ vars.addAll(hashVariables);
+ vars.add("\'" + method.get() + "\'");
+ return "GEOHASH_DIST(" + Joiner.on(",").skipNulls().join(vars) + ")";
+ }
+ else {
+ return "GEOHASH_DIST(" + Joiner.on(",").skipNulls().join(hashVariables) + ")";
+ }
+ }
+
+ @Test
+ public void testCentroid_List() throws Exception {
+ //happy path
+ {
+ double expectedLong = -98.740087 //calculated via http://www.geomidpoint.com/ using the center of gravity or geographic midpoint.
+ , expectedLat = 41.86921
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState, mosconeCenter]))"
+ , ImmutableMap.of("empireState", empireStateHash, "mosconeCenter", mosconeCenterHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //same point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState, empireState]))"
+ , ImmutableMap.of("empireState", empireStateHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //one point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([empireState]))"
+ , ImmutableMap.of("empireState", empireStateHash)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ //no points
+ {
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID([]))"
+ , new HashMap<>()
+ );
+ Assert.assertNull(centroid);
+ }
+ }
+
+ @Test
+ public void testCentroid_weighted() throws Exception {
+ //happy path
+ {
+ double expectedLong = -98.740087 //calculated via http://www.geomidpoint.com/ using the center of gravity or geographic midpoint.
+ , expectedLat = 41.86921
+ ;
+ for(int weight = 1;weight < 10;++weight) {
+ Map<Object, Integer> weightedPoints = ImmutableMap.of(empireStateHash, weight, mosconeCenterHash, weight);
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ }
+ //same point
+ {
+ double expectedLong = empireStatePoint.getLongitude()
+ , expectedLat = empireStatePoint.getLatitude()
+ ;
+ for(int weight = 1;weight < 10;++weight) {
+ Map<Object, Integer> weightedPoints = ImmutableMap.of(empireStateHash, weight);
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertEquals(expectedLong, centroid.get("longitude"), 1e-3);
+ Assert.assertEquals(expectedLat, centroid.get("latitude"), 1e-3);
+ }
+ }
+ //no points
+ {
+ Map<Object, Integer> weightedPoints = new HashMap<>();
+ Map<String, Double> centroid = (Map) StellarProcessorUtils.run("GEOHASH_TO_LATLONG(GEOHASH_CENTROID(weightedPoints))"
+ , ImmutableMap.of("weightedPoints", weightedPoints)
+ );
+ Assert.assertNull(centroid);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-hbase-client/pom.xml b/metron-platform/metron-hbase-client/pom.xml
index 5dd6127..1237be7 100644
--- a/metron-platform/metron-hbase-client/pom.xml
+++ b/metron-platform/metron-hbase-client/pom.xml
@@ -80,6 +80,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>org.apache.commons.logging</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index c64c374..7d07665 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -222,6 +222,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-management/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/pom.xml b/metron-platform/metron-management/pom.xml
index 4117d69..a5cae38 100644
--- a/metron-platform/metron-management/pom.xml
+++ b/metron-platform/metron-management/pom.xml
@@ -205,6 +205,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index b7c21ff..85c6218 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -266,6 +266,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-pcap-backend/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/pom.xml b/metron-platform/metron-pcap-backend/pom.xml
index 388b1e0..5878873 100644
--- a/metron-platform/metron-pcap-backend/pom.xml
+++ b/metron-platform/metron-pcap-backend/pom.xml
@@ -221,6 +221,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index be1fe33..97132c4 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -232,6 +232,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-platform/metron-writer/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/pom.xml b/metron-platform/metron-writer/pom.xml
index 7d3152c..de6b3b8 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -238,6 +238,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.google.common</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 4dc7d8d..340a7ae 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -135,6 +135,12 @@ In the core language functions, we support basic functional programming primitiv
| [ `FUZZY_SCORE`](#fuzzy_score) |
| [ `FORMAT`](#format) |
| [ `GEO_GET`](#geo_get) |
+| [ `GEOHASH_CENTROID`](#geohash_centroid) |
+| [ `GEOHASH_DIST`](#geohash_dist) |
+| [ `GEOHASH_FROM_LATLONG`](#geohash_from_latlong) |
+| [ `GEOHASH_FROM_LOC`](#geohash_from_loc) |
+| [ `GEOHASH_MAX_DIST`](#geohash_max_dist) |
+| [ `GEOHASH_TO_LATLONG`](#geohash_to_latlong) |
| [ `GET`](#get) |
| [ `GET_FIRST`](#get_first) |
| [ `GET_LAST`](#get_last) |
@@ -443,6 +449,50 @@ In the core language functions, we support basic functional programming primitiv
* fields - Optional list of GeoIP fields to grab. Options are locID, country, city postalCode, dmaCode, latitude, longitude, location_point
* Returns: If a Single field is requested a string of the field, If multiple fields a map of string of the fields, and null otherwise
+### `GEOHASH_CENTROID`
+ * Description: Compute the centroid (geographic midpoint or center of gravity) of a set of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hashes - A collection of [geohashes](https://en.wikipedia.org/wiki/Geohash) or a map associating geohashes to numeric weights
+ * character_precision? - The number of characters to use in the hash. Default is 12
+ * Returns: The geohash of the centroid
+
+### `GEOHASH_DIST`
+ * Description: Compute the distance between [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hash1 - The first point as a geohash
+ * hash2 - The second point as a geohash
+ * strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default.
+ * Returns: The distance in kilometers between the hashes.
+
+### `GEOHASH_FROM_LATLONG`
+ * Description: Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a lat/long
+ * Input:
+ * latitude - The latitude
+ * longitude - The longitude
+ * character_precision? - The number of characters to use in the hash. Default is 12
+ * Returns: A [geohash](https://en.wikipedia.org/wiki/Geohash) of the lat/long
+
+### `GEOHASH_FROM_LOC`
+ * Description: Compute [geohash](https://en.wikipedia.org/wiki/Geohash) given a geo enrichment location
+ * Input:
+ * map - the latitude and logitude in a map (the output of [GEO_GET](#geo_get) )
+ * longitude - The longitude
+ * character_precision? - The number of characters to use in the hash. Default is `12`
+ * Returns: A [geohash](https://en.wikipedia.org/wiki/Geohash) of the location
+
+### `GEOHASH_MAX_DIST`
+ * Description: Compute the maximum distance among a list of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hashes - A set of [geohashes](https://en.wikipedia.org/wiki/Geohash)
+ * strategy? - The great circle distance strategy to use. One of [HAVERSINE](https://en.wikipedia.org/wiki/Haversine_formula), [LAW_OF_COSINES](https://en.wikipedia.org/wiki/Law_of_cosines#Using_the_distance_formula), or [VICENTY](https://en.wikipedia.org/wiki/Vincenty%27s_formulae). Haversine is default.
+ * Returns: The maximum distance in kilometers between any two locations
+
+### `GEOHASH_TO_LATLONG`
+ * Description: Compute the lat/long of a given [geohash](https://en.wikipedia.org/wiki/Geohash)
+ * Input:
+ * hash - The [geohash](https://en.wikipedia.org/wiki/Geohash)
+ * Returns: A map containing the latitude and longitude of the hash (keys "latitude" and "longitude")
+
### `GET`
* Description: Returns the i'th element of the list
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index 5945bbd..9ec29b8 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -257,6 +257,16 @@
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
<relocations>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/use-cases/README.md
----------------------------------------------------------------------
diff --git a/use-cases/README.md b/use-cases/README.md
new file mode 100644
index 0000000..02be32d
--- /dev/null
+++ b/use-cases/README.md
@@ -0,0 +1,4 @@
+# Worked Examples
+
+The following are worked examples of use-cases that showcase some (or
+many) component(s) of Metron.
http://git-wip-us.apache.org/repos/asf/metron/blob/dd711819/use-cases/geographic_login_outliers/README.md
----------------------------------------------------------------------
diff --git a/use-cases/geographic_login_outliers/README.md b/use-cases/geographic_login_outliers/README.md
new file mode 100644
index 0000000..99e9a5b
--- /dev/null
+++ b/use-cases/geographic_login_outliers/README.md
@@ -0,0 +1,267 @@
+# Problem Statement
+
+One way to find anomalous behavior in a network is by inspecting user
+login behavior. In particular, if a user is logging in via vastly
+differing geographic locations in a short period of time, this may be
+evidence of malicious behavior.
+
+More formally, we can encode this potentially malicious event in terms
+of how far from the geographic centroid of the user's historic logins
+as compared to all users. For instance, if we track all users and the
+median distance from the central geographic location of all of their
+logins for the last 2 hours is 3 km and the standard deviation is 1 km,
+if we see a user logging in 1700 km from the central geographic location of
+their logins for the last 2 hours, then they MAY be exhibiting a
+deviation that we want to monitor since it would be hard to travel that
+distance in 4 hours. On the other hand, the user may have
+just used a VPN or proxy. Ultimately, this sort of analytic must be
+considered only one piece of evidence in addition to many others before
+we want to indicate an alert.
+
+# Demonstration Design
+For the purposes of demonstration, we will construct synthetic data
+whereby 2 users are logging into a system rather quickly (once per
+second) from various hosts. Each user's locations share the same first
+2 octets, but will choose the last 2 randomly. We will then inject a
+data point indicating `user1` is logging in via a russian IP address.
+
+## Preliminaries
+We assume that the following environment variables are set:
+* `METRON_HOME` - the home directory for metron
+* `ZOOKEEPER` - The zookeeper quorum (comma separated with port specified: e.g. `node1:2181` for full-dev)
+* `BROKERLIST` - The Kafka broker list (comma separated with port specified: e.g. `node1:6667` for full-dev)
+* `ES_HOST` - The elasticsearch master (and port) e.g. `node1:9200` for full-dev.
+
+Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.
+
+Before editing configurations, be sure to pull the configs from zookeeper locally via
+```
+$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z $ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f
+```
+
+## Configure the Profiler
+First, we'll configure the profiler to emit a profiler every 1 minute:
+* In Ambari, set the profiler period duration to `1` minute via the Profiler config section.
+* Adjust `$METRON_HOME/config/zookeeper/global.json` to adjust the capture duration:
+```
+ "profiler.client.period.duration" : "1",
+ "profiler.client.period.duration.units" : "MINUTES"
+```
+
+## Create the Data Generator
+We want to create a new sensor for our synthetic data called `auth`. To
+feed it, we need a synthetic data generator. In particular, we want a
+process which will feed authentication events per second for a set of
+users where the IPs are randomly chosen, but each user's login ip
+addresses share the same first 2 octets.
+
+Edit `~/gen_data.py` and paste the following into it:
+```
+#!/usr/bin/python
+
+import random
+import sys
+import time
+
+domains = { 'user1' : '173.90', 'user2' : '156.33' }
+
+def get_ip(base):
+ return base + '.' + str(random.randint(1,255)) + '.' + str(random.randint(1, 255))
+
+def main():
+ freq_s = 1
+ while True:
+ user='user' + str(random.randint(1,len(domains)))
+ epoch_time = int(time.time())
+ ip=get_ip(domains[user])
+ print user + ',' + ip + ',' + str(epoch_time)
+ sys.stdout.flush()
+ time.sleep(freq_s)
+
+if __name__ == '__main__':
+ main()
+```
+
+## Create the `auth` Parser
+
+The message format for our simple synthetic data is a CSV with:
+* username
+* login ip address
+* timestamp
+
+We will need to parse this via our `CSVParser` and add the geohash of the login ip address.
+
+* To create this parser, edit `$METRON_HOME/config/zookeeper/parsers/auth.json` and paste the following:
+```
+{
+ "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+ ,"sensorTopic" : "auth"
+ ,"parserConfig" : {
+ "columns" : {
+ "user" : 0,
+ "ip" : 1,
+ "timestamp" : 2
+ }
+ }
+ ,"fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : [ "hash" ]
+ ,"config" : {
+ "hash" : "GEOHASH_FROM_LOC(GEO_GET(ip))"
+ }
+ }
+ ]
+}
+```
+* Create the kafka topic via:
+```
+/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic auth --partitions 1 --replication-factor 1
+```
+
+## Create the Profiles for Enrichment
+
+We will need to track 2 profiles to accomplish this task:
+* `locations_by_user` - The geohashes of the locations the user has logged in from. This is a multiset of geohashes per user. Note that the multiset in this case is effectively a map of geohashes to occurrance counts.
+* `geo_distribution_from_centroid` - The statistical distribution of the distance between a login location and the geographic centroid of the user's previous logins from the last 2 minutes. Note, in a real installation this would be a larger temporal lookback.
+
+We can represent these in the `$METRON_HOME/config/zookeeper/profiler.json` via the following:
+```
+{
+ "profiles": [
+ {
+ "profile": "geo_distribution_from_centroid",
+ "foreach": "'global'",
+ "onlyif": "exists(geo_distance) && geo_distance != null",
+ "init" : {
+ "s": "STATS_INIT()"
+ },
+ "update": {
+ "s": "STATS_ADD(s, geo_distance)"
+ },
+ "result": "s"
+ },
+ {
+ "profile": "locations_by_user",
+ "foreach": "user",
+ "onlyif": "exists(hash) && hash != null && LENGTH(hash) > 0",
+ "init" : {
+ "s": "MULTISET_INIT()"
+ },
+ "update": {
+ "s": "MULTISET_ADD(s, hash)"
+ },
+ "result": "s"
+ }
+ ]
+}
+```
+
+## Enrich authentication Events
+
+We will need to enrich the authentication records in a couple of ways to use in the threat triage section as well as the profiles:
+* `geo_distance`: representing the distance between the current geohash and the geographic centroid for the last 2 minutes.
+* `geo_centroid`: representing the geographic centroid for the last 2 minutes
+
+Beyond that, we will need to determine if the authentication event is a geographic outlier by computing the following fields:
+* `dist_median` : representing the median distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the median of the `geo_distance` values across all users).
+* `dist_sd` : representing the standard deviation of the distance between a user's login location and the geographic centroid for the last 2 minutes (essentially the standard deviation of the `geo_distance` values across all users).
+* `geo_outlier` : whether `geo_distance` is more than 5 standard deviations from the median across all users.
+
+We also want to set up a triage rule associating a score and setting an alert if `geo_outlier` is true. In reality, this would be more complex as this metric is at best circumstantial and would need supporting evidence, but for simplicity we'll deal with the false positives.
+
+* Edit `$METRON_HOME/config/zookeeper/enrichments/auth.json` and paste the following:
+```
+{
+ "enrichment": {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "geo_locations := MULTISET_MERGE( PROFILE_GET( 'locations_by_user', user, PROFILE_FIXED( 2, 'MINUTES')))",
+ "geo_centroid := GEOHASH_CENTROID(geo_locations)",
+ "geo_distance := TO_INTEGER(GEOHASH_DIST(geo_centroid, hash))",
+ "geo_locations := null"
+ ]
+ }
+ }
+ ,"fieldToTypeMap": { }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "stellar" : {
+ "config" : [
+ "geo_distance_distr:= STATS_MERGE( PROFILE_GET( 'geo_distribution_from_centroid', 'global', PROFILE_FIXED( 2, 'MINUTES')))",
+ "dist_median := STATS_PERCENTILE(geo_distance_distr, 50.0)",
+ "dist_sd := STATS_SD(geo_distance_distr)",
+ "geo_outlier := ABS(dist_median - geo_distance) >= 5*dist_sd",
+ "is_alert := exists(is_alert) && is_alert",
+ "is_alert := is_alert || (geo_outlier != null && geo_outlier == true)",
+ "geo_distance_distr := null"
+ ]
+ }
+
+ },
+ "fieldToTypeMap": { },
+ "triageConfig" : {
+ "riskLevelRules" : [
+ {
+ "name" : "Geographic Outlier",
+ "comment" : "Determine if the user's geographic distance from the centroid of the historic logins is an outlier as compared to all users.",
+ "rule" : "geo_outlier != null && geo_outlier",
+ "score" : 10,
+ "reason" : "FORMAT('user %s has a distance (%d) from the centroid of their last login is 5 std deviations (%f) from the median (%f)', user, geo_distance, dist_sd, dist_median)"
+ }
+ ],
+ "aggregator" : "MAX"
+ }
+ }
+}
+```
+
+## Execute Demonstration
+
+From here, we've set up our configuration and can push the configs:
+* Push the configs to zookeeper via
+```
+$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z node1:2181 -i $METRON_HOME/config/zookeeper/
+```
+* Start the parser via:
+```
+$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s auth
+```
+* Push synthetic data into the `auth` topic via
+```
+python ~/gen_data.py | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic auth
+```
+* Wait for about `5` minutes and kill the previous command
+* Push a synthetic record indicating `user1` has logged in from a russian IP (`109.252.227.173`):
+```
+echo -e "import time\nprint 'user1,109.252.227.173,'+str(int(time.time()))" | python | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic auth
+```
+* Execute the following to search elasticsearch for our geographic login outliers:
+```
+curl -XPOST "http://$ES_HOST/auth*/_search?pretty" -d '
+{
+ "_source" : [ "is_alert", "threat:triage:rules:0:reason", "user", "ip", "geo_distance" ],
+ "query": { "exists" : { "field" : "threat:triage:rules:0:reason" } }
+}
+'
+```
+
+You should see, among a few other false positive results, something like the following:
+```
+{
+ "_index" : "auth_index_2017.09.07.20",
+ "_type" : "auth_doc",
+ "_id" : "f5bdbf76-9d78-48cc-b21d-bc434c96e62e",
+ "_score" : 1.0,
+ "_source" : {
+ "geo_distance" : 7879,
+ "threat:triage:rules:0:reason" : "user user1 has a distance (7879) from the centroid of their last login is 5 std deviations (334.814719) from the median (128.000000)",
+ "ip" : "109.252.227.173",
+ "is_alert" : "true",
+ "user" : "user1"
+ }
+}
+```
+