You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2017/11/20 22:39:55 UTC
metron git commit: METRON-1321 Metaalert Threat Score Type Does Not
Match Sensor Indices (nickwallen) closes apache/metron#845
Repository: metron
Updated Branches:
refs/heads/master 4a089900a -> 995b20170
METRON-1321 Metaalert Threat Score Type Does Not Match Sensor Indices (nickwallen) closes apache/metron#845
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/995b2017
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/995b2017
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/995b2017
Branch: refs/heads/master
Commit: 995b20170f460df4fc121fec0f3946857d748426
Parents: 4a08990
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Nov 20 17:39:00 2017 -0500
Committer: nickallen <ni...@apache.org>
Committed: Mon Nov 20 17:39:00 2017 -0500
----------------------------------------------------------------------
.../package/files/metaalert_index.template | 9 ++++
.../dao/ElasticsearchMetaAlertDao.java | 28 +++++++++++--
.../dao/ElasticsearchMetaAlertDaoTest.java | 43 +++++++++++++++-----
.../indexing/dao/metaalert/MetaScores.java | 3 +-
4 files changed, 68 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
index 964a480..5af9251 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
@@ -15,6 +15,15 @@
"index": "not_analyzed"
}
}
+ },
+ {
+ "threat_triage_score": {
+ "mapping": {
+ "type": "float"
+ },
+ "match": "threat:triage:*score",
+ "match_mapping_type": "*"
+ }
}
],
"properties": {
http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index 90d5410..f8fb145 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -61,6 +61,7 @@ import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@@ -77,6 +78,14 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
private ElasticsearchDao elasticsearchDao;
private String index = METAALERTS_INDEX;
private String threatTriageField = THREAT_FIELD_DEFAULT;
+
+ /**
+ * Defines which summary aggregation is used to represent the overall threat triage score for
+ * the metaalert. The summary aggregation is applied to the threat triage score of all child alerts.
+ *
+ * This overall score is primarily used for sorting; hence it is called the 'threatSort'. This
+ * can be either max, min, average, count, median, or sum.
+ */
private String threatSort = THREAT_SORT_DEFAULT;
private int pageSize = 500;
@@ -92,9 +101,11 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
* Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts.
* @param indexDao The Dao to wrap
* @param triageLevelField The field name to use as the threat scoring field
+ * @param threatSort The summary aggregation of all child threat triage scores used
+ * as the overall threat triage score for the metaalert. This
+ * can be either max, min, average, count, median, or sum.
*/
- public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField,
- String threatSort) {
+ public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField, String threatSort) {
init(indexDao, Optional.of(threatSort));
this.index = index;
this.threatTriageField = triageLevelField;
@@ -108,7 +119,9 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
* Initializes this implementation by setting the supplied IndexDao and also setting a separate ElasticsearchDao.
* This is needed for some specific Elasticsearch functions (looking up an index from a GUID for example).
* @param indexDao The DAO to wrap for our queries
- * @param threatSort The aggregation to use as the threat field. E.g. "sum", "median", etc.
+ * @param threatSort The summary aggregation of the child threat triage scores used
+ * as the overall threat triage score for the metaalert. This
+ * can be either max, min, average, count, median, or sum.
*/
@Override
public void init(IndexDao indexDao, Optional<String> threatSort) {
@@ -608,8 +621,15 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
}
metaScores = new MetaScores(scores);
}
+
+ // add a summary (max, min, avg, ...) of all the threat scores from the child alerts
metaAlert.getDocument().putAll(metaScores.getMetaScores());
- metaAlert.getDocument().put(threatTriageField, metaScores.getMetaScores().get(threatSort));
+
+ // add the overall threat score for the metaalert; one of the summary aggregations as defined by `threatSort`
+ Object threatScore = metaScores.getMetaScores().get(threatSort);
+
+ // add the threat score as a float; type needs to match the threat score field from each of the sensor indices
+ metaAlert.getDocument().put(threatTriageField, ConversionUtils.convert(threatScore, Float.class));
}
private Double parseThreatField(Object threatRaw) {
http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
index ffafe52..1bfa9d6 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.elasticsearch.dao;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
@@ -50,6 +51,8 @@ import org.junit.Test;
public class ElasticsearchMetaAlertDaoTest {
+
+
@Test(expected = IllegalArgumentException.class)
public void testInvalidInit() {
IndexDao dao = new IndexDao() {
@@ -207,20 +210,40 @@ public class ElasticsearchMetaAlertDaoTest {
@Test
public void testCalculateMetaScoresList() {
+ final double delta = 0.001;
List<Map<String, Object>> alertList = new ArrayList<>();
- Map<String, Object> alertMap = new HashMap<>();
- alertMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d);
- alertList.add(alertMap);
+
+ // add an alert with a threat score
+ alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f));
+
+ // add a second alert with a threat score
+ alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 20.0f));
+
+ // add a third alert with NO threat score
+ alertList.add( Collections.singletonMap("alert3", "has no threat score"));
+
+ // create the metaalert
Map<String, Object> docMap = new HashMap<>();
docMap.put(MetaAlertDao.ALERT_FIELD, alertList);
+ Document metaalert = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L);
- Document doc = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L);
-
+ // calculate the threat score for the metaalert
ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
- metaAlertDao.calculateMetaScores(doc);
- assertEquals(1L, doc.getDocument().get("count"));
- assertEquals(10.0d,
- doc.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT)
- );
+ metaAlertDao.calculateMetaScores(metaalert);
+ Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT);
+
+ // the metaalert must contain a summary of all child threat scores
+ assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta);
+ assertEquals(10D, (Double) metaalert.getDocument().get("min"), delta);
+ assertEquals(15D, (Double) metaalert.getDocument().get("average"), delta);
+ assertEquals(2L, metaalert.getDocument().get("count"));
+ assertEquals(30D, (Double) metaalert.getDocument().get("sum"), delta);
+ assertEquals(15D, (Double) metaalert.getDocument().get("median"), delta);
+
+ // it must contain an overall threat score; a float to match the type of the threat score of the other sensor indices
+ assertTrue(threatScore instanceof Float);
+
+ // by default, the overall threat score is the sum of all child threat scores
+ assertEquals(30.0F, threatScore);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
index 632cfd2..07285d6 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
@@ -18,11 +18,12 @@
package org.apache.metron.indexing.dao.metaalert;
+import org.apache.commons.math3.stat.descriptive.rank.Median;
+
import java.util.DoubleSummaryStatistics;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.math3.stat.descriptive.rank.Median;
public class MetaScores {