You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/06/11 13:32:06 UTC
metron git commit: METRON-1608 Add configuration for
threat.triage.field name (merrimanr) closes apache/metron#1055
Repository: metron
Updated Branches:
refs/heads/master 8202cd242 -> 32d132b3f
METRON-1608 Add configuration for threat.triage.field name (merrimanr) closes apache/metron#1055
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/32d132b3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/32d132b3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/32d132b3
Branch: refs/heads/master
Commit: 32d132b3f8ac7d915180882934d38504263d86ae
Parents: 8202cd2
Author: merrimanr <me...@gmail.com>
Authored: Mon Jun 11 08:31:51 2018 -0500
Committer: merrimanr <me...@apache.org>
Committed: Mon Jun 11 08:31:51 2018 -0500
----------------------------------------------------------------------
.../rest/service/impl/SearchServiceImpl.java | 10 +--
.../service/impl/SearchServiceImplTest.java | 3 +-
.../org/apache/metron/common/Constants.java | 1 +
.../configuration/ConfigurationsUtils.java | 7 ++
.../elasticsearch/dao/ElasticsearchDao.java | 8 ++
.../dao/ElasticsearchMetaAlertDao.java | 78 ++++++++++----------
.../dao/ElasticsearchMetaAlertDaoTest.java | 33 ++++++++-
.../ElasticsearchMetaAlertIntegrationTest.java | 64 ++++++++++++++--
.../metron/indexing/dao/MetaAlertDao.java | 3 +-
9 files changed, 150 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
index 21d158f..1c92fcb 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service.impl;
import static org.apache.metron.common.Constants.ERROR_TYPE;
+import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE;
import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY;
@@ -31,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
@@ -146,13 +148,9 @@ public class SearchServiceImpl implements SearchService {
if (!alertUserSettings.isPresent() || alertUserSettings.get().getFacetFields() == null) {
String facetFieldsProperty = environment
.getProperty(SEARCH_FACET_FIELDS_SPRING_PROPERTY, String.class, "");
-
- Map<String, Object> globalConfig = globalConfigService.get();
- String sourceTypeField = Constants.SENSOR_TYPE.replace('.', ':');
+ String sourceTypeField = ConfigurationsUtils.getFieldName(globalConfigService.get(), SENSOR_TYPE_FIELD_PROPERTY,
+ Constants.SENSOR_TYPE.replace('.', ':'));
List<String> facetFields = new ArrayList<>();
- if (globalConfig != null) {
- sourceTypeField = (String) globalConfig.getOrDefault("source.type.field", sourceTypeField);
- }
facetFields.add(sourceTypeField);
if (facetFieldsProperty != null) {
facetFields.addAll(Arrays.asList(facetFieldsProperty.split(",")));
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java
index 4c63c6d..82e7221 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.service.impl;
+import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY;
import static org.junit.Assert.assertEquals;
@@ -183,7 +184,7 @@ public class SearchServiceImplTest {
when(environment.getProperty(SEARCH_FACET_FIELDS_SPRING_PROPERTY, String.class, ""))
.thenReturn("ip_src_addr");
Map<String, Object> globalConfig = new HashMap<>();
- globalConfig.put("source.type.field", "source.type");
+ globalConfig.put(SENSOR_TYPE_FIELD_PROPERTY, "source.type");
when(globalConfigService.get()).thenReturn(globalConfig);
when(alertsUIService.getAlertsUIUserSettings()).thenReturn(Optional.empty());
List<String> defaultFields = searchService.getDefaultFacetFields();
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index 12b541c..f74660c 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -27,6 +27,7 @@ public class Constants {
public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology";
public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
public static final String SENSOR_TYPE = "source.type";
+ public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field";
public static final String ENRICHMENT_TOPIC = "enrichments";
public static final String INDEXING_TOPIC = "indexing";
public static final String ERROR_STREAM = "error";
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
index c7b39f0..4550a41 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
@@ -799,5 +799,12 @@ public class ConfigurationsUtils {
out.println(type + " Config: " + name + System.lineSeparator() + data);
}, configType, configName);
}
+
+ public static String getFieldName(Map<String, Object> globalConfig, String globalConfigKey, String defaultFieldName) {
+ if (globalConfig == null) {
+ return defaultFieldName;
+ }
+ return (String) globalConfig.getOrDefault(globalConfigKey, defaultFieldName);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index cb5bb58..181cb87 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -125,6 +125,14 @@ public class ElasticsearchDao implements IndexDao {
//uninitialized.
}
+ public AccessConfig getAccessConfig() {
+ return accessConfig;
+ }
+
+ public void setAccessConfig(AccessConfig accessConfig) {
+ this.accessConfig = accessConfig;
+ }
+
private static Map<String, FieldType> elasticsearchSearchTypeMap;
static {
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index d12e40c..ee3ca89 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -18,21 +18,9 @@
package org.apache.metron.elasticsearch.dao;
-import static org.apache.metron.common.Constants.GUID;
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
-import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
-import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.io.IOException;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.stream.Collectors;
-import org.apache.commons.collections4.SetUtils;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.MetaAlertDao;
@@ -51,40 +39,48 @@ import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.update.Document;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.MultiGetItemResponse;
-import org.elasticsearch.action.get.MultiGetRequest.Item;
-import org.elasticsearch.action.get.MultiGetRequestBuilder;
-import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.InnerHitBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.QueryStringQueryBuilder;
-import org.elasticsearch.search.SearchHit;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
+import org.elasticsearch.search.SearchHit;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import static org.apache.metron.common.Constants.GUID;
+import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
public class ElasticsearchMetaAlertDao implements MetaAlertDao {
public static final String SOURCE_TYPE = Constants.SENSOR_TYPE.replace('.', ':');
+ public static final String THREAT_TRIAGE_FIELD = THREAT_FIELD_DEFAULT.replace('.', ':');
private static final String STATUS_PATH = "/status";
private static final String ALERT_PATH = "/alert";
private IndexDao indexDao;
private ElasticsearchDao elasticsearchDao;
private String index = METAALERTS_INDEX;
- private String threatTriageField = THREAT_FIELD_DEFAULT;
/**
* Defines which summary aggregation is used to represent the overall threat triage score for
@@ -101,21 +97,19 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
* @param indexDao The Dao to wrap
*/
public ElasticsearchMetaAlertDao(IndexDao indexDao) {
- this(indexDao, METAALERTS_INDEX, THREAT_FIELD_DEFAULT, THREAT_SORT_DEFAULT);
+ this(indexDao, METAALERTS_INDEX, THREAT_SORT_DEFAULT);
}
/**
* Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts.
* @param indexDao The Dao to wrap
- * @param triageLevelField The field name to use as the threat scoring field
* @param threatSort The summary aggregation of all child threat triage scores used
* as the overall threat triage score for the metaalert. This
* can be either max, min, average, count, median, or sum.
*/
- public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField, String threatSort) {
+ public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String threatSort) {
init(indexDao, Optional.of(threatSort));
this.index = index;
- this.threatTriageField = triageLevelField;
}
public ElasticsearchMetaAlertDao() {
@@ -196,7 +190,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
Document metaAlert = buildCreateDocument(alerts, request.getGroups());
calculateMetaScores(metaAlert);
// Add source type to be consistent with other sources and allow filtering
- metaAlert.getDocument().put(SOURCE_TYPE, MetaAlertDao.METAALERT_TYPE);
+ metaAlert.getDocument().put(getFieldName(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE), MetaAlertDao.METAALERT_TYPE);
// Start a list of updates / inserts we need to run
Map<Document, Optional<String>> updates = new HashMap<>();
@@ -353,7 +347,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
List<Map<String, Object>> currentAlerts = (List<Map<String, Object>>) metaAlert.getDocument()
.get(MetaAlertDao.ALERT_FIELD);
currentAlerts.stream().forEach(currentAlert -> {
- getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(SOURCE_TYPE)));
+ getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(getFieldName(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE))));
});
Iterable<Document> alerts = indexDao.getAllLatest(getRequests);
List<Map<String, Object>> updatedAlerts = new ArrayList<>();
@@ -685,7 +679,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
ArrayList<Double> scores = new ArrayList<>();
for (Object alertRaw : alertsRaw) {
Map<String, Object> alert = (Map<String, Object>) alertRaw;
- Double scoreNum = parseThreatField(alert.get(threatTriageField));
+ Double scoreNum = parseThreatField(alert.get(getFieldName(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD)));
if (scoreNum != null) {
scores.add(scoreNum);
}
@@ -700,7 +694,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
Object threatScore = metaScores.getMetaScores().get(threatSort);
// add the threat score as a float; type needs to match the threat score field from each of the sensor indices
- metaAlert.getDocument().put(threatTriageField, ConversionUtils.convert(threatScore, Float.class));
+ metaAlert.getDocument().put(getFieldName(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD), ConversionUtils.convert(threatScore, Float.class));
}
private Double parseThreatField(Object threatRaw) {
@@ -720,4 +714,12 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
+
+ private String getFieldName(String globalConfigKey, String defaultFieldName) {
+ if (this.elasticsearchDao == null || this.elasticsearchDao.getAccessConfig() == null) {
+ return defaultFieldName;
+ }
+ Map<String, Object> globalConfig = this.elasticsearchDao.getAccessConfig().getGlobalConfigSupplier().get();
+ return ConfigurationsUtils.getFieldName(globalConfig, globalConfigKey, defaultFieldName);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
index 1bfa9d6..44defb3 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
@@ -214,10 +214,10 @@ public class ElasticsearchMetaAlertDaoTest {
List<Map<String, Object>> alertList = new ArrayList<>();
// add an alert with a threat score
- alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f));
+ alertList.add( Collections.singletonMap(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, 10.0f));
// add a second alert with a threat score
- alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 20.0f));
+ alertList.add( Collections.singletonMap(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, 20.0f));
// add a third alert with NO threat score
alertList.add( Collections.singletonMap("alert3", "has no threat score"));
@@ -230,7 +230,7 @@ public class ElasticsearchMetaAlertDaoTest {
// calculate the threat score for the metaalert
ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
metaAlertDao.calculateMetaScores(metaalert);
- Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT);
+ Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD);
// the metaalert must contain a summary of all child threat scores
assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta);
@@ -246,4 +246,31 @@ public class ElasticsearchMetaAlertDaoTest {
// by default, the overall threat score is the sum of all child threat scores
assertEquals(30.0F, threatScore);
}
+
+ @Test
+ public void testCalculateMetaScoresWithDifferentFieldName() {
+ List<Map<String, Object>> alertList = new ArrayList<>();
+
+ // add an alert with a threat score
+ alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f));
+
+ // create the metaalert
+ Map<String, Object> docMap = new HashMap<>();
+ docMap.put(MetaAlertDao.ALERT_FIELD, alertList);
+ Document metaalert = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L);
+
+ // Configure a different threat triage score field name
+ AccessConfig accessConfig = new AccessConfig();
+ accessConfig.setGlobalConfigSupplier(() -> new HashMap<String, Object>() {{
+ put(MetaAlertDao.THREAT_FIELD_PROPERTY, MetaAlertDao.THREAT_FIELD_DEFAULT);
+ }});
+ ElasticsearchDao elasticsearchDao = new ElasticsearchDao();
+ elasticsearchDao.setAccessConfig(accessConfig);
+
+ // calculate the threat score for the metaalert
+ ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
+ metaAlertDao.init(elasticsearchDao);
+ metaAlertDao.calculateMetaScores(metaalert);
+ assertNotNull(metaalert.getDocument().get(MetaAlertDao.THREAT_FIELD_DEFAULT));
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index b001050..adc1a27 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -18,11 +18,14 @@
package org.apache.metron.elasticsearch.integration;
+import static org.apache.metron.common.Constants.SENSOR_TYPE;
+import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
import static org.apache.metron.indexing.dao.MetaAlertDao.ALERT_FIELD;
import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX;
import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD;
import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE;
import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD;
+import static org.apache.metron.indexing.dao.MetaAlertDao.THREAT_FIELD_PROPERTY;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Joiner;
@@ -85,7 +88,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
private static final String NEW_FIELD = "new-field";
private static final String NAME_FIELD = "name";
- private static IndexDao esDao;
+ private static ElasticsearchDao esDao;
private static MetaAlertDao metaDao;
private static ElasticSearchComponent es;
@@ -193,6 +196,12 @@ public class ElasticsearchMetaAlertIntegrationTest {
.withIndexDir(new File(INDEX_DIR))
.build();
es.start();
+ }
+
+ @Before
+ public void setup() throws IOException {
+ es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert"));
+ es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index"));
AccessConfig accessConfig = new AccessConfig();
Map<String, Object> globalConfig = new HashMap<String, Object>() {
@@ -212,12 +221,6 @@ public class ElasticsearchMetaAlertIntegrationTest {
metaDao = new ElasticsearchMetaAlertDao(esDao);
}
- @Before
- public void setup() throws IOException {
- es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert"));
- es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index"));
- }
-
@AfterClass
public static void teardown() {
if (es != null) {
@@ -320,6 +323,12 @@ public class ElasticsearchMetaAlertIntegrationTest {
findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE);
}
{
+ // Verify metaalert has the default field names
+ Document metaAlert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE);
+ Assert.assertTrue(metaAlert.getDocument().containsKey(ElasticsearchMetaAlertDao.SOURCE_TYPE));
+ Assert.assertTrue(metaAlert.getDocument().containsKey(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD));
+ }
+ {
// Verify alert 0 was not updated with metaalert field
Document alert = metaDao.getLatest("message_0", SENSOR_NAME);
Assert.assertEquals(4, alert.getDocument().size());
@@ -343,6 +352,45 @@ public class ElasticsearchMetaAlertIntegrationTest {
}
@Test
+ public void shouldCreateMetaAlertWithConfiguredFieldNames() throws Exception {
+ // Configure field names
+ AccessConfig accessConfig = esDao.getAccessConfig();
+ accessConfig.setGlobalConfigSupplier(() -> new HashMap<String, Object>() {{
+ put("es.date.format", DATE_FORMAT);
+ put(SENSOR_TYPE_FIELD_PROPERTY, SENSOR_TYPE);
+ put(THREAT_FIELD_PROPERTY, MetaAlertDao.THREAT_FIELD_DEFAULT);
+ }});
+
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(1);
+ elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
+
+ // Verify load was successful
+ findCreatedDocs(Collections.singletonList(
+ new GetRequest("message_0", SENSOR_NAME)));
+
+ {
+ MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest() {{
+ setAlerts(new ArrayList<GetRequest>() {{
+ add(new GetRequest("message_0", SENSOR_NAME));
+ }});
+ setGroups(Collections.singletonList("group"));
+ }};
+ MetaAlertCreateResponse metaAlertCreateResponse = metaDao.createMetaAlert(metaAlertCreateRequest);
+ {
+ // Verify metaAlert was created
+ findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE);
+ }
+ {
+ // Verify alert 0 was not updated with metaalert field
+ Document metaAlert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE);
+ Assert.assertTrue(metaAlert.getDocument().containsKey(SENSOR_TYPE));
+ Assert.assertTrue(metaAlert.getDocument().containsKey(MetaAlertDao.THREAT_FIELD_DEFAULT));
+ }
+ }
+ }
+
+ @Test
public void shouldAddAlertsToMetaAlert() throws Exception {
// Load alerts
List<Map<String, Object>> alerts = buildAlerts(4);
@@ -1173,7 +1221,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
Map<String, Object> alerts = new HashMap<>();
alerts.put(Constants.GUID, guid);
alerts.put("source:type", SENSOR_NAME);
- alerts.put(MetaAlertDao.THREAT_FIELD_DEFAULT, i);
+ alerts.put(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, i);
alerts.put("timestamp", System.currentTimeMillis());
inputData.add(alerts);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
index 8807849..93c791b 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
@@ -68,7 +68,8 @@ public interface MetaAlertDao extends IndexDao {
String METAALERT_TYPE = "metaalert";
String METAALERT_FIELD = "metaalerts";
String METAALERT_DOC = METAALERT_TYPE + "_doc";
- String THREAT_FIELD_DEFAULT = "threat:triage:score";
+ String THREAT_FIELD_DEFAULT = "threat.triage.score";
+ String THREAT_FIELD_PROPERTY = "threat.triage.score.field";
String THREAT_SORT_DEFAULT = "sum";
String ALERT_FIELD = "metron_alert";
String STATUS_FIELD = "status";