You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2017/10/18 21:21:51 UTC
metron git commit: METRON-1255 MetaAlert search is not filtering on
status (merrimanr) closes apache/metron#802
Repository: metron
Updated Branches:
refs/heads/master fef8833c1 -> 73623ec74
METRON-1255 MetaAlert search is not filtering on status (merrimanr) closes apache/metron#802
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73623ec7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73623ec7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73623ec7
Branch: refs/heads/master
Commit: 73623ec74232765849f40480add9117da67c11d5
Parents: fef8833
Author: merrimanr <me...@gmail.com>
Authored: Wed Oct 18 16:21:37 2017 -0500
Committer: merrimanr <me...@apache.org>
Committed: Wed Oct 18 16:21:37 2017 -0500
----------------------------------------------------------------------
.../dao/ElasticsearchMetaAlertDao.java | 13 +-
.../ElasticsearchMetaAlertIntegrationTest.java | 163 ++++++++++++++++++-
.../metron/indexing/dao/MetaAlertDao.java | 2 +-
3 files changed, 168 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index a64f5ad..44c983e 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -21,6 +21,7 @@ package org.apache.metron.elasticsearch.dao;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -177,15 +178,19 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
// Wrap the query to also get any meta-alerts.
QueryBuilder qb = constantScoreQuery(boolQuery()
- .should(new QueryStringQueryBuilder(searchRequest.getQuery()))
- .should(boolQuery()
- .must(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()))
- .must(nestedQuery(
+ .must(boolQuery()
+ .should(new QueryStringQueryBuilder(searchRequest.getQuery()))
+ .should(nestedQuery(
ALERT_FIELD,
new QueryStringQueryBuilder(searchRequest.getQuery())
)
)
)
+ // Ensures that it's a meta alert with active status or that it's an alert (signified by having no status field)
+ .must(boolQuery()
+ .should(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()))
+ .should(boolQuery().mustNot(existsQuery(MetaAlertDao.STATUS_FIELD)))
+ )
);
return elasticsearchDao.search(searchRequest, qb);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index c562e0b..12ac888 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -21,16 +21,20 @@ package org.apache.metron.elasticsearch.integration;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
+import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
@@ -40,6 +44,9 @@ import org.apache.metron.elasticsearch.integration.components.ElasticSearchCompo
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SortField;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
import org.junit.AfterClass;
@@ -83,6 +90,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
put("es.date.format", DATE_FORMAT);
}
};
+ accessConfig.setMaxSearchResults(1000);
accessConfig.setGlobalConfigSupplier(() -> globalConfig);
esDao = new ElasticsearchDao();
@@ -198,7 +206,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
{
//ensure alerts in ES are up-to-date
- boolean found = findUpdatedDoc(message0, guid);
+ boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME);
Assert.assertTrue("Unable to find updated document", found);
long cnt = 0;
for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
@@ -257,7 +265,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
}
}, Optional.empty());
- boolean found = findUpdatedDoc(message0, guid);
+ boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME);
Assert.assertTrue("Unable to find updated document", found);
{
//ensure ES is up-to-date
@@ -302,12 +310,157 @@ public class ElasticsearchMetaAlertIntegrationTest {
}
}
- protected boolean findUpdatedDoc(Map<String, Object> message0, String guid)
+ /**
+ {
+ "guid": "active_metaalert",
+ "source:type": "metaalert",
+ "alert": [],
+ "status": "active"
+ }
+ */
+ @Multiline
+ public static String activeMetaAlert;
+
+ /**
+ {
+ "guid": "inactive_metaalert",
+ "source:type": "metaalert",
+ "alert": [],
+ "status": "inactive"
+ }
+ */
+ @Multiline
+ public static String inactiveMetaAlert;
+
+ @Test
+ public void shouldSearchByStatus() throws Exception {
+ List<Map<String, Object>> metaInputData = new ArrayList<>();
+ Map<String, Object> activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert, new TypeReference<Map<String, Object>>() {});
+ metaInputData.add(activeMetaAlertJSON);
+ Map<String, Object> inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert, new TypeReference<Map<String, Object>>() {});
+ metaInputData.add(inactiveMetaAlertJSON);
+
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ elasticsearchAdd(metaInputData, MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
+ // Wait for updates to persist
+ findUpdatedDoc(inactiveMetaAlertJSON, "inactive_metaalert", MetaAlertDao.METAALERT_TYPE);
+
+ SearchResponse searchResponse = metaDao.search(new SearchRequest() {
+ {
+ setQuery("*");
+ setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
+ setFrom(0);
+ setSize(5);
+ setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }}));
+ }
+ });
+ Assert.assertEquals(1, searchResponse.getTotal());
+ Assert.assertEquals(MetaAlertStatus.ACTIVE.getStatusString(), searchResponse.getResults().get(0).getSource().get(MetaAlertDao.STATUS_FIELD));
+ }
+
+ /**
+ {
+ "guid": "search_by_nested_alert_active_0",
+ "source:type": "test",
+ "ip_src_addr": "192.168.1.1",
+ "ip_src_port": 8010
+ }
+ */
+ @Multiline
+ public static String searchByNestedAlertActive0;
+
+ /**
+ {
+ "guid": "search_by_nested_alert_inactive_1",
+ "source:type": "test",
+ "ip_src_addr": "192.168.1.2",
+ "ip_src_port": 8009
+ }
+ */
+ @Multiline
+ public static String searchByNestedAlertActive1;
+
+ /**
+ {
+ "guid": "search_by_nested_alert_inactive_0",
+ "source:type": "test",
+ "ip_src_addr": "192.168.1.3",
+ "ip_src_port": 8008
+ }
+ */
+ @Multiline
+ public static String searchByNestedAlertInactive0;
+
+ /**
+ {
+ "guid": "search_by_nested_alert_inactive_1",
+ "source:type": "test",
+ "ip_src_addr": "192.168.1.4",
+ "ip_src_port": 8007
+ }
+ */
+ @Multiline
+ public static String searchByNestedAlertInactive1;
+
+ @Test
+ public void shouldSearchByNestedAlert() throws Exception {
+ // Create alerts
+ List<Map<String, Object>> alerts = new ArrayList<>();
+ Map<String, Object> searchByNestedAlertActive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive0, new TypeReference<Map<String, Object>>() {});
+ alerts.add(searchByNestedAlertActive0JSON);
+ Map<String, Object> searchByNestedAlertActive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive1, new TypeReference<Map<String, Object>>() {});
+ alerts.add(searchByNestedAlertActive1JSON);
+ Map<String, Object> searchByNestedAlertInactive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive0, new TypeReference<Map<String, Object>>() {});
+ alerts.add(searchByNestedAlertInactive0JSON);
+ Map<String, Object> searchByNestedAlertInactive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive1, new TypeReference<Map<String, Object>>() {});
+ alerts.add(searchByNestedAlertInactive1JSON);
+ elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
+ // Wait for updates to persist
+ findUpdatedDoc(searchByNestedAlertInactive1JSON, "search_by_nested_alert_inactive_1", SENSOR_NAME);
+
+ // Create metaalerts
+ Map<String, Object> activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert, new TypeReference<Map<String, Object>>() {});
+ activeMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertActive0JSON, searchByNestedAlertActive1JSON));
+ Map<String, Object> inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert, new TypeReference<Map<String, Object>>() {});
+ inactiveMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertInactive0JSON, searchByNestedAlertInactive1JSON));
+
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ elasticsearchAdd(Arrays.asList(activeMetaAlertJSON, inactiveMetaAlertJSON), MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
+ // Wait for updates to persist
+ findUpdatedDoc(activeMetaAlertJSON, "active_metaalert", MetaAlertDao.METAALERT_TYPE);
+
+ SearchResponse searchResponse = metaDao.search(new SearchRequest() {
+ {
+ setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)");
+ setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
+ setFrom(0);
+ setSize(5);
+ setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }}));
+ }
+ });
+ // Should not have results because nested alerts shouldn't be flattened
+ Assert.assertEquals(0, searchResponse.getTotal());
+
+ searchResponse = metaDao.search(new SearchRequest() {
+ {
+ setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)");
+ setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
+ setFrom(0);
+ setSize(5);
+ setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }}));
+ }
+ });
+ // Nested query should match a nested alert
+ Assert.assertEquals(1, searchResponse.getTotal());
+ Assert.assertEquals("active_metaalert", searchResponse.getResults().get(0).getSource().get("guid"));
+ }
+
+ protected boolean findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType)
throws InterruptedException, IOException {
boolean found = false;
for (int t = 0; t < MAX_RETRIES && !found; ++t, Thread.sleep(SLEEP_MS)) {
- Document doc = metaDao.getLatest(guid, SENSOR_NAME);
- if (message0.equals(doc.getDocument())) {
+ Document doc = metaDao.getLatest(guid, sensorType);
+ if (doc != null && message0.equals(doc.getDocument())) {
found = true;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
index 05746c4..e9f047b 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
@@ -27,7 +27,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
public interface MetaAlertDao extends IndexDao {
- String METAALERTS_INDEX = "metaalert";
+ String METAALERTS_INDEX = "metaalert_index";
String METAALERT_TYPE = "metaalert";
String METAALERT_DOC = METAALERT_TYPE + "_doc";
String THREAT_FIELD_DEFAULT = "threat:triage:score";