You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2017/10/20 18:46:50 UTC
metron git commit: METRON-1262 Unable to add comment for a alert in a
meta-alert (merrimanr) closes apache/metron#806
Repository: metron
Updated Branches:
refs/heads/master 243ff1f72 -> 32a0a9da8
METRON-1262 Unable to add comment for a alert in a meta-alert (merrimanr) closes apache/metron#806
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/32a0a9da
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/32a0a9da
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/32a0a9da
Branch: refs/heads/master
Commit: 32a0a9da89f7480fdc6a15a93eebf8df05fb0ecf
Parents: 243ff1f
Author: merrimanr <me...@gmail.com>
Authored: Fri Oct 20 13:46:39 2017 -0500
Committer: merrimanr <me...@apache.org>
Committed: Fri Oct 20 13:46:39 2017 -0500
----------------------------------------------------------------------
metron-platform/metron-elasticsearch/pom.xml | 10 --
.../dao/ElasticsearchMetaAlertDao.java | 16 ++-
.../dao/ElasticsearchMetaAlertDaoTest.java | 63 +++-------
.../ElasticsearchMetaAlertIntegrationTest.java | 119 ++++++++++++++++++-
.../dao/metaalert/MetaAlertCreateResponse.java | 9 ++
5 files changed, 148 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index bdd37f9..f9f05ea 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -164,16 +164,6 @@
<version>3.10.5.Final</version>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>2.2.3</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>2.2.3</version>
- </dependency>
- <dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-integration-test</artifactId>
<version>${project.parent.version}</version>
http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index 44c983e..3409973 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -66,7 +66,6 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.support.QueryInnerHitBuilder;
import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
public class ElasticsearchMetaAlertDao implements MetaAlertDao {
@@ -168,6 +167,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
handleMetaUpdate(createDoc, Optional.of(METAALERTS_INDEX));
MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse();
createResponse.setCreated(true);
+ createResponse.setGuid(createDoc.getGuid());
return createResponse;
} catch (IOException ioe) {
throw new InvalidCreateException("Unable to create meta alert", ioe);
@@ -404,16 +404,14 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
// Run through the nested alerts of the meta alert and either use the new or old versions
builder.startArray(ALERT_FIELD);
- Map<String, SearchHits> innerHits = hit.getInnerHits();
+ Map<String, Object> hitAlerts = hit.sourceAsMap();
- SearchHits alertHits = innerHits.get(ALERT_FIELD);
- for (SearchHit alertHit : alertHits.getHits()) {
- Map<String, Object> docMap;
- // If we're at the update use it, otherwise use the original
- if (alertHit.sourceAsMap().get(Constants.GUID).equals(update.getGuid())) {
+ List<Map<String, Object>> alertHits = (List<Map<String, Object>>) hitAlerts.get(ALERT_FIELD);
+ for (Map<String, Object> alertHit : alertHits) {
+ Map<String, Object> docMap = alertHit;
+ // If we're at the update use it instead of the original
+ if (alertHit.get(Constants.GUID).equals(update.getGuid())) {
docMap = update.getDocument();
- } else {
- docMap = alertHit.getSource();
}
builder.map(docMap);
http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
index ad2e7f3..5d6f4e0 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
@@ -27,6 +27,8 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -54,14 +56,11 @@ import org.elasticsearch.action.get.MultiGetItemResponse;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchHits;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.junit.Test;
-import org.mockito.Mock;
public class ElasticsearchMetaAlertDaoTest {
@@ -96,22 +95,13 @@ public class ElasticsearchMetaAlertDaoTest {
when(metaHit.getSource()).thenReturn(metaSource);
// Construct the inner alert
- SearchHit innerAlertHit = mock(SearchHit.class);
HashMap<String, Object> innerAlertSource = new HashMap<>();
innerAlertSource.put(Constants.GUID, "a1");
- when(innerAlertHit.sourceAsMap()).thenReturn(innerAlertSource);
- SearchHitField field = mock(SearchHitField.class);
- when(field.getValue()).thenReturn(10d);
- when(innerAlertHit.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(field);
- SearchHit[] innerHitArray = new SearchHit[1];
- innerHitArray[0] = innerAlertHit;
-
- // Construct the inner hits that contains the alert
- SearchHits searchHits = mock(SearchHits.class);
- when(searchHits.getHits()).thenReturn(innerHitArray);
- Map<String, SearchHits> innerHits = new HashMap<>();
- innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits);
- when(metaHit.getInnerHits()).thenReturn(innerHits);
+ innerAlertSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10d);
+
+ Map<String, Object> innerHits = new HashMap<>();
+ innerHits.put(MetaAlertDao.ALERT_FIELD, Collections.singletonList(innerAlertSource));
+ when(metaHit.sourceAsMap()).thenReturn(innerHits);
// Construct the updated Document
Map<String, Object> updateMap = new HashMap<>();
@@ -169,43 +159,18 @@ public class ElasticsearchMetaAlertDaoTest {
when(metaHit.getSource()).thenReturn(metaSource);
// Construct the inner alerts
- SearchHit innerAlertHitOne = mock(SearchHit.class);
HashMap<String, Object> innerAlertSourceOne = new HashMap<>();
String guidOne = "a1";
innerAlertSourceOne.put(Constants.GUID, guidOne);
- when(innerAlertHitOne.sourceAsMap()).thenReturn(innerAlertSourceOne);
- when(innerAlertHitOne.getId()).thenReturn(guidOne);
- SearchHitField triageOne = mock(SearchHitField.class);
- when(triageOne.getValue()).thenReturn(threatValueOne);
- Map<String, Object> innerAlertHitOneSource = new HashMap<>();
- innerAlertHitOneSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo);
- innerAlertHitOneSource.put(Constants.GUID, guidOne);
- when(innerAlertHitOne.getSource()).thenReturn(innerAlertHitOneSource);
- when(innerAlertHitOne.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageOne);
-
- SearchHit innerAlertHitTwo = mock(SearchHit.class);
+ innerAlertSourceOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo);
+
HashMap<String, Object> innerAlertSourceTwo = new HashMap<>();
innerAlertSourceTwo.put(Constants.GUID, guidTwo);
- when(innerAlertHitTwo.sourceAsMap()).thenReturn(innerAlertSourceTwo);
- when(innerAlertHitOne.getId()).thenReturn(guidTwo);
- SearchHitField triageTwo = mock(SearchHitField.class);
- when(triageTwo.getValue()).thenReturn(threatValueTwo);
- Map<String, Object> innerAlertHitTwoSource = new HashMap<>();
- innerAlertHitTwoSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo);
- innerAlertHitTwoSource.put(Constants.GUID, guidTwo);
- when(innerAlertHitTwo.getSource()).thenReturn(innerAlertHitTwoSource);
- when(innerAlertHitTwo.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageTwo);
-
- SearchHit[] innerHitArray = new SearchHit[2];
- innerHitArray[0] = innerAlertHitOne;
- innerHitArray[1] = innerAlertHitTwo;
-
- // Construct the inner hits that contains the alert
- SearchHits searchHits = mock(SearchHits.class);
- when(searchHits.getHits()).thenReturn(innerHitArray);
- Map<String, SearchHits> innerHits = new HashMap<>();
- innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits);
- when(metaHit.getInnerHits()).thenReturn(innerHits);
+ innerAlertSourceTwo.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo);
+
+ Map<String, Object> innerHits = new HashMap<>();
+ innerHits.put(MetaAlertDao.ALERT_FIELD, Arrays.asList(innerAlertSourceOne, innerAlertSourceTwo));
+ when(metaHit.sourceAsMap()).thenReturn(innerHits);
// Construct the updated Document
Map<String, Object> updateMap = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index 12ac888..b13032f 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -44,10 +44,13 @@ import org.apache.metron.elasticsearch.integration.components.ElasticSearchCompo
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SortField;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -66,7 +69,7 @@ public class ElasticsearchMetaAlertIntegrationTest {
private static final String NEW_FIELD = "new-field";
private static IndexDao esDao;
- private static IndexDao metaDao;
+ private static MetaAlertDao metaDao;
private static ElasticSearchComponent es;
@BeforeClass
@@ -455,6 +458,108 @@ public class ElasticsearchMetaAlertIntegrationTest {
Assert.assertEquals("active_metaalert", searchResponse.getResults().get(0).getSource().get("guid"));
}
+ /**
+ {
+ "guid": "update_metaalert_alert_0",
+ "source:type": "test",
+ "field": "value 0"
+ }
+ */
+ @Multiline
+ public static String updateMetaAlertAlert0;
+
+ /**
+ {
+ "guid": "update_metaalert_alert_1",
+ "source:type": "test",
+ "field":"value 1"
+ }
+ */
+ @Multiline
+ public static String updateMetaAlertAlert1;
+
+ /**
+ {
+ "guid": "update_metaalert_alert_0",
+ "patch": [
+ {
+ "op": "add",
+ "path": "/field",
+ "value": "patched value 0"
+ }
+ ],
+ "sensorType": "test"
+ }
+ */
+ @Multiline
+ public static String updateMetaAlertPatchRequest;
+
+ /**
+ {
+ "guid": "update_metaalert_alert_0",
+ "replacement": {
+ "guid": "update_metaalert_alert_0",
+ "source:type": "test",
+ "field": "replaced value 0"
+ },
+ "sensorType": "test"
+ }
+ */
+ @Multiline
+ public static String updateMetaAlertReplaceRequest;
+
+ @Test
+ public void shouldUpdateMetaAlertOnAlertPatchOrReplace() throws Exception {
+ List<Map<String, Object>> inputData = new ArrayList<>();
+ Map<String, Object> updateMetaAlertAlert0JSON = JSONUtils.INSTANCE.load(updateMetaAlertAlert0, new TypeReference<Map<String, Object>>() {});
+ inputData.add(updateMetaAlertAlert0JSON);
+ Map<String, Object> updateMetaAlertAlert1JSON = JSONUtils.INSTANCE.load(updateMetaAlertAlert1, new TypeReference<Map<String, Object>>() {});
+ inputData.add(updateMetaAlertAlert1JSON);
+ elasticsearchAdd(inputData, INDEX, SENSOR_NAME);
+ // Wait for updates to persist
+ findUpdatedDoc(updateMetaAlertAlert1JSON, "update_metaalert_alert_1", SENSOR_NAME);
+
+ MetaAlertCreateResponse metaAlertCreateResponse = metaDao.createMetaAlert(new MetaAlertCreateRequest() {{
+ setGuidToIndices(new HashMap<String, String>() {{
+ put("update_metaalert_alert_0", INDEX);
+ put("update_metaalert_alert_1", INDEX);
+ }});
+ setGroups(Collections.singletonList("group"));
+ }});
+ // Wait for updates to persist
+ findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE);
+
+ // Patch alert
+ metaDao.patch(JSONUtils.INSTANCE.load(updateMetaAlertPatchRequest, PatchRequest.class), Optional.empty());
+
+ // Wait for updates to persist
+ updateMetaAlertAlert0JSON.put("field", "patched value 0");
+ findUpdatedDoc(updateMetaAlertAlert0JSON, "update_metaalert_alert_0", SENSOR_NAME);
+
+ Map<String, Object> metaalert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE).getDocument();
+ List<Map<String, Object>> alerts = (List<Map<String, Object>>) metaalert.get("alert");
+ Assert.assertEquals(2, alerts.size());
+ Assert.assertEquals("update_metaalert_alert_1", alerts.get(0).get("guid"));
+ Assert.assertEquals("value 1", alerts.get(0).get("field"));
+ Assert.assertEquals("update_metaalert_alert_0", alerts.get(1).get("guid"));
+ Assert.assertEquals("patched value 0", alerts.get(1).get("field"));
+
+ // Replace alert
+ metaDao.replace(JSONUtils.INSTANCE.load(updateMetaAlertReplaceRequest, ReplaceRequest.class), Optional.empty());
+
+ // Wait for updates to persist
+ updateMetaAlertAlert0JSON.put("field", "replaced value 0");
+ findUpdatedDoc(updateMetaAlertAlert0JSON, "update_metaalert_alert_0", SENSOR_NAME);
+
+ metaalert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE).getDocument();
+ alerts = (List<Map<String, Object>>) metaalert.get("alert");
+ Assert.assertEquals(2, alerts.size());
+ Assert.assertEquals("update_metaalert_alert_1", alerts.get(0).get("guid"));
+ Assert.assertEquals("value 1", alerts.get(0).get("field"));
+ Assert.assertEquals("update_metaalert_alert_0", alerts.get(1).get("guid"));
+ Assert.assertEquals("replaced value 0", alerts.get(1).get("field"));
+ }
+
protected boolean findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType)
throws InterruptedException, IOException {
boolean found = false;
@@ -467,6 +572,18 @@ public class ElasticsearchMetaAlertIntegrationTest {
return found;
}
+ protected boolean findCreatedDoc(String guid, String sensorType)
+ throws InterruptedException, IOException {
+ boolean found = false;
+ for (int t = 0; t < MAX_RETRIES && !found; ++t, Thread.sleep(SLEEP_MS)) {
+ Document doc = metaDao.getLatest(guid, sensorType);
+ if (doc != null) {
+ found = true;
+ }
+ }
+ return found;
+ }
+
protected void elasticsearchAdd(List<Map<String, Object>> inputData, String index, String docType)
throws IOException {
es.add(index, docType, inputData.stream().map(m -> {
http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
index e84286e..0bdf332 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
@@ -20,6 +20,7 @@ package org.apache.metron.indexing.dao.metaalert;
public class MetaAlertCreateResponse {
private boolean created;
+ private String guid;
public boolean isCreated() {
return created;
@@ -28,4 +29,12 @@ public class MetaAlertCreateResponse {
public void setCreated(boolean created) {
this.created = created;
}
+
+ public String getGuid() {
+ return guid;
+ }
+
+ public void setGuid(String guid) {
+ this.guid = guid;
+ }
}