You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:25 UTC
[09/50] [abbrv] metron git commit: METRON-1421 Create a
SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 56406f4..7fca764 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -39,8 +39,6 @@ import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.integration.InMemoryComponent;
import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -472,25 +470,15 @@ public abstract class SearchIntegrationTest {
@Multiline
public static String differentTypeFilterQuery;
- protected static IndexDao dao;
protected static InMemoryComponent indexComponent;
- @Before
- public synchronized void setup() throws Exception {
- if(dao == null && indexComponent == null) {
- indexComponent = startIndex();
- loadTestData();
- dao = createDao();
- }
- }
-
@Rule
public ExpectedException thrown = ExpectedException.none();
@Test
public void all_query_returns_all_results() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List<SearchResult> results = response.getResults();
Assert.assertEquals(10, results.size());
@@ -507,7 +495,7 @@ public abstract class SearchIntegrationTest {
@Test
public void find_one_guid() throws Exception {
GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
- Optional<Map<String, Object>> response = dao.getLatestResult(request);
+ Optional<Map<String, Object>> response = getIndexDao().getLatestResult(request);
Assert.assertTrue(response.isPresent());
Map<String, Object> doc = response.get();
Assert.assertEquals("bro", doc.get(getSourceTypeField()));
@@ -519,7 +507,7 @@ public abstract class SearchIntegrationTest {
List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){});
Map<String, Document> docs = new HashMap<>();
- for(Document doc : dao.getAllLatest(request)) {
+ for(Document doc : getIndexDao().getAllLatest(request)) {
docs.put(doc.getGuid(), doc);
}
Assert.assertEquals(2, docs.size());
@@ -532,7 +520,7 @@ public abstract class SearchIntegrationTest {
@Test
public void filter_query_filters_results() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(3, response.getTotal());
List<SearchResult> results = response.getResults();
Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
@@ -546,7 +534,7 @@ public abstract class SearchIntegrationTest {
@Test
public void sort_query_sorts_results_ascending() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List<SearchResult> results = response.getResults();
for (int i = 8001; i < 8011; ++i) {
@@ -557,7 +545,7 @@ public abstract class SearchIntegrationTest {
@Test
public void sort_ascending_with_missing_fields() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List<SearchResult> results = response.getResults();
Assert.assertEquals(10, results.size());
@@ -575,7 +563,7 @@ public abstract class SearchIntegrationTest {
@Test
public void sort_descending_with_missing_fields() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List<SearchResult> results = response.getResults();
Assert.assertEquals(10, results.size());
@@ -593,7 +581,7 @@ public abstract class SearchIntegrationTest {
@Test
public void results_are_paginated() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List<SearchResult> results = response.getResults();
Assert.assertEquals(3, results.size());
@@ -608,7 +596,7 @@ public abstract class SearchIntegrationTest {
@Test
public void returns_results_only_for_specified_indices() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(5, response.getTotal());
List<SearchResult> results = response.getResults();
for (int i = 5, j = 0; i > 0; i--, j++) {
@@ -621,7 +609,7 @@ public abstract class SearchIntegrationTest {
public void facet_query_yields_field_types() throws Exception {
String facetQuery = facetQueryRaw.replace("source:type", getSourceTypeField());
SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
Assert.assertEquals(8, facetCounts.size());
@@ -696,14 +684,14 @@ public abstract class SearchIntegrationTest {
@Test
public void disabled_facet_query_returns_null_count() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertNull(response.getFacetCounts());
}
@Test
public void missing_type_facet_query() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(missingTypeFacetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
@@ -723,7 +711,7 @@ public abstract class SearchIntegrationTest {
public void different_type_facet_query() throws Exception {
thrown.expect(Exception.class);
SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(3, response.getTotal());
}
@@ -732,14 +720,14 @@ public abstract class SearchIntegrationTest {
thrown.expect(InvalidSearchException.class);
thrown.expectMessage("Search result size must be less than 100");
SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class);
- dao.search(request);
+ getIndexDao().search(request);
}
@Test
public void column_metadata_for_missing_index() throws Exception {
// getColumnMetadata with an index that doesn't exist
{
- Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex"));
+ Map<String, FieldType> fieldTypes = getIndexDao().getColumnMetadata(Collections.singletonList("someindex"));
Assert.assertEquals(0, fieldTypes.size());
}
}
@@ -747,14 +735,14 @@ public abstract class SearchIntegrationTest {
@Test
public void no_results_returned_when_query_does_not_match() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(0, response.getTotal());
}
@Test
public void group_by_ip_query() throws Exception {
GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class);
- GroupResponse response = dao.group(request);
+ GroupResponse response = getIndexDao().group(request);
// expect only 1 group for 'ip_src_addr'
Assert.assertEquals("ip_src_addr", response.getGroupedBy());
@@ -778,7 +766,7 @@ public abstract class SearchIntegrationTest {
public void group_by_returns_results_in_groups() throws Exception {
// Group by test case, default order is count descending
GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class);
- GroupResponse response = dao.group(request);
+ GroupResponse response = getIndexDao().group(request);
Assert.assertEquals("is_alert", response.getGroupedBy());
List<GroupResult> isAlertGroups = response.getGroupResults();
Assert.assertEquals(2, isAlertGroups.size());
@@ -830,7 +818,7 @@ public abstract class SearchIntegrationTest {
public void group_by_returns_results_in_sorted_groups() throws Exception {
// Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending
GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class);
- GroupResponse response = dao.group(request);
+ GroupResponse response = getIndexDao().group(request);
Assert.assertEquals("is_alert", response.getGroupedBy());
List<GroupResult> isAlertGroups = response.getGroupResults();
Assert.assertEquals(2, isAlertGroups.size());
@@ -909,7 +897,7 @@ public abstract class SearchIntegrationTest {
@Test
public void queries_fields() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(10, response.getTotal());
List<SearchResult> results = response.getResults();
for (int i = 0; i < 5; ++i) {
@@ -927,7 +915,7 @@ public abstract class SearchIntegrationTest {
@Test
public void sort_by_guid() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
+ SearchResponse response = getIndexDao().search(request);
Assert.assertEquals(5, response.getTotal());
List<SearchResult> results = response.getResults();
for (int i = 0; i < 5; ++i) {
@@ -938,7 +926,7 @@ public abstract class SearchIntegrationTest {
}
@AfterClass
- public static void stop() throws Exception {
+ public static void stop() {
indexComponent.stop();
}
@@ -949,9 +937,7 @@ public abstract class SearchIntegrationTest {
@Test
public abstract void different_type_filter_query() throws Exception;
+ protected abstract IndexDao getIndexDao();
- protected abstract IndexDao createDao() throws Exception;
- protected abstract InMemoryComponent startIndex() throws Exception;
- protected abstract void loadTestData() throws Exception;
protected abstract String getSourceTypeField();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
index 471acf6..eebf0bb 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
@@ -20,86 +20,43 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Optional;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
-import org.apache.metron.integration.InMemoryComponent;
-import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
public abstract class UpdateIntegrationTest {
private static final int MAX_RETRIES = 10;
private static final int SLEEP_MS = 500;
- protected static final String SENSOR_NAME= "test";
- private static final String TABLE_NAME = "modifications";
+ protected static final String SENSOR_NAME = "test";
private static final String CF = "p";
- private static String index;
- private static MockHTable table;
- private static IndexDao hbaseDao;
- private static AccessConfig accessConfig;
protected static MultiIndexDao dao;
- protected static InMemoryComponent indexComponent;
-
- @Before
- public void setup() throws Exception {
- if(dao == null && indexComponent == null) {
- index = getIndexName();
- indexComponent = startIndex();
- loadTestData();
- Configuration config = HBaseConfiguration.create();
- MockHBaseTableProvider tableProvider = new MockHBaseTableProvider();
- tableProvider.addToCache(TABLE_NAME, CF);
- table = (MockHTable)tableProvider.getTable(config, TABLE_NAME);
-
- hbaseDao = new HBaseDao();
- accessConfig = new AccessConfig();
- accessConfig.setTableProvider(tableProvider);
- Map<String, Object> globalConfig = createGlobalConfig();
- globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME);
- globalConfig.put(HBaseDao.HBASE_CF, CF);
- accessConfig.setGlobalConfigSupplier(() -> globalConfig);
- }
- }
-
- protected AccessConfig getAccessConfig() {
- return accessConfig;
- }
@Test
public void test() throws Exception {
- dao = new MultiIndexDao(hbaseDao, createDao());
- dao.init(getAccessConfig());
-
List<Map<String, Object>> inputData = new ArrayList<>();
for(int i = 0; i < 10;++i) {
final String name = "message" + i;
inputData.add(
new HashMap<String, Object>() {{
- put("source:type", SENSOR_NAME);
+ put("source.type", SENSOR_NAME);
put("name" , name);
put("timestamp", System.currentTimeMillis());
put(Constants.GUID, name);
}}
);
}
- addTestData(index, SENSOR_NAME, inputData);
+ addTestData(getIndexName(), SENSOR_NAME, inputData);
List<Map<String,Object>> docs = null;
for(int t = 0;t < MAX_RETRIES;++t, Thread.sleep(SLEEP_MS)) {
- docs = getIndexedTestData(index, SENSOR_NAME);
+ docs = getIndexedTestData(getIndexName(), SENSOR_NAME);
if(docs.size() >= 10) {
break;
}
@@ -115,16 +72,16 @@ public abstract class UpdateIntegrationTest {
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
- setIndex(index);
+ setIndex(getIndexName());
}}, Optional.empty());
- Assert.assertEquals(1, table.size());
+ Assert.assertEquals(1, getMockHTable().size());
Document doc = dao.getLatest(guid, SENSOR_NAME);
Assert.assertEquals(message0, doc.getDocument());
{
//ensure hbase is up to date
Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
- Result r = table.get(g);
+ Result r = getMockHTable().get(g);
NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
Assert.assertEquals(1, columns.size());
Assert.assertEquals(message0
@@ -136,7 +93,7 @@ public abstract class UpdateIntegrationTest {
//ensure ES is up-to-date
long cnt = 0;
for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
- docs = getIndexedTestData(index, SENSOR_NAME);
+ docs = getIndexedTestData(getIndexName(), SENSOR_NAME);
cnt = docs
.stream()
.filter(d -> message0.get("new-field").equals(d.get("new-field")))
@@ -155,15 +112,15 @@ public abstract class UpdateIntegrationTest {
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
- setIndex(index);
+ setIndex(getIndexName());
}}, Optional.empty());
- Assert.assertEquals(1, table.size());
+ Assert.assertEquals(1, getMockHTable().size());
Document doc = dao.getLatest(guid, SENSOR_NAME);
Assert.assertEquals(message0, doc.getDocument());
{
//ensure hbase is up to date
Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
- Result r = table.get(g);
+ Result r = getMockHTable().get(g);
NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
Assert.assertEquals(2, columns.size());
Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue())
@@ -177,36 +134,20 @@ public abstract class UpdateIntegrationTest {
//ensure ES is up-to-date
long cnt = 0;
for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t,Thread.sleep(SLEEP_MS)) {
- docs = getIndexedTestData(index, SENSOR_NAME);
+ docs = getIndexedTestData(getIndexName(), SENSOR_NAME);
cnt = docs
.stream()
.filter(d -> message0.get("new-field").equals(d.get("new-field")))
.count();
}
- Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
+ Assert.assertNotEquals("Index is not updated!", cnt, 0);
}
}
}
- @After
- public void reset() throws Exception {
- indexComponent.reset();
- }
-
- @AfterClass
- public static void teardown() {
- if(indexComponent != null) {
- indexComponent.stop();
- }
- }
-
protected abstract String getIndexName();
- protected abstract Map<String, Object> createGlobalConfig() throws Exception;
- protected abstract IndexDao createDao() throws Exception;
- protected abstract InMemoryComponent startIndex() throws Exception;
- protected abstract void loadTestData() throws Exception;
+ protected abstract MockHTable getMockHTable();
protected abstract void addTestData(String indexName, String sensorType, List<Map<String,Object>> docs) throws Exception;
protected abstract List<Map<String,Object>> getIndexedTestData(String indexName, String sensorType) throws Exception;
-
}
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
new file mode 100644
index 0000000..b4f7d38
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
@@ -0,0 +1,1012 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.STATUS_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_FIELD_DEFAULT;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.Group;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
+import org.apache.metron.indexing.dao.search.GroupResult;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.search.SortField;
+import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
+import org.apache.metron.indexing.dao.update.PatchRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+public abstract class MetaAlertIntegrationTest {
+
+ private static final String META_INDEX_FLAG = "%META_INDEX%";
+ // To change back after testing
+ protected static int MAX_RETRIES = 10;
+ protected static final int SLEEP_MS = 500;
+ protected static final String SENSOR_NAME = "test";
+
+ protected static final String NEW_FIELD = "new-field";
+ protected static final String NAME_FIELD = "name";
+ protected static final String DATE_FORMAT = "yyyy.MM.dd.HH";
+
+ // Separate the raw indices from the query indices. ES for example, modifies the indices to
+ // have a separator
+ protected ArrayList<String> allIndices = new ArrayList<String>() {
+ {
+ add(getTestIndexName());
+ add(getMetaAlertIndex());
+ }
+ };
+
+ protected ArrayList<String> queryIndices = allIndices;
+
+ protected static MetaAlertDao metaDao;
+
+ /**
+ {
+ "guid": "meta_alert",
+ "index": "%META_INDEX%",
+ "patch": [
+ {
+ "op": "add",
+ "path": "/name",
+ "value": "New Meta Alert"
+ }
+ ],
+ "sensorType": "metaalert"
+ }
+ */
+ @Multiline
+ public static String namePatchRequest;
+
+ /**
+ {
+ "guid": "meta_alert",
+ "index": "%META_INDEX%",
+ "patch": [
+ {
+ "op": "add",
+ "path": "/name",
+ "value": "New Meta Alert"
+ },
+ {
+ "op": "add",
+ "path": "/alert",
+ "value": []
+ }
+ ],
+ "sensorType": "metaalert"
+ }
+ */
+ @Multiline
+ public static String alertPatchRequest;
+
+ /**
+ {
+ "guid": "meta_alert",
+ "index": "%META_INDEX%",
+ "patch": [
+ {
+ "op": "add",
+ "path": "/status",
+ "value": "inactive"
+ },
+ {
+ "op": "add",
+ "path": "/name",
+ "value": "New Meta Alert"
+ }
+ ],
+ "sensorType": "metaalert"
+ }
+ */
+ @Multiline
+ public static String statusPatchRequest;
+
+
+ @Test
+ public void shouldGetAllMetaAlertsForAlert() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(3);
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Load metaAlerts
+ List<Map<String, Object>> metaAlerts = buildMetaAlerts(12, MetaAlertStatus.ACTIVE,
+ Optional.of(Collections.singletonList(alerts.get(0))));
+ metaAlerts.add(buildMetaAlert("meta_active_12", MetaAlertStatus.ACTIVE,
+ Optional.of(Arrays.asList(alerts.get(0), alerts.get(2)))));
+ metaAlerts.add(buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE,
+ Optional.of(Arrays.asList(alerts.get(0), alerts.get(2)))));
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ addRecords(metaAlerts, getMetaAlertIndex(), METAALERT_TYPE);
+
+ // Verify load was successful
+ List<GetRequest> createdDocs = metaAlerts.stream().map(metaAlert ->
+ new GetRequest((String) metaAlert.get(Constants.GUID), METAALERT_TYPE))
+ .collect(Collectors.toList());
+ createdDocs.addAll(alerts.stream().map(alert ->
+ new GetRequest((String) alert.get(Constants.GUID), SENSOR_NAME))
+ .collect(Collectors.toList()));
+ findCreatedDocs(createdDocs);
+
+ {
+ // Verify searches successfully return more than 10 results
+ SearchResponse searchResponse0 = metaDao.getAllMetaAlertsForAlert("message_0");
+ List<SearchResult> searchResults0 = searchResponse0.getResults();
+ Assert.assertEquals(13, searchResults0.size());
+ Set<Map<String, Object>> resultSet = new HashSet<>();
+ Iterables.addAll(resultSet, Iterables.transform(searchResults0, r -> r.getSource()));
+ StringBuffer reason = new StringBuffer("Unable to find " + metaAlerts.get(0) + "\n");
+ reason.append(Joiner.on("\n").join(resultSet));
+ Assert.assertTrue(reason.toString(), resultSet.contains(metaAlerts.get(0)));
+
+ // Verify no meta alerts are returned because message_1 was not added to any
+ SearchResponse searchResponse1 = metaDao.getAllMetaAlertsForAlert("message_1");
+ List<SearchResult> searchResults1 = searchResponse1.getResults();
+ Assert.assertEquals(0, searchResults1.size());
+
+ // Verify only the meta alert message_2 was added to is returned
+ SearchResponse searchResponse2 = metaDao.getAllMetaAlertsForAlert("message_2");
+ List<SearchResult> searchResults2 = searchResponse2.getResults();
+ Assert.assertEquals(1, searchResults2.size());
+ Assert.assertEquals(metaAlerts.get(12), searchResults2.get(0).getSource());
+ }
+ }
+
+ @Test
+ public void getAllMetaAlertsForAlertShouldThrowExceptionForEmptyGuid() throws Exception {
+ try {
+ metaDao.getAllMetaAlertsForAlert("");
+ Assert.fail("An exception should be thrown for empty guid");
+ } catch (InvalidSearchException ise) {
+ Assert.assertEquals("Guid cannot be empty", ise.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldCreateMetaAlert() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(3);
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("message_2", SENSOR_NAME)));
+
+ {
+ MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest() {{
+ setAlerts(new ArrayList<GetRequest>() {{
+ add(new GetRequest("message_1", SENSOR_NAME));
+ add(new GetRequest("message_2", SENSOR_NAME, getTestIndexFullName()));
+ }});
+ setGroups(Collections.singletonList("group"));
+ }};
+ MetaAlertCreateResponse metaAlertCreateResponse = metaDao
+ .createMetaAlert(metaAlertCreateRequest);
+ {
+ // Verify metaAlert was created
+ findCreatedDoc(metaAlertCreateResponse.getGuid(), METAALERT_TYPE);
+ }
+ {
+ // Verify alert 0 was not updated with metaalert field
+ Document alert = metaDao.getLatest("message_0", SENSOR_NAME);
+ Assert.assertEquals(4, alert.getDocument().size());
+ Assert.assertNull(alert.getDocument().get(METAALERT_FIELD));
+ }
+ {
+ // Verify alert 1 was properly updated with metaalert field
+ Map<String, Object> expectedAlert = new HashMap<>(alerts.get(1));
+ expectedAlert
+ .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid()));
+ findUpdatedDoc(expectedAlert, "message_1", SENSOR_NAME);
+ }
+ {
+ // Verify alert 2 was properly updated with metaalert field
+ Map<String, Object> expectedAlert = new HashMap<>(alerts.get(2));
+ expectedAlert
+ .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid()));
+ findUpdatedDoc(expectedAlert, "message_2", SENSOR_NAME);
+ }
+ }
+ }
+
+ @Test
+ public void shouldAddAlertsToMetaAlert() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(4);
+ alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Load metaAlert
+ Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+ Optional.of(Collections.singletonList(alerts.get(0))));
+ addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("message_2", SENSOR_NAME),
+ new GetRequest("message_3", SENSOR_NAME),
+ new GetRequest("meta_alert", METAALERT_TYPE)
+ ));
+
+ // Build expected metaAlert after alerts are added
+ Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+
+ // Verify the proper alerts were added
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> metaAlertAlerts = new ArrayList<>(
+ (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+ // Alert 0 is already in the metaalert. Add alerts 1 and 2.
+ Map<String, Object> expectedAlert1 = alerts.get(1);
+ expectedAlert1.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ metaAlertAlerts.add(expectedAlert1);
+ Map<String, Object> expectedAlert2 = alerts.get(2);
+ expectedAlert2.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ metaAlertAlerts.add(expectedAlert2);
+ expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+ // Verify the counts were properly updated
+ expectedMetaAlert.put("average", 1.0d);
+ expectedMetaAlert.put("min", 0.0d);
+ expectedMetaAlert.put("median", 1.0d);
+ expectedMetaAlert.put("max", 2.0d);
+ expectedMetaAlert.put("count", 3);
+ expectedMetaAlert.put("sum", 3.0d);
+ expectedMetaAlert.put(getThreatTriageField(), 3.0d);
+
+ {
+ // Verify alerts were successfully added to the meta alert
+ Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("message_2", SENSOR_NAME))));
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+
+ {
+ // Verify False when alerts are already in a meta alert and no new alerts are added
+ Assert.assertFalse(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME))));
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+
+ {
+ // Verify only 1 alert is added when a list of alerts only contains 1 alert that is not in the meta alert
+ metaAlertAlerts = (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD);
+ Map<String, Object> expectedAlert3 = alerts.get(3);
+ expectedAlert3.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ metaAlertAlerts.add(expectedAlert3);
+ expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+ expectedMetaAlert.put("average", 1.5d);
+ expectedMetaAlert.put("min", 0.0d);
+ expectedMetaAlert.put("median", 1.5d);
+ expectedMetaAlert.put("max", 3.0d);
+ expectedMetaAlert.put("count", 4);
+ expectedMetaAlert.put("sum", 6.0d);
+ expectedMetaAlert.put(getThreatTriageField(), 6.0d);
+
+ Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_2", SENSOR_NAME),
+ new GetRequest("message_3", SENSOR_NAME))));
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void shouldRemoveAlertsFromMetaAlert() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(4);
+ alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ alerts.get(2).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ alerts.get(3).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Load metaAlert
+ Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+ Optional.of(Arrays.asList(alerts.get(0), alerts.get(1), alerts.get(2), alerts.get(3))));
+ addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("message_2", SENSOR_NAME),
+ new GetRequest("message_3", SENSOR_NAME),
+ new GetRequest("meta_alert", METAALERT_TYPE)));
+
+ // Build expected metaAlert after alerts are added
+ Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+
+ // Verify the proper alerts were added
+ List<Map<String, Object>> metaAlertAlerts = new ArrayList<>(
+ (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+ metaAlertAlerts.remove(0);
+ metaAlertAlerts.remove(0);
+ expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+ // Verify the counts were properly updated
+ expectedMetaAlert.put("average", 2.5d);
+ expectedMetaAlert.put("min", 2.0d);
+ expectedMetaAlert.put("median", 2.5d);
+ expectedMetaAlert.put("max", 3.0d);
+ expectedMetaAlert.put("count", 2);
+ expectedMetaAlert.put("sum", 5.0d);
+ expectedMetaAlert.put(getThreatTriageField(), 5.0d);
+
+ {
+ // Verify a list of alerts are removed from a meta alert
+ Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME))));
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+
+ {
+ // Verify False when alerts are not present in a meta alert and no alerts are removed
+ Assert.assertFalse(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME))));
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+
+ {
+ // Verify only 1 alert is removed when a list of alerts only contains 1 alert that is in the meta alert
+ metaAlertAlerts = new ArrayList<>(
+ (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+ metaAlertAlerts.remove(0);
+ expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+
+ expectedMetaAlert.put("average", 3.0d);
+ expectedMetaAlert.put("min", 3.0d);
+ expectedMetaAlert.put("median", 3.0d);
+ expectedMetaAlert.put("max", 3.0d);
+ expectedMetaAlert.put("count", 1);
+ expectedMetaAlert.put("sum", 3.0d);
+ expectedMetaAlert.put(getThreatTriageField(), 3.0d);
+
+ Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays
+ .asList(new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_2", SENSOR_NAME))));
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+
+ {
+ // Verify all alerts are removed from a metaAlert
+ metaAlertAlerts = new ArrayList<>(
+ (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD));
+ metaAlertAlerts.remove(0);
+ if (isEmptyMetaAlertList()) {
+ expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts);
+ } else {
+ expectedMetaAlert.remove(ALERT_FIELD);
+ }
+
+ expectedMetaAlert.put("average", 0.0d);
+ expectedMetaAlert.put("count", 0);
+ expectedMetaAlert.put("sum", 0.0d);
+ expectedMetaAlert.put(getThreatTriageField(), 0.0d);
+
+ // Handle the cases with non-finite Double values on a per store basis
+ if (isFiniteDoubleOnly()) {
+ expectedMetaAlert.put("min", String.valueOf(Double.POSITIVE_INFINITY));
+ expectedMetaAlert.put("median", String.valueOf(Double.NaN));
+ expectedMetaAlert.put("max", String.valueOf(Double.NEGATIVE_INFINITY));
+ } else {
+ expectedMetaAlert.put("min", Double.POSITIVE_INFINITY);
+ expectedMetaAlert.put("median", Double.NaN);
+ expectedMetaAlert.put("max", Double.NEGATIVE_INFINITY);
+ }
+
+ // Verify removing alerts cannot result in an empty meta alert
+ try {
+ metaDao.removeAlertsFromMetaAlert("meta_alert",
+ Collections.singletonList(new GetRequest("message_3", SENSOR_NAME)));
+ Assert.fail("Removing these alerts will result in an empty meta alert. Empty meta alerts are not allowed.");
+ } catch (IllegalStateException ise) {
+ Assert.assertEquals("Removing these alerts will result in an empty meta alert. Empty meta alerts are not allowed.",
+ ise.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void addRemoveAlertsShouldThrowExceptionForInactiveMetaAlert() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(2);
+ alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Load metaAlert
+ Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.INACTIVE,
+ Optional.of(Collections.singletonList(alerts.get(0))));
+ addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("meta_alert", METAALERT_TYPE)));
+
+ {
+ // Verify alerts cannot be added to an INACTIVE meta alert
+ try {
+ metaDao.addAlertsToMetaAlert("meta_alert",
+ Collections.singletonList(new GetRequest("message_1", SENSOR_NAME)));
+ Assert.fail("Adding alerts to an inactive meta alert should throw an exception");
+ } catch (IllegalStateException ise) {
+ Assert.assertEquals("Adding alerts to an INACTIVE meta alert is not allowed",
+ ise.getMessage());
+ }
+ }
+
+ {
+ // Verify alerts cannot be removed from an INACTIVE meta alert
+ try {
+ metaDao.removeAlertsFromMetaAlert("meta_alert",
+ Collections.singletonList(new GetRequest("message_0", SENSOR_NAME)));
+ Assert.fail("Removing alerts from an inactive meta alert should throw an exception");
+ } catch (IllegalStateException ise) {
+ Assert.assertEquals("Removing alerts from an INACTIVE meta alert is not allowed",
+ ise.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void shouldUpdateMetaAlertStatus() throws Exception {
+ int numChildAlerts = 25;
+ int numUnrelatedAlerts = 25;
+ int totalAlerts = numChildAlerts + numUnrelatedAlerts;
+
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(totalAlerts);
+ List<Map<String, Object>> childAlerts = alerts.subList(0, numChildAlerts);
+ List<Map<String, Object>> unrelatedAlerts = alerts.subList(numChildAlerts, totalAlerts);
+ for (Map<String, Object> alert : childAlerts) {
+ alert.put(METAALERT_FIELD, Collections.singletonList("meta_alert"));
+ }
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Load metaAlerts
+ Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+ Optional.of(childAlerts));
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(),
+ METAALERT_TYPE);
+
+ List<GetRequest> requests = new ArrayList<>();
+ for (int i = 0; i < numChildAlerts; ++i) {
+ requests.add(new GetRequest("message_" + i, SENSOR_NAME));
+ }
+ requests.add(new GetRequest("meta_alert", METAALERT_TYPE));
+
+ // Verify load was successful
+ findCreatedDocs(requests);
+
+ {
+ // Verify status changed to inactive and child alerts are updated
+ Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.INACTIVE));
+
+ Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+ expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString());
+
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+
+ for (int i = 0; i < numChildAlerts; ++i) {
+ Map<String, Object> expectedAlert = new HashMap<>(childAlerts.get(i));
+ setEmptiedMetaAlertField(expectedAlert);
+ findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME);
+ }
+
+ // Ensure unrelated alerts are unaffected
+ for (int i = 0; i < numUnrelatedAlerts; ++i) {
+ Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i));
+ // Make sure to handle the guid offset from creation
+ findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME);
+ }
+ }
+
+ {
+ // Verify status changed to active and child alerts are updated
+ Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE));
+
+ Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+ expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+
+ for (int i = 0; i < numChildAlerts; ++i) {
+ Map<String, Object> expectedAlert = new HashMap<>(alerts.get(i));
+ expectedAlert.put("metaalerts", Collections.singletonList("meta_alert"));
+ findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME);
+ }
+
+ // Ensure unrelated alerts are unaffected
+ for (int i = 0; i < numUnrelatedAlerts; ++i) {
+ Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i));
+ // Make sure to handle the guid offset from creation
+ findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME);
+ }
+
+ {
+ // Verify status changed to current status has no effect
+ Assert.assertFalse(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE));
+
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+
+ for (int i = 0; i < numChildAlerts; ++i) {
+ Map<String, Object> expectedAlert = new HashMap<>(alerts.get(i));
+ expectedAlert.put("metaalerts", Collections.singletonList("meta_alert"));
+ findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME);
+ }
+
+ // Ensure unrelated alerts are unaffected
+ for (int i = 0; i < numUnrelatedAlerts; ++i) {
+ Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i));
+ // Make sure to handle the guid offset from creation
+ findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void shouldSearchByStatus() throws Exception {
+ // Load alert
+ List<Map<String, Object>> alerts = buildAlerts(1);
+ alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+ alerts.get(0).put("ip_src_addr", "192.168.1.1");
+ alerts.get(0).put("ip_src_port", 8010);
+
+ // Load metaAlerts
+ Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE,
+ Optional.of(Collections.singletonList(alerts.get(0))));
+ Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive",
+ MetaAlertStatus.INACTIVE,
+ Optional.empty());
+
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), getMetaAlertIndex(),
+ METAALERT_TYPE);
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("meta_active", METAALERT_TYPE),
+ new GetRequest("meta_inactive", METAALERT_TYPE)));
+
+ SearchResponse searchResponse = metaDao.search(new SearchRequest() {
+ {
+ setQuery("*:*");
+ setIndices(Collections.singletonList(METAALERT_TYPE));
+ setFrom(0);
+ setSize(5);
+ setSort(Collections.singletonList(new SortField() {{
+ setField(Constants.GUID);
+ }}));
+ }
+ });
+
+ // Verify only active meta alerts are returned
+ Assert.assertEquals(1, searchResponse.getTotal());
+ Assert.assertEquals(MetaAlertStatus.ACTIVE.getStatusString(),
+ searchResponse.getResults().get(0).getSource().get(STATUS_FIELD));
+ }
+
+
+ @Test
+ public void shouldHidesAlertsOnGroup() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(2);
+ alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+ alerts.get(0).put("ip_src_addr", "192.168.1.1");
+ alerts.get(0).put("score", 1);
+ alerts.get(1).put("ip_src_addr", "192.168.1.1");
+ alerts.get(1).put("score", 10);
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Put the nested type into the test index, so that it'll match appropriately
+ setupTypings();
+
+ // Don't need any meta alerts to actually exist, since we've populated the field on the alerts.
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME)));
+
+ // Build our group request
+ Group searchGroup = new Group();
+ searchGroup.setField("ip_src_addr");
+ List<Group> groupList = new ArrayList<>();
+ groupList.add(searchGroup);
+ GroupResponse groupResponse = metaDao.group(new GroupRequest() {
+ {
+ setQuery("ip_src_addr:192.168.1.1");
+ setIndices(queryIndices);
+ setScoreField("score");
+ setGroups(groupList);
+ }
+ });
+
+ // Should only return the standalone alert in the group
+ GroupResult result = groupResponse.getGroupResults().get(0);
+ Assert.assertEquals(1, result.getTotal());
+ Assert.assertEquals("192.168.1.1", result.getKey());
+ // No delta, since no ops happen
+ Assert.assertEquals(10.0d, result.getScore(), 0.0d);
+ }
+
+ // This test is important enough that everyone should implement it, but is pretty specific to
+ // implementation
+ @Test
+ public abstract void shouldSearchByNestedAlert() throws Exception;
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void shouldUpdateMetaAlertOnAlertUpdate() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(2);
+ alerts.get(0).put(METAALERT_FIELD, Arrays.asList("meta_active", "meta_inactive"));
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Load metaAlerts
+ Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE,
+ Optional.of(Collections.singletonList(alerts.get(0))));
+ Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive",
+ MetaAlertStatus.INACTIVE,
+ Optional.of(Collections.singletonList(alerts.get(0))));
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), getMetaAlertIndex(),
+ METAALERT_TYPE);
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("meta_active", METAALERT_TYPE),
+ new GetRequest("meta_inactive", METAALERT_TYPE)));
+
+ {
+ // Modify the first message and add a new field
+ Map<String, Object> message0 = new HashMap<String, Object>(alerts.get(0)) {
+ {
+ put(NEW_FIELD, "metron");
+ put(THREAT_FIELD_DEFAULT, 10.0d);
+ }
+ };
+ String guid = "" + message0.get(Constants.GUID);
+ metaDao.update(new Document(message0, guid, SENSOR_NAME, null),
+ Optional.of(getTestIndexFullName()));
+
+ {
+ // Verify alerts are up-to-date
+ findUpdatedDoc(message0, guid, SENSOR_NAME);
+ long cnt = getMatchingAlertCount(NEW_FIELD, message0.get(NEW_FIELD));
+ if (cnt == 0) {
+ Assert.fail("Alert not updated!");
+ }
+ }
+
+ {
+ // Verify meta alerts are up-to-date
+ long cnt = getMatchingMetaAlertCount(NEW_FIELD, "metron");
+ if (cnt == 0) {
+ Assert.fail("Active metaalert was not updated!");
+ }
+ if (cnt != 1) {
+ Assert.fail("Metaalerts not updated correctly!");
+ }
+ }
+ }
+ //modify the same message and modify the new field
+ {
+ Map<String, Object> message0 = new HashMap<String, Object>(alerts.get(0)) {
+ {
+ put(NEW_FIELD, "metron2");
+ }
+ };
+ String guid = "" + message0.get(Constants.GUID);
+ metaDao.update(new Document(message0, guid, SENSOR_NAME, null), Optional.empty());
+
+ {
+ // Verify index is up-to-date
+ findUpdatedDoc(message0, guid, SENSOR_NAME);
+ long cnt = getMatchingAlertCount(NEW_FIELD, message0.get(NEW_FIELD));
+ if (cnt == 0) {
+ Assert.fail("Alert not updated!");
+ }
+ }
+ {
+ // Verify meta alerts are up-to-date
+ long cnt = getMatchingMetaAlertCount(NEW_FIELD, "metron2");
+ if (cnt == 0) {
+ Assert.fail("Active metaalert was not updated!");
+ }
+ if (cnt != 1) {
+ Assert.fail("Metaalerts not updated correctly!");
+ }
+ }
+ }
+ }
+
+ @Test
+ public void shouldThrowExceptionOnMetaAlertUpdate() throws Exception {
+ Document metaAlert = new Document(new HashMap<>(), "meta_alert", METAALERT_TYPE, 0L);
+ try {
+ // Verify a meta alert cannot be updated in the meta alert dao
+ metaDao.update(metaAlert, Optional.empty());
+ Assert.fail("Direct meta alert update should throw an exception");
+ } catch (UnsupportedOperationException uoe) {
+ Assert.assertEquals("Meta alerts cannot be directly updated", uoe.getMessage());
+ }
+ }
+
+ @Test
+ public void shouldPatchAllowedMetaAlerts() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(2);
+ alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+ alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Put the nested type into the test index, so that it'll match appropriately
+ setupTypings();
+
+ // Load metaAlerts
+ Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE,
+ Optional.of(Arrays.asList(alerts.get(0), alerts.get(1))));
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE);
+
+ // Verify load was successful
+ findCreatedDocs(Arrays.asList(
+ new GetRequest("message_0", SENSOR_NAME),
+ new GetRequest("message_1", SENSOR_NAME),
+ new GetRequest("meta_alert", METAALERT_TYPE)));
+
+ Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert);
+ expectedMetaAlert.put(NAME_FIELD, "New Meta Alert");
+ {
+ // Verify a patch to a field other than "status" or "alert" can be patched
+ String namePatch = namePatchRequest.replace(META_INDEX_FLAG, getMetaAlertIndex());
+ PatchRequest patchRequest = JSONUtils.INSTANCE.load(namePatch, PatchRequest.class);
+ metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis()));
+
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+
+ {
+ // Verify a patch to an alert field should throw an exception
+ try {
+ String alertPatch = alertPatchRequest.replace(META_INDEX_FLAG, getMetaAlertIndex());
+ PatchRequest patchRequest = JSONUtils.INSTANCE.load(alertPatch, PatchRequest.class);
+ metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis()));
+
+ Assert.fail("A patch on the alert field should throw an exception");
+ } catch (IllegalArgumentException iae) {
+ Assert.assertEquals("Meta alert patches are not allowed for /alert or /status paths. "
+ + "Please use the add/remove alert or update status functions instead.",
+ iae.getMessage());
+ }
+
+ // Verify the metaAlert was not updated
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+
+ {
+ // Verify a patch to a status field should throw an exception
+ try {
+ String statusPatch = statusPatchRequest
+ .replace(META_INDEX_FLAG, getMetaAlertIndex());
+ PatchRequest patchRequest = JSONUtils.INSTANCE.load(statusPatch, PatchRequest.class);
+ metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis()));
+
+ Assert.fail("A patch on the status field should throw an exception");
+ } catch (IllegalArgumentException iae) {
+ Assert.assertEquals("Meta alert patches are not allowed for /alert or /status paths. "
+ + "Please use the add/remove alert or update status functions instead.",
+ iae.getMessage());
+ }
+
+ // Verify the metaAlert was not updated
+ findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE);
+ }
+ }
+
+ protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType)
+ throws InterruptedException, IOException, OriginalNotFoundException {
+ commit();
+ for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+ Document doc = metaDao.getLatest(guid, sensorType);
+ // Change the underlying document alerts lists to sets to avoid ordering issues.
+ convertAlertsFieldToSet(doc.getDocument());
+ convertAlertsFieldToSet(message0);
+
+ if (doc.getDocument() != null && message0.equals(doc.getDocument())) {
+ convertAlertsFieldToList(doc.getDocument());
+ convertAlertsFieldToList(message0);
+ return;
+ }
+ }
+
+ throw new OriginalNotFoundException(
+ "Count not find " + guid + " after " + MAX_RETRIES + " tries");
+ }
+
+ protected void convertAlertsFieldToSet(Map<String, Object> document) {
+ if (document.get(ALERT_FIELD) instanceof List) {
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> message0AlertField = (List<Map<String, Object>>) document
+ .get(ALERT_FIELD);
+ Set<Map<String, Object>> message0AlertSet = new HashSet<>(message0AlertField);
+ document.put(ALERT_FIELD, message0AlertSet);
+ }
+ }
+
+ protected void convertAlertsFieldToList(Map<String, Object> document) {
+ if (document.get(ALERT_FIELD) instanceof Set) {
+ @SuppressWarnings("unchecked")
+ Set<Map<String, Object>> message0AlertField = (Set<Map<String, Object>>) document
+ .get(ALERT_FIELD);
+ List<Map<String, Object>> message0AlertList = new ArrayList<>(message0AlertField);
+ message0AlertList.sort(Comparator.comparing(o -> ((String) o.get(Constants.GUID))));
+ document.put(ALERT_FIELD, message0AlertList);
+ }
+ }
+
+ protected boolean findCreatedDoc(String guid, String sensorType)
+ throws InterruptedException, IOException, OriginalNotFoundException {
+ for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+ Document doc = metaDao.getLatest(guid, sensorType);
+ if (doc != null) {
+ return true;
+ }
+ }
+ throw new OriginalNotFoundException(
+ "Count not find " + guid + " after " + MAX_RETRIES + "tries");
+ }
+
+ protected boolean findCreatedDocs(List<GetRequest> getRequests)
+ throws InterruptedException, IOException, OriginalNotFoundException {
+ for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+ Iterable<Document> docs = metaDao.getAllLatest(getRequests);
+ if (docs != null) {
+ int docCount = 0;
+ for (Document doc : docs) {
+ docCount++;
+ }
+ if (getRequests.size() == docCount) {
+ return true;
+ }
+ }
+ }
+ throw new OriginalNotFoundException("Count not find guids after " + MAX_RETRIES + "tries");
+ }
+
+ protected List<Map<String, Object>> buildAlerts(int count) {
+ List<Map<String, Object>> inputData = new ArrayList<>();
+ for (int i = 0; i < count; ++i) {
+ final String guid = "message_" + i;
+ Map<String, Object> alerts = new HashMap<>();
+ alerts.put(Constants.GUID, guid);
+ alerts.put(getSourceTypeField(), SENSOR_NAME);
+ alerts.put(THREAT_FIELD_DEFAULT, (double) i);
+ alerts.put("timestamp", System.currentTimeMillis());
+ inputData.add(alerts);
+ }
+ return inputData;
+ }
+
+ protected List<Map<String, Object>> buildMetaAlerts(int count, MetaAlertStatus status,
+ Optional<List<Map<String, Object>>> alerts) {
+ List<Map<String, Object>> inputData = new ArrayList<>();
+ for (int i = 0; i < count; ++i) {
+ final String guid = "meta_" + status.getStatusString() + "_" + i;
+ inputData.add(buildMetaAlert(guid, status, alerts));
+ }
+ return inputData;
+ }
+
+ protected Map<String, Object> buildMetaAlert(String guid, MetaAlertStatus status,
+ Optional<List<Map<String, Object>>> alerts) {
+ Map<String, Object> metaAlert = new HashMap<>();
+ metaAlert.put(Constants.GUID, guid);
+ metaAlert.put(getSourceTypeField(), METAALERT_TYPE);
+ metaAlert.put(STATUS_FIELD, status.getStatusString());
+ if (alerts.isPresent()) {
+ List<Map<String, Object>> alertsList = alerts.get();
+ metaAlert.put(ALERT_FIELD, alertsList);
+ }
+ return metaAlert;
+ }
+
+ protected abstract long getMatchingAlertCount(String fieldName, Object fieldValue)
+ throws IOException, InterruptedException;
+
+ protected abstract void addRecords(List<Map<String, Object>> inputData, String index,
+ String docType) throws IOException;
+
+ protected abstract long getMatchingMetaAlertCount(String fieldName, String fieldValue)
+ throws IOException, InterruptedException;
+
+ protected abstract void setupTypings();
+
+ // Get the base index name without any adjustments (e.g. without ES's "_index")
+ protected abstract String getTestIndexName();
+
+ // Get the full name of the test index. E.g. Elasticsearch appends "_index"
+ protected String getTestIndexFullName() {
+ return getTestIndexName();
+ }
+
+ protected abstract String getMetaAlertIndex();
+
+ protected abstract String getSourceTypeField();
+
+ protected String getThreatTriageField() {
+ return THREAT_FIELD_DEFAULT;
+ }
+
+ // Allow for impls to do any commit they need to do.
+ protected void commit() throws IOException {
+ }
+
+ // Different stores can have different representations of empty metaalerts field.
+ // E.g. Solr expects the field to not be present, ES expects it to be empty.
+ protected abstract void setEmptiedMetaAlertField(Map<String, Object> docMap);
+
+ // Different stores may choose to store non finite double values as Strings.
+ // E.g. NaN may be a string, not a double value.
+ protected abstract boolean isFiniteDoubleOnly();
+
+ // Different stores may choose to return empty alerts lists differently.
+ // E.g. It may be missing completely, or may be an empty list
+ protected abstract boolean isEmptyMetaAlertList();
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
new file mode 100644
index 0000000..1359ba9
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_FIELD_DEFAULT;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_SORT_DEFAULT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.indexing.dao.update.Document;
+import org.junit.Test;
+
+public class MetaScoresTest {
+ @Test
+ public void testCalculateMetaScoresList() {
+ final double delta = 0.001;
+ List<Map<String, Object>> alertList = new ArrayList<>();
+
+ // add an alert with a threat score
+ alertList.add(Collections.singletonMap(THREAT_FIELD_DEFAULT, 10.0f));
+
+ // add a second alert with a threat score
+ alertList.add(Collections.singletonMap(THREAT_FIELD_DEFAULT, 20.0f));
+
+ // add a third alert with NO threat score
+ alertList.add(Collections.singletonMap("alert3", "has no threat score"));
+
+ // create the metaalert
+ Map<String, Object> docMap = new HashMap<>();
+ docMap.put(ALERT_FIELD, alertList);
+ Document metaalert = new Document(docMap, "guid", METAALERT_TYPE, 0L);
+
+ // calculate the threat score for the metaalert
+ MetaScores.calculateMetaScores(metaalert, THREAT_FIELD_DEFAULT, THREAT_SORT_DEFAULT);
+
+ // the metaalert must contain a summary of all child threat scores
+ assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta);
+ assertEquals(10D, (Double) metaalert.getDocument().get("min"), delta);
+ assertEquals(15D, (Double) metaalert.getDocument().get("average"), delta);
+ assertEquals(2L, metaalert.getDocument().get("count"));
+ assertEquals(30D, (Double) metaalert.getDocument().get("sum"), delta);
+ assertEquals(15D, (Double) metaalert.getDocument().get("median"), delta);
+
+ // it must contain an overall threat score; a float to match the type of the threat score of
+ // the other sensor indices
+ Object threatScore = metaalert.getDocument().get(THREAT_FIELD_DEFAULT);
+ assertTrue(threatScore instanceof Float);
+
+ // by default, the overall threat score is the sum of all child threat scores
+ assertEquals(30.0F, threatScore);
+ }
+}