You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2017/09/13 15:39:22 UTC
[1/2] metron git commit: METRON-1158 Build backend for grouping
alerts into meta alerts (justinleet) closes apache/metron#734
Repository: metron
Updated Branches:
refs/heads/master 309d3757d -> 40c93527e
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
new file mode 100644
index 0000000..02ea795
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
@@ -0,0 +1,427 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.elasticsearch.dao;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.Constants.Fields;
+import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.MultiIndexDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaScores;
+import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
+import org.apache.metron.indexing.dao.search.InvalidCreateException;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.Document;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.SearchHits;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Test;
+
+public class ElasticsearchMetaAlertDaoTest {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testBuildUpdatedMetaAlertSingleAlert() throws IOException, ParseException {
+ // Construct the expected result
+ JSONObject expected = new JSONObject();
+ expected.put("average", 5.0);
+ expected.put("min", 5.0);
+ expected.put("median", 5.0);
+ expected.put("max", 5.0);
+ expected.put("count", 1L);
+ expected.put(Constants.GUID, "m1");
+ expected.put("sum", 5.0);
+ expected.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ JSONArray expectedAlerts = new JSONArray();
+ JSONObject expectedAlert = new JSONObject();
+ expectedAlert.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5L);
+ expectedAlert.put("fakekey", "fakevalue");
+ expectedAlerts.add(expectedAlert);
+ expected.put(MetaAlertDao.ALERT_FIELD, expectedAlerts);
+
+ // Construct the meta alert object
+ Map<String, Object> metaSource = new HashMap<>();
+ metaSource.put(Constants.GUID, "m1");
+ metaSource.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ List<Double> alertScores = new ArrayList<>();
+ alertScores.add(10d);
+ metaSource.putAll(new MetaScores(alertScores).getMetaScores());
+ SearchHit metaHit = mock(SearchHit.class);
+ when(metaHit.getSource()).thenReturn(metaSource);
+
+ // Construct the inner alert
+ SearchHit innerAlertHit = mock(SearchHit.class);
+ HashMap<String, Object> innerAlertSource = new HashMap<>();
+ innerAlertSource.put(Constants.GUID, "a1");
+ when(innerAlertHit.sourceAsMap()).thenReturn(innerAlertSource);
+ SearchHitField field = mock(SearchHitField.class);
+ when(field.getValue()).thenReturn(10d);
+ when(innerAlertHit.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(field);
+ SearchHit[] innerHitArray = new SearchHit[1];
+ innerHitArray[0] = innerAlertHit;
+
+ // Construct the inner hits that contains the alert
+ SearchHits searchHits = mock(SearchHits.class);
+ when(searchHits.getHits()).thenReturn(innerHitArray);
+ Map<String, SearchHits> innerHits = new HashMap<>();
+ innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits);
+ when(metaHit.getInnerHits()).thenReturn(innerHits);
+
+ // Construct the updated Document
+ Map<String, Object> updateMap = new HashMap<>();
+ updateMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5);
+ updateMap.put("fakekey", "fakevalue");
+ Document update = new Document(updateMap, "a1", "bro_doc", 0L);
+
+ ElasticsearchDao esDao = new ElasticsearchDao();
+ ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao();
+ emaDao.init(esDao);
+ XContentBuilder builder = emaDao.buildUpdatedMetaAlert(update, metaHit);
+ JSONParser parser = new JSONParser();
+ Object obj = parser.parse(builder.string());
+ JSONObject actual = (JSONObject) obj;
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testBuildUpdatedMetaAlertMultipleAlerts() throws IOException, ParseException {
+ // Construct the expected result
+ JSONObject expected = new JSONObject();
+ expected.put("average", 7.5);
+ expected.put("min", 5.0);
+ expected.put("median", 7.5);
+ expected.put("max", 10.0);
+ expected.put("count", 2L);
+ expected.put(Constants.GUID, "m1");
+ expected.put("sum", 15.0);
+ expected.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ JSONArray expectedAlerts = new JSONArray();
+ JSONObject expectedAlertOne = new JSONObject();
+ expectedAlertOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5d);
+ expectedAlertOne.put("fakekey", "fakevalue");
+ expectedAlerts.add(expectedAlertOne);
+ JSONObject expectedAlertTwo = new JSONObject();
+ expectedAlertTwo.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10d);
+ String guidTwo = "a2";
+ expectedAlertTwo.put(Constants.GUID, guidTwo);
+ expectedAlerts.add(expectedAlertTwo);
+ expected.put(MetaAlertDao.ALERT_FIELD, expectedAlerts);
+
+ // Construct the meta alert object
+ Map<String, Object> metaSource = new HashMap<>();
+ metaSource.put(Constants.GUID, "m1");
+ metaSource.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ double threatValueOne = 5d;
+ double threatValueTwo = 10d;
+ List<Double> alertScores = new ArrayList<>();
+ alertScores.add(threatValueOne);
+ alertScores.add(threatValueTwo);
+ metaSource.putAll(new MetaScores(alertScores).getMetaScores());
+ SearchHit metaHit = mock(SearchHit.class);
+ when(metaHit.getSource()).thenReturn(metaSource);
+
+ // Construct the inner alerts
+ SearchHit innerAlertHitOne = mock(SearchHit.class);
+ HashMap<String, Object> innerAlertSourceOne = new HashMap<>();
+ String guidOne = "a1";
+ innerAlertSourceOne.put(Constants.GUID, guidOne);
+ when(innerAlertHitOne.sourceAsMap()).thenReturn(innerAlertSourceOne);
+ when(innerAlertHitOne.getId()).thenReturn(guidOne);
+ SearchHitField triageOne = mock(SearchHitField.class);
+ when(triageOne.getValue()).thenReturn(threatValueOne);
+ Map<String, Object> innerAlertHitOneSource = new HashMap<>();
+ innerAlertHitOneSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo);
+ innerAlertHitOneSource.put(Constants.GUID, guidOne);
+ when(innerAlertHitOne.getSource()).thenReturn(innerAlertHitOneSource);
+ when(innerAlertHitOne.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageOne);
+
+ SearchHit innerAlertHitTwo = mock(SearchHit.class);
+ HashMap<String, Object> innerAlertSourceTwo = new HashMap<>();
+ innerAlertSourceTwo.put(Constants.GUID, guidTwo);
+ when(innerAlertHitTwo.sourceAsMap()).thenReturn(innerAlertSourceTwo);
+ when(innerAlertHitOne.getId()).thenReturn(guidTwo);
+ SearchHitField triageTwo = mock(SearchHitField.class);
+ when(triageTwo.getValue()).thenReturn(threatValueTwo);
+ Map<String, Object> innerAlertHitTwoSource = new HashMap<>();
+ innerAlertHitTwoSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo);
+ innerAlertHitTwoSource.put(Constants.GUID, guidTwo);
+ when(innerAlertHitTwo.getSource()).thenReturn(innerAlertHitTwoSource);
+ when(innerAlertHitTwo.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageTwo);
+
+ SearchHit[] innerHitArray = new SearchHit[2];
+ innerHitArray[0] = innerAlertHitOne;
+ innerHitArray[1] = innerAlertHitTwo;
+
+ // Construct the inner hits that contains the alert
+ SearchHits searchHits = mock(SearchHits.class);
+ when(searchHits.getHits()).thenReturn(innerHitArray);
+ Map<String, SearchHits> innerHits = new HashMap<>();
+ innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits);
+ when(metaHit.getInnerHits()).thenReturn(innerHits);
+
+ // Construct the updated Document
+ Map<String, Object> updateMap = new HashMap<>();
+ updateMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueOne);
+ updateMap.put("fakekey", "fakevalue");
+ Document update = new Document(updateMap, guidOne, "bro_doc", 0L);
+
+ ElasticsearchDao esDao = new ElasticsearchDao();
+ ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao();
+ MultiIndexDao multiIndexDao = new MultiIndexDao(esDao);
+ emaDao.init(multiIndexDao);
+ XContentBuilder builder = emaDao.buildUpdatedMetaAlert(update, metaHit);
+
+ JSONParser parser = new JSONParser();
+ Object obj = parser.parse(builder.string());
+ JSONObject actual = (JSONObject) obj;
+
+ assertEquals(expected, actual);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidInit() {
+ IndexDao dao = new IndexDao() {
+ @Override
+ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
+ return null;
+ }
+
+ @Override
+ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+ return null;
+ }
+
+ @Override
+ public void init(AccessConfig config) {
+ }
+
+ @Override
+ public Document getLatest(String guid, String sensorType) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void update(Document update, Optional<String> index) throws IOException {
+ }
+
+ @Override
+ public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public Map<String, FieldType> getCommonColumnMetadata(List<String> indices)
+ throws IOException {
+ return null;
+ }
+ };
+ ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
+ metaAlertDao.init(dao);
+ }
+
+ @Test
+ public void testBuildCreateDocumentSingleAlert() throws InvalidCreateException, IOException {
+ ElasticsearchDao esDao = new ElasticsearchDao();
+ ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao();
+ emaDao.init(esDao);
+
+ List<String> groups = new ArrayList<>();
+ groups.add("group_one");
+ groups.add("group_two");
+
+ // Build the first response from the multiget
+ Map<String, Object> alertOne = new HashMap<>();
+ alertOne.put(Constants.GUID, "alert_one");
+ alertOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d);
+ GetResponse getResponseOne = mock(GetResponse.class);
+ when(getResponseOne.isExists()).thenReturn(true);
+ when(getResponseOne.getSource()).thenReturn(alertOne);
+ MultiGetItemResponse multiGetItemResponseOne = mock(MultiGetItemResponse.class);
+ when(multiGetItemResponseOne.getResponse()).thenReturn(getResponseOne);
+
+ // Add it to the iterator
+ @SuppressWarnings("unchecked")
+ Iterator<MultiGetItemResponse> mockIterator = mock(Iterator.class);
+ when(mockIterator.hasNext()).thenReturn(true, false);
+ when(mockIterator.next()).thenReturn(multiGetItemResponseOne);
+
+ // Add it to the response
+ MultiGetResponse mockResponse = mock(MultiGetResponse.class);
+ when(mockResponse.iterator()).thenReturn(mockIterator);
+
+ // Actually build the doc
+ Document actual = emaDao.buildCreateDocument(mockResponse, groups);
+
+ ArrayList<Map<String, Object>> alertList = new ArrayList<>();
+ alertList.add(alertOne);
+
+ Map<String, Object> actualDocument = actual.getDocument();
+ assertEquals(
+ MetaAlertStatus.ACTIVE.getStatusString(),
+ actualDocument.get(MetaAlertDao.STATUS_FIELD)
+ );
+ assertArrayEquals(
+ alertList.toArray(),
+ (Object[]) actualDocument.get(MetaAlertDao.ALERT_FIELD)
+ );
+ assertArrayEquals(
+ groups.toArray(),
+ (Object[]) actualDocument.get(MetaAlertDao.GROUPS_FIELD)
+ );
+
+ // Don't care about the result, just that it's a UUID. Exception will be thrown if not.
+ UUID.fromString((String) actualDocument.get(Constants.GUID));
+ }
+
+ @Test
+ public void testBuildCreateDocumentMultipleAlerts() throws InvalidCreateException, IOException {
+ ElasticsearchDao esDao = new ElasticsearchDao();
+ ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao();
+ emaDao.init(esDao);
+
+ List<String> groups = new ArrayList<>();
+ groups.add("group_one");
+ groups.add("group_two");
+
+ // Build the first response from the multiget
+ Map<String, Object> alertOne = new HashMap<>();
+ alertOne.put(Constants.GUID, "alert_one");
+ alertOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d);
+ GetResponse getResponseOne = mock(GetResponse.class);
+ when(getResponseOne.isExists()).thenReturn(true);
+ when(getResponseOne.getSource()).thenReturn(alertOne);
+ MultiGetItemResponse multiGetItemResponseOne = mock(MultiGetItemResponse.class);
+ when(multiGetItemResponseOne.getResponse()).thenReturn(getResponseOne);
+
+ // Build the second response from the multiget
+ Map<String, Object> alertTwo = new HashMap<>();
+ alertTwo.put(Constants.GUID, "alert_one");
+ alertTwo.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5.0d);
+ GetResponse getResponseTwo = mock(GetResponse.class);
+ when(getResponseTwo.isExists()).thenReturn(true);
+ when(getResponseTwo.getSource()).thenReturn(alertTwo);
+ MultiGetItemResponse multiGetItemResponseTwo = mock(MultiGetItemResponse.class);
+ when(multiGetItemResponseTwo.getResponse()).thenReturn(getResponseTwo);
+
+ // Add it to the iterator
+ @SuppressWarnings("unchecked")
+ Iterator<MultiGetItemResponse> mockIterator = mock(Iterator.class);
+ when(mockIterator.hasNext()).thenReturn(true, true, false);
+ when(mockIterator.next()).thenReturn(multiGetItemResponseOne, multiGetItemResponseTwo);
+
+ // Add them to the response
+ MultiGetResponse mockResponse = mock(MultiGetResponse.class);
+ when(mockResponse.iterator()).thenReturn(mockIterator);
+
+ // Actually build the doc
+ Document actual = emaDao.buildCreateDocument(mockResponse, groups);
+
+ ArrayList<Map<String, Object>> alertList = new ArrayList<>();
+ alertList.add(alertOne);
+ alertList.add(alertTwo);
+
+ Map<String, Object> actualDocument = actual.getDocument();
+ assertNotNull(actualDocument.get(Fields.TIMESTAMP.getName()));
+ assertArrayEquals(
+ alertList.toArray(),
+ (Object[]) actualDocument.get(MetaAlertDao.ALERT_FIELD)
+ );
+ assertArrayEquals(
+ groups.toArray(),
+ (Object[]) actualDocument.get(MetaAlertDao.GROUPS_FIELD)
+ );
+
+ // Don't care about the result, just that it's a UUID. Exception will be thrown if not.
+ UUID.fromString((String) actualDocument.get(Constants.GUID));
+ }
+
+ @Test(expected = InvalidCreateException.class)
+ public void testCreateMetaAlertEmptyGuids() throws InvalidCreateException, IOException {
+ ElasticsearchDao esDao = new ElasticsearchDao();
+ ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao();
+ emaDao.init(esDao);
+
+ MetaAlertCreateRequest createRequest = new MetaAlertCreateRequest();
+ emaDao.createMetaAlert(createRequest);
+ }
+
+ @Test(expected = InvalidCreateException.class)
+ public void testCreateMetaAlertEmptyGroups() throws InvalidCreateException, IOException {
+ ElasticsearchDao esDao = new ElasticsearchDao();
+ ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao();
+ emaDao.init(esDao);
+
+ MetaAlertCreateRequest createRequest = new MetaAlertCreateRequest();
+ HashMap<String, String> guidsToGroups = new HashMap<>();
+ guidsToGroups.put("don't", "care");
+ createRequest.setGuidToIndices(guidsToGroups);
+ emaDao.createMetaAlert(createRequest);
+ }
+
+ @Test
+ public void testCalculateMetaScores() {
+ List<Map<String, Object>> alertList = new ArrayList<>();
+ Map<String, Object> alertMap = new HashMap<>();
+ alertMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d);
+ alertList.add(alertMap);
+ Map<String, Object> docMap = new HashMap<>();
+ docMap.put(MetaAlertDao.ALERT_FIELD, alertList);
+
+ Document doc = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L);
+
+ List<Double> scores = new ArrayList<>();
+ scores.add(10.0d);
+ MetaScores expected = new MetaScores(scores);
+
+ ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
+ MetaScores actual = metaAlertDao.calculateMetaScores(doc);
+ assertEquals(expected.getMetaScores(), actual.getMetaScores());
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
new file mode 100644
index 0000000..fda62ab
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.elasticsearch.integration;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
+import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao;
+import org.apache.metron.elasticsearch.dao.MetaAlertStatus;
+import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
+import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.ReplaceRequest;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ElasticsearchMetaAlertIntegrationTest {
+
+ private static final int MAX_RETRIES = 10;
+ private static final int SLEEP_MS = 500;
+ private static final String SENSOR_NAME = "test";
+ private static final String INDEX_DIR = "target/elasticsearch_meta";
+ private static final String DATE_FORMAT = "yyyy.MM.dd.HH";
+ private static final String INDEX =
+ SENSOR_NAME + "_index_" + new SimpleDateFormat(DATE_FORMAT).format(new Date());
+ private static final String NEW_FIELD = "new-field";
+
+ private static IndexDao esDao;
+ private static IndexDao metaDao;
+ private static ElasticSearchComponent es;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ // setup the client
+ es = new ElasticSearchComponent.Builder()
+ .withHttpPort(9211)
+ .withIndexDir(new File(INDEX_DIR))
+ .build();
+ es.start();
+
+ es.createIndexWithMapping(MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC,
+ buildMetaMappingSource());
+
+ AccessConfig accessConfig = new AccessConfig();
+ Map<String, Object> globalConfig = new HashMap<String, Object>() {
+ {
+ put("es.clustername", "metron");
+ put("es.port", "9300");
+ put("es.ip", "localhost");
+ put("es.date.format", DATE_FORMAT);
+ }
+ };
+ accessConfig.setGlobalConfigSupplier(() -> globalConfig);
+
+ esDao = new ElasticsearchDao();
+ esDao.init(accessConfig);
+ metaDao = new ElasticsearchMetaAlertDao(esDao);
+ }
+
+ @AfterClass
+ public static void teardown() {
+ if (es != null) {
+ es.stop();
+ }
+ }
+
+ protected static String buildMetaMappingSource() throws IOException {
+ return jsonBuilder().prettyPrint()
+ .startObject()
+ .startObject(MetaAlertDao.METAALERT_DOC)
+ .startObject("properties")
+ .startObject("guid")
+ .field("type", "string")
+ .field("index", "not_analyzed")
+ .endObject()
+ .startObject("score")
+ .field("type", "integer")
+ .field("index", "not_analyzed")
+ .endObject()
+ .startObject("alert")
+ .field("type", "nested")
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject()
+ .string();
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void test() throws Exception {
+ List<Map<String, Object>> inputData = new ArrayList<>();
+ for (int i = 0; i < 2; ++i) {
+ final String name = "message" + i;
+ int finalI = i;
+ inputData.add(
+ new HashMap<String, Object>() {
+ {
+ put("source:type", SENSOR_NAME);
+ put("name", name);
+ put(MetaAlertDao.THREAT_FIELD_DEFAULT, finalI);
+ put("timestamp", System.currentTimeMillis());
+ put(Constants.GUID, name);
+ }
+ }
+ );
+ }
+
+ elasticsearchAdd(inputData, INDEX, SENSOR_NAME);
+
+ List<Map<String, Object>> metaInputData = new ArrayList<>();
+ final String name = "meta_message";
+ Map<String, Object>[] alertArray = new Map[1];
+ alertArray[0] = inputData.get(0);
+ metaInputData.add(
+ new HashMap<String, Object>() {
+ {
+ put("source:type", SENSOR_NAME);
+ put("alert", alertArray);
+ put(Constants.GUID, name + "_active");
+ put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+ }
+ }
+ );
+ // Add an inactive message
+ metaInputData.add(
+ new HashMap<String, Object>() {
+ {
+ put("source:type", SENSOR_NAME);
+ put("alert", alertArray);
+ put(Constants.GUID, name + "_inactive");
+ put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString());
+ }
+ }
+ );
+
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ elasticsearchAdd(metaInputData, MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
+
+ List<Map<String, Object>> docs = null;
+ for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+ docs = es.getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc");
+ if (docs.size() >= 10) {
+ break;
+ }
+ }
+ Assert.assertEquals(2, docs.size());
+ {
+ //modify the first message and add a new field
+ Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) {
+ {
+ put(NEW_FIELD, "metron");
+ put(MetaAlertDao.THREAT_FIELD_DEFAULT, "10");
+ }
+ };
+ String guid = "" + message0.get(Constants.GUID);
+ metaDao.replace(new ReplaceRequest() {
+ {
+ setReplacement(message0);
+ setGuid(guid);
+ setSensorType(SENSOR_NAME);
+ }
+ }, Optional.empty());
+
+ {
+ //ensure alerts in ES are up-to-date
+ Document doc = metaDao.getLatest(guid, SENSOR_NAME);
+ Assert.assertEquals(message0, doc.getDocument());
+ long cnt = 0;
+ for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
+ docs = es.getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc");
+ cnt = docs
+ .stream()
+ .filter(d -> {
+ Object newfield = d.get(NEW_FIELD);
+ return newfield != null && newfield.equals(message0.get(NEW_FIELD));
+ }).count();
+ }
+ if (cnt == 0) {
+ Assert.fail("Elasticsearch is not updated!");
+ }
+ }
+
+ {
+ //ensure meta alerts in ES are up-to-date
+ long cnt = 0;
+ for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
+ docs = es.getAllIndexedDocs(MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC);
+ cnt = docs
+ .stream()
+ .filter(d -> {
+ List<Map<String, Object>> alerts = (List<Map<String, Object>>) d
+ .get(MetaAlertDao.ALERT_FIELD);
+
+ for (Map<String, Object> alert : alerts) {
+ Object newField = alert.get(NEW_FIELD);
+ if (newField != null && newField.equals(message0.get(NEW_FIELD))) {
+ return true;
+ }
+ }
+
+ return false;
+ }).count();
+ }
+ if (cnt == 0) {
+ Assert.fail("Elasticsearch metaalerts not updated!");
+ }
+ }
+ }
+ //modify the same message and modify the new field
+ {
+ Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) {
+ {
+ put(NEW_FIELD, "metron2");
+ }
+ };
+ String guid = "" + message0.get(Constants.GUID);
+ metaDao.replace(new ReplaceRequest() {
+ {
+ setReplacement(message0);
+ setGuid(guid);
+ setSensorType(SENSOR_NAME);
+ }
+ }, Optional.empty());
+
+ Document doc = metaDao.getLatest(guid, SENSOR_NAME);
+ Assert.assertEquals(message0, doc.getDocument());
+ {
+ //ensure ES is up-to-date
+ long cnt = 0;
+ for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
+ docs = es.getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc");
+ cnt = docs
+ .stream()
+ .filter(d -> message0.get(NEW_FIELD).equals(d.get(NEW_FIELD)))
+ .count();
+ }
+ Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
+ if (cnt == 0) {
+ Assert.fail("Elasticsearch is not updated!");
+ }
+ }
+ {
+ //ensure meta alerts in ES are up-to-date
+ long cnt = 0;
+ for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) {
+ docs = es.getAllIndexedDocs(MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC);
+ cnt = docs
+ .stream()
+ .filter(d -> {
+ List<Map<String, Object>> alerts = (List<Map<String, Object>>) d
+ .get(MetaAlertDao.ALERT_FIELD);
+
+ for (Map<String, Object> alert : alerts) {
+ Object newField = alert.get(NEW_FIELD);
+ if (newField != null && newField.equals(message0.get(NEW_FIELD))) {
+ return true;
+ }
+ }
+
+ return false;
+ }).count();
+ }
+ if (cnt == 0) {
+ Assert.fail("Elasticsearch metaalerts not updated!");
+ }
+ }
+ }
+ }
+
+ protected void elasticsearchAdd(List<Map<String, Object>> inputData, String index, String docType)
+ throws IOException {
+ es.add(index, docType, inputData.stream().map(m -> {
+ try {
+ return JSONUtils.INSTANCE.toJSON(m, true);
+ } catch (JsonProcessingException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ ).collect(Collectors.toList())
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 5de9fd2..adb69ee 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -20,9 +20,11 @@ package org.apache.metron.elasticsearch.integration;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
+import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao;
import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MetaAlertDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.apache.metron.integration.InMemoryComponent;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -87,8 +89,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
@Override
protected IndexDao createDao() throws Exception {
- IndexDao ret = new ElasticsearchDao();
- ret.init(
+ IndexDao elasticsearchDao = new ElasticsearchDao();
+ elasticsearchDao.init(
new AccessConfig() {{
setMaxSearchResults(100);
setMaxSearchGroups(100);
@@ -102,7 +104,9 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
);
}}
);
- return ret;
+ MetaAlertDao ret = new ElasticsearchMetaAlertDao();
+ ret.init(elasticsearchDao);
+ return elasticsearchDao;
}
@Override
@@ -140,6 +144,14 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
bulkRequest.add(indexRequestBuilder);
}
+ JSONArray metaAlertArray = (JSONArray) new JSONParser().parse(metaAlertData);
+ for(Object o: metaAlertArray) {
+ JSONObject jsonObject = (JSONObject) o;
+ IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalerts", "metaalert_doc");
+ indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
+// indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
+ bulkRequest.add(indexRequestBuilder);
+ }
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
throw new RuntimeException("Failed to index test data");
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
index 9a1d7a7..fddf056 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
@@ -33,6 +33,8 @@ import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.indexing.dao.*;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.QueryBuilders;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -144,6 +146,7 @@ public class ElasticsearchUpdateIntegrationTest {
setGuid(guid);
setSensorType(SENSOR_NAME);
}}, Optional.empty());
+
Assert.assertEquals(1, table.size());
Document doc = dao.getLatest(guid, SENSOR_NAME);
Assert.assertEquals(message0, doc.getDocument());
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
index 7facff5..171b6ab 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
@@ -19,6 +19,7 @@ package org.apache.metron.elasticsearch.integration.components;
import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.commons.io.FileUtils;
+import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.integration.UnableToStartException;
@@ -26,6 +27,8 @@ import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -112,6 +115,7 @@ public class ElasticSearchComponent implements InMemoryComponent {
indexRequestBuilder = indexRequestBuilder.setSource(doc);
Map<String, Object> esDoc = JSONUtils.INSTANCE.load(doc, new TypeReference<Map<String, Object>>() {
});
+ indexRequestBuilder.setId((String) esDoc.get(Constants.GUID));
Object ts = esDoc.get("timestamp");
if(ts != null) {
indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
@@ -126,6 +130,17 @@ public class ElasticSearchComponent implements InMemoryComponent {
return response;
}
+ public void createIndexWithMapping(String indexName, String mappingType, String mappingSource)
+ throws IOException {
+ CreateIndexResponse cir = client.admin().indices().prepareCreate(indexName)
+ .addMapping(mappingType, mappingSource)
+ .get();
+
+ if (!cir.isAcknowledged()) {
+ throw new IOException("Create index was not acknowledged");
+ }
+ }
+
@Override
public void start() throws UnableToStartException {
File logDir= new File(indexDir, "/logs");
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/README.md
index aea670c..e65152c 100644
--- a/metron-platform/metron-indexing/README.md
+++ b/metron-platform/metron-indexing/README.md
@@ -146,6 +146,23 @@ in parallel. This enables a flexible strategy for specifying your backing store
For instance, currently the REST API supports the update functionality and may be configured with a list of
IndexDao implementations to use to support the updates.
+### The `MetaAlertDao`
+
+The goal of meta alerts is to be able to group together a set of alerts while being able to transparently perform actions
+like searches, as if meta alerts were normal alerts. `org.apache.metron.indexing.dao.MetaAlertDao` extends `IndexDao` and
+enables a couple extra features: creation of a meta alert and the ability to get all meta alerts associated with an alert.
+
+The implementation of this is to denormalize the relationship between alerts and meta alerts, and store alerts as a nested field within a meta alert.
+The use of nested fields is to avoid the limitations of parent-child relationships (one-to-many) and merely linking by IDs
+(which causes issues with pagination as a result of being unable to join indices).
+
+The search functionality of `IndexDao` is wrapped by the `MetaAlertDao` in order to provide both regular and meta alerts side-by-side with sorting.
+The updating capabilities are similarly wrapped, in order to ensure updates are carried through both the alerts and associated meta alerts.
+Both of these functions are handled under the hood.
+
+In addition, an API endpoint is added for the meta alert specific features of creation and going from meta alert to alert.
+The denormalization handles the case of going from meta alert to alert automatically.
+
# Notes on Performance Tuning
Default installed Metron is untuned for production deployment. By far
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
new file mode 100644
index 0000000..4e0851b
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao;
+
+import java.io.IOException;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.search.InvalidCreateException;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+
+public interface MetaAlertDao extends IndexDao {
+
+ String METAALERTS_INDEX = "metaalerts";
+ String METAALERT_TYPE = "metaalert";
+ String METAALERT_DOC = METAALERT_TYPE + "_doc";
+ String THREAT_FIELD_DEFAULT = "threat:triage:score";
+ String THREAT_SORT_DEFAULT = "sum";
+ String ALERT_FIELD = "alert";
+ String STATUS_FIELD = "status";
+ String GROUPS_FIELD = "groups";
+
+ /**
+ * Given an alert GUID, retrieve all associated meta alerts.
+ * @param guid The alert GUID to be searched for
+ * @return All meta alerts with a child alert having the GUID
+ * @throws InvalidSearchException If a problem occurs with the search
+ */
+ SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException;
+
+ /**
+ * Create a meta alert.
+ * @param request The parameters for creating the new meta alert
+ * @return A response indicating success or failure
+ * @throws InvalidCreateException If a malformed create request is provided
+ * @throws IOException If a problem occurs during communication
+ */
+ MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ throws InvalidCreateException, IOException;
+
+ /**
+ * Initializes a Meta Alert DAO with default "sum" meta alert threat sorting.
+ * @param indexDao The DAO to wrap for our queries.
+ */
+ default void init(IndexDao indexDao) {
+ init(indexDao, null);
+ }
+
+ /**
+ * Initializes a Meta Alert DAO.
+ * @param indexDao The DAO to wrap for our queries
+ * @param threatSort The aggregation to use as the threat field. E.g. "sum", "median", etc.
+ * null is "sum"
+ */
+ void init(IndexDao indexDao, String threatSort);
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
index 61c6231..2df06fc 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
@@ -169,4 +169,8 @@ public class MultiIndexDao implements IndexDao {
}
return ret;
}
+
+ public List<IndexDao> getIndices() {
+ return indices;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java
new file mode 100644
index 0000000..388527a
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MetaAlertCreateRequest {
+ // A map from the alert GUID to the Document index
+ private Map<String, String> guidToIndices;
+ private List<String> groups;
+
+ public MetaAlertCreateRequest() {
+ this.guidToIndices = new HashMap<>();
+ this.groups = new ArrayList<>();
+ }
+
+ public Map<String, String> getGuidToIndices() {
+ return guidToIndices;
+ }
+
+ public void setGuidToIndices(Map<String, String> guidToIndices) {
+ this.guidToIndices = guidToIndices;
+ }
+
+ public List<String> getGroups() {
+ return groups;
+ }
+
+ public void setGroups(List<String> groups) {
+ this.groups = groups;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
new file mode 100644
index 0000000..e84286e
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+public class MetaAlertCreateResponse {
+ private boolean created;
+
+ public boolean isCreated() {
+ return created;
+ }
+
+ public void setCreated(boolean created) {
+ this.created = created;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
new file mode 100644
index 0000000..632cfd2
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+import java.util.DoubleSummaryStatistics;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.math3.stat.descriptive.rank.Median;
+
+public class MetaScores {
+
+ protected Map<String, Object> metaScores = new HashMap<>();
+
+ public MetaScores(List<Double> scores) {
+ // A meta alert could be entirely alerts with no values.
+ DoubleSummaryStatistics stats = scores
+ .stream()
+ .mapToDouble(a -> a)
+ .summaryStatistics();
+ metaScores.put("max", stats.getMax());
+ metaScores.put("min", stats.getMin());
+ metaScores.put("average", stats.getAverage());
+ metaScores.put("count", stats.getCount());
+ metaScores.put("sum", stats.getSum());
+
+ // median isn't in the stats summary
+ double[] arr = scores
+ .stream()
+ .mapToDouble(d -> d)
+ .toArray();
+ metaScores.put("median", new Median().evaluate(arr));
+ }
+
+ public Map<String, Object> getMetaScores() {
+ return metaScores;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java
index 5848cb3..1f00cf5 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java
@@ -36,6 +36,8 @@ public enum FieldType {
DOUBLE("double"),
@JsonProperty("boolean")
BOOLEAN("boolean"),
+ @JsonProperty("nested")
+ NESTED("nested"),
@JsonProperty("other")
OTHER("other");
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java
new file mode 100644
index 0000000..be32cee
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.search;
+
+public class InvalidCreateException extends Exception {
+ public InvalidCreateException(String message) {
+ super(message);
+ }
+ public InvalidCreateException(String message, Throwable t) {
+ super(message, t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
index 9c00bea..da4fac1 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java
@@ -73,4 +73,14 @@ public class SearchResult {
public void setScore(float score) {
this.score = score;
}
+
+ @Override
+ public String toString() {
+ return "SearchResult{" +
+ "id='" + id + '\'' +
+ ", source=" + source +
+ ", score=" + score +
+ ", index='" + index + '\'' +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
index 85c079f..461ce3e 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
@@ -18,14 +18,11 @@
package org.apache.metron.indexing.dao.update;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonNode;
import org.apache.metron.common.utils.JSONUtils;
import java.io.IOException;
import java.util.Map;
-import java.util.Optional;
public class Document {
Long timestamp;
@@ -85,4 +82,14 @@ public class Document {
public void setGuid(String guid) {
this.guid = guid;
}
+
+ @Override
+ public String toString() {
+ return "Document{" +
+ "timestamp=" + timestamp +
+ ", document=" + document +
+ ", guid='" + guid + '\'' +
+ ", sensorType='" + sensorType + '\'' +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
index 6e48b58..c83f6aa 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
@@ -30,6 +30,7 @@ import java.io.IOException;
import java.util.*;
public class InMemoryDao implements IndexDao {
+ // Map from index to list of documents as JSON strings
public static Map<String, List<String>> BACKING_STORE = new HashMap<>();
public static Map<String, Map<String, FieldType>> COLUMN_METADATA;
private AccessConfig config;
@@ -123,6 +124,9 @@ public class InMemoryDao implements IndexDao {
}
private static boolean isMatch(String query, Map<String, Object> doc) {
+ if (query == null) {
+ return false;
+ }
if(query.equals("*")) {
return true;
}
@@ -130,12 +134,36 @@ public class InMemoryDao implements IndexDao {
Iterable<String> splits = Splitter.on(":").split(query.trim());
String field = Iterables.getFirst(splits, "");
String val = Iterables.getLast(splits, "");
- Object o = doc.get(field);
- if(o == null) {
+
+ // Immediately quit if there's no value ot find
+ if (val == null) {
return false;
}
- else {
- return o.equals(val);
+
+ // Check if we're looking into a nested field. The '|' is arbitrarily chosen.
+ String nestingField = null;
+ if (field.contains("|")) {
+ Iterable<String> fieldSplits = Splitter.on('|').split(field);
+ nestingField = Iterables.getFirst(fieldSplits, null);
+ field = Iterables.getLast(fieldSplits, null);
+ }
+ if (nestingField == null) {
+ // Just grab directly
+ Object o = doc.get(field);
+ return val.equals(o);
+ } else {
+ // We need to look into a nested field for the value
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> nestedList = (List<Map<String, Object>>) doc.get(nestingField);
+ if (nestedList == null) {
+ return false;
+ } else {
+ for (Map<String, Object> nestedEntry : nestedList) {
+ if (val.equals(nestedEntry.get(field))) {
+ return true;
+ }
+ }
+ }
}
}
return false;
@@ -185,7 +213,7 @@ public class InMemoryDao implements IndexDao {
}
}
}
-
+
public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws IOException {
Map<String, Map<String, FieldType>> columnMetadata = new HashMap<>();
for(String index: indices) {
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
new file mode 100644
index 0000000..8807bbc
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao;
+
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.metaalert.MetaScores;
+import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
+import org.apache.metron.indexing.dao.search.InvalidCreateException;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
+import org.apache.metron.indexing.dao.update.PatchRequest;
+import org.apache.metron.indexing.dao.update.ReplaceRequest;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+public class InMemoryMetaAlertDao implements MetaAlertDao {
+
+ private IndexDao indexDao;
+
+ /**
+ * {
+ * "indices": ["metaalerts"],
+ * "query": "alert|guid:${GUID}",
+ * "from": 0,
+ * "size": 10,
+ * "sort": [
+ * {
+ * "field": "guid",
+ * "sortOrder": "desc"
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ public static String metaAlertsForAlertQuery;
+
+ @Override
+ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
+ return indexDao.search(searchRequest);
+ }
+
+ @Override
+ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+ return indexDao.group(groupRequest);
+ }
+
+ @Override
+ public void init(AccessConfig config) {
+ // Do nothing
+ }
+
+ @Override
+ public void init(IndexDao indexDao, String threatSort) {
+ this.indexDao = indexDao;
+ // Ignore threatSort for test.
+ }
+
+ @Override
+ public Document getLatest(String guid, String sensorType) throws IOException {
+ return indexDao.getLatest(guid, sensorType);
+ }
+
+ @Override
+ public void update(Document update, Optional<String> index) throws IOException {
+ indexDao.update(update, index);
+ }
+
+ @Override
+ public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices)
+ throws IOException {
+ return indexDao.getColumnMetadata(indices);
+ }
+
+ @Override
+ public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws IOException {
+ return indexDao.getCommonColumnMetadata(indices);
+ }
+
+ @Override
+ public Optional<Map<String, Object>> getLatestResult(GetRequest request) throws IOException {
+ return indexDao.getLatestResult(request);
+ }
+
+ @Override
+ public void patch(PatchRequest request, Optional<Long> timestamp)
+ throws OriginalNotFoundException, IOException {
+ indexDao.patch(request, timestamp);
+ }
+
+ @Override
+ public void replace(ReplaceRequest request, Optional<Long> timestamp) throws IOException {
+ indexDao.replace(request, timestamp);
+ }
+
+ @Override
+ public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException {
+ SearchRequest request;
+ try {
+ String replacedQuery = metaAlertsForAlertQuery.replace("${GUID}", guid);
+ request = JSONUtils.INSTANCE.load(replacedQuery, SearchRequest.class);
+ } catch (IOException e) {
+ throw new InvalidSearchException("Unable to process query:", e);
+ }
+ return search(request);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ throws InvalidCreateException, IOException {
+ if (request.getGuidToIndices().isEmpty()) {
+ MetaAlertCreateResponse response = new MetaAlertCreateResponse();
+ response.setCreated(false);
+ return response;
+ }
+ // Build meta alert json. Give it a reasonable GUID
+ JSONObject metaAlert = new JSONObject();
+ metaAlert.put(Constants.GUID,
+ "meta_" + (InMemoryDao.BACKING_STORE.get(MetaAlertDao.METAALERTS_INDEX).size() + 1));
+
+ JSONArray groupsArray = new JSONArray();
+ groupsArray.addAll(request.getGroups());
+ metaAlert.put(MetaAlertDao.GROUPS_FIELD, groupsArray);
+
+ // Retrieve the alert for each guid
+ // For the purpose of testing, we're just using guids for the alerts field and grabbing the scores.
+ JSONArray alertArray = new JSONArray();
+ List<Double> threatScores = new ArrayList<>();
+ for (Map.Entry<String, String> entry : request.getGuidToIndices().entrySet()) {
+ SearchRequest searchRequest = new SearchRequest();
+ searchRequest.setIndices(ImmutableList.of(entry.getValue()));
+ searchRequest.setQuery("guid:" + entry.getKey());
+ try {
+ SearchResponse searchResponse = search(searchRequest);
+ List<SearchResult> searchResults = searchResponse.getResults();
+ if (searchResults.size() > 1) {
+ throw new InvalidCreateException(
+ "Found more than one result for: " + entry.getKey() + ". Values: " + searchResults
+ );
+ }
+
+ if (searchResults.size() == 1) {
+ SearchResult result = searchResults.get(0);
+ alertArray.add(result.getSource());
+ Double threatScore = Double
+ .parseDouble(result.getSource().getOrDefault(THREAT_FIELD_DEFAULT, "0").toString());
+
+ threatScores.add(threatScore);
+ }
+ } catch (InvalidSearchException e) {
+ throw new InvalidCreateException("Unable to find guid: " + entry.getKey(), e);
+ }
+ }
+
+ metaAlert.put(MetaAlertDao.ALERT_FIELD, alertArray);
+ metaAlert.putAll(new MetaScores(threatScores).getMetaScores());
+
+ // Add the alert to the store, but make sure not to overwrite existing results
+ InMemoryDao.BACKING_STORE.get(MetaAlertDao.METAALERTS_INDEX).add(metaAlert.toJSONString());
+
+ MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse();
+ createResponse.setCreated(true);
+ return createResponse;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 0db8e37..26d1a75 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -43,11 +43,11 @@ import java.util.Map;
public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5"}
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2", "guid":"bro_2"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3", "guid":"bro_3"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4", "guid":"bro_4"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5", "guid":"bro_5"}
* ]
*/
@Multiline
@@ -55,17 +55,26 @@ public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5}
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1, "guid":"snort_1"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2, "guid":"snort_2"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3, "guid":"snort_3"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4, "guid":"snort_4"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5, "guid":"snort_5"}
* ]
*/
@Multiline
public static String snortData;
/**
+ * [
+ *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
+ *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
+ * ]
+ */
+ @Multiline
+ public static String metaAlertData;
+
+ /**
* {
* "indices": ["bro", "snort"],
* "query": "*",
@@ -258,6 +267,25 @@ public abstract class SearchIntegrationTest {
/**
* {
+ * "fields": ["guid"],
+ * "indices": ["metaalerts"],
+ * "query": "*",
+ * "from": 0,
+ * "size": 10,
+ * "sort": [
+ * {
+ * "field": "guid",
+ * "sortOrder": "asc"
+ * }
+ * ]
+ * }
+ * }
+ */
+ @Multiline
+ public static String metaAlertsFieldQuery;
+
+ /**
+ * {
* "groups": [
* {
* "field":"is_alert"
@@ -497,7 +525,7 @@ public abstract class SearchIntegrationTest {
Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
Assert.assertEquals(2, fieldTypes.size());
Map<String, FieldType> broTypes = fieldTypes.get("bro");
- Assert.assertEquals(11, broTypes.size());
+ Assert.assertEquals(12, broTypes.size());
Assert.assertEquals(FieldType.STRING, broTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, broTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, broTypes.get("ip_src_port"));
@@ -509,8 +537,9 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point"));
Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field"));
Assert.assertEquals(FieldType.STRING, broTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.STRING, broTypes.get("guid"));
Map<String, FieldType> snortTypes = fieldTypes.get("snort");
- Assert.assertEquals(11, snortTypes.size());
+ Assert.assertEquals(12, snortTypes.size());
Assert.assertEquals(FieldType.STRING, snortTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, snortTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, snortTypes.get("ip_src_port"));
@@ -522,13 +551,14 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point"));
Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field"));
Assert.assertEquals(FieldType.INTEGER, snortTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.STRING, broTypes.get("guid"));
}
// getColumnMetadata with only bro
{
Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
Assert.assertEquals(1, fieldTypes.size());
Map<String, FieldType> broTypes = fieldTypes.get("bro");
- Assert.assertEquals(11, broTypes.size());
+ Assert.assertEquals(12, broTypes.size());
Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field"));
}
// getColumnMetadata with only snort
@@ -536,14 +566,14 @@ public abstract class SearchIntegrationTest {
Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
Assert.assertEquals(1, fieldTypes.size());
Map<String, FieldType> snortTypes = fieldTypes.get("snort");
- Assert.assertEquals(11, snortTypes.size());
+ Assert.assertEquals(12, snortTypes.size());
Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field"));
}
// getCommonColumnMetadata with multiple Indices
{
Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Arrays.asList("bro", "snort"));
// Should only return fields in both
- Assert.assertEquals(9, fieldTypes.size());
+ Assert.assertEquals(10, fieldTypes.size());
Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
@@ -553,18 +583,19 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
+ Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid"));
}
// getCommonColumnMetadata with only bro
{
Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("bro"));
- Assert.assertEquals(11, fieldTypes.size());
+ Assert.assertEquals(12, fieldTypes.size());
Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field"));
Assert.assertEquals(FieldType.STRING, fieldTypes.get("duplicate_name_field"));
}
// getCommonColumnMetadata with only snort
{
Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("snort"));
- Assert.assertEquals(11, fieldTypes.size());
+ Assert.assertEquals(12, fieldTypes.size());
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
}
@@ -585,6 +616,18 @@ public abstract class SearchIntegrationTest {
Assert.assertNotNull(source.get("ip_src_addr"));
}
}
+ //Meta Alerts Fields query
+ {
+ SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(2, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ for (int i = 0;i < 2;++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertEquals(source.get("guid"), "meta_" + (i + 1));
+ }
+ }
//No results fields query
{
SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index af86902..2b20feb 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -548,6 +548,11 @@ public class BasicStellarTest {
}
@Test
+ public void testToStringNull() {
+ Assert.assertEquals("null", run("TO_STRING(\"null\")", ImmutableMap.of("foo", "null")));
+ }
+
+ @Test
public void testToInteger() {
Assert.assertEquals(5, run("TO_INTEGER(foo)", ImmutableMap.of("foo", "5")));
Assert.assertEquals(5, run("TO_INTEGER(foo)", ImmutableMap.of("foo", 5)));
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6e92772..3f0af7e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -307,6 +307,7 @@
<exclude>**/*.tokens</exclude>
<exclude>**/*.log</exclude>
<exclude>**/*.template</exclude>
+ <exclude>**/*.mapping</exclude>
<exclude>**/.*</exclude>
<exclude>**/.*/**</exclude>
<exclude>**/*.seed</exclude>
[2/2] metron git commit: METRON-1158 Build backend for grouping
alerts into meta alerts (justinleet) closes apache/metron#734
Posted by le...@apache.org.
METRON-1158 Build backend for grouping alerts into meta alerts (justinleet) closes apache/metron#734
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/40c93527
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/40c93527
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/40c93527
Branch: refs/heads/master
Commit: 40c93527e2a693ec6580dc0d09356dfa3b525aa4
Parents: 309d375
Author: justinleet <ju...@gmail.com>
Authored: Wed Sep 13 11:38:05 2017 -0400
Committer: leet <le...@apache.org>
Committed: Wed Sep 13 11:38:05 2017 -0400
----------------------------------------------------------------------
.../CURRENT/package/files/bro_index.template | 3 +
.../CURRENT/package/files/error_index.template | 3 +
.../CURRENT/package/files/meta_index.mapping | 42 ++
.../CURRENT/package/files/snort_index.template | 3 +
.../CURRENT/package/files/yaf_index.template | 3 +
.../CURRENT/package/scripts/indexing_master.py | 8 +
.../package/scripts/params/params_linux.py | 1 +
metron-interface/metron-rest/README.md | 18 +
.../apache/metron/rest/MetronRestConstants.java | 3 +
.../apache/metron/rest/config/IndexConfig.java | 16 +-
.../rest/controller/MetaAlertController.java | 64 +++
.../metron/rest/service/MetaAlertService.java | 31 ++
.../rest/service/impl/MetaAlertServiceImpl.java | 66 +++
.../rest/service/impl/SearchServiceImpl.java | 1 +
.../src/main/resources/application-test.yml | 5 +
.../src/main/resources/application.yml | 4 +
.../rest/controller/DaoControllerTest.java | 20 +-
.../MetaAlertControllerIntegrationTest.java | 174 ++++++++
.../SearchControllerIntegrationTest.java | 8 +-
.../UpdateControllerIntegrationTest.java | 20 +-
.../elasticsearch/dao/ElasticsearchDao.java | 57 ++-
.../dao/ElasticsearchMetaAlertDao.java | 446 +++++++++++++++++++
.../elasticsearch/dao/MetaAlertStatus.java | 34 ++
.../dao/ElasticsearchMetaAlertDaoTest.java | 427 ++++++++++++++++++
.../ElasticsearchMetaAlertIntegrationTest.java | 317 +++++++++++++
.../ElasticsearchSearchIntegrationTest.java | 18 +-
.../ElasticsearchUpdateIntegrationTest.java | 3 +
.../components/ElasticSearchComponent.java | 15 +
metron-platform/metron-indexing/README.md | 17 +
.../metron/indexing/dao/MetaAlertDao.java | 72 +++
.../metron/indexing/dao/MultiIndexDao.java | 4 +
.../dao/metaalert/MetaAlertCreateRequest.java | 51 +++
.../dao/metaalert/MetaAlertCreateResponse.java | 31 ++
.../indexing/dao/metaalert/MetaScores.java | 54 +++
.../metron/indexing/dao/search/FieldType.java | 2 +
.../dao/search/InvalidCreateException.java | 28 ++
.../indexing/dao/search/SearchResult.java | 10 +
.../metron/indexing/dao/update/Document.java | 13 +-
.../apache/metron/indexing/dao/InMemoryDao.java | 38 +-
.../indexing/dao/InMemoryMetaAlertDao.java | 198 ++++++++
.../indexing/dao/SearchIntegrationTest.java | 77 +++-
.../stellar/dsl/functions/BasicStellarTest.java | 5 +
pom.xml | 1 +
43 files changed, 2357 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 18c5d9b..7db006e 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -151,6 +151,9 @@
"type": "string",
"index": "not_analyzed"
},
+ "alert": {
+ "type": "nested"
+ },
"ip_src_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
index 3bb4633..e79d482 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template
@@ -50,6 +50,9 @@
"error_type": {
"type": "string",
"index": "not_analyzed"
+ },
+ "alert": {
+ "type": "nested"
}
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping
new file mode 100644
index 0000000..c42343e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/meta_index.mapping
@@ -0,0 +1,42 @@
+{
+ "mappings": {
+ "metaalert_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "dynamic_templates": [
+ {
+ "alert_template": {
+ "path_match": "alert.*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ ],
+ "properties": {
+ "guid": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "score": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "status": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "alert": {
+ "type": "nested"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index 2311cf2..f13a9ee 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -203,6 +203,9 @@
},
"ttl": {
"type": "integer"
+ },
+ "alert": {
+ "type": "nested"
}
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index bd90929..d84235d 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -225,6 +225,9 @@
},
"end-reason": {
"type": "string"
+ },
+ "alert": {
+ "type": "nested"
}
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 71dcc74..68e238a 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -124,6 +124,11 @@ class Indexing(Script):
content=StaticFile('error_index.template')
)
+ File(params.meta_index_path,
+ mode=0755,
+ content=StaticFile('meta_index.mapping')
+ )
+
bro_cmd = ambari_format(
'curl -s -XPOST http://{es_http_url}/_template/bro_index -d @{bro_index_path}')
Execute(bro_cmd, logoutput=True)
@@ -136,6 +141,9 @@ class Indexing(Script):
error_cmd = ambari_format(
'curl -s -XPOST http://{es_http_url}/_template/error_index -d @{error_index_path}')
Execute(error_cmd, logoutput=True)
+ error_cmd = ambari_format(
+ 'curl -s -XPOST http://{es_http_url}/metaalerts -d @{meta_index_path}')
+ Execute(error_cmd, logoutput=True)
def elasticsearch_template_delete(self, env):
from params import params
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index a9d00dd..72f295b 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -188,6 +188,7 @@ bro_index_path = tmp_dir + "/bro_index.template"
snort_index_path = tmp_dir + "/snort_index.template"
yaf_index_path = tmp_dir + "/yaf_index.template"
error_index_path = tmp_dir + "/error_index.template"
+meta_index_path = tmp_dir + "/meta_index.mapping"
# Zeppelin Notebooks
metron_config_zeppelin_path = format("{metron_config_path}/zeppelin")
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 27e04a3..97ab95c 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -200,6 +200,9 @@ Request and Response objects are JSON formatted. The JSON schemas are available
| [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopicname)|
| [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopicname)|
| [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopicnamesample)|
+| [ `GET /api/v1/metaalert/searchByAlert`](#get-apiv1metaalertsearchbyalert)|
+| [ `GET /api/v1/metaalert/create`](#get-apiv1metaalertcreate)|
+| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
| [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)|
@@ -365,6 +368,21 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* 200 - Returns sample message
* 404 - Either Kafka topic is missing or contains no messages
+### `POST /api/v1/metaalert/searchByAlert`
+ * Description: Searches meta alerts to find any containing an alert for the provided GUID
+ * Input:
+ * guid - GUID of the alert
+ * Returns:
+ * 200 - Returns the meta alerts associated with this alert
+ * 404 - The child alert isn't found
+
+### `POST /api/v1/metaalert/create`
+ * Description: Creates a meta alert containing the provide alerts
+ * Input:
+ * request - Meta Alert Create Request
+ * Returns:
+ * 200 - The meta alert was created
+
### `POST /api/v1/search/search`
* Description: Searches the indexing store
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index c5b3c13..b0f553f 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -59,4 +59,7 @@ public class MetronRestConstants {
public static final String SEARCH_MAX_GROUPS = "search.max.groups";
public static final String INDEX_DAO_IMPL = "index.dao.impl";
public static final String INDEX_HBASE_TABLE_PROVIDER_IMPL = "index.hbase.provider";
+
+ public static final String META_DAO_IMPL = "meta.dao.impl";
+ public static final String META_DAO_SORT = "meta.dao.sort";
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
index b6ac5e7..8eabb2e 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
@@ -24,6 +24,7 @@ import org.apache.metron.hbase.TableProvider;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.IndexDaoFactory;
+import org.apache.metron.indexing.dao.MetaAlertDao;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.service.GlobalConfigService;
@@ -53,6 +54,8 @@ public class IndexConfig {
String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null);
int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, 1000);
int searchMaxGroups = environment.getProperty(MetronRestConstants.SEARCH_MAX_GROUPS, Integer.class, 1000);
+ String metaDaoImpl = environment.getProperty(MetronRestConstants.META_DAO_IMPL, String.class, null);
+ String metaDaoSort = environment.getProperty(MetronRestConstants.META_DAO_SORT, String.class, null);
AccessConfig config = new AccessConfig();
config.setMaxSearchResults(searchMaxResults);
config.setMaxSearchGroups(searchMaxGroups);
@@ -67,10 +70,18 @@ public class IndexConfig {
if (indexDaoImpl == null) {
throw new IllegalStateException("You must provide an index DAO implementation via the " + INDEX_DAO_IMPL + " config");
}
- IndexDao ret = IndexDaoFactory.combine(IndexDaoFactory.create(indexDaoImpl, config));
- if (ret == null) {
+ IndexDao indexDao = IndexDaoFactory.combine(IndexDaoFactory.create(indexDaoImpl, config));
+ if (indexDao == null) {
throw new IllegalStateException("IndexDao is unable to be created.");
}
+ if (metaDaoImpl == null) {
+ // We're not using meta alerts.
+ return indexDao;
+ }
+
+ // Create the meta alert dao and wrap it around the index dao.
+ MetaAlertDao ret = (MetaAlertDao) IndexDaoFactory.create(metaDaoImpl, config).get(0);
+ ret.init(indexDao, metaDaoSort);
return ret;
}
catch(RuntimeException re) {
@@ -80,5 +91,4 @@ public class IndexConfig {
throw new IllegalStateException("Unable to create index DAO: " + e.getMessage(), e);
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
new file mode 100644
index 0000000..e9cff8b
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.MetaAlertService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/v1/metaalert")
+public class MetaAlertController {
+
+ @Autowired
+ private MetaAlertService metaAlertService;
+
+ @ApiOperation(value = "Get all meta alerts for alert")
+ @ApiResponse(message = "Search results", code = 200)
+ @RequestMapping(value = "/searchByAlert", method = RequestMethod.POST)
+ ResponseEntity<SearchResponse> searchByAlert(
+ @ApiParam(name = "guid", value = "GUID", required = true)
+ @RequestBody final String guid
+ ) throws RestException {
+ return new ResponseEntity<>(metaAlertService.getAllMetaAlertsForAlert(guid), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Create a meta alert")
+ @ApiResponse(message = "Created meta alert", code = 200)
+ @RequestMapping(value = "/create", method = RequestMethod.POST)
+ ResponseEntity<MetaAlertCreateResponse> create(
+ @ApiParam(name = "request", value = "Meta Alert Create Request", required = true)
+ @RequestBody final MetaAlertCreateRequest createRequest
+ ) throws RestException {
+ return new ResponseEntity<>(metaAlertService.create(createRequest), HttpStatus.OK);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java
new file mode 100644
index 0000000..c339506
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/MetaAlertService.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.rest.service;
+
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.rest.RestException;
+
+public interface MetaAlertService {
+
+ MetaAlertCreateResponse create(MetaAlertCreateRequest createRequest) throws RestException;
+
+ SearchResponse getAllMetaAlertsForAlert(String guid) throws RestException;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java
new file mode 100644
index 0000000..f120c9e
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/MetaAlertServiceImpl.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.rest.service.impl;
+
+import java.io.IOException;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.search.InvalidCreateException;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.MetaAlertService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Service;
+
+@Service
+public class MetaAlertServiceImpl implements MetaAlertService {
+ private MetaAlertDao dao;
+ private Environment environment;
+
+ @Autowired
+ public MetaAlertServiceImpl(IndexDao indexDao, Environment environment) {
+ // By construction this is always a meta alert dao
+ this.dao = (MetaAlertDao) indexDao;
+ this.environment = environment;
+ }
+
+
+ @Override
+ public MetaAlertCreateResponse create(MetaAlertCreateRequest createRequest) throws RestException {
+ try {
+ return dao.createMetaAlert(createRequest);
+ } catch (InvalidCreateException | IOException e) {
+ throw new RestException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public SearchResponse getAllMetaAlertsForAlert(String guid) throws RestException {
+ try {
+ return dao.getAllMetaAlertsForAlert(guid);
+ } catch (InvalidSearchException ise) {
+ throw new RestException(ise.getMessage(), ise);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
index d865e0e..326ee02 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
@@ -76,6 +76,7 @@ public class SearchServiceImpl implements SearchService {
}
}
+ @Override
public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException {
try {
return dao.getColumnMetadata(indices);
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/resources/application-test.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml
index b5e65a7..749dec4 100644
--- a/metron-interface/metron-rest/src/main/resources/application-test.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-test.yml
@@ -51,3 +51,8 @@ index:
hbase:
# HBase is provided via a mock provider, so no actual HBase infrastructure is started.
provider: org.apache.metron.hbase.mock.MockHBaseTableProvider
+
+meta:
+ dao:
+ # By default, we use the InMemoryMetaAlertDao for our tests
+ impl: org.apache.metron.indexing.dao.InMemoryMetaAlertDao
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index 3aa5fd9..764bd40 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -54,3 +54,7 @@ index:
# By default, we use the ElasticsearchDao and HBaseDao for backing updates.
impl: org.apache.metron.elasticsearch.dao.ElasticsearchDao,org.apache.metron.indexing.dao.HBaseDao
+meta:
+ dao:
+ # By default, we use the ElasticsearchMetaAlertDao
+ impl: org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java
index 096f1be..bd3f5bd 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/DaoControllerTest.java
@@ -17,10 +17,8 @@
*/
package org.apache.metron.rest.controller;
-import com.google.common.collect.ImmutableMap;
import org.apache.metron.common.Constants;
import org.apache.metron.indexing.dao.InMemoryDao;
-import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -34,22 +32,20 @@ import java.util.Map;
public class DaoControllerTest {
public static final String TABLE = "updates";
public static final String CF = "t";
- public void loadTestData() throws ParseException {
+ public void loadTestData(Map<String, String> indicesToDataMap) throws ParseException {
Map<String, List<String>> backingStore = new HashMap<>();
- for(Map.Entry<String, String> indices :
- ImmutableMap.of(
- "bro_index_2017.01.01.01", SearchIntegrationTest.broData,
- "snort_index_2017.01.01.01", SearchIntegrationTest.snortData
- ).entrySet()
- )
+ for(Map.Entry<String, String> indices : indicesToDataMap.entrySet())
{
List<String> results = new ArrayList<>();
backingStore.put(indices.getKey(), results);
- JSONArray broArray = (JSONArray) new JSONParser().parse(indices.getValue());
+ JSONArray docArray = (JSONArray) new JSONParser().parse(indices.getValue());
int i = 0;
- for(Object o: broArray) {
+ for(Object o: docArray) {
JSONObject jsonObject = (JSONObject) o;
- jsonObject.put(Constants.GUID, indices.getKey() + ":" + i++);
+ // Don't replace the GUID if we've already provided one
+ if (!jsonObject.containsKey(Constants.GUID)) {
+ jsonObject.put(Constants.GUID, indices.getKey() + ":" + i++);
+ }
results.add(jsonObject.toJSONString());
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
new file mode 100644
index 0000000..983c207
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.rest.controller;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.SearchIntegrationTest;
+import org.apache.metron.rest.service.MetaAlertService;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.ResultActions;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
+
+ @Autowired
+ private MetaAlertService metaAlertService;
+ @Autowired
+ public CuratorFramework client;
+
+ @Autowired
+ private WebApplicationContext wac;
+
+ private MockMvc mockMvc;
+
+ private String metaalertUrl = "/api/v1/metaalert";
+ private String user = "user";
+ private String password = "password";
+
+ /**
+ {
+ "guidToIndices" : {
+ "bro_1":"bro_index_2017.01.01.01",
+ "snort_2":"snort_index_2017.01.01.01"
+ },
+ "groups" : ["group_one", "group_two"]
+ }
+ */
+ @Multiline
+ public static String create;
+
+ @Before
+ public void setup() throws Exception {
+ this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+ ImmutableMap<String, String> testData = ImmutableMap.of(
+ "bro_index_2017.01.01.01", SearchIntegrationTest.broData,
+ "snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
+ MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
+ );
+ loadTestData(testData);
+ }
+
+ @Test
+ public void test() throws Exception {
+ // Testing searching by alert
+ // Test no meta alert
+ String guid = "missing_1";
+ ResultActions result = this.mockMvc.perform(
+ post(metaalertUrl + "/searchByAlert")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))
+ .content(guid));
+ result.andExpect(status().isOk())
+ .andExpect(
+ content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.total").value(0));
+
+ // Test single meta alert
+ guid = "snort_1";
+ result = this.mockMvc.perform(
+ post(metaalertUrl + "/searchByAlert")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))
+ .content(guid));
+ result.andExpect(status().isOk())
+ .andExpect(
+ content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.total").value(1))
+ .andExpect(jsonPath("$.results[0].source.guid").value("meta_2"))
+ .andExpect(jsonPath("$.results[0].source.count").value(3.0));
+
+ // Test multiple meta alerts
+ guid = "bro_1";
+ result = this.mockMvc.perform(
+ post(metaalertUrl + "/searchByAlert")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))
+ .content(guid));
+ result.andExpect(status().isOk())
+ .andExpect(
+ content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.total").value(2))
+ .andExpect(jsonPath("$.results[0].source.guid").value("meta_2"))
+ .andExpect(jsonPath("$.results[0].source.count").value(3.0))
+ .andExpect(jsonPath("$.results[1].source.guid").value("meta_1"))
+ .andExpect(jsonPath("$.results[1].source.count").value(1.0));
+
+ result = this.mockMvc.perform(
+ post(metaalertUrl + "/create")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
+ .content(create));
+ result.andExpect(status().isOk());
+
+ // Test that we can find the newly created meta alert by the sub alerts
+ guid = "bro_1";
+ result = this.mockMvc.perform(
+ post(metaalertUrl + "/searchByAlert")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))
+ .content(guid));
+ result.andExpect(status().isOk())
+ .andExpect(
+ content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.total").value(3))
+ .andExpect(jsonPath("$.results[0].source.guid").value("meta_3"))
+ .andExpect(jsonPath("$.results[0].source.count").value(2.0))
+ .andExpect(jsonPath("$.results[1].source.guid").value("meta_2"))
+ .andExpect(jsonPath("$.results[1].source.count").value(3.0))
+ .andExpect(jsonPath("$.results[2].source.guid").value("meta_1"))
+ .andExpect(jsonPath("$.results[2].source.count").value(1.0));
+
+ guid = "snort_2";
+ result = this.mockMvc.perform(
+ post(metaalertUrl + "/searchByAlert")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))
+ .content(guid));
+ result.andExpect(status().isOk())
+ .andExpect(
+ content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.total").value(1))
+ .andExpect(jsonPath("$.results[0].source.guid").value("meta_3"))
+ .andExpect(jsonPath("$.results[0].source.count").value(2.0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
index 645e525..ca7f209 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.metron.rest.controller;
+import com.google.common.collect.ImmutableMap;
+import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.indexing.dao.InMemoryDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.apache.metron.indexing.dao.search.FieldType;
@@ -70,7 +72,11 @@ public class SearchControllerIntegrationTest extends DaoControllerTest {
@Before
public void setup() throws Exception {
this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
- loadTestData();
+ ImmutableMap<String, String> testData = ImmutableMap.of(
+ "bro_index_2017.01.01.01", SearchIntegrationTest.broData,
+ "snort_index_2017.01.01.01", SearchIntegrationTest.snortData
+ );
+ loadTestData(testData);
loadColumnTypes();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
index 8955980..4708bc4 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
@@ -17,12 +17,15 @@
*/
package org.apache.metron.rest.controller;
+import com.google.common.collect.ImmutableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.apache.metron.rest.service.UpdateService;
import org.junit.Assert;
import org.junit.Before;
@@ -71,7 +74,7 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
/**
{
- "guid" : "bro_index_2017.01.01.01:1",
+ "guid" : "bro_2",
"sensorType" : "bro"
}
*/
@@ -80,7 +83,7 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
/**
{
- "guid" : "bro_index_2017.01.01.01:1",
+ "guid" : "bro_2",
"sensorType" : "bro",
"patch" : [
{
@@ -96,11 +99,11 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
/**
{
- "guid" : "bro_index_2017.01.01.01:1",
+ "guid" : "bro_2",
"sensorType" : "bro",
"replacement" : {
"source:type": "bro",
- "guid" : "bro_index_2017.01.01.01:1",
+ "guid" : "bro_2",
"ip_src_addr":"192.168.1.2",
"ip_src_port": 8009,
"timestamp":200,
@@ -114,12 +117,17 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
@Before
public void setup() throws Exception {
this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
- loadTestData();
+ ImmutableMap<String, String> testData = ImmutableMap.of(
+ "bro_index_2017.01.01.01", SearchIntegrationTest.broData,
+ "snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
+ MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
+ );
+ loadTestData(testData);
}
@Test
public void test() throws Exception {
- String guid = "bro_index_2017.01.01.01:1";
+ String guid = "bro_2";
ResultActions result = this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0));
try {
result.andExpect(status().isOk())
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 0d7a76c..0a06c80 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -45,6 +45,10 @@ import org.apache.metron.indexing.dao.search.GroupResult;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.elasticsearch.action.ActionWriteResponse.ShardInfo;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.*;
+import org.elasticsearch.action.update.UpdateRequest;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.search.SortOrder;
import org.apache.metron.indexing.dao.update.Document;
@@ -52,6 +56,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.transport.TransportClient;
@@ -72,6 +77,24 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.*;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Date;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
public class ElasticsearchDao implements IndexDao {
private transient TransportClient client;
@@ -105,6 +128,17 @@ public class ElasticsearchDao implements IndexDao {
@Override
public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
+ return search(searchRequest, new QueryStringQueryBuilder(searchRequest.getQuery()));
+ }
+
+ /**
+ * Defers to a provided {@link org.elasticsearch.index.query.QueryBuilder} for the query.
+ * @param searchRequest The request defining the parameters of the search
+ * @param queryBuilder The actual query to be run. Intended for if the SearchRequest requires wrapping
+ * @return The results of the query
+ * @throws InvalidSearchException When the query is malformed or the current state doesn't allow search
+ */
+ protected SearchResponse search(SearchRequest searchRequest, QueryBuilder queryBuilder) throws InvalidSearchException {
if(client == null) {
throw new InvalidSearchException("Uninitialized Dao! You must call init() prior to use.");
}
@@ -114,10 +148,10 @@ public class ElasticsearchDao implements IndexDao {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.size(searchRequest.getSize())
.from(searchRequest.getFrom())
- .query(new QueryStringQueryBuilder(searchRequest.getQuery()))
+ .query(queryBuilder)
+
.trackScores(true);
- searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder())));
- Optional<List<String>> fields = searchRequest.getFields();
+ searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder())));Optional<List<String>> fields = searchRequest.getFields();
if (fields.isPresent()) {
searchSourceBuilder.fields(fields.get());
} else {
@@ -264,8 +298,19 @@ public class ElasticsearchDao implements IndexDao {
.upsert(indexRequest)
;
+ org.elasticsearch.action.search.SearchResponse result = client.prepareSearch("test*").setFetchSource(true).setQuery(QueryBuilders.matchAllQuery()).get();
+ result.getHits();
try {
- client.update(updateRequest).get();
+ UpdateResponse response = client.update(updateRequest).get();
+
+ ShardInfo shardInfo = response.getShardInfo();
+ int failed = shardInfo.getFailed();
+ if (failed > 0) {
+ throw new IOException("ElasticsearchDao upsert failed: " + Arrays.toString(shardInfo.getFailures()));
+ }
+ Thread.sleep(10000);
+ org.elasticsearch.action.search.SearchResponse resultAfter = client.prepareSearch("test*").setFetchSource(true).setQuery(QueryBuilders.matchAllQuery()).get();
+ resultAfter.getHits();
} catch (Exception e) {
throw new IOException(e.getMessage(), e);
}
@@ -438,6 +483,10 @@ public class ElasticsearchDao implements IndexDao {
return String.format("%s_count", field);
}
+ public TransportClient getClient() {
+ return client;
+ }
+
private String getGroupByAggregationName(String field) {
return String.format("%s_group", field);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
new file mode 100644
index 0000000..cd6ed75
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.elasticsearch.dao;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
+import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.metron.common.Constants;
+import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.MultiIndexDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.metaalert.MetaScores;
+import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
+import org.apache.metron.indexing.dao.search.InvalidCreateException;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.Document;
+import org.elasticsearch.action.ActionWriteResponse.ShardInfo;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetRequest.Item;
+import org.elasticsearch.action.get.MultiGetRequestBuilder;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
+import org.elasticsearch.index.query.support.QueryInnerHitBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+
+public class ElasticsearchMetaAlertDao implements MetaAlertDao {
+
+ private IndexDao indexDao;
+ private ElasticsearchDao elasticsearchDao;
+ private String index = METAALERTS_INDEX;
+ private String threatTriageField = THREAT_FIELD_DEFAULT;
+ private String threatSort = THREAT_SORT_DEFAULT;
+
+ /**
+ * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts.
+ * @param indexDao The Dao to wrap
+ */
+ public ElasticsearchMetaAlertDao(IndexDao indexDao) {
+ this(indexDao, METAALERTS_INDEX, THREAT_FIELD_DEFAULT, THREAT_SORT_DEFAULT);
+ }
+
+ /**
+ * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts.
+ * @param indexDao The Dao to wrap
+ * @param triageLevelField The field name to use as the threat scoring field
+ */
+ public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField,
+ String threatSort) {
+ init(indexDao, threatSort);
+ this.index = index;
+ this.threatTriageField = triageLevelField;
+ }
+
+ public ElasticsearchMetaAlertDao() {
+ //uninitialized.
+ }
+
+ @Override
+ public void init(IndexDao indexDao, String threatSort) {
+ if (indexDao instanceof MultiIndexDao) {
+ this.indexDao = indexDao;
+ MultiIndexDao multiIndexDao = (MultiIndexDao) indexDao;
+ for (IndexDao childDao : multiIndexDao.getIndices()) {
+ if (childDao instanceof ElasticsearchDao) {
+ this.elasticsearchDao = (ElasticsearchDao) childDao;
+ }
+ }
+ } else if (indexDao instanceof ElasticsearchDao) {
+ this.indexDao = indexDao;
+ this.elasticsearchDao = (ElasticsearchDao) indexDao;
+ } else {
+ throw new IllegalArgumentException(
+ "Need an ElasticsearchDao when using ElasticsearchMetaAlertDao"
+ );
+ }
+
+ if (threatSort != null) {
+ this.threatSort = threatSort;
+ }
+ }
+
+ @Override
+ public void init(AccessConfig config) {
+ // Do nothing. We're just wrapping a child dao
+ }
+
+ @Override
+ public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException {
+ if (guid == null || guid.trim().isEmpty()) {
+ throw new InvalidSearchException("Guid cannot be empty");
+ }
+ org.elasticsearch.action.search.SearchResponse esResponse = getMetaAlertsForAlert(guid.trim());
+ SearchResponse searchResponse = new SearchResponse();
+ searchResponse.setTotal(esResponse.getHits().getTotalHits());
+ searchResponse.setResults(
+ Arrays.stream(esResponse.getHits().getHits()).map(searchHit -> {
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(searchHit.getId());
+ searchResult.setSource(searchHit.getSource());
+ searchResult.setScore(searchHit.getScore());
+ searchResult.setIndex(searchHit.getIndex());
+ return searchResult;
+ }
+ ).collect(Collectors.toList()));
+ return searchResponse;
+ }
+
+ @Override
+ public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request)
+ throws InvalidCreateException, IOException {
+ if (request.getGuidToIndices().isEmpty()) {
+ throw new InvalidCreateException("MetaAlertCreateRequest must contain alert GUIDs");
+ }
+ if (request.getGroups().isEmpty()) {
+ throw new InvalidCreateException("MetaAlertCreateRequest must contain UI groups");
+ }
+
+ // Retrieve the documents going into the meta alert
+ MultiGetResponse multiGetResponse = getDocumentsByGuid(request);
+ Document createDoc = buildCreateDocument(multiGetResponse, request.getGroups());
+
+ try {
+ handleMetaUpdate(createDoc, Optional.of(METAALERTS_INDEX));
+ MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse();
+ createResponse.setCreated(true);
+ return createResponse;
+ } catch (IOException ioe) {
+ throw new InvalidCreateException("Unable to create meta alert", ioe);
+ }
+ }
+
+ @Override
+ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
+ // Wrap the query to also get any meta-alerts.
+ QueryBuilder qb = constantScoreQuery(boolQuery()
+ .should(new QueryStringQueryBuilder(searchRequest.getQuery()))
+ .should(boolQuery()
+ .must(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()))
+ .must(nestedQuery(
+ ALERT_FIELD,
+ new QueryStringQueryBuilder(searchRequest.getQuery())
+ )
+ )
+ )
+ );
+ return elasticsearchDao.search(searchRequest, qb);
+ }
+
+ @Override
+ public Document getLatest(String guid, String sensorType) throws IOException {
+ return indexDao.getLatest(guid, sensorType);
+ }
+
+ @Override
+ public void update(Document update, Optional<String> index) throws IOException {
+ if (METAALERT_TYPE.equals(update.getSensorType())) {
+ // We've been passed an update to the meta alert.
+ handleMetaUpdate(update, index);
+ } else {
+ // We need to update an alert itself. Only that portion of the update can be delegated.
+ // We still need to get meta alerts potentially associated with it and update.
+ org.elasticsearch.action.search.SearchResponse response = getMetaAlertsForAlert(
+ update.getGuid()
+ );
+
+ // Each hit, if any, is a metaalert that needs to be updated
+ for (SearchHit hit : response.getHits()) {
+ handleAlertUpdate(update, hit);
+ }
+
+ // Run the alert's update
+ indexDao.update(update, index);
+ }
+ }
+
+ /**
+ * Given an alert GUID, retrieve all associated meta alerts.
+ * @param guid The GUID of the child alert
+ * @return The Elasticsearch response containing the meta alerts
+ */
+ protected org.elasticsearch.action.search.SearchResponse getMetaAlertsForAlert(String guid) {
+ QueryBuilder qb = boolQuery()
+ .must(
+ nestedQuery(
+ ALERT_FIELD,
+ boolQuery()
+ .must(termQuery(ALERT_FIELD + "." + Constants.GUID, guid))
+ ).innerHit(new QueryInnerHitBuilder())
+ )
+ .must(termQuery(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()));
+ SearchRequest sr = new SearchRequest();
+ ArrayList<String> indices = new ArrayList<>();
+ indices.add(index);
+ sr.setIndices(indices);
+ return elasticsearchDao
+ .getClient()
+ .prepareSearch(index)
+ .addFields("*")
+ .setFetchSource(true)
+ .setQuery(qb)
+ .execute()
+ .actionGet();
+ }
+
+ /**
+ * Return child documents after retrieving them from Elasticsearch.
+ * @param request The request detailing which child alerts we need
+ * @return The Elasticsearch response to our request for alerts
+ */
+ protected MultiGetResponse getDocumentsByGuid(MetaAlertCreateRequest request) {
+ MultiGetRequestBuilder multiGet = elasticsearchDao.getClient().prepareMultiGet();
+ for (Entry<String, String> entry : request.getGuidToIndices().entrySet()) {
+ multiGet.add(new Item(entry.getValue(), null, entry.getKey()));
+ }
+ return multiGet.get();
+ }
+
+ /**
+ * Build the Document representing a meta alert to be created.
+ * @param multiGetResponse The Elasticsearch results for the meta alerts child documents
+ * @param groups The groups used to create this meta alert
+ * @return A Document representing the new meta alert
+ */
+ protected Document buildCreateDocument(MultiGetResponse multiGetResponse, List<String> groups) {
+ // Need to create a Document from the multiget. Scores will be calculated later
+ Map<String, Object> metaSource = new HashMap<>();
+ List<Map<String, Object>> alertList = new ArrayList<>();
+ for (MultiGetItemResponse itemResponse : multiGetResponse) {
+ GetResponse response = itemResponse.getResponse();
+ if (response.isExists()) {
+ alertList.add(response.getSource());
+ }
+ }
+ metaSource.put(ALERT_FIELD, alertList.toArray());
+
+ // Add any meta fields and score calculation.
+ String guid = UUID.randomUUID().toString();
+ metaSource.put(Constants.GUID, guid);
+ metaSource.put(Constants.Fields.TIMESTAMP.getName(), System.currentTimeMillis());
+ metaSource.put(GROUPS_FIELD, groups.toArray());
+ metaSource.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString());
+
+ return new Document(metaSource, guid, METAALERT_TYPE, System.currentTimeMillis());
+ }
+
+ /**
+ * Process an update to a meta alert itself.
+ * @param update The update Document to be applied
+ * @param index The optional index to update to
+ * @throws IOException If there's a problem running the update
+ */
+ protected void handleMetaUpdate(Document update, Optional<String> index) throws IOException {
+ // We have an update to a meta alert itself (e.g. adding a document, etc.) Calculate scores
+ // and defer the final result to the Elasticsearch DAO.
+ MetaScores metaScores = calculateMetaScores(update);
+ update.getDocument().putAll(metaScores.getMetaScores());
+ update.getDocument().put(threatTriageField, metaScores.getMetaScores().get(threatSort));
+ indexDao.update(update, index);
+ }
+
+ /**
+ * Takes care of upserting a child alert to a meta alert.
+ * @param update The update Document to be applied
+ * @param hit The meta alert to be updated
+ * @throws IOException If there's an issue running the upsert
+ */
+ protected void handleAlertUpdate(Document update, SearchHit hit) throws IOException {
+ XContentBuilder builder = buildUpdatedMetaAlert(update, hit);
+
+ // Run the meta alert's update
+ IndexRequest indexRequest = new IndexRequest(
+ METAALERTS_INDEX,
+ METAALERT_DOC,
+ hit.getId()
+ ).source(builder);
+ UpdateRequest updateRequest = new UpdateRequest(
+ METAALERTS_INDEX,
+ METAALERT_DOC,
+ hit.getId()
+ ).doc(builder).upsert(indexRequest);
+ try {
+ UpdateResponse updateResponse = elasticsearchDao.getClient().update(updateRequest).get();
+
+ ShardInfo shardInfo = updateResponse.getShardInfo();
+ int failed = shardInfo.getFailed();
+ if (failed > 0) {
+ throw new IOException(
+ "ElasticsearchMetaAlertDao upsert failed: "
+ + Arrays.toString(shardInfo.getFailures())
+ );
+ }
+ } catch (Exception e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices)
+ throws IOException {
+ return indexDao.getColumnMetadata(indices);
+ }
+
+ @Override
+ public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws
+ IOException {
+ return indexDao.getCommonColumnMetadata(indices);
+ }
+
+ @Override
+ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+ return indexDao.group(groupRequest);
+ }
+
+ /**
+ * Calculate the meta alert scores for a Document.
+ * @param document The Document containing scores
+ * @return Set of score statistics
+ */
+ @SuppressWarnings("unchecked")
+ protected MetaScores calculateMetaScores(Document document) {
+ List<Object> alertsRaw = ((List<Object>) document.getDocument().get(ALERT_FIELD));
+ if (alertsRaw == null || alertsRaw.isEmpty()) {
+ throw new IllegalArgumentException("No alerts to use in calculation for doc GUID: "
+ + document.getDocument().get(Constants.GUID));
+ }
+
+ ArrayList<Double> scores = new ArrayList<>();
+ for (Object alertRaw : alertsRaw) {
+ Map<String, Object> alert = (Map<String, Object>) alertRaw;
+ Double scoreNum = parseThreatField(alert.get(threatTriageField));
+ if (scoreNum != null) {
+ scores.add(scoreNum);
+ }
+ }
+
+ return new MetaScores(scores);
+ }
+
+ /**
+ * Builds the updated meta alert based on the update.
+ * @param update The update Document for the meta alert
+ * @param hit The meta alert to be updated
+ * @return A builder for Elasticsearch to use
+ * @throws IOException If we have an issue building the result
+ */
+ protected XContentBuilder buildUpdatedMetaAlert(Document update, SearchHit hit)
+ throws IOException {
+ // Make sure to get all the threat scores while we're going through the docs
+ List<Double> scores = new ArrayList<>();
+ // Start building the new version of the metaalert
+ XContentBuilder builder = jsonBuilder().startObject();
+
+ // Run through the nested alerts of the meta alert and either use the new or old versions
+ builder.startArray(ALERT_FIELD);
+ Map<String, SearchHits> innerHits = hit.getInnerHits();
+
+ SearchHits alertHits = innerHits.get(ALERT_FIELD);
+ for (SearchHit alertHit : alertHits.getHits()) {
+ Map<String, Object> docMap;
+ // If we're at the update use it, otherwise use the original
+ if (alertHit.sourceAsMap().get(Constants.GUID).equals(update.getGuid())) {
+ docMap = update.getDocument();
+ } else {
+ docMap = alertHit.getSource();
+ }
+ builder.map(docMap);
+
+ // Handle either String or Number values in the threatTriageField
+ Object threatRaw = docMap.get(threatTriageField);
+ Double threat = parseThreatField(threatRaw);
+
+ if (threat != null) {
+ scores.add(threat);
+ }
+ }
+ builder.endArray();
+
+ // Add all the meta alert fields, and score calculation
+ Map<String, Object> updatedMeta = new HashMap<>();
+ updatedMeta.putAll(hit.getSource());
+ updatedMeta.putAll(new MetaScores(scores).getMetaScores());
+ for (Entry<String, Object> entry : updatedMeta.entrySet()) {
+ // The alerts field is being added separately, so ignore the original
+ if (!(entry.getKey().equals(ALERT_FIELD))) {
+ builder.field(entry.getKey(), entry.getValue());
+ }
+ }
+ builder.endObject();
+
+ return builder;
+ }
+
+ private Double parseThreatField(Object threatRaw) {
+ Double threat = null;
+ if (threatRaw instanceof Number) {
+ threat = ((Number) threatRaw).doubleValue();
+ } else if (threatRaw instanceof String) {
+ threat = Double.parseDouble((String) threatRaw);
+ }
+ return threat;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/MetaAlertStatus.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/MetaAlertStatus.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/MetaAlertStatus.java
new file mode 100644
index 0000000..6c8e858
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/MetaAlertStatus.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.elasticsearch.dao;
+
+public enum MetaAlertStatus {
+ ACTIVE("active"),
+ INACTIVE("inactive");
+
+ private String statusString;
+
+ MetaAlertStatus(String statusString) {
+ this.statusString = statusString;
+ }
+
+ public String getStatusString() {
+ return statusString;
+ }
+}