You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2018/06/05 19:00:08 UTC
[1/2] metron git commit: METRON-1547 Solr Comment Fields (justinleet)
closes apache/metron#1037
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1416-upgrade-solr 9348c6081 -> a68d031b0
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index d461792..7db0ab5 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -22,13 +22,16 @@ import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
import com.google.common.base.Splitter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.function.Function;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -36,6 +39,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -88,12 +92,21 @@ public class SolrDao implements IndexDao {
this.accessConfig = config;
this.client = getSolrClient(getZkHosts());
this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig);
- this.solrUpdateDao = new SolrUpdateDao(this.client, this.accessConfig);
this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client);
+ this.solrUpdateDao = new SolrUpdateDao(this.client, this.solrRetrieveLatestDao, this.accessConfig);
this.solrColumnMetadataDao = new SolrColumnMetadataDao(this.client);
}
}
+ public Optional<String> getIndex(String sensorName, Optional<String> index) {
+ if (index.isPresent()) {
+ return index;
+ } else {
+ String realIndex = accessConfig.getIndexSupplier().apply(sensorName);
+ return Optional.ofNullable(realIndex);
+ }
+ }
+
@Override
public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
return this.solrSearchDao.search(searchRequest);
@@ -125,6 +138,16 @@ public class SolrDao implements IndexDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ this.solrUpdateDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ this.solrUpdateDao.removeCommentFromAlert(request);
+ }
+
+ @Override
public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
@@ -136,6 +159,18 @@ public class SolrDao implements IndexDao {
return this.solrColumnMetadataDao.getColumnMetadata(indices);
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ this.solrUpdateDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ this.solrUpdateDao.removeCommentFromAlert(request, latest);
+ }
+
/**
* Builds a Solr client using the ZK hosts from the global config.
* @return SolrClient
@@ -170,7 +205,7 @@ public class SolrDao implements IndexDao {
return solrSearchDao;
}
- public SolrSearchDao getSolrUpdateDao() {
- return solrSearchDao;
+ public SolrUpdateDao getSolrUpdateDao() {
+ return solrUpdateDao;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
index e65700f..8b37a49 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
@@ -41,6 +41,7 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -222,4 +223,24 @@ public class SolrMetaAlertDao implements MetaAlertDao {
throws IOException {
return metaAlertUpdateDao.updateMetaAlertStatus(metaAlertGuid, status);
}
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ solrDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ solrDao.removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ solrDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ solrDao.removeCommentFromAlert(request, latest);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
index 2a66d47..b96bbc6 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
@@ -42,6 +42,7 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.UpdateDao;
import org.apache.solr.client.solrj.SolrClient;
@@ -182,6 +183,28 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
}
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().removeCommentFromAlert(request, latest);
+ }
+
protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) {
boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert,
Collections.singleton(alert.getGuid()));
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
index 8578bfb..b3bc564 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
@@ -45,6 +45,7 @@ public class SolrRetrieveLatestDao implements RetrieveLatestDao {
@Override
public Document getLatest(String guid, String collection) throws IOException {
+
try {
SolrDocument solrDocument = client.getById(collection, guid);
if (solrDocument == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index 4a8d482..d978ec9 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -17,10 +17,24 @@
*/
package org.apache.metron.solr.dao;
+import static org.apache.metron.common.Constants.SENSOR_TYPE;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.metron.common.Constants;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -29,6 +43,8 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.Group;
import org.apache.metron.indexing.dao.search.GroupOrder;
import org.apache.metron.indexing.dao.search.GroupOrderType;
@@ -53,6 +69,7 @@ import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
+import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
index 7c56169..7f5a4ed 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
@@ -17,19 +17,24 @@
*/
package org.apache.metron.solr.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
-
+import java.util.stream.Collectors;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.UpdateDao;
import org.apache.solr.client.solrj.SolrClient;
@@ -39,23 +44,24 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SolrUpdateDao implements UpdateDao {
-
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private transient SolrClient client;
private AccessConfig config;
private Function<String, String> indexSupplier;
- public SolrUpdateDao(SolrClient client, AccessConfig config) {
+ private transient SolrRetrieveLatestDao retrieveLatestDao;
+
+ public SolrUpdateDao(SolrClient client, SolrRetrieveLatestDao retrieveLatestDao, AccessConfig config) {
this.client = client;
+ this.retrieveLatestDao = retrieveLatestDao;
this.config = config;
this.indexSupplier = config.getIndexSupplier();
}
private Optional<String> getIndex(String sensorName, Optional<String> index) {
- if(index.isPresent()) {
- return Optional.ofNullable(index.get());
- }
- else {
+ if (index.isPresent()) {
+ return index;
+ } else {
String realIndex = indexSupplier.apply(sensorName);
return Optional.ofNullable(realIndex);
}
@@ -63,8 +69,17 @@ public class SolrUpdateDao implements UpdateDao {
@Override
public void update(Document update, Optional<String> rawIndex) throws IOException {
+ Document newVersion = update;
+ // Handle any case where we're given comments in Map form, instead of raw String
+ Object commentsObj = update.getDocument().get(COMMENTS_FIELD);
+ if ( commentsObj instanceof List &&
+ ((List<Object>) commentsObj).size() > 0 &&
+ ((List<Object>) commentsObj).get(0) instanceof Map) {
+ newVersion = new Document(update);
+ convertCommentsToRaw(newVersion.getDocument());
+ }
try {
- SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(update);
+ SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(newVersion);
Optional<String> index = getIndex(update.getSensorType(), rawIndex);
if (index.isPresent()) {
this.client.add(index.get(), solrInputDocument);
@@ -113,4 +128,87 @@ public class SolrUpdateDao implements UpdateDao {
throw new IOException(e);
}
}
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null) {
+ return;
+ }
+
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> comments = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(comments);
+
+ // Convert all comments back to raw JSON before updating.
+ List<String> commentStrs = new ArrayList<>();
+ for (Map<String, Object> comment : originalComments) {
+ commentStrs.add(new AlertComment(comment).asJson());
+ }
+ commentStrs.add(new AlertComment(
+ request.getComment(),
+ request.getUsername(),
+ request.getTimestamp()
+ ).asJson());
+
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, commentStrs);
+ update(newVersion, Optional.empty());
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ if (latest == null) {
+ return;
+ }
+
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument()
+ .get(COMMENTS_FIELD);
+ // Can't remove anything if there's nothing there
+ if (commentMap == null) {
+ LOG.debug("Provided alert had no comments to be able to remove from");
+ return;
+ }
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentMap);
+ List<AlertComment> comments = new ArrayList<>();
+ for (Map<String, Object> commentStr : originalComments) {
+ comments.add(new AlertComment(commentStr));
+ }
+
+ comments.remove(
+ new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()));
+ List<String> commentsAsJson = comments.stream().map(AlertComment::asJson)
+ .collect(Collectors.toList());
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsAsJson);
+ update(newVersion, Optional.empty());
+ }
+
+ public void convertCommentsToRaw(Map<String,Object> source) {
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> comments = (List<Map<String, Object>>) source.get(COMMENTS_FIELD);
+ if (comments == null || comments.isEmpty()) {
+ return;
+ }
+ List<String> asJson = new ArrayList<>();
+ for (Map<String, Object> comment : comments) {
+ asJson.add((new AlertComment(comment)).asJson());
+ }
+ source.put(COMMENTS_FIELD, asJson);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
index 616bd1a..88146b0 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
@@ -18,18 +18,23 @@
package org.apache.metron.solr.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.metron.common.Constants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
+import org.json.simple.parser.ParseException;
public class SolrUtilities {
@@ -53,6 +58,34 @@ public class SolrUtilities {
solrDocument.getFieldNames().stream()
.filter(name -> !name.equals(SolrDao.VERSION_FIELD))
.forEach(name -> document.put(name, solrDocument.getFieldValue(name)));
+
+ reformatComments(solrDocument, document);
+ insertChildAlerts(solrDocument, document);
+
+ return new Document(document,
+ (String) solrDocument.getFieldValue(Constants.GUID),
+ (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), 0L);
+ }
+
+ protected static void reformatComments(SolrDocument solrDocument, Map<String, Object> document) {
+ // Make sure comments are in the proper format
+ @SuppressWarnings("unchecked")
+ List<String> commentStrs = (List<String>) solrDocument.get(COMMENTS_FIELD);
+ if (commentStrs != null) {
+ try {
+ List<AlertComment> comments = new ArrayList<>();
+ for (String commentStr : commentStrs) {
+ comments.add(new AlertComment(commentStr));
+ }
+ document.put(COMMENTS_FIELD,
+ comments.stream().map(AlertComment::asMap).collect(Collectors.toList()));
+ } catch (ParseException e) {
+ throw new IllegalStateException("Unable to parse comment", e);
+ }
+ }
+ }
+
+ protected static void insertChildAlerts(SolrDocument solrDocument, Map<String, Object> document) {
// Make sure to put child alerts in
if (solrDocument.hasChildDocuments() && solrDocument
.getFieldValue(Constants.SENSOR_TYPE)
@@ -68,9 +101,6 @@ public class SolrUtilities {
document.put(MetaAlertConstants.ALERT_FIELD, childDocuments);
}
- return new Document(document,
- (String) solrDocument.getFieldValue(Constants.GUID),
- (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), 0L);
}
public static SolrInputDocument toSolrInputDocument(Document document) {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
index 076fb20..85523b2 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
@@ -35,6 +35,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
import org.apache.metron.indexing.dao.search.SearchRequest;
@@ -107,9 +109,10 @@ public class SolrDaoTest {
solrDao = spy(new SolrDao());
doReturn(client).when(solrDao).getSolrClient(Collections.singletonList("zookeeper:2181"));
whenNew(SolrSearchDao.class).withArguments(client, accessConfig).thenReturn(solrSearchDao);
- whenNew(SolrUpdateDao.class).withArguments(client, accessConfig).thenReturn(solrUpdateDao);
whenNew(SolrRetrieveLatestDao.class).withArguments(client)
.thenReturn(solrRetrieveLatestDao);
+ whenNew(SolrUpdateDao.class).withArguments(client, solrRetrieveLatestDao, accessConfig)
+ .thenReturn(solrUpdateDao);
whenNew(SolrColumnMetadataDao.class).withArguments(client)
.thenReturn(solrColumnMetadataDao);
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
index 4571b15..43bf1b1 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
@@ -39,8 +39,8 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
-import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -94,6 +94,22 @@ public class SolrMetaAlertDaoTest {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
public void patch(RetrieveLatestDao dao, PatchRequest request, Optional<Long> timestamp) {
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
index 41fe7f9..7de02c1 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
@@ -17,11 +17,32 @@
*/
package org.apache.metron.solr.dao;
-import org.apache.metron.common.configuration.Configurations;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.doReturn;
+import static org.powermock.api.mockito.PowerMockito.spy;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
+import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.util.IndexingCacheUtil;
import org.apache.metron.solr.matcher.SolrInputDocumentListMatcher;
import org.apache.metron.solr.matcher.SolrInputDocumentMatcher;
@@ -34,22 +55,9 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
-import org.mockito.Matchers;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
@RunWith(PowerMockRunner.class)
@PrepareForTest({CollectionAdminRequest.class})
public class SolrUpdateDaoTest {
@@ -58,6 +66,7 @@ public class SolrUpdateDaoTest {
public final ExpectedException exception = ExpectedException.none();
private SolrClient client;
+ private SolrRetrieveLatestDao solrRetrieveLatestDao;
private SolrUpdateDao solrUpdateDao;
private static AccessConfig accessConfig = new AccessConfig();
@@ -86,7 +95,8 @@ public class SolrUpdateDaoTest {
@Before
public void setUp() throws Exception {
client = mock(SolrClient.class);
- solrUpdateDao = new SolrUpdateDao(client, accessConfig);
+ solrRetrieveLatestDao = new SolrRetrieveLatestDao(client);
+ solrUpdateDao = new SolrUpdateDao(client, solrRetrieveLatestDao, accessConfig);
}
@Test
@@ -174,4 +184,58 @@ public class SolrUpdateDaoTest {
verify(client).add(eq("snort"), argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2))));
}
+ @Test
+ public void testConvertCommentsToRaw() {
+ List<Map<String, Object>> commentList = new ArrayList<>();
+ Map<String, Object> comments = new HashMap<>();
+ comments.put("comment", "test comment");
+ comments.put("username", "test username");
+ comments.put("timestamp", 1526424323279L);
+ commentList.add(comments);
+
+ Map<String, Object> document = new HashMap<>();
+ document.put("testField", "testValue");
+ document.put(COMMENTS_FIELD, commentList);
+ solrUpdateDao.convertCommentsToRaw(document);
+
+ @SuppressWarnings("unchecked")
+ List<String> actualComments = (List<String>) document.get(COMMENTS_FIELD);
+ String expectedComment = "{\"comment\":\"test comment\",\"username\":\"test username\",\"timestamp\":1526424323279}";
+ assertEquals(expectedComment, actualComments.get(0));
+ assertEquals(1, actualComments.size());
+ assertEquals("testValue", document.get("testField"));
+ }
+
+ @Test
+ public void getPatchedDocument() throws IOException, OriginalNotFoundException {
+ // Create the document to be patched. Including comments
+ Map<String, Object> latestDoc = new HashMap<>();
+ latestDoc.put(Constants.GUID, "guid");
+ List<Map<String, Object>> comments = new ArrayList<>();
+ comments.add(new AlertComment("comment", "user", 0L).asMap());
+ comments.add(new AlertComment("comment_2", "user_2", 0L).asMap());
+ latestDoc.put(COMMENTS_FIELD, comments);
+ Document latest = new Document(latestDoc, "guid", "bro", 0L);
+
+ SolrRetrieveLatestDao retrieveLatestDao = spy(new SolrRetrieveLatestDao(null));
+ doReturn(latest).when(retrieveLatestDao).getLatest("guid", "bro");
+
+ // Create the patch
+ PatchRequest request = new PatchRequest();
+ request.setIndex("bro");
+ request.setSensorType("bro");
+ request.setGuid("guid");
+ List<Map<String, Object>> patchList = new ArrayList<>();
+ Map<String, Object> patch = new HashMap<>();
+ patch.put("op", "add");
+ patch.put("path", "/project");
+ patch.put("value", "metron");
+ patchList.add(patch);
+ request.setPatch(patchList);
+ Document actual = solrUpdateDao.getPatchedDocument(retrieveLatestDao, request, Optional.of(0L));
+
+ // Add the patch to our original document
+ latest.getDocument().put("project", "metron");
+ assertEquals(actual, latest);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index f09d7a8..4390fd1 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -110,8 +110,8 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
// getColumnMetadata with only bro
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
- // Don't test all 256, just test a sample of different fields
- Assert.assertEquals(262, fieldTypes.size());
+ // Don't test all fields, just test a sample of different fields
+ Assert.assertEquals(263, fieldTypes.size());
// Fields present in both with same type
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
@@ -147,7 +147,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
// getColumnMetadata with only snort
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
- Assert.assertEquals(32, fieldTypes.size());
+ Assert.assertEquals(33, fieldTypes.size());
// Fields present in both with same type
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index 7faeeb9..c0697b8 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -26,8 +26,11 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.AccessConfig;
@@ -36,6 +39,7 @@ import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.MultiIndexDao;
import org.apache.metron.indexing.dao.UpdateIntegrationTest;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.util.IndexingCacheUtil;
import org.apache.metron.solr.dao.SolrDao;
import org.apache.metron.solr.integration.components.SolrComponent;
import org.junit.After;
@@ -81,8 +85,16 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
globalConfig.put(HBaseDao.HBASE_CF, CF);
accessConfig.setGlobalConfigSupplier(() -> globalConfig);
- dao = new MultiIndexDao(hbaseDao, new SolrDao());
+ CuratorFramework client = ConfigurationsUtils
+ .getClient(solrComponent.getZookeeperUrl());
+ client.start();
+ ZKConfigurationsCache cache = new ZKConfigurationsCache(client);
+ cache.start();
+ accessConfig.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, "solr"));
+
+ MultiIndexDao dao = new MultiIndexDao(hbaseDao, new SolrDao());
dao.init(accessConfig);
+ setDao(dao);
}
@After
@@ -136,9 +148,9 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
fields.put("field.location_point", "48.5839,7.7455");
Document document = new Document(fields, "bro_1", SENSOR_NAME, 0L);
- dao.update(document, Optional.of(SENSOR_NAME));
+ getDao().update(document, Optional.of(SENSOR_NAME));
- Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME);
+ Document indexedDocument = getDao().getLatest("bro_1", SENSOR_NAME);
// assert no extra expanded fields are included
assertEquals(8, indexedDocument.getDocument().size());
@@ -155,10 +167,10 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
documentMap.put("raw_message", hugeString);
documentMap.put("raw_message_1", hugeStringTwo);
Document errorDoc = new Document(documentMap, "error", "error", 0L);
- dao.update(errorDoc, Optional.of("error"));
+ getDao().update(errorDoc, Optional.of("error"));
// Ensure that the huge string is returned when not a string field
- Document latest = dao.getLatest("error_guid", "error");
+ Document latest = getDao().getLatest("error_guid", "error");
@SuppressWarnings("unchecked")
String actual = (String) latest.getDocument().get("raw_message");
assertEquals(actual, hugeString);
@@ -171,6 +183,6 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
exception.expect(IOException.class);
exception.expectMessage("Document contains at least one immense term in field=\"error_hash\"");
- dao.update(errorDoc, Optional.of("error"));
+ getDao().update(errorDoc, Optional.of("error"));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
index bb2de59..8340a36 100644
--- a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
+++ b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
@@ -44,6 +44,9 @@
<field name="score" type="pdouble" indexed="true" stored="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
[2/2] metron git commit: METRON-1547 Solr Comment Fields (justinleet)
closes apache/metron#1037
Posted by le...@apache.org.
METRON-1547 Solr Comment Fields (justinleet) closes apache/metron#1037
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a68d031b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a68d031b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a68d031b
Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: a68d031b01fe677d84abb0d25aca4f2ceaf90c53
Parents: 9348c60
Author: justinleet <ju...@gmail.com>
Authored: Tue Jun 5 14:59:29 2018 -0400
Committer: leet <le...@apache.org>
Committed: Tue Jun 5 14:59:29 2018 -0400
----------------------------------------------------------------------
.../alert-details/alert-details.component.ts | 25 ++-
.../src/app/model/comment-add-remove-request.ts | 25 +++
.../src/app/service/update.service.ts | 27 +++
.../rest/controller/UpdateController.java | 23 +++
.../metron/rest/service/UpdateService.java | 3 +
.../rest/service/impl/UpdateServiceImpl.java | 19 +++
.../UpdateControllerIntegrationTest.java | 92 ++++++++--
.../elasticsearch/dao/ElasticsearchDao.java | 21 +++
.../dao/ElasticsearchMetaAlertDao.java | 22 +++
.../dao/ElasticsearchMetaAlertUpdateDao.java | 23 +++
.../dao/ElasticsearchUpdateDao.java | 70 ++++++++
.../dao/ElasticsearchMetaAlertDaoTest.java | 17 ++
.../ElasticsearchUpdateIntegrationTest.java | 3 +-
.../apache/metron/indexing/dao/HBaseDao.java | 93 +++++++++-
.../apache/metron/indexing/dao/IndexDao.java | 2 +
.../metron/indexing/dao/MultiIndexDao.java | 47 ++++++
.../indexing/dao/search/AlertComment.java | 130 ++++++++++++++
.../dao/update/CommentAddRemoveRequest.java | 78 +++++++++
.../metron/indexing/dao/update/Document.java | 14 +-
.../metron/indexing/dao/update/PatchUtil.java | 50 ------
.../metron/indexing/dao/update/UpdateDao.java | 33 +++-
.../apache/metron/indexing/dao/InMemoryDao.java | 17 ++
.../indexing/dao/InMemoryMetaAlertDao.java | 17 ++
.../indexing/dao/UpdateIntegrationTest.java | 169 ++++++++++++++++++-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 17 ++
.../integration/HBaseDaoIntegrationTest.java | 79 ++++++++-
.../src/main/config/schema/bro/schema.xml | 3 +
.../src/main/config/schema/snort/schema.xml | 3 +
.../src/main/config/schema/yaf/schema.xml | 3 +
.../org/apache/metron/solr/dao/SolrDao.java | 41 ++++-
.../metron/solr/dao/SolrMetaAlertDao.java | 21 +++
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 23 +++
.../metron/solr/dao/SolrRetrieveLatestDao.java | 1 +
.../apache/metron/solr/dao/SolrSearchDao.java | 17 ++
.../apache/metron/solr/dao/SolrUpdateDao.java | 114 ++++++++++++-
.../apache/metron/solr/dao/SolrUtilities.java | 36 +++-
.../org/apache/metron/solr/dao/SolrDaoTest.java | 5 +-
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 18 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 94 +++++++++--
.../integration/SolrSearchIntegrationTest.java | 6 +-
.../integration/SolrUpdateIntegrationTest.java | 24 ++-
.../resources/config/test/conf/managed-schema | 3 +
42 files changed, 1400 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index e1c1685..6a07e08 100644
--- a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -30,6 +30,7 @@ import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
+import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
@@ -204,10 +205,15 @@ export class AlertDetailsComponent implements OnInit {
}
onAddComment() {
- let alertComment = new AlertComment(this.alertCommentStr, this.authenticationService.getCurrentUserName(), new Date().getTime());
- let tAlertComments = this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment);
- tAlertComments.unshift(alertComment);
- this.patchAlert(new Patch('add', '/comments', tAlertComments));
+ let commentRequest = new CommentAddRemoveRequest();
+ commentRequest.guid = this.alertSource.guid;
+ commentRequest.comment = this.alertCommentStr;
+ commentRequest.username = this.authenticationService.getCurrentUserName();
+ commentRequest.timestamp = new Date().getTime();
+ commentRequest.sensorType = this.alertSourceType;
+ this.updateService.addComment(commentRequest).subscribe( () => {
+ this.getData(true);
+ });
}
patchAlert(patch: Patch) {
@@ -232,8 +238,15 @@ export class AlertDetailsComponent implements OnInit {
this.metronDialogBox.showConfirmationMessage(commentText).subscribe(response => {
if (response) {
- this.alertCommentsWrapper.splice(index, 1);
- this.patchAlert(new Patch('add', '/comments', this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment)));
+ let commentRequest = new CommentAddRemoveRequest();
+ commentRequest.guid = this.alertSource.guid;
+ commentRequest.comment = this.alertCommentsWrapper[index].alertComment.comment;
+ commentRequest.username = this.alertCommentsWrapper[index].alertComment.username;
+ commentRequest.timestamp = this.alertCommentsWrapper[index].alertComment.timestamp;
+ commentRequest.sensorType = this.alertSourceType;
+ this.updateService.removeComment(commentRequest).subscribe( () => {
+ this.getData(true);
+ });
}
});
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts
new file mode 100644
index 0000000..35f5d86
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export class CommentAddRemoveRequest {
+ guid: string;
+ comment: string;
+ username: string;
+ sensorType: string;
+ timestamp: number;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/service/update.service.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/update.service.ts b/metron-interface/metron-alerts/src/app/service/update.service.ts
index 24b55f0..42a4944 100644
--- a/metron-interface/metron-alerts/src/app/service/update.service.ts
+++ b/metron-interface/metron-alerts/src/app/service/update.service.ts
@@ -31,6 +31,7 @@ import {Utils} from '../utils/utils';
import {Patch} from '../model/patch';
import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../utils/constants';
import { GlobalConfigService } from './global-config.service';
+import {CommentAddRemoveRequest} from "../model/comment-add-remove-request";
@Injectable()
export class UpdateService {
@@ -40,6 +41,8 @@ export class UpdateService {
alertChangedSource = new Subject<PatchRequest>();
alertChanged$ = this.alertChangedSource.asObservable();
sourceType = 'source:type';
+ alertCommentChangedSource = new Subject<CommentAddRemoveRequest>();
+ alertCommentChanged$ = this.alertCommentChangedSource.asObservable();
constructor(private http: Http, private globalConfigService: GlobalConfigService) {
this.globalConfigService.get().subscribe((config: {}) => {
@@ -47,6 +50,30 @@ export class UpdateService {
});
}
+ public addComment(commentRequest: CommentAddRemoveRequest, fireChangeListener = true): Observable<{}> {
+ let url = '/api/v1/update/add/comment';
+ return this.http.post(url, commentRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ .catch(HttpUtil.handleError)
+ .map(result => {
+ if (fireChangeListener) {
+ this.alertCommentChangedSource.next(commentRequest);
+ }
+ return result;
+ });
+ }
+
+ public removeComment(commentRequest: CommentAddRemoveRequest, fireChangeListener = true): Observable<{}> {
+ let url = '/api/v1/update/remove/comment';
+ return this.http.post(url, commentRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ .catch(HttpUtil.handleError)
+ .map(result => {
+ if (fireChangeListener) {
+ this.alertCommentChangedSource.next(commentRequest);
+ }
+ return result;
+ });
+ }
+
public patch(patchRequest: PatchRequest, fireChangeListener = true): Observable<{}> {
let url = '/api/v1/update/patch';
return this.http.patch(url, patchRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
index 56b0b7b..609442b 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
@@ -20,6 +20,7 @@ package org.apache.metron.rest.controller;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -67,4 +68,26 @@ public class UpdateController {
service.replace(request);
return new ResponseEntity<>(HttpStatus.OK);
}
+
+ @ApiOperation(value = "Add a comment to an alert")
+ @ApiResponse(message = "Nothing", code = 200)
+ @RequestMapping(value = "/add/comment", method = RequestMethod.POST)
+ ResponseEntity<Void> addCommentToAlert(
+ @RequestBody @ApiParam(name = "request", value = "Comment add request", required = true) final
+ CommentAddRemoveRequest request
+ ) throws RestException {
+ service.addComment(request);
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Remove a comment to an alert")
+ @ApiResponse(message = "Nothing", code = 200)
+ @RequestMapping(value = "/remove/comment", method = RequestMethod.POST)
+ ResponseEntity<Void> removeCommentFromAlert(
+ @RequestBody @ApiParam(name = "request", value = "Comment remove request", required = true) final
+ CommentAddRemoveRequest request
+ ) throws RestException {
+ service.removeComment(request);
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
index 4cdf4b3..bd59f39 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.service;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -26,4 +27,6 @@ public interface UpdateService {
void patch(PatchRequest request) throws RestException, OriginalNotFoundException;
void replace(ReplaceRequest request) throws RestException;
+ void addComment(CommentAddRemoveRequest request) throws RestException;
+ void removeComment(CommentAddRemoveRequest request) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
index 6a42248..49490fd 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service.impl;
import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -59,4 +60,22 @@ public class UpdateServiceImpl implements UpdateService {
throw new RestException(e.getMessage(), e);
}
}
+
+ @Override
+ public void addComment(CommentAddRemoveRequest request) throws RestException {
+ try {
+ dao.addCommentToAlert(request);
+ } catch (Exception e) {
+ throw new RestException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void removeComment(CommentAddRemoveRequest request) throws RestException {
+ try {
+ dao.removeCommentFromAlert(request);
+ } catch (Exception e) {
+ throw new RestException(e.getMessage(), e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
index e437325..6b8d5d3 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
@@ -17,15 +17,29 @@
*/
package org.apache.metron.rest.controller;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
import com.google.common.collect.ImmutableMap;
+import java.util.NavigableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.metron.hbase.mock.MockHTable;
+import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.HBaseDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.rest.service.UpdateService;
import org.junit.Assert;
import org.junit.Before;
@@ -37,28 +51,17 @@ import org.springframework.http.MediaType;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.ResultActions;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
-import java.util.NavigableMap;
-
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles(TEST_PROFILE)
public class UpdateControllerIntegrationTest extends DaoControllerTest {
@Autowired
- private UpdateService searchService;
+ private UpdateService updateService;
@Autowired
public CuratorFramework client;
@@ -115,6 +118,30 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
@Multiline
public static String replace;
+ /**
+ {
+ "guid" : "bro_2",
+ "sensorType" : "bro",
+ "comment": "test_comment",
+ "username" : "test_username",
+ "timestamp":0
+ }
+ */
+ @Multiline
+ public static String addComment;
+
+ /**
+ {
+ "guid" : "bro_2",
+ "sensorType" : "bro",
+ "comment": "test_comment",
+ "username" : "test_username",
+ "timestamp":0
+ }
+ */
+ @Multiline
+ public static String removeComment;
+
@Before
public void setup() throws Exception {
this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
@@ -191,4 +218,41 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
}
}
+ @Test
+ public void shouldAddComment() throws Exception {
+ CommentAddRemoveRequest commentAddRemoveRequest = new CommentAddRemoveRequest();
+ commentAddRemoveRequest.setGuid("bro_1");
+ commentAddRemoveRequest.setSensorType("bro");
+ commentAddRemoveRequest.setComment("test_comment");
+ commentAddRemoveRequest.setUsername("test_username");
+ commentAddRemoveRequest.setTimestamp(0L);
+
+ updateService.addComment(commentAddRemoveRequest);
+
+ ResultActions result = this.mockMvc.perform(
+ post(updateUrl + "/add/comment")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
+ .content(addComment));
+ result.andExpect(status().isOk());
+ }
+
+ @Test
+ public void shouldRemoveComment() throws Exception {
+ CommentAddRemoveRequest commentAddRemoveRequest = new CommentAddRemoveRequest();
+ commentAddRemoveRequest.setGuid("bro_1");
+ commentAddRemoveRequest.setSensorType("bro");
+ commentAddRemoveRequest.setComment("test_comment");
+ commentAddRemoveRequest.setUsername("test_username");
+ commentAddRemoveRequest.setTimestamp(0L);
+
+ updateService.removeComment(commentAddRemoveRequest);
+
+ ResultActions result = this.mockMvc.perform(
+ post(updateUrl + "/remove/comment")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
+ .content(removeComment));
+ result.andExpect(status().isOk());
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 246de6a..eae0a39 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -33,6 +33,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -150,6 +151,16 @@ public class ElasticsearchDao implements IndexDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ updateDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ updateDao.removeCommentFromAlert(request);
+ }
+
+ @Override
public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
return this.columnMetadataDao.getColumnMetadata(indices);
}
@@ -159,6 +170,16 @@ public class ElasticsearchDao implements IndexDao {
return retrieveLatestDao.getLatestResult(request);
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ this.updateDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ this.updateDao.removeCommentFromAlert(request, latest);
+ }
+
protected Optional<String> getIndexName(String guid, String sensorType) {
return updateDao.getIndexName(guid, sensorType);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index faec939..ab6c40c 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -41,6 +41,8 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -216,6 +218,26 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ indexDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ indexDao.removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ indexDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ indexDao.removeCommentFromAlert(request, latest);
+ }
+
+ @Override
public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
index 6c709a6..d3bdcbb 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
@@ -44,6 +44,7 @@ import org.apache.metron.indexing.dao.metaalert.lucene.AbstractLuceneMetaAlertUp
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.QueryBuilder;
@@ -185,6 +186,28 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda
}
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().removeCommentFromAlert(request, latest);
+ }
+
/**
* Given an alert GUID, retrieve all associated meta alerts.
* @param alertGuid The GUID of the child alert
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
index c4d7412..f2b08d2 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
@@ -17,14 +17,21 @@
*/
package org.apache.metron.elasticsearch.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.stream.Collectors;
import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.UpdateDao;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -103,6 +110,69 @@ public class ElasticsearchUpdateDao implements UpdateDao {
}
}
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null) {
+ return;
+ }
+ List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentsField);
+
+ originalComments.add(
+ new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp())
+ .asMap());
+
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, originalComments);
+ update(newVersion, Optional.empty());
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null) {
+ return;
+ }
+ List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentsField);
+
+ List<AlertComment> alertComments = new ArrayList<>();
+ for (Map<String, Object> commentRaw : originalComments) {
+ alertComments.add(new AlertComment(commentRaw));
+ }
+
+ alertComments.remove(
+ new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()));
+ List<Map<String, Object>> commentsFinal = alertComments.stream().map(AlertComment::asMap)
+ .collect(Collectors.toList());
+ Document newVersion = new Document(latest);
+ if (commentsFinal.size() > 0) {
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsFinal);
+ update(newVersion, Optional.empty());
+ } else {
+ newVersion.getDocument().remove(COMMENTS_FIELD);
+ }
+
+ update(newVersion, Optional.empty());
+ }
+
protected String getIndexName(Document update, Optional<String> index, String indexPostFix) {
return index.orElse(getIndexName(update.getGuid(), update.getSensorType())
.orElse(ElasticsearchUtils.getIndexName(update.getSensorType(), indexPostFix, null))
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
index 25799ad..a3a5f16 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
@@ -35,6 +35,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.junit.Test;
@@ -81,6 +82,22 @@ public class ElasticsearchMetaAlertDaoTest {
public Map<String, FieldType> getColumnMetadata(List<String> indices) {
return null;
}
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
};
ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
metaAlertDao.init(dao);
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
index 97993ff..c5c0bc1 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
@@ -85,8 +85,9 @@ public class ElasticsearchUpdateIntegrationTest extends UpdateIntegrationTest {
globalConfig.put(HBaseDao.HBASE_CF, CF);
accessConfig.setGlobalConfigSupplier(() -> globalConfig);
- dao = new MultiIndexDao(hbaseDao, createDao());
+ MultiIndexDao dao = new MultiIndexDao(hbaseDao, createDao());
dao.init(accessConfig);
+ setDao(dao);
}
@After
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
index ebb9907..f22372e 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
@@ -28,8 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Optional;
-
-import com.google.common.hash.Hasher;
+import java.util.stream.Collectors;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -38,6 +37,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.common.utils.KeyUtil;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -45,6 +45,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
/**
@@ -210,7 +211,21 @@ public class HBaseDao implements IndexDao {
if(entry.getValue()!= null) {
Map<String, Object> json = JSONUtils.INSTANCE.load(new String(entry.getValue()),
JSONUtils.MAP_SUPPLIER);
+
+ // Make sure comments are in the proper format
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> commentsMap = (List<Map<String, Object>>) json.get(COMMENTS_FIELD);
try {
+ if (commentsMap != null) {
+ List<AlertComment> comments = new ArrayList<>();
+ for (Map<String, Object> commentMap : commentsMap) {
+ comments.add(new AlertComment(commentMap));
+ }
+ if (comments.size() > 0) {
+ json.put(COMMENTS_FIELD,
+ comments.stream().map(AlertComment::asMap).collect(Collectors.toList()));
+ }
+ }
Key k = Key.fromBytes(result.getRow());
return new Document(json, k.getGuid(), k.getSensorType(), ts);
} catch (IOException e) {
@@ -262,4 +277,78 @@ public class HBaseDao implements IndexDao {
public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
return null;
}
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException("Unable to add comment to document that doesn't exist");
+ }
+
+ List<Map<String, Object>> comments = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(comments);
+
+ // Convert all comments back to raw JSON before updating.
+ List<Map<String, Object>> commentsMap = new ArrayList<>();
+ for (Map<String, Object> comment : originalComments) {
+ commentsMap.add(new AlertComment(comment).asMap());
+ }
+ commentsMap.add(new AlertComment(
+ request.getComment(),
+ request.getUsername(),
+ request.getTimestamp())
+ .asMap());
+
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsMap);
+ update(newVersion, Optional.empty());
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException("Unable to remove comment document that doesn't exist");
+ }
+ List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument().get(COMMENTS_FIELD);
+ // Can't remove anything if there's nothing there
+ if (commentMap == null) {
+ return;
+ }
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentMap);
+ List<AlertComment> comments = new ArrayList<>();
+ for (Map<String, Object> commentStr : originalComments) {
+ comments.add(new AlertComment(commentStr));
+ }
+
+ comments.remove(new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()));
+ Document newVersion = new Document(latest);
+ if (comments.size() > 0) {
+ List<Map<String, Object>> commentsAsMap = comments.stream().map(AlertComment::asMap)
+ .collect(Collectors.toList());
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsAsMap);
+ update(newVersion, Optional.empty());
+ } else {
+ newVersion.getDocument().remove(COMMENTS_FIELD);
+ }
+
+ update(newVersion, Optional.empty());
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index 4187428..11b2ff0 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -26,6 +26,8 @@ import org.apache.metron.indexing.dao.update.UpdateDao;
*/
public interface IndexDao extends UpdateDao, SearchDao, RetrieveLatestDao, ColumnMetadataDao {
+ String COMMENTS_FIELD = "comments";
+
/**
* Initialize the DAO with the AccessConfig object.
* @param config The config to use for initialization
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
index dad08d6..420c775 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -36,6 +37,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
public class MultiIndexDao implements IndexDao {
@@ -98,6 +100,51 @@ public class MultiIndexDao implements IndexDao {
return null;
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ List<String> exceptions =
+ indices.parallelStream().map(dao -> {
+ try {
+ dao.addCommentToAlert(request, latest);
+ return null;
+ } catch (Throwable e) {
+ return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e);
+ }
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+ if (exceptions.size() > 0) {
+ throw new IOException(Joiner.on("\n").join(exceptions));
+ }
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ List<String> exceptions =
+ indices.parallelStream().map(dao -> {
+ try {
+ dao.removeCommentFromAlert(request, latest);
+ return null;
+ } catch (Throwable e) {
+ return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e);
+ }
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+ if (exceptions.size() > 0) {
+ throw new IOException(Joiner.on("\n").join(exceptions));
+ }
+ }
+
private static class DocumentContainer {
private Optional<Document> d = Optional.empty();
private Optional<Throwable> t = Optional.empty();
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
new file mode 100644
index 0000000..04aac60
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.search;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+public class AlertComment {
+
+ private static final String COMMENT_FIELD = "comment";
+ private static final String COMMENT_USERNAME_FIELD = "username";
+ private static final String COMMENT_TIMESTAMP_FIELD = "timestamp";
+ private String comment;
+ private String username;
+ private long timestamp;
+
+ private JSONParser parser = new JSONParser();
+
+ public AlertComment(String comment, String username, long timestamp) {
+ this.comment = comment;
+ this.username = username;
+ this.timestamp = timestamp;
+ }
+
+ public AlertComment(String json) throws ParseException {
+ JSONObject parsed = (JSONObject) parser.parse(json);
+ this.comment = (String) parsed.get(COMMENT_FIELD);
+ this.username = (String) parsed.get(COMMENT_USERNAME_FIELD);
+ this.timestamp = (long) parsed.get(COMMENT_TIMESTAMP_FIELD);
+ }
+
+ public AlertComment(Map<String, Object> comment) {
+ this.comment = (String) comment.get(COMMENT_FIELD);
+ this.username = (String) comment.get(COMMENT_USERNAME_FIELD);
+ this.timestamp = (long) comment.get(COMMENT_TIMESTAMP_FIELD);
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @SuppressWarnings("unchecked")
+ public String asJson() {
+ return asJSONObject().toJSONString();
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, Object> asMap() {
+ Map<String, Object> map = new HashMap<>();
+ map.put(COMMENT_FIELD, comment);
+ map.put(COMMENT_USERNAME_FIELD, username);
+ map.put(COMMENT_TIMESTAMP_FIELD, timestamp);
+ return map;
+ }
+
+ @SuppressWarnings("unchecked")
+ public JSONObject asJSONObject() {
+ JSONObject json = new JSONObject();
+ json.put(COMMENT_FIELD, comment);
+ json.put(COMMENT_USERNAME_FIELD, username);
+ json.put(COMMENT_TIMESTAMP_FIELD, timestamp);
+ return json;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ AlertComment that = (AlertComment) o;
+
+ if (getTimestamp() != that.getTimestamp()) {
+ return false;
+ }
+ if (getComment() != null ? !getComment().equals(that.getComment())
+ : that.getComment() != null) {
+ return false;
+ }
+ return getUsername() != null ? getUsername().equals(that.getUsername())
+ : that.getUsername() == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = getComment() != null ? getComment().hashCode() : 0;
+ result = 31 * result + (getUsername() != null ? getUsername().hashCode() : 0);
+ result = 31 * result + (int) (getTimestamp() ^ (getTimestamp() >>> 32));
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "AlertComment{" +
+ "comment='" + comment + '\'' +
+ ", username='" + username + '\'' +
+ ", timestamp=" + timestamp +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
new file mode 100644
index 0000000..8e8bde7
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.update;
+
+public class CommentAddRemoveRequest {
+ private String guid;
+ private String sensorType;
+ private String comment;
+ private String username;
+ private long timestamp;
+
+ public String getGuid() {
+ return guid;
+ }
+
+ public void setGuid(String guid) {
+ this.guid = guid;
+ }
+
+ public String getSensorType() {
+ return sensorType;
+ }
+
+ public void setSensorType(String sensorType) {
+ this.sensorType = sensorType;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public String toString() {
+ return "CommentAddRemoveRequest{" +
+ "guid='" + guid + '\'' +
+ ", sensorType='" + sensorType + '\'' +
+ ", comment='" + comment + '\'' +
+ ", username='" + username + '\'' +
+ ", timestamp=" + timestamp +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
index 6f2f779..3686b19 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
@@ -18,10 +18,10 @@
package org.apache.metron.indexing.dao.update;
-import org.apache.metron.common.utils.JSONUtils;
-
import java.io.IOException;
+import java.util.HashMap;
import java.util.Map;
+import org.apache.metron.common.utils.JSONUtils;
public class Document {
Long timestamp;
@@ -36,7 +36,6 @@ public class Document {
setSensorType(sensorType);
}
-
public Document(String document, String guid, String sensorType, Long timestamp) throws IOException {
this(convertDoc(document), guid, sensorType, timestamp);
}
@@ -45,6 +44,15 @@ public class Document {
this( document, guid, sensorType, null);
}
+ /**
+ * Copy constructor
+ * @param other The document to be copied.
+ */
+ public Document(Document other) {
+ this(new HashMap<>(other.getDocument()), other.getGuid(), other.getSensorType(),
+ other.getTimestamp());
+ }
+
private static Map<String, Object> convertDoc(String document) throws IOException {
return JSONUtils.INSTANCE.load(document, JSONUtils.MAP_SUPPLIER);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java
deleted file mode 100644
index 5a4ef27..0000000
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.indexing.dao.update;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Optional;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.indexing.dao.RetrieveLatestDao;
-
-public class PatchUtil {
-
- public static Document getPatchedDocument(
- RetrieveLatestDao retrieveLatestDao,
- PatchRequest request
- , Optional<Long> timestamp
- ) throws OriginalNotFoundException, IOException {
- Map<String, Object> latest = request.getSource();
- if (latest == null) {
- Document latestDoc = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
- if (latestDoc != null && latestDoc.getDocument() != null) {
- latest = latestDoc.getDocument();
- } else {
- throw new OriginalNotFoundException(
- "Unable to patch an document that doesn't exist and isn't specified.");
- }
- }
- Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest);
- return new Document(updated
- , request.getGuid()
- , request.getSensorType()
- , timestamp.orElse(System.currentTimeMillis()));
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
index 6f136ea..b5f38e4 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
@@ -18,8 +18,10 @@
package org.apache.metron.indexing.dao.update;
import java.io.IOException;
+import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
public interface UpdateDao {
@@ -43,6 +45,15 @@ public interface UpdateDao {
*/
void batchUpdate(Map<Document, Optional<String>> updates) throws IOException;
+ void addCommentToAlert(CommentAddRemoveRequest request) throws IOException;
+
+ void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException;
+
+ void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
+
+ void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
+
+
/**
* Update a document in an index given a JSON Patch (see RFC 6902 at
* https://tools.ietf.org/html/rfc6902)
@@ -54,10 +65,30 @@ public interface UpdateDao {
default void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request
, Optional<Long> timestamp
) throws OriginalNotFoundException, IOException {
- Document d = PatchUtil.getPatchedDocument(retrieveLatestDao, request, timestamp);
+ Document d = getPatchedDocument(retrieveLatestDao, request, timestamp);
update(d, Optional.ofNullable(request.getIndex()));
}
+ default Document getPatchedDocument(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ Optional<Long> timestamp
+ ) throws OriginalNotFoundException, IOException {
+ Map<String, Object> latest = request.getSource();
+ if (latest == null) {
+ Document latestDoc = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ if (latestDoc != null && latestDoc.getDocument() != null) {
+ latest = latestDoc.getDocument();
+ } else {
+ throw new OriginalNotFoundException(
+ "Unable to patch an document that doesn't exist and isn't specified.");
+ }
+ }
+
+ Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest);
+ return new Document(updated,
+ request.getGuid(),
+ request.getSensorType(),
+ timestamp.orElse(System.currentTimeMillis()));
+ }
/**
* Replace a document in an index.
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
index d6e1521..e306567 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
@@ -35,6 +35,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.search.SortField;
import org.apache.metron.indexing.dao.search.SortOrder;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import java.io.IOException;
@@ -291,6 +292,22 @@ public class InMemoryDao implements IndexDao {
return indexColumnMetadata;
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
public static void setColumnMetadata(Map<String, Map<String, FieldType>> columnMetadata) {
Map<String, Map<String, FieldType>> columnMetadataMap = new HashMap<>();
for (Map.Entry<String, Map<String, FieldType>> e: columnMetadata.entrySet()) {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
index 803d320..9e95ee9 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
@@ -47,6 +47,7 @@ import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -127,6 +128,22 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
public Optional<Map<String, Object>> getLatestResult(GetRequest request) throws IOException {
return indexDao.getLatestResult(request);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
index eebf0bb..1e35523 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
@@ -14,30 +14,61 @@
*/
package org.apache.metron.indexing.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Optional;
+import java.util.stream.Collectors;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.collections.MapUtils;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.hbase.mock.MockHTable;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
+import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
import org.junit.Assert;
import org.junit.Test;
public abstract class UpdateIntegrationTest {
+ /**
+ * {
+ * "comment":"New Comment",
+ * "username":"test_user",
+ * "timestamp":1526401584951
+ * }
+ */
+ @Multiline
+ protected String commentOne;
+
+ /**
+ * {
+ * "comment":"New Comment 2",
+ * "username":"test_user_2",
+ * "timestamp":1526401584952
+ * }
+ */
+ @Multiline
+ protected String commentTwo;
+
private static final int MAX_RETRIES = 10;
private static final int SLEEP_MS = 500;
protected static final String SENSOR_NAME = "test";
private static final String CF = "p";
- protected static MultiIndexDao dao;
+ private MultiIndexDao dao;
@Test
public void test() throws Exception {
@@ -68,7 +99,7 @@ public abstract class UpdateIntegrationTest {
put("new-field", "metron");
}};
String guid = "" + message0.get(Constants.GUID);
- dao.replace(new ReplaceRequest(){{
+ getDao().replace(new ReplaceRequest(){{
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
@@ -76,8 +107,7 @@ public abstract class UpdateIntegrationTest {
}}, Optional.empty());
Assert.assertEquals(1, getMockHTable().size());
- Document doc = dao.getLatest(guid, SENSOR_NAME);
- Assert.assertEquals(message0, doc.getDocument());
+ findUpdatedDoc(message0, guid, SENSOR_NAME);
{
//ensure hbase is up to date
Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
@@ -99,7 +129,7 @@ public abstract class UpdateIntegrationTest {
.filter(d -> message0.get("new-field").equals(d.get("new-field")))
.count();
}
- Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
+ Assert.assertNotEquals("Data store is not updated!", cnt, 0);
}
}
//modify the same message and modify the new field
@@ -108,15 +138,16 @@ public abstract class UpdateIntegrationTest {
put("new-field", "metron2");
}};
String guid = "" + message0.get(Constants.GUID);
- dao.replace(new ReplaceRequest(){{
+ getDao().replace(new ReplaceRequest(){{
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
setIndex(getIndexName());
}}, Optional.empty());
Assert.assertEquals(1, getMockHTable().size());
- Document doc = dao.getLatest(guid, SENSOR_NAME);
+ Document doc = getDao().getLatest(guid, SENSOR_NAME);
Assert.assertEquals(message0, doc.getDocument());
+ findUpdatedDoc(message0, guid, SENSOR_NAME);
{
//ensure hbase is up to date
Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
@@ -141,11 +172,133 @@ public abstract class UpdateIntegrationTest {
.count();
}
- Assert.assertNotEquals("Index is not updated!", cnt, 0);
+ Assert.assertNotEquals("Data store is not updated!", cnt, 0);
}
}
}
+ @Test
+ public void testAddCommentAndPatch() throws Exception {
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("guid", "add_comment");
+ fields.put("source.type", SENSOR_NAME);
+
+ Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526306463050L);
+ getDao().update(document, Optional.of(SENSOR_NAME));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment", "test_user", 1526306463050L);
+ // Ensure we have the first comment
+ ArrayList<AlertComment> comments = new ArrayList<>();
+ comments.add(new AlertComment("New Comment", "test_user", 1526306463050L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ List<Map<String, Object>> patchList = new ArrayList<>();
+ Map<String, Object> patch = new HashMap<>();
+ patch.put("op", "add");
+ patch.put("path", "/project");
+ patch.put("value", "metron");
+ patchList.add(patch);
+
+ PatchRequest pr = new PatchRequest();
+ pr.setGuid("add_comment");
+ pr.setIndex(SENSOR_NAME);
+ pr.setSensorType(SENSOR_NAME);
+ pr.setPatch(patchList);
+ getDao().patch(getDao(), pr, Optional.of(new Date().getTime()));
+
+ document.getDocument().put("project", "metron");
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRemoveComments() throws Exception {
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("guid", "add_comment");
+ fields.put("source.type", SENSOR_NAME);
+
+ Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L);
+ getDao().update(document, Optional.of(SENSOR_NAME));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have the first comment
+ ArrayList<AlertComment> comments = new ArrayList<>();
+ comments.add(new AlertComment("New Comment", "test_user", 1526401584951L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we have the second comment
+ comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we only have the first comments
+ comments = new ArrayList<>();
+ comments.add(new AlertComment(commentOne));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have no comments
+ document.getDocument().remove(COMMENTS_FIELD);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+
+ protected void addAlertComment(String guid, String comment, String username, long timestamp)
+ throws IOException {
+ CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp);
+ getDao().addCommentToAlert(request);
+ }
+
+ protected void removeAlertComment(String guid, String comment, String username, long timestamp)
+ throws IOException {
+ CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp);
+ getDao().removeCommentFromAlert(request);
+ }
+
+ private CommentAddRemoveRequest buildAlertRequest(String guid, String comment, String username,
+ long timestamp) {
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid(guid);
+ request.setComment(comment);
+ request.setUsername(username);
+ request.setTimestamp(timestamp);
+ request.setSensorType(SENSOR_NAME);
+ return request;
+ }
+
+ protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType)
+ throws InterruptedException, IOException, OriginalNotFoundException {
+ for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+ Document doc = getDao().getLatest(guid, sensorType);
+ if (doc != null && message0.equals(doc.getDocument())) {
+ return;
+ }
+ if (t == MAX_RETRIES -1) {
+ MapUtils.debugPrint(System.out, "Expected", message0);
+ MapUtils.debugPrint(System.out, "actual", doc.getDocument());
+ }
+ }
+ throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES + " tries");
+ }
+
+ protected IndexDao getDao() {
+ return dao;
+ }
+
+ protected void setDao(MultiIndexDao dao) {
+ this.dao = dao;
+ }
+
protected abstract String getIndexName();
protected abstract MockHTable getMockHTable();
protected abstract void addTestData(String indexName, String sensorType, List<Map<String,Object>> docs) throws Exception;
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
index 2d620d9..7028b75 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
@@ -58,6 +58,7 @@ import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.metaalert.MetaScores;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.json.simple.JSONArray;
@@ -146,6 +147,22 @@ public class AbstractLuceneMetaAlertUpdateDaoTest {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp) {
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
index aa32aa0..da74d46 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.indexing.integration;
import static org.apache.metron.indexing.dao.HBaseDao.HBASE_CF;
import static org.apache.metron.indexing.dao.HBaseDao.HBASE_TABLE;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,12 +30,14 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
-
-import org.apache.commons.codec.binary.Hex;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.HBaseDao;
import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MultiIndexDao;
+import org.apache.metron.indexing.dao.UpdateIntegrationTest;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.junit.After;
@@ -42,7 +45,7 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-public class HBaseDaoIntegrationTest {
+public class HBaseDaoIntegrationTest extends UpdateIntegrationTest {
private static final String TABLE_NAME = "metron_update";
private static final String COLUMN_FAMILY = "cf";
@@ -150,6 +153,13 @@ public class HBaseDaoIntegrationTest {
Assert.assertFalse("Result size should be 12 but was greater", results.hasNext());
}
+ @Override
+ public void test() {
+ // The main test ensures a variety of things not implemented by HBase run alongside
+ // HBaseDao itself.
+ // Therefore, just don't do anything for this test.
+ }
+
protected List<Document> buildAlerts(int count) throws IOException {
List<Document> alerts = new ArrayList<>();
for (int i = 0; i < count; ++i) {
@@ -161,4 +171,67 @@ public class HBaseDaoIntegrationTest {
return alerts;
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRemoveComments() throws Exception {
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("guid", "add_comment");
+ fields.put("source.type", SENSOR_NAME);
+
+ Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L);
+ hbaseDao.update(document, Optional.of(SENSOR_NAME));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have the first comment
+ ArrayList<AlertComment> comments = new ArrayList<>();
+ comments.add(new AlertComment("New Comment", "test_user", 1526401584951L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we have the second comment
+ comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we only have the first comments
+ comments = new ArrayList<>();
+ comments.add(new AlertComment(commentOne));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have no comments
+ document.getDocument().remove(COMMENTS_FIELD);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+
+ @Override
+ protected IndexDao getDao() {
+ return hbaseDao;
+ }
+
+ @Override
+ protected String getIndexName() {
+ return null;
+ }
+
+ @Override
+ protected MockHTable getMockHTable() {
+ return null;
+ }
+
+ @Override
+ protected void addTestData(String indexName, String sensorType, List<Map<String, Object>> docs) {
+ }
+
+ @Override
+ protected List<Map<String, Object>> getIndexedTestData(String indexName, String sensorType) {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
index ca69304..1326dfc 100644
--- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
@@ -677,6 +677,9 @@
<dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.name" type="string" multiValued="false" docValues="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<!-- Metaalerts Field -->
<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
index 82d0320..84855df 100644
--- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
@@ -70,6 +70,9 @@
<dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.name" type="string" multiValued="false" docValues="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<!-- Metaalerts Field -->
<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
index fc8e641..5555a14 100644
--- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
@@ -76,6 +76,9 @@
<dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.name" type="string" multiValued="false" docValues="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<!-- Metaalerts Field -->
<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>