You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:31 UTC
[15/50] [abbrv] metron git commit: METRON-1547 Solr Comment Fields
(justinleet) closes apache/metron#1037
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index d461792..7db0ab5 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -22,13 +22,16 @@ import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
import com.google.common.base.Splitter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.function.Function;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -36,6 +39,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -88,12 +92,21 @@ public class SolrDao implements IndexDao {
this.accessConfig = config;
this.client = getSolrClient(getZkHosts());
this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig);
- this.solrUpdateDao = new SolrUpdateDao(this.client, this.accessConfig);
this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client);
+ this.solrUpdateDao = new SolrUpdateDao(this.client, this.solrRetrieveLatestDao, this.accessConfig);
this.solrColumnMetadataDao = new SolrColumnMetadataDao(this.client);
}
}
+ public Optional<String> getIndex(String sensorName, Optional<String> index) {
+ if (index.isPresent()) {
+ return index;
+ } else {
+ String realIndex = accessConfig.getIndexSupplier().apply(sensorName);
+ return Optional.ofNullable(realIndex);
+ }
+ }
+
@Override
public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
return this.solrSearchDao.search(searchRequest);
@@ -125,6 +138,16 @@ public class SolrDao implements IndexDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ this.solrUpdateDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ this.solrUpdateDao.removeCommentFromAlert(request);
+ }
+
+ @Override
public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
@@ -136,6 +159,18 @@ public class SolrDao implements IndexDao {
return this.solrColumnMetadataDao.getColumnMetadata(indices);
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ this.solrUpdateDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ this.solrUpdateDao.removeCommentFromAlert(request, latest);
+ }
+
/**
* Builds a Solr client using the ZK hosts from the global config.
* @return SolrClient
@@ -170,7 +205,7 @@ public class SolrDao implements IndexDao {
return solrSearchDao;
}
- public SolrSearchDao getSolrUpdateDao() {
- return solrSearchDao;
+ public SolrUpdateDao getSolrUpdateDao() {
+ return solrUpdateDao;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
index e65700f..8b37a49 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
@@ -41,6 +41,7 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -222,4 +223,24 @@ public class SolrMetaAlertDao implements MetaAlertDao {
throws IOException {
return metaAlertUpdateDao.updateMetaAlertStatus(metaAlertGuid, status);
}
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ solrDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ solrDao.removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ solrDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ solrDao.removeCommentFromAlert(request, latest);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
index 2a66d47..b96bbc6 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
@@ -42,6 +42,7 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.UpdateDao;
import org.apache.solr.client.solrj.SolrClient;
@@ -182,6 +183,28 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
}
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().removeCommentFromAlert(request, latest);
+ }
+
protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) {
boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert,
Collections.singleton(alert.getGuid()));
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
index 8578bfb..b3bc564 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
@@ -45,6 +45,7 @@ public class SolrRetrieveLatestDao implements RetrieveLatestDao {
@Override
public Document getLatest(String guid, String collection) throws IOException {
+
try {
SolrDocument solrDocument = client.getById(collection, guid);
if (solrDocument == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index 4a8d482..d978ec9 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -17,10 +17,24 @@
*/
package org.apache.metron.solr.dao;
+import static org.apache.metron.common.Constants.SENSOR_TYPE;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.metron.common.Constants;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -29,6 +43,8 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.Group;
import org.apache.metron.indexing.dao.search.GroupOrder;
import org.apache.metron.indexing.dao.search.GroupOrderType;
@@ -53,6 +69,7 @@ import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
+import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
index 7c56169..7f5a4ed 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
@@ -17,19 +17,24 @@
*/
package org.apache.metron.solr.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
-
+import java.util.stream.Collectors;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.UpdateDao;
import org.apache.solr.client.solrj.SolrClient;
@@ -39,23 +44,24 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SolrUpdateDao implements UpdateDao {
-
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private transient SolrClient client;
private AccessConfig config;
private Function<String, String> indexSupplier;
- public SolrUpdateDao(SolrClient client, AccessConfig config) {
+ private transient SolrRetrieveLatestDao retrieveLatestDao;
+
+ public SolrUpdateDao(SolrClient client, SolrRetrieveLatestDao retrieveLatestDao, AccessConfig config) {
this.client = client;
+ this.retrieveLatestDao = retrieveLatestDao;
this.config = config;
this.indexSupplier = config.getIndexSupplier();
}
private Optional<String> getIndex(String sensorName, Optional<String> index) {
- if(index.isPresent()) {
- return Optional.ofNullable(index.get());
- }
- else {
+ if (index.isPresent()) {
+ return index;
+ } else {
String realIndex = indexSupplier.apply(sensorName);
return Optional.ofNullable(realIndex);
}
@@ -63,8 +69,17 @@ public class SolrUpdateDao implements UpdateDao {
@Override
public void update(Document update, Optional<String> rawIndex) throws IOException {
+ Document newVersion = update;
+ // Handle any case where we're given comments in Map form, instead of raw String
+ Object commentsObj = update.getDocument().get(COMMENTS_FIELD);
+ if ( commentsObj instanceof List &&
+ ((List<Object>) commentsObj).size() > 0 &&
+ ((List<Object>) commentsObj).get(0) instanceof Map) {
+ newVersion = new Document(update);
+ convertCommentsToRaw(newVersion.getDocument());
+ }
try {
- SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(update);
+ SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(newVersion);
Optional<String> index = getIndex(update.getSensorType(), rawIndex);
if (index.isPresent()) {
this.client.add(index.get(), solrInputDocument);
@@ -113,4 +128,87 @@ public class SolrUpdateDao implements UpdateDao {
throw new IOException(e);
}
}
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null) {
+ return;
+ }
+
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> comments = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(comments);
+
+ // Convert all comments back to raw JSON before updating.
+ List<String> commentStrs = new ArrayList<>();
+ for (Map<String, Object> comment : originalComments) {
+ commentStrs.add(new AlertComment(comment).asJson());
+ }
+ commentStrs.add(new AlertComment(
+ request.getComment(),
+ request.getUsername(),
+ request.getTimestamp()
+ ).asJson());
+
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, commentStrs);
+ update(newVersion, Optional.empty());
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ if (latest == null) {
+ return;
+ }
+
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument()
+ .get(COMMENTS_FIELD);
+ // Can't remove anything if there's nothing there
+ if (commentMap == null) {
+ LOG.debug("Provided alert had no comments to be able to remove from");
+ return;
+ }
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentMap);
+ List<AlertComment> comments = new ArrayList<>();
+ for (Map<String, Object> commentStr : originalComments) {
+ comments.add(new AlertComment(commentStr));
+ }
+
+ comments.remove(
+ new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()));
+ List<String> commentsAsJson = comments.stream().map(AlertComment::asJson)
+ .collect(Collectors.toList());
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsAsJson);
+ update(newVersion, Optional.empty());
+ }
+
+ public void convertCommentsToRaw(Map<String,Object> source) {
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> comments = (List<Map<String, Object>>) source.get(COMMENTS_FIELD);
+ if (comments == null || comments.isEmpty()) {
+ return;
+ }
+ List<String> asJson = new ArrayList<>();
+ for (Map<String, Object> comment : comments) {
+ asJson.add((new AlertComment(comment)).asJson());
+ }
+ source.put(COMMENTS_FIELD, asJson);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
index 616bd1a..88146b0 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
@@ -18,18 +18,23 @@
package org.apache.metron.solr.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
+import java.util.stream.Collectors;
import org.apache.metron.common.Constants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
+import org.json.simple.parser.ParseException;
public class SolrUtilities {
@@ -53,6 +58,34 @@ public class SolrUtilities {
solrDocument.getFieldNames().stream()
.filter(name -> !name.equals(SolrDao.VERSION_FIELD))
.forEach(name -> document.put(name, solrDocument.getFieldValue(name)));
+
+ reformatComments(solrDocument, document);
+ insertChildAlerts(solrDocument, document);
+
+ return new Document(document,
+ (String) solrDocument.getFieldValue(Constants.GUID),
+ (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), 0L);
+ }
+
+ protected static void reformatComments(SolrDocument solrDocument, Map<String, Object> document) {
+ // Make sure comments are in the proper format
+ @SuppressWarnings("unchecked")
+ List<String> commentStrs = (List<String>) solrDocument.get(COMMENTS_FIELD);
+ if (commentStrs != null) {
+ try {
+ List<AlertComment> comments = new ArrayList<>();
+ for (String commentStr : commentStrs) {
+ comments.add(new AlertComment(commentStr));
+ }
+ document.put(COMMENTS_FIELD,
+ comments.stream().map(AlertComment::asMap).collect(Collectors.toList()));
+ } catch (ParseException e) {
+ throw new IllegalStateException("Unable to parse comment", e);
+ }
+ }
+ }
+
+ protected static void insertChildAlerts(SolrDocument solrDocument, Map<String, Object> document) {
// Make sure to put child alerts in
if (solrDocument.hasChildDocuments() && solrDocument
.getFieldValue(Constants.SENSOR_TYPE)
@@ -68,9 +101,6 @@ public class SolrUtilities {
document.put(MetaAlertConstants.ALERT_FIELD, childDocuments);
}
- return new Document(document,
- (String) solrDocument.getFieldValue(Constants.GUID),
- (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), 0L);
}
public static SolrInputDocument toSolrInputDocument(Document document) {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
index 076fb20..85523b2 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
@@ -35,6 +35,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
import org.apache.metron.indexing.dao.search.SearchRequest;
@@ -107,9 +109,10 @@ public class SolrDaoTest {
solrDao = spy(new SolrDao());
doReturn(client).when(solrDao).getSolrClient(Collections.singletonList("zookeeper:2181"));
whenNew(SolrSearchDao.class).withArguments(client, accessConfig).thenReturn(solrSearchDao);
- whenNew(SolrUpdateDao.class).withArguments(client, accessConfig).thenReturn(solrUpdateDao);
whenNew(SolrRetrieveLatestDao.class).withArguments(client)
.thenReturn(solrRetrieveLatestDao);
+ whenNew(SolrUpdateDao.class).withArguments(client, solrRetrieveLatestDao, accessConfig)
+ .thenReturn(solrUpdateDao);
whenNew(SolrColumnMetadataDao.class).withArguments(client)
.thenReturn(solrColumnMetadataDao);
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
index 4571b15..43bf1b1 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
@@ -39,8 +39,8 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
-import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -94,6 +94,22 @@ public class SolrMetaAlertDaoTest {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
public void patch(RetrieveLatestDao dao, PatchRequest request, Optional<Long> timestamp) {
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
index 41fe7f9..7de02c1 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
@@ -17,11 +17,32 @@
*/
package org.apache.metron.solr.dao;
-import org.apache.metron.common.configuration.Configurations;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.doReturn;
+import static org.powermock.api.mockito.PowerMockito.spy;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
+import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.util.IndexingCacheUtil;
import org.apache.metron.solr.matcher.SolrInputDocumentListMatcher;
import org.apache.metron.solr.matcher.SolrInputDocumentMatcher;
@@ -34,22 +55,9 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
-import org.mockito.Matchers;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
@RunWith(PowerMockRunner.class)
@PrepareForTest({CollectionAdminRequest.class})
public class SolrUpdateDaoTest {
@@ -58,6 +66,7 @@ public class SolrUpdateDaoTest {
public final ExpectedException exception = ExpectedException.none();
private SolrClient client;
+ private SolrRetrieveLatestDao solrRetrieveLatestDao;
private SolrUpdateDao solrUpdateDao;
private static AccessConfig accessConfig = new AccessConfig();
@@ -86,7 +95,8 @@ public class SolrUpdateDaoTest {
@Before
public void setUp() throws Exception {
client = mock(SolrClient.class);
- solrUpdateDao = new SolrUpdateDao(client, accessConfig);
+ solrRetrieveLatestDao = new SolrRetrieveLatestDao(client);
+ solrUpdateDao = new SolrUpdateDao(client, solrRetrieveLatestDao, accessConfig);
}
@Test
@@ -174,4 +184,58 @@ public class SolrUpdateDaoTest {
verify(client).add(eq("snort"), argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2))));
}
+ @Test
+ public void testConvertCommentsToRaw() {
+ List<Map<String, Object>> commentList = new ArrayList<>();
+ Map<String, Object> comments = new HashMap<>();
+ comments.put("comment", "test comment");
+ comments.put("username", "test username");
+ comments.put("timestamp", 1526424323279L);
+ commentList.add(comments);
+
+ Map<String, Object> document = new HashMap<>();
+ document.put("testField", "testValue");
+ document.put(COMMENTS_FIELD, commentList);
+ solrUpdateDao.convertCommentsToRaw(document);
+
+ @SuppressWarnings("unchecked")
+ List<String> actualComments = (List<String>) document.get(COMMENTS_FIELD);
+ String expectedComment = "{\"comment\":\"test comment\",\"username\":\"test username\",\"timestamp\":1526424323279}";
+ assertEquals(expectedComment, actualComments.get(0));
+ assertEquals(1, actualComments.size());
+ assertEquals("testValue", document.get("testField"));
+ }
+
+ @Test
+ public void getPatchedDocument() throws IOException, OriginalNotFoundException {
+ // Create the document to be patched. Including comments
+ Map<String, Object> latestDoc = new HashMap<>();
+ latestDoc.put(Constants.GUID, "guid");
+ List<Map<String, Object>> comments = new ArrayList<>();
+ comments.add(new AlertComment("comment", "user", 0L).asMap());
+ comments.add(new AlertComment("comment_2", "user_2", 0L).asMap());
+ latestDoc.put(COMMENTS_FIELD, comments);
+ Document latest = new Document(latestDoc, "guid", "bro", 0L);
+
+ SolrRetrieveLatestDao retrieveLatestDao = spy(new SolrRetrieveLatestDao(null));
+ doReturn(latest).when(retrieveLatestDao).getLatest("guid", "bro");
+
+ // Create the patch
+ PatchRequest request = new PatchRequest();
+ request.setIndex("bro");
+ request.setSensorType("bro");
+ request.setGuid("guid");
+ List<Map<String, Object>> patchList = new ArrayList<>();
+ Map<String, Object> patch = new HashMap<>();
+ patch.put("op", "add");
+ patch.put("path", "/project");
+ patch.put("value", "metron");
+ patchList.add(patch);
+ request.setPatch(patchList);
+ Document actual = solrUpdateDao.getPatchedDocument(retrieveLatestDao, request, Optional.of(0L));
+
+ // Add the patch to our original document
+ latest.getDocument().put("project", "metron");
+ assertEquals(actual, latest);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index f09d7a8..4390fd1 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -110,8 +110,8 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
// getColumnMetadata with only bro
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
- // Don't test all 256, just test a sample of different fields
- Assert.assertEquals(262, fieldTypes.size());
+ // Don't test all fields, just test a sample of different fields
+ Assert.assertEquals(263, fieldTypes.size());
// Fields present in both with same type
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
@@ -147,7 +147,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
// getColumnMetadata with only snort
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
- Assert.assertEquals(32, fieldTypes.size());
+ Assert.assertEquals(33, fieldTypes.size());
// Fields present in both with same type
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index 7faeeb9..c0697b8 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -26,8 +26,11 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.AccessConfig;
@@ -36,6 +39,7 @@ import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.MultiIndexDao;
import org.apache.metron.indexing.dao.UpdateIntegrationTest;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.util.IndexingCacheUtil;
import org.apache.metron.solr.dao.SolrDao;
import org.apache.metron.solr.integration.components.SolrComponent;
import org.junit.After;
@@ -81,8 +85,16 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
globalConfig.put(HBaseDao.HBASE_CF, CF);
accessConfig.setGlobalConfigSupplier(() -> globalConfig);
- dao = new MultiIndexDao(hbaseDao, new SolrDao());
+ CuratorFramework client = ConfigurationsUtils
+ .getClient(solrComponent.getZookeeperUrl());
+ client.start();
+ ZKConfigurationsCache cache = new ZKConfigurationsCache(client);
+ cache.start();
+ accessConfig.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, "solr"));
+
+ MultiIndexDao dao = new MultiIndexDao(hbaseDao, new SolrDao());
dao.init(accessConfig);
+ setDao(dao);
}
@After
@@ -136,9 +148,9 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
fields.put("field.location_point", "48.5839,7.7455");
Document document = new Document(fields, "bro_1", SENSOR_NAME, 0L);
- dao.update(document, Optional.of(SENSOR_NAME));
+ getDao().update(document, Optional.of(SENSOR_NAME));
- Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME);
+ Document indexedDocument = getDao().getLatest("bro_1", SENSOR_NAME);
// assert no extra expanded fields are included
assertEquals(8, indexedDocument.getDocument().size());
@@ -155,10 +167,10 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
documentMap.put("raw_message", hugeString);
documentMap.put("raw_message_1", hugeStringTwo);
Document errorDoc = new Document(documentMap, "error", "error", 0L);
- dao.update(errorDoc, Optional.of("error"));
+ getDao().update(errorDoc, Optional.of("error"));
// Ensure that the huge string is returned when not a string field
- Document latest = dao.getLatest("error_guid", "error");
+ Document latest = getDao().getLatest("error_guid", "error");
@SuppressWarnings("unchecked")
String actual = (String) latest.getDocument().get("raw_message");
assertEquals(actual, hugeString);
@@ -171,6 +183,6 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
exception.expect(IOException.class);
exception.expectMessage("Document contains at least one immense term in field=\"error_hash\"");
- dao.update(errorDoc, Optional.of("error"));
+ getDao().update(errorDoc, Optional.of("error"));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
index bb2de59..8340a36 100644
--- a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
+++ b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
@@ -44,6 +44,9 @@
<field name="score" type="pdouble" indexed="true" stored="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>