You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:34 UTC
[18/50] [abbrv] metron git commit: METRON-1585 SolrRetrieveLatestDao
does not use the collection lookup (justinleet via merrimanr) closes
apache/metron#1050
METRON-1585 SolrRetrieveLatestDao does not use the collection lookup (justinleet via merrimanr) closes apache/metron#1050
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/aed4ffcb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/aed4ffcb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/aed4ffcb
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: aed4ffcb24cf9a95c1347cccd5dd79fce7c77a6b
Parents: 95e6528
Author: justinleet <ju...@gmail.com>
Authored: Fri Jun 8 12:16:59 2018 -0500
Committer: merrimanr <me...@gmail.com>
Committed: Fri Jun 8 12:16:59 2018 -0500
----------------------------------------------------------------------
.../org/apache/metron/solr/dao/SolrDao.java | 10 +-
.../metron/solr/dao/SolrRetrieveLatestDao.java | 37 +++-
.../apache/metron/solr/dao/SolrSearchDao.java | 17 --
.../apache/metron/solr/dao/SolrUpdateDao.java | 18 +-
.../apache/metron/solr/dao/SolrUtilities.java | 18 ++
.../org/apache/metron/solr/dao/SolrDaoTest.java | 4 +-
.../metron/solr/dao/SolrSearchDaoTest.java | 2 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 4 +-
.../SolrRetrieveLatestIntegrationTest.java | 207 +++++++++++++++++++
.../integration/SolrUpdateIntegrationTest.java | 1 +
10 files changed, 266 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index 7db0ab5..a840bb4 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -22,16 +22,13 @@ import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
import com.google.common.base.Splitter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.function.Function;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
-import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -43,7 +40,6 @@ import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
-import org.apache.metron.solr.SolrConstants;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -92,7 +88,7 @@ public class SolrDao implements IndexDao {
this.accessConfig = config;
this.client = getSolrClient(getZkHosts());
this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig);
- this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client);
+ this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client, this.accessConfig);
this.solrUpdateDao = new SolrUpdateDao(this.client, this.solrRetrieveLatestDao, this.accessConfig);
this.solrColumnMetadataDao = new SolrColumnMetadataDao(this.client);
}
@@ -118,8 +114,8 @@ public class SolrDao implements IndexDao {
}
@Override
- public Document getLatest(String guid, String collection) throws IOException {
- return this.solrRetrieveLatestDao.getLatest(guid, collection);
+ public Document getLatest(String guid, String sensorType) throws IOException {
+ return this.solrRetrieveLatestDao.getLatest(guid, sensorType);
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
index b3bc564..40262ba 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
@@ -19,13 +19,16 @@
package org.apache.metron.solr.dao;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.stream.Collectors;
+import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.update.Document;
@@ -34,21 +37,34 @@ import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SolrRetrieveLatestDao implements RetrieveLatestDao {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
private transient SolrClient client;
+ private AccessConfig config;
- public SolrRetrieveLatestDao(SolrClient client) {
+ public SolrRetrieveLatestDao(SolrClient client, AccessConfig config) {
this.client = client;
+ this.config = config;
}
@Override
- public Document getLatest(String guid, String collection) throws IOException {
-
+ public Document getLatest(String guid, String sensorType) throws IOException {
try {
- SolrDocument solrDocument = client.getById(collection, guid);
+ Optional<String> index = SolrUtilities
+ .getIndex(config.getIndexSupplier(), sensorType, Optional.empty());
+ if (!index.isPresent()) {
+ LOG.debug("Unable to find index for sensorType {}", sensorType);
+ return null;
+ }
+
+ SolrDocument solrDocument = client.getById(index.get(), guid);
if (solrDocument == null) {
+ LOG.debug("Unable to find document for sensorType {} and guid {}", sensorType, guid);
return null;
}
return SolrUtilities.toDocument(solrDocument);
@@ -61,10 +77,15 @@ public class SolrRetrieveLatestDao implements RetrieveLatestDao {
public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException {
Map<String, Collection<String>> collectionIdMap = new HashMap<>();
for (GetRequest getRequest : getRequests) {
- Collection<String> ids = collectionIdMap
- .getOrDefault(getRequest.getSensorType(), new HashSet<>());
- ids.add(getRequest.getGuid());
- collectionIdMap.put(getRequest.getSensorType(), ids);
+ Optional<String> index = SolrUtilities
+ .getIndex(config.getIndexSupplier(), getRequest.getSensorType(), getRequest.getIndex());
+ if (index.isPresent()) {
+ Collection<String> ids = collectionIdMap.getOrDefault(index.get(), new HashSet<>());
+ ids.add(getRequest.getGuid());
+ collectionIdMap.put(index.get(), ids);
+ } else {
+ LOG.debug("Unable to find index for sensorType {}", getRequest.getSensorType());
+ }
}
try {
List<Document> documents = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index d978ec9..4a8d482 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -17,24 +17,10 @@
*/
package org.apache.metron.solr.dao;
-import static org.apache.metron.common.Constants.SENSOR_TYPE;
-import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import org.apache.metron.common.Constants;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -43,8 +29,6 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.AccessConfig;
-import org.apache.metron.indexing.dao.search.AlertComment;
-import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.Group;
import org.apache.metron.indexing.dao.search.GroupOrder;
import org.apache.metron.indexing.dao.search.GroupOrderType;
@@ -69,7 +53,6 @@ import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
-import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
index 7f5a4ed..2f83921 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.search.AlertComment;
@@ -48,23 +47,12 @@ public class SolrUpdateDao implements UpdateDao {
private transient SolrClient client;
private AccessConfig config;
- private Function<String, String> indexSupplier;
private transient SolrRetrieveLatestDao retrieveLatestDao;
public SolrUpdateDao(SolrClient client, SolrRetrieveLatestDao retrieveLatestDao, AccessConfig config) {
this.client = client;
this.retrieveLatestDao = retrieveLatestDao;
this.config = config;
- this.indexSupplier = config.getIndexSupplier();
- }
-
- private Optional<String> getIndex(String sensorName, Optional<String> index) {
- if (index.isPresent()) {
- return index;
- } else {
- String realIndex = indexSupplier.apply(sensorName);
- return Optional.ofNullable(realIndex);
- }
}
@Override
@@ -80,7 +68,8 @@ public class SolrUpdateDao implements UpdateDao {
}
try {
SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(newVersion);
- Optional<String> index = getIndex(update.getSensorType(), rawIndex);
+ Optional<String> index = SolrUtilities
+ .getIndex(config.getIndexSupplier(), newVersion.getSensorType(), rawIndex);
if (index.isPresent()) {
this.client.add(index.get(), solrInputDocument);
this.client.commit(index.get());
@@ -100,7 +89,8 @@ public class SolrUpdateDao implements UpdateDao {
for (Entry<Document, Optional<String>> entry : updates.entrySet()) {
SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(entry.getKey());
- Optional<String> index = getIndex(entry.getKey().getSensorType(), entry.getValue());
+ Optional<String> index = SolrUtilities
+ .getIndex(config.getIndexSupplier(), entry.getKey().getSensorType(), entry.getValue());
if (index.isPresent()) {
Collection<SolrInputDocument> solrInputDocuments = solrCollectionUpdates
.getOrDefault(index.get(), new ArrayList<>());
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
index 88146b0..d41b7e4 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -122,4 +123,21 @@ public class SolrUtilities {
}
return solrInputDocument;
}
+
+ /**
+ * Gets the actual collection for the given sensor type
+ * @param indexSupplier The function to employ in the lookup
+ * @param sensorName The sensor type to be looked up
+ * @param index An index to use, if present.
+ * @return An Optional containing the actual collection
+ */
+ public static Optional<String> getIndex(Function<String, String> indexSupplier, String sensorName,
+ Optional<String> index) {
+ if (index.isPresent()) {
+ return index;
+ } else {
+ String realIndex = indexSupplier.apply(sensorName);
+ return Optional.ofNullable(realIndex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
index 85523b2..9d84669 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java
@@ -35,8 +35,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.metron.indexing.dao.AccessConfig;
-import org.apache.metron.indexing.dao.IndexDao;
-import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
import org.apache.metron.indexing.dao.search.SearchRequest;
@@ -109,7 +107,7 @@ public class SolrDaoTest {
solrDao = spy(new SolrDao());
doReturn(client).when(solrDao).getSolrClient(Collections.singletonList("zookeeper:2181"));
whenNew(SolrSearchDao.class).withArguments(client, accessConfig).thenReturn(solrSearchDao);
- whenNew(SolrRetrieveLatestDao.class).withArguments(client)
+ whenNew(SolrRetrieveLatestDao.class).withArguments(client, accessConfig)
.thenReturn(solrRetrieveLatestDao);
whenNew(SolrUpdateDao.class).withArguments(client, solrRetrieveLatestDao, accessConfig)
.thenReturn(solrUpdateDao);
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
index d17bf72..fe27a55 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
@@ -92,7 +92,7 @@ public class SolrSearchDaoTest {
accessConfig = mock(AccessConfig.class);
when(accessConfig.getIndexSupplier()).thenReturn(sensorType -> sensorType);
solrSearchDao = new SolrSearchDao(client, accessConfig);
- solrRetrieveLatestDao = new SolrRetrieveLatestDao(client);
+ solrRetrieveLatestDao = new SolrRetrieveLatestDao(client, accessConfig);
mockStatic(CollectionAdminRequest.class);
when(CollectionAdminRequest.listCollections(client)).thenReturn(Arrays.asList("bro", "snort"));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
index 7de02c1..bed43ae 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
@@ -95,7 +95,7 @@ public class SolrUpdateDaoTest {
@Before
public void setUp() throws Exception {
client = mock(SolrClient.class);
- solrRetrieveLatestDao = new SolrRetrieveLatestDao(client);
+ solrRetrieveLatestDao = new SolrRetrieveLatestDao(client, accessConfig);
solrUpdateDao = new SolrUpdateDao(client, solrRetrieveLatestDao, accessConfig);
}
@@ -217,7 +217,7 @@ public class SolrUpdateDaoTest {
latestDoc.put(COMMENTS_FIELD, comments);
Document latest = new Document(latestDoc, "guid", "bro", 0L);
- SolrRetrieveLatestDao retrieveLatestDao = spy(new SolrRetrieveLatestDao(null));
+ SolrRetrieveLatestDao retrieveLatestDao = spy(new SolrRetrieveLatestDao(null, accessConfig));
doReturn(latest).when(retrieveLatestDao).getLatest("guid", "bro");
// Create the patch
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
new file mode 100644
index 0000000..f7c2e86
--- /dev/null
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.solr.integration;
+
+import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Iterables;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.metron.common.Constants;
+import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.solr.dao.SolrDao;
+import org.apache.metron.solr.integration.components.SolrComponent;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class SolrRetrieveLatestIntegrationTest {
+
+ private static SolrComponent solrComponent;
+
+ protected static final String TEST_COLLECTION = "test";
+ protected static final String TEST_SENSOR = "test_sensor";
+ protected static final String BRO_SENSOR = "bro";
+
+ private static IndexDao dao;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ solrComponent = new SolrComponent.Builder().build();
+ solrComponent.start();
+ }
+
+ @Before
+ public void setup() throws Exception {
+ solrComponent
+ .addCollection(TEST_COLLECTION, "../metron-solr/src/test/resources/config/test/conf");
+ solrComponent.addCollection(BRO_SENSOR, "../metron-solr/src/main/config/schema/bro");
+
+ AccessConfig accessConfig = new AccessConfig();
+ Map<String, Object> globalConfig = new HashMap<>();
+ globalConfig.put(SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl());
+ accessConfig.setGlobalConfigSupplier(() -> globalConfig);
+ // Map the sensor name to the collection name for test.
+ accessConfig.setIndexSupplier(s -> s.equals(TEST_SENSOR) ? TEST_COLLECTION : s);
+
+ dao = new SolrDao();
+ dao.init(accessConfig);
+ addData(BRO_SENSOR, BRO_SENSOR);
+ addData(TEST_COLLECTION, TEST_SENSOR);
+ }
+
+ @After
+ public void reset() {
+ solrComponent.reset();
+ }
+
+ @AfterClass
+ public static void teardown() {
+ solrComponent.stop();
+ }
+
+ @Test
+ public void testGetLatest() throws IOException {
+ Document actual = dao.getLatest("message_1_bro", BRO_SENSOR);
+ assertEquals(buildExpectedDocument(BRO_SENSOR, 1), actual);
+ }
+
+ @Test
+ public void testGetMissing() throws IOException {
+ Document actual = dao.getLatest("message_1_bro", TEST_SENSOR);
+ assertNull(actual);
+ }
+
+ @Test
+ public void testGetBrokenMapping() throws IOException {
+ AccessConfig accessConfig = new AccessConfig();
+ Map<String, Object> globalConfig = new HashMap<>();
+ globalConfig.put(SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl());
+ accessConfig.setGlobalConfigSupplier(() -> globalConfig);
+ // Map the sensor name to the collection name for test.
+ accessConfig.setIndexSupplier(s -> null);
+
+ dao = new SolrDao();
+ dao.init(accessConfig);
+
+ Document actual = dao.getLatest("message_1_bro", TEST_SENSOR);
+ assertNull(actual);
+ }
+
+ @Test
+ public void testGetLatestCollectionSensorDiffer() throws IOException {
+ Document actual = dao.getLatest("message_1_test_sensor", TEST_SENSOR);
+ assertEquals(buildExpectedDocument(TEST_SENSOR, 1), actual);
+ }
+
+ @Test
+ public void testGetAllLatest() throws IOException {
+ List<GetRequest> requests = new ArrayList<>();
+ requests.add(buildGetRequest(BRO_SENSOR, 1));
+ requests.add(buildGetRequest(BRO_SENSOR, 2));
+
+ Iterable<Document> actual = dao.getAllLatest(requests);
+ assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 1)));
+ assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 2)));
+ assertEquals(2, Iterables.size(actual));
+ }
+
+ @Test
+ public void testGetAllLatestCollectionExplicitIndex() throws IOException {
+ List<GetRequest> requests = new ArrayList<>();
+ GetRequest getRequestOne = buildGetRequest(TEST_SENSOR, 1);
+ // Explicitly use the incorrect index. This forces it to prefer the explicit index over the
+ // implicit one.
+ getRequestOne.setIndex(BRO_SENSOR);
+ requests.add(getRequestOne);
+
+ Iterable<Document> actual = dao.getAllLatest(requests);
+ // Expect 0 because the explicit index was incorrect.
+ assertEquals(0, Iterables.size(actual));
+ }
+
+ @Test
+ public void testGetAllLatestCollectionSensorMixed() throws IOException {
+ List<GetRequest> requests = new ArrayList<>();
+ requests.add(buildGetRequest(TEST_SENSOR, 1));
+ requests.add(buildGetRequest(BRO_SENSOR, 2));
+
+ Iterable<Document> actual = dao.getAllLatest(requests);
+ assertTrue(Iterables.contains(actual, buildExpectedDocument(TEST_SENSOR, 1)));
+ assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 2)));
+ assertEquals(2, Iterables.size(actual));
+ }
+
+ @Test
+ public void testGetAllLatestCollectionOneMissing() throws IOException {
+ List<GetRequest> requests = new ArrayList<>();
+ requests.add(buildGetRequest(TEST_SENSOR, 1));
+ GetRequest brokenRequest= new GetRequest();
+ brokenRequest.setGuid(buildGuid(BRO_SENSOR, 2));
+ brokenRequest.setSensorType(TEST_SENSOR);
+ requests.add(brokenRequest);
+
+ Iterable<Document> actual = dao.getAllLatest(requests);
+ assertTrue(Iterables.contains(actual, buildExpectedDocument(TEST_SENSOR, 1)));
+ assertEquals(1, Iterables.size(actual));
+ }
+
+ protected Document buildExpectedDocument(String sensor, int i) {
+ Map<String, Object> expectedMapOne = new HashMap<>();
+ expectedMapOne.put("source.type", sensor);
+ expectedMapOne.put(Constants.GUID, buildGuid(sensor, i));
+ return new Document(expectedMapOne, buildGuid(sensor, i), sensor, 0L);
+ }
+
+ protected GetRequest buildGetRequest(String sensor, int i) {
+ GetRequest requestOne = new GetRequest();
+ requestOne.setGuid(buildGuid(sensor, i));
+ requestOne.setSensorType(sensor);
+ return requestOne;
+ }
+
+ protected static void addData(String collection, String sensorName)
+ throws IOException, SolrServerException {
+ List<Map<String, Object>> inputData = new ArrayList<>();
+ for (int i = 0; i < 3; ++i) {
+ final String name = buildGuid(sensorName, i);
+ HashMap<String, Object> inputMap = new HashMap<>();
+ inputMap.put("source.type", sensorName);
+ inputMap.put(Constants.GUID, name);
+ inputData.add(inputMap);
+ }
+ solrComponent.addDocs(collection, inputData);
+ }
+
+ protected static String buildGuid(String sensorName, int i) {
+ return "message_" + i + "_" + sensorName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index c0697b8..5b96559 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -84,6 +84,7 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME);
globalConfig.put(HBaseDao.HBASE_CF, CF);
accessConfig.setGlobalConfigSupplier(() -> globalConfig);
+ accessConfig.setIndexSupplier(s -> s);
CuratorFramework client = ConfigurationsUtils
.getClient(solrComponent.getZookeeperUrl());