You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:29 UTC
[13/50] [abbrv] metron git commit: METRON-1577 Solr searches don'
t include the index of the result (merrimanr) closes apache/metron#1031
METRON-1577 Solr searches don't include the index of the result (merrimanr) closes apache/metron#1031
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7a071f6d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7a071f6d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7a071f6d
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 7a071f6d64794267f40a0ef1810bd7deaf044eeb
Parents: 49f851e
Author: merrimanr <me...@gmail.com>
Authored: Fri May 25 08:41:44 2018 -0500
Committer: merrimanr <me...@gmail.com>
Committed: Fri May 25 08:41:44 2018 -0500
----------------------------------------------------------------------
.../src/app/service/alerts.service.ts | 2 +-
.../apache/metron/rest/config/IndexConfig.java | 3 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 3 +
.../ElasticsearchSearchIntegrationTest.java | 9 +++
.../metron/indexing/util/IndexingCacheUtil.java | 14 ++++-
.../indexing/dao/SearchIntegrationTest.java | 4 ++
.../indexing/util/IndexingCacheUtilTest.java | 66 ++++++++++++++++++++
.../org/apache/metron/solr/SolrConstants.java | 1 +
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 3 +-
.../apache/metron/solr/dao/SolrSearchDao.java | 7 ++-
.../apache/metron/solr/dao/SolrUpdateDao.java | 23 +++++--
.../apache/metron/solr/dao/SolrUtilities.java | 5 +-
.../apache/metron/solr/writer/SolrWriter.java | 4 +-
.../metron/solr/dao/SolrSearchDaoTest.java | 8 ++-
.../metron/solr/dao/SolrUpdateDaoTest.java | 34 ++++++++--
.../integration/SolrSearchIntegrationTest.java | 6 ++
16 files changed, 172 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-alerts/src/app/service/alerts.service.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/alerts.service.ts b/metron-interface/metron-alerts/src/app/service/alerts.service.ts
index 7324a72..56939d8 100644
--- a/metron-interface/metron-alerts/src/app/service/alerts.service.ts
+++ b/metron-interface/metron-alerts/src/app/service/alerts.service.ts
@@ -30,7 +30,7 @@ export class AlertsService {
}
public escalate(alerts: Alert[]): Observable<null> {
- return this.http.post('/api/v1/alert/escalate', alerts, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ return this.http.post('/api/v1/alerts/ui/escalate', alerts, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
.catch(HttpUtil.handleError);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
index c432c6c..53b10f9 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.config;
import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL;
+import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
import java.util.Optional;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
@@ -73,7 +74,7 @@ public class IndexConfig {
throw new IllegalStateException("Unable to retrieve the global config.", e);
}
});
- config.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache));
+ config.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, environment.getProperty(INDEX_WRITER_NAME)));
config.setTableProvider(TableProvider.create(hbaseProviderImpl, () -> new HTableProvider()));
config.setKerberosEnabled(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false));
if (indexDaoImpl == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index 20b535e..21e8128 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -113,10 +113,13 @@ echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}"
if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then
METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao"
METRON_METAALERT_DAO=" --meta.dao.impl=org.apache.metron.solr.dao.SolrMetaAlertDao"
+ METRON_WRITER_NAME=" --index.writer.name=solr"
echo "METRON_INDEX_DAO=${METRON_INDEX_DAO}"
echo "METRON_METAALERT_DAO=${METRON_METAALERT_DAO}"
+ echo "METRON_WRITER_NAME=${METRON_WRITER_NAME}"
METRON_SPRING_OPTIONS+=${METRON_INDEX_DAO}
METRON_SPRING_OPTIONS+=${METRON_METAALERT_DAO}
+ METRON_SPRING_OPTIONS+=${METRON_WRITER_NAME}
fi
echo "Starting application"
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 6f76093..8071e68 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -361,4 +361,13 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
protected IndexDao getIndexDao() {
return dao;
}
+
+ @Override
+ protected String getIndexName(String sensorType) {
+ if ("bro".equals(sensorType)) {
+ return "bro_index_2017.01.01.01";
+ } else {
+ return "snort_index_2017.01.01.02";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java
index 3ff3a20..86a0642 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/util/IndexingCacheUtil.java
@@ -24,12 +24,20 @@ import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
public class IndexingCacheUtil {
- public static Function<String, String> getIndexLookupFunction(ConfigurationsCache cache) {
+
+ @SuppressWarnings("unchecked")
+ public static Function<String, String> getIndexLookupFunction(ConfigurationsCache cache, String writerName) {
return sensorType -> {
+ String indexingTopic = sensorType;
IndexingConfigurations indexingConfigs = cache.get( IndexingConfigurations.class);
Map<String, Object> indexingSensorConfigs = indexingConfigs.getSensorIndexingConfig(sensorType);
- String indexingTopic = (String) indexingSensorConfigs.get(IndexingConfigurations.INDEX_CONF);
- return indexingTopic != null ? indexingTopic : sensorType;
+ if (indexingSensorConfigs != null) {
+ Map<String, Object> writerConfigs = (Map<String, Object>) indexingSensorConfigs.get(writerName);
+ if (writerConfigs != null) {
+ indexingTopic = (String) writerConfigs.getOrDefault(IndexingConfigurations.INDEX_CONF, indexingTopic);
+ }
+ }
+ return indexingTopic;
};
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 7fca764..2e1968a 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -484,10 +484,12 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(10, results.size());
for(int i = 0;i < 5;++i) {
Assert.assertEquals("snort", results.get(i).getSource().get(getSourceTypeField()));
+ Assert.assertEquals(getIndexName("snort"), results.get(i).getIndex());
Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
}
for (int i = 5; i < 10; ++i) {
Assert.assertEquals("bro", results.get(i).getSource().get(getSourceTypeField()));
+ Assert.assertEquals(getIndexName("bro"), results.get(i).getIndex());
Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
}
}
@@ -940,4 +942,6 @@ public abstract class SearchIntegrationTest {
protected abstract IndexDao getIndexDao();
protected abstract String getSourceTypeField();
+
+ protected abstract String getIndexName(String sensorType);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java
new file mode 100644
index 0000000..3d6ee85
--- /dev/null
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/util/IndexingCacheUtilTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.util;
+
+import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.common.zookeeper.ConfigurationsCache;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IndexingCacheUtilTest {
+
+ @Test
+ public void getIndexLookupFunctionShouldReturnConfiguredIndex() {
+ IndexingConfigurations indexingConfigs = mock(IndexingConfigurations.class);
+ ConfigurationsCache cache = mock(ConfigurationsCache.class);
+
+ Map<String, Object> broIndexingConfig = new HashMap<String, Object>() {{
+ put("writer", new HashMap<String, Object>() {{
+ put("index", "bro_index");
+ }});
+ }};
+ when(indexingConfigs.getSensorIndexingConfig("bro")).thenReturn(broIndexingConfig);
+ when(cache.get(IndexingConfigurations.class)).thenReturn(indexingConfigs);
+
+ assertEquals("bro_index", IndexingCacheUtil.getIndexLookupFunction(cache, "writer").apply("bro"));
+ }
+
+ @Test
+ public void getIndexLookupFunctionShouldDefaultToSensorType() {
+ IndexingConfigurations indexingConfigs = mock(IndexingConfigurations.class);
+ ConfigurationsCache cache = mock(ConfigurationsCache.class);
+
+ Map<String, Object> broIndexingConfig = new HashMap<String, Object>() {{
+ put("writer", new HashMap<String, Object>() {{
+ put("index", "bro_index");
+ }});
+ }};
+ when(indexingConfigs.getSensorIndexingConfig("bro")).thenReturn(broIndexingConfig);
+ when(cache.get(IndexingConfigurations.class)).thenReturn(indexingConfigs);
+
+ assertEquals("Should default to sensor type on missing sensor config", "snort", IndexingCacheUtil.getIndexLookupFunction(cache, "writer").apply("snort"));
+ assertEquals("Should default to sensor type on missing writer config", "bro", IndexingCacheUtil.getIndexLookupFunction(cache, "someWriter").apply("bro"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
index d5dc7a0..879b983 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
@@ -26,4 +26,5 @@ public class SolrConstants {
public static final String REQUEST_COLLECTION_CONFIG_NAME = "collection.configName";
public static final String REQUEST_COLLECTIONS_PATH = "/admin/collections";
public static final String RESPONSE_COLLECTIONS = "collections";
+ public static final String SOLR_WRITER_NAME = "solr";
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
index 6b5b3a8..c6f7124 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
@@ -93,7 +93,8 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao {
QueryResponse rsp = solrClient.query(METAALERTS_COLLECTION, solrQuery);
String nextCursorMark = rsp.getNextCursorMark();
rsp.getResults().stream()
- .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, null))
+ .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, null,
+ solrSearchDao.getAccessConfig().getIndexSupplier()))
.forEachOrdered(allResults::add);
if (cursorMark.equals(nextCursorMark)) {
done = true;
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index 53ac083..4a8d482 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -68,6 +68,10 @@ public class SolrSearchDao implements SearchDao {
this.accessConfig = accessConfig;
}
+ protected AccessConfig getAccessConfig() {
+ return accessConfig;
+ }
+
@Override
public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
return search(searchRequest, null);
@@ -181,7 +185,8 @@ public class SolrSearchDao implements SearchDao {
// search hits --> search results
List<SearchResult> results = solrDocumentList.stream()
- .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, searchRequest.getFields()))
+ .map(solrDocument -> SolrUtilities.getSearchResult(solrDocument, searchRequest.getFields(),
+ accessConfig.getIndexSupplier()))
.collect(Collectors.toList());
searchResponse.setResults(results);
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
index 54b10ad..7c56169 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
@@ -27,6 +27,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Function;
+
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.UpdateDao;
@@ -42,22 +44,33 @@ public class SolrUpdateDao implements UpdateDao {
private transient SolrClient client;
private AccessConfig config;
-
+ private Function<String, String> indexSupplier;
public SolrUpdateDao(SolrClient client, AccessConfig config) {
this.client = client;
this.config = config;
+ this.indexSupplier = config.getIndexSupplier();
+ }
+
+ private Optional<String> getIndex(String sensorName, Optional<String> index) {
+ if(index.isPresent()) {
+ return Optional.ofNullable(index.get());
+ }
+ else {
+ String realIndex = indexSupplier.apply(sensorName);
+ return Optional.ofNullable(realIndex);
+ }
}
@Override
- public void update(Document update, Optional<String> index) throws IOException {
+ public void update(Document update, Optional<String> rawIndex) throws IOException {
try {
SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(update);
+ Optional<String> index = getIndex(update.getSensorType(), rawIndex);
if (index.isPresent()) {
this.client.add(index.get(), solrInputDocument);
this.client.commit(index.get());
} else {
- this.client.add(solrInputDocument);
- this.client.commit();
+ throw new IllegalStateException("Index must be specified or inferred.");
}
} catch (SolrServerException e) {
throw new IOException(e);
@@ -72,7 +85,7 @@ public class SolrUpdateDao implements UpdateDao {
for (Entry<Document, Optional<String>> entry : updates.entrySet()) {
SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(entry.getKey());
- Optional<String> index = entry.getValue();
+ Optional<String> index = getIndex(entry.getKey().getSensorType(), entry.getValue());
if (index.isPresent()) {
Collection<SolrInputDocument> solrInputDocuments = solrCollectionUpdates
.getOrDefault(index.get(), new ArrayList<>());
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
index ce0ae84..616bd1a 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java
@@ -22,6 +22,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
+
import org.apache.metron.common.Constants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.search.SearchResult;
@@ -31,9 +33,10 @@ import org.apache.solr.common.SolrInputDocument;
public class SolrUtilities {
- public static SearchResult getSearchResult(SolrDocument solrDocument, List<String> fields) {
+ public static SearchResult getSearchResult(SolrDocument solrDocument, List<String> fields, Function<String, String> indexSupplier) {
SearchResult searchResult = new SearchResult();
searchResult.setId((String) solrDocument.getFieldValue(Constants.GUID));
+ searchResult.setIndex(indexSupplier.apply((String) solrDocument.getFieldValue(Constants.SENSOR_TYPE)));
Map<String, Object> docSource = toDocument(solrDocument).getDocument();
final Map<String, Object> source = new HashMap<>();
if (fields != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
index 6147966..93bd8fe 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
@@ -50,6 +50,8 @@ import java.util.Set;
import java.util.UUID;
import java.util.function.Supplier;
+import static org.apache.metron.solr.SolrConstants.SOLR_WRITER_NAME;
+
public class SolrWriter implements BulkMessageWriter<JSONObject>, Serializable {
public static final String JAVA_SECURITY_CONFIG_PROPERTY = "java.security.auth.login.config";
@@ -237,7 +239,7 @@ public class SolrWriter implements BulkMessageWriter<JSONObject>, Serializable {
@Override
public String getName() {
- return "solr";
+ return SOLR_WRITER_NAME;
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
index 49a40ef..d17bf72 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
@@ -90,6 +90,7 @@ public class SolrSearchDaoTest {
public void setUp() throws Exception {
client = mock(SolrClient.class);
accessConfig = mock(AccessConfig.class);
+ when(accessConfig.getIndexSupplier()).thenReturn(sensorType -> sensorType);
solrSearchDao = new SolrSearchDao(client, accessConfig);
solrRetrieveLatestDao = new SolrRetrieveLatestDao(client);
mockStatic(CollectionAdminRequest.class);
@@ -312,28 +313,31 @@ public class SolrSearchDaoTest {
SolrDocument solrDocument = mock(SolrDocument.class);
when(solrDocument.getFieldValue(Constants.GUID)).thenReturn("guid");
+ when(solrDocument.getFieldValue(Constants.SENSOR_TYPE)).thenReturn("sensorType");
when(solrDocument.getFieldValue("field1")).thenReturn("value1");
when(solrDocument.getFieldValue("field2")).thenReturn("value2");
when(solrDocument.getFieldNames()).thenReturn(Arrays.asList("field1", "field2"));
SearchResult expectedSearchResult = new SearchResult();
expectedSearchResult.setId("guid");
+ expectedSearchResult.setIndex("sensorType");
expectedSearchResult.setSource(new HashMap<String, Object>() {{
put("field1", "value1");
}});
assertEquals(expectedSearchResult, SolrUtilities.getSearchResult(solrDocument,
- Collections.singletonList("field1")));
+ Collections.singletonList("field1"), solrSearchDao.getAccessConfig().getIndexSupplier()));
SearchResult expectedSearchResultAllFields = new SearchResult();
expectedSearchResultAllFields.setId("guid");
+ expectedSearchResultAllFields.setIndex("sensorType");
expectedSearchResultAllFields.setSource(new HashMap<String, Object>() {{
put("field1", "value1");
put("field2", "value2");
}});
assertEquals(expectedSearchResultAllFields,
- SolrUtilities.getSearchResult(solrDocument, null));
+ SolrUtilities.getSearchResult(solrDocument, null, solrSearchDao.getAccessConfig().getIndexSupplier()));
}
@Test
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
index b9ed0b5..0011aa8 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java
@@ -17,8 +17,12 @@
*/
package org.apache.metron.solr.dao;
+import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.util.IndexingCacheUtil;
import org.apache.metron.solr.matcher.SolrInputDocumentListMatcher;
import org.apache.metron.solr.matcher.SolrInputDocumentMatcher;
import org.apache.solr.client.solrj.SolrClient;
@@ -43,6 +47,7 @@ import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
@PrepareForTest({CollectionAdminRequest.class})
@@ -63,7 +68,17 @@ public class SolrUpdateDaoTest {
put("solr.zookeeper", "zookeeper:2181");
}}
);
- accessConfig.setIndexSupplier(s -> null);
+ IndexingConfigurations indexingConfigs = mock(IndexingConfigurations.class);
+ ConfigurationsCache cache = mock(ConfigurationsCache.class);
+
+ Map<String, Object> broIndexingConfig = new HashMap<String, Object>() {{
+ put("solr", new HashMap<String, Object>() {{
+ }});
+ }};
+ when(indexingConfigs.getSensorIndexingConfig("bro")).thenReturn(broIndexingConfig);
+ when(cache.get(IndexingConfigurations.class)).thenReturn(indexingConfigs);
+
+ accessConfig.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, "solr"));
}
@SuppressWarnings("unchecked")
@@ -74,7 +89,7 @@ public class SolrUpdateDaoTest {
}
@Test
- public void updateShouldProperlyUpdateDocument() throws Exception {
+ public void updateShouldProperlyUpdateDocumentImplicitIndex() throws Exception {
Document document = new Document(new HashMap<String, Object>(){{
put("field", "value");
}}, "guid", "bro", 0L);
@@ -84,7 +99,18 @@ public class SolrUpdateDaoTest {
solrUpdateDao.update(document, Optional.empty());
- verify(client).add(argThat(new SolrInputDocumentMatcher(solrInputDocument)));
+ verify(client).add(eq("bro"), argThat(new SolrInputDocumentMatcher(solrInputDocument)));
+
+ }
+
+ @Test
+ public void updateShouldProperlyUpdateDocumentExplicitIndex() throws Exception {
+ Document document = new Document(new HashMap<String, Object>(){{
+ put("field", "value");
+ }}, "guid", "bro", 0L);
+
+ SolrInputDocument solrInputDocument = new SolrInputDocument();
+ solrInputDocument.addField("field", "value");
solrUpdateDao.update(document, Optional.of("bro"));
@@ -144,7 +170,7 @@ public class SolrUpdateDaoTest {
solrUpdateDao.batchUpdate(updates);
- verify(client).add((String) Matchers.isNull(), argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2))));
+ verify(client).add(eq("snort"), argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2))));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/7a071f6d/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index 2aa65ae..2fb3086 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -82,6 +82,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
}}
);
+ config.setIndexSupplier( sensorType -> sensorType);
IndexDao dao = new SolrDao();
dao.init(config);
return dao;
@@ -231,4 +232,9 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
protected String getSourceTypeField() {
return Constants.SENSOR_TYPE;
}
+
+ @Override
+ protected String getIndexName(String sensorType) {
+ return sensorType;
+ }
}