You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:37 UTC
[21/50] [abbrv] metron git commit: Merge branch 'master' into
feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0717cfc2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0717cfc2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0717cfc2
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 0717cfc25acc80b80121b424bed93db7014a1376
Parents: 4170887
Author: cstella <ce...@gmail.com>
Authored: Mon Jun 11 21:41:06 2018 -0400
Committer: cstella <ce...@gmail.com>
Committed: Mon Jun 11 21:41:06 2018 -0400
----------------------------------------------------------------------
.../org/apache/metron/common/Constants.java | 1 +
.../elasticsearch/dao/ElasticsearchDao.java | 4 ++
.../dao/ElasticsearchMetaAlertDao.java | 30 +++++++++-----
.../dao/ElasticsearchMetaAlertUpdateDao.java | 4 +-
.../indexing/dao/metaalert/MetaAlertConfig.java | 43 ++++++++++++--------
.../AbstractLuceneMetaAlertUpdateDao.java | 3 ++
.../dao/metaalert/MetaAlertIntegrationTest.java | 2 +-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 26 +++++++++---
.../metron/solr/dao/SolrMetaAlertDao.java | 29 +++++++++----
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 15 ++++---
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 2 +-
.../SolrMetaAlertIntegrationTest.java | 28 +++++++++----
12 files changed, 128 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index f74660c..4a8bea2 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -28,6 +28,7 @@ public class Constants {
public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
public static final String SENSOR_TYPE = "source.type";
public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field";
+ public static final String THREAT_SCORE_FIELD_PROPERTY = "threat.triage.score.field";
public static final String ENRICHMENT_TOPIC = "enrichments";
public static final String INDEXING_TOPIC = "indexing";
public static final String ERROR_STREAM = "error";
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index eae0a39..3eb86ce 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -85,6 +85,10 @@ public class ElasticsearchDao implements IndexDao {
//uninitialized.
}
+ public AccessConfig getAccessConfig() {
+ return accessConfig;
+ }
+
@Override
public synchronized void init(AccessConfig config) {
if (this.client == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index f73a640..649077e 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -68,6 +68,7 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.apache.metron.common.Constants.GUID;
@@ -83,10 +84,8 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
public static final String THREAT_TRIAGE_FIELD = MetaAlertConstants.THREAT_FIELD_DEFAULT
.replace('.', ':');
public static final String METAALERTS_INDEX = "metaalert_index";
-
public static final String SOURCE_TYPE_FIELD = Constants.SENSOR_TYPE.replace('.', ':');
protected String metaAlertsIndex = METAALERTS_INDEX;
- protected String threatTriageField = THREAT_TRIAGE_FIELD;
protected String threatSort = MetaAlertConstants.THREAT_SORT_DEFAULT;
private ElasticsearchDao elasticsearchDao;
@@ -102,8 +101,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
* @param indexDao The Dao to wrap
*/
public ElasticsearchMetaAlertDao(IndexDao indexDao) {
- this(indexDao, METAALERTS_INDEX, MetaAlertConstants.THREAT_FIELD_DEFAULT,
- MetaAlertConstants.THREAT_SORT_DEFAULT);
+ this(indexDao, METAALERTS_INDEX, MetaAlertConstants.THREAT_SORT_DEFAULT);
}
/**
@@ -114,10 +112,8 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
* can be either max, min, average, count, median, or sum.
*/
public ElasticsearchMetaAlertDao(IndexDao indexDao, String metaAlertsIndex,
- String triageLevelField,
String threatSort) {
init(indexDao, Optional.of(threatSort));
- this.threatTriageField = triageLevelField;
this.threatSort = threatSort;
this.metaAlertsIndex = metaAlertsIndex;
}
@@ -158,13 +154,25 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
if (threatSort.isPresent()) {
this.threatSort = threatSort.get();
}
-
+ Supplier<Map<String, Object>> globalConfigSupplier = () -> new HashMap<>();
+ if(elasticsearchDao != null && elasticsearchDao.getAccessConfig() != null) {
+ globalConfigSupplier = elasticsearchDao.getAccessConfig().getGlobalConfigSupplier();
+ }
MetaAlertConfig config = new MetaAlertConfig(
metaAlertsIndex,
- threatTriageField,
- this.threatSort,
- ElasticsearchMetaAlertDao.SOURCE_TYPE_FIELD
- );
+ this.threatSort,
+ globalConfigSupplier
+ ) {
+ @Override
+ protected String getDefaultThreatTriageField() {
+ return THREAT_TRIAGE_FIELD;
+ }
+
+ @Override
+ protected String getDefaultSourceTypeField() {
+ return SOURCE_TYPE_FIELD;
+ }
+ };
this.metaAlertSearchDao = new ElasticsearchMetaAlertSearchDao(
elasticsearchDao,
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
index d3bdcbb..d757dfe 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.metron.common.Constants;
@@ -37,6 +38,7 @@ import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.metaalert.MetaScores;
@@ -96,7 +98,7 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda
getConfig().getThreatSort());
// Add source type to be consistent with other sources and allow filtering
metaAlert.getDocument()
- .put(ElasticsearchMetaAlertDao.SOURCE_TYPE_FIELD, MetaAlertConstants.METAALERT_TYPE);
+ .put(getConfig().getSourceTypeField(), MetaAlertConstants.METAALERT_TYPE);
// Start a list of updates / inserts we need to run
Map<Document, Optional<String>> updates = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java
index 9254425..b538bc2 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConfig.java
@@ -18,26 +18,30 @@
package org.apache.metron.indexing.dao.metaalert;
-public class MetaAlertConfig {
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+public abstract class MetaAlertConfig {
private String metaAlertIndex;
- private String threatTriageField;
private String threatSort;
- private String sourceTypeField;
+ private Supplier<Map<String, Object>> globalConfigSupplier;
/**
* Simple object for storing and retrieving configs, primarily to make passing all the info to
* the sub DAOs easier.
* @param metaAlertIndex The metaalert index or collection we're using
- * @param threatTriageField The threat triage field's name
* @param threatSort The sorting operation on the threat triage field
- * @param sourceTypeField The source type field
*/
- public MetaAlertConfig(String metaAlertIndex, String threatTriageField,
- String threatSort, String sourceTypeField) {
+ public MetaAlertConfig( String metaAlertIndex
+ , String threatSort
+ , Supplier<Map<String, Object>> globalConfigSupplier) {
this.metaAlertIndex = metaAlertIndex;
- this.threatTriageField = threatTriageField;
this.threatSort = threatSort;
- this.sourceTypeField = sourceTypeField;
+ this.globalConfigSupplier = globalConfigSupplier;
}
public String getMetaAlertIndex() {
@@ -49,12 +53,14 @@ public class MetaAlertConfig {
}
public String getThreatTriageField() {
- return threatTriageField;
+ Optional<Map<String, Object>> globalConfig = Optional.ofNullable(globalConfigSupplier.get());
+ if(!globalConfig.isPresent()) {
+ return getDefaultThreatTriageField();
+ }
+ return ConfigurationsUtils.getFieldName(globalConfig.get(), Constants.THREAT_SCORE_FIELD_PROPERTY, getDefaultThreatTriageField());
}
- public void setThreatTriageField(String threatTriageField) {
- this.threatTriageField = threatTriageField;
- }
+ protected abstract String getDefaultThreatTriageField();
public String getThreatSort() {
return threatSort;
@@ -65,10 +71,13 @@ public class MetaAlertConfig {
}
public String getSourceTypeField() {
- return sourceTypeField;
+ Optional<Map<String, Object>> globalConfig = Optional.ofNullable(globalConfigSupplier.get());
+ if(!globalConfig.isPresent()) {
+ return getDefaultSourceTypeField();
+ }
+ return ConfigurationsUtils.getFieldName(globalConfig.get(), Constants.SENSOR_TYPE_FIELD_PROPERTY, getDefaultSourceTypeField());
}
- public void setSourceTypeField(String sourceTypeField) {
- this.sourceTypeField = sourceTypeField;
- }
+ protected abstract String getDefaultSourceTypeField();
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
index b47d648..4d48075 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
@@ -30,8 +30,10 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
@@ -331,4 +333,5 @@ public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdat
updateDao.batchUpdate(updates);
} // else we have no updates, so don't do anything
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
index b4f7d38..6f96fb5 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
@@ -111,7 +111,7 @@ public abstract class MetaAlertIntegrationTest {
},
{
"op": "add",
- "path": "/alert",
+ "path": "/metron_alert",
"value": []
}
],
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
index 7028b75..b5965e8 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
@@ -43,6 +43,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.UUID;
+
+import com.google.common.collect.ImmutableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.math.util.MathUtils;
import org.apache.metron.common.Constants;
@@ -88,12 +90,24 @@ public class AbstractLuceneMetaAlertUpdateDaoTest {
private static final String METAALERT_INDEX = "metaalert_index";
private static final String METAALERT_GUID = "meta_0";
private static final String DEFAULT_PREFIX = "child_";
- private static final MetaAlertConfig TEST_CONFIG = new MetaAlertConfig(
- METAALERT_INDEX,
- THREAT_FIELD_DEFAULT,
- THREAT_SORT_DEFAULT,
- Constants.SENSOR_TYPE
- );
+ private static final MetaAlertConfig TEST_CONFIG =
+ new MetaAlertConfig(METAALERT_INDEX
+ , THREAT_SORT_DEFAULT
+ , () -> ImmutableMap.of(Constants.SENSOR_TYPE_FIELD_PROPERTY, Constants.SENSOR_TYPE
+ , Constants.THREAT_SCORE_FIELD_PROPERTY, THREAT_FIELD_DEFAULT
+ )
+ ) {
+
+ @Override
+ protected String getDefaultThreatTriageField() {
+ return THREAT_FIELD_DEFAULT.replace(':', '.');
+ }
+
+ @Override
+ protected String getDefaultSourceTypeField() {
+ return Constants.SENSOR_TYPE;
+ }
+ };
private static Map<String, Document> documents = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
index 8b37a49..4748315 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
@@ -19,9 +19,12 @@
package org.apache.metron.solr.dao;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.function.Supplier;
+
import org.apache.metron.common.Constants;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
@@ -57,7 +60,6 @@ public class SolrMetaAlertDao implements MetaAlertDao {
private SolrMetaAlertUpdateDao metaAlertUpdateDao;
private SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao;
protected String metaAlertsCollection = METAALERTS_COLLECTION;
- protected String threatTriageField = MetaAlertConstants.THREAT_FIELD_DEFAULT;
protected String threatSort = MetaAlertConstants.THREAT_SORT_DEFAULT;
/**
@@ -69,14 +71,12 @@ public class SolrMetaAlertDao implements MetaAlertDao {
SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao) {
this(indexDao, metaAlertSearchDao, metaAlertUpdateDao, metaAlertRetrieveLatestDao,
METAALERTS_COLLECTION,
- MetaAlertConstants.THREAT_FIELD_DEFAULT,
MetaAlertConstants.THREAT_SORT_DEFAULT);
}
/**
* Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts.
* @param indexDao The Dao to wrap
- * @param triageLevelField The field name to use as the threat scoring field
* @param threatSort The summary aggregation of all child threat triage scores used
* as the overall threat triage score for the metaalert. This
* can be either max, min, average, count, median, or sum.
@@ -85,14 +85,12 @@ public class SolrMetaAlertDao implements MetaAlertDao {
SolrMetaAlertUpdateDao metaAlertUpdateDao,
SolrMetaAlertRetrieveLatestDao metaAlertRetrieveLatestDao,
String metaAlertsCollection,
- String triageLevelField,
String threatSort) {
init(indexDao, Optional.of(threatSort));
this.metaAlertSearchDao = metaAlertSearchDao;
this.metaAlertUpdateDao = metaAlertUpdateDao;
this.metaAlertRetrieveLatestDao = metaAlertRetrieveLatestDao;
this.metaAlertsCollection = metaAlertsCollection;
- this.threatTriageField = triageLevelField;
this.threatSort = threatSort;
}
@@ -126,16 +124,29 @@ public class SolrMetaAlertDao implements MetaAlertDao {
"Need a SolrDao when using SolrMetaAlertDao"
);
}
+ Supplier<Map<String, Object>> globalConfigSupplier = () -> new HashMap<>();
+ if(metaAlertSearchDao != null && metaAlertSearchDao.solrSearchDao != null && metaAlertSearchDao.solrSearchDao.getAccessConfig() != null) {
+ globalConfigSupplier = metaAlertSearchDao.solrSearchDao.getAccessConfig().getGlobalConfigSupplier();
+ }
MetaAlertConfig config = new MetaAlertConfig(
metaAlertsCollection,
- threatTriageField,
this.threatSort,
- Constants.SENSOR_TYPE
- );
+ globalConfigSupplier
+ ) {
+ @Override
+ protected String getDefaultThreatTriageField() {
+ return MetaAlertConstants.THREAT_FIELD_DEFAULT.replace(':', '.');
+ }
+
+ @Override
+ protected String getDefaultSourceTypeField() {
+ return Constants.SENSOR_TYPE;
+ }
+ };
SolrClient solrClient = solrDao.getSolrClient(solrDao.getZkHosts());
- this.metaAlertSearchDao = new SolrMetaAlertSearchDao(solrClient, solrDao.getSolrSearchDao());
+ this.metaAlertSearchDao = new SolrMetaAlertSearchDao(solrClient, solrDao.getSolrSearchDao(), config);
this.metaAlertRetrieveLatestDao = new SolrMetaAlertRetrieveLatestDao(solrDao);
this.metaAlertUpdateDao = new SolrMetaAlertUpdateDao(
solrDao,
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
index c6f7124..c1e3af6 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertSearchDao.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.Constants;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertSearchDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
@@ -57,10 +58,12 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao {
transient SolrSearchDao solrSearchDao;
transient SolrClient solrClient;
+ private MetaAlertConfig config;
- public SolrMetaAlertSearchDao(SolrClient solrClient, SolrSearchDao solrSearchDao) {
+ public SolrMetaAlertSearchDao(SolrClient solrClient, SolrSearchDao solrSearchDao, MetaAlertConfig config) {
this.solrClient = solrClient;
this.solrSearchDao = solrSearchDao;
+ this.config = config;
}
@Override
@@ -76,7 +79,7 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao {
MetaAlertConstants.STATUS_FIELD + ":" + MetaAlertStatus.ACTIVE.getStatusString();
String guidClause = Constants.GUID + ":" + guid;
String fullClause = "{!parent which=" + activeClause + "}" + guidClause;
- String metaalertTypeClause = Constants.SENSOR_TYPE + ":" + MetaAlertConstants.METAALERT_TYPE;
+ String metaalertTypeClause = config.getSourceTypeField() + ":" + MetaAlertConstants.METAALERT_TYPE;
SolrQuery solrQuery = new SolrQuery()
.setQuery(fullClause)
.setFields("*", "[child parentFilter=" + metaalertTypeClause + " limit=999]")
@@ -120,7 +123,7 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao {
String activeStatusClause =
MetaAlertConstants.STATUS_FIELD + ":" + MetaAlertStatus.ACTIVE.getStatusString();
- String metaalertTypeClause = Constants.SENSOR_TYPE + ":" + MetaAlertConstants.METAALERT_TYPE;
+ String metaalertTypeClause = config.getSourceTypeField() + ":" + MetaAlertConstants.METAALERT_TYPE;
// Use the 'v=' form in order to ensure complex clauses are properly handled.
// Per the docs, the 'which=' clause should be used to identify all metaalert parents, not to
// filter
@@ -157,10 +160,10 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao {
// Get them in a second query.
// However, we can only retrieve them if we have the source type field (either explicit or
// wildcard).
- if (fieldList.contains("*") || fieldList.contains(Constants.SENSOR_TYPE)) {
+ if (fieldList.contains("*") || fieldList.contains(config.getSourceTypeField())) {
List<String> metaalertGuids = new ArrayList<>();
for (SearchResult result : results.getResults()) {
- if (result.getSource().get(Constants.SENSOR_TYPE)
+ if (result.getSource().get(config.getSourceTypeField())
.equals(MetaAlertConstants.METAALERT_TYPE)) {
// Then we need to add it to the list to retrieve child alerts in a second query.
metaalertGuids.add(result.getId());
@@ -201,7 +204,7 @@ public class SolrMetaAlertSearchDao implements MetaAlertSearchDao {
@Override
public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
// Make sure to escape any problematic characters here
- String sourceType = ClientUtils.escapeQueryChars(Constants.SENSOR_TYPE);
+ String sourceType = ClientUtils.escapeQueryChars(config.getSourceTypeField());
String baseQuery = groupRequest.getQuery();
String adjustedQuery = baseQuery + " -" + MetaAlertConstants.METAALERT_FIELD + ":[* TO *]"
+ " -" + sourceType + ":" + MetaAlertConstants.METAALERT_TYPE;
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
index b96bbc6..132d872 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertUpdateDao.java
@@ -90,7 +90,7 @@ public class SolrMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpdateDao imp
getConfig().getThreatSort());
// Add source type to be consistent with other sources and allow filtering
- metaAlert.getDocument().put(Constants.SENSOR_TYPE, MetaAlertConstants.METAALERT_TYPE);
+ metaAlert.getDocument().put(getConfig().getSourceTypeField(), MetaAlertConstants.METAALERT_TYPE);
// Start a list of updates / inserts we need to run
Map<Document, Optional<String>> updates = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/metron/blob/0717cfc2/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
index f7dd02c..6687e9a 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
@@ -33,6 +33,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+
+import com.google.common.collect.ImmutableMap;
import org.apache.metron.common.Constants;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
@@ -93,16 +95,28 @@ public class SolrMetaAlertIntegrationTest extends MetaAlertIntegrationTest {
solrDao = new SolrDao();
solrDao.init(accessConfig);
- MetaAlertConfig config = new MetaAlertConfig(
- METAALERTS_COLLECTION,
- THREAT_FIELD_DEFAULT,
- THREAT_SORT_DEFAULT,
- Constants.SENSOR_TYPE
- );
+ MetaAlertConfig config = new MetaAlertConfig(METAALERTS_COLLECTION
+ , THREAT_SORT_DEFAULT
+ , () -> ImmutableMap.of(Constants.SENSOR_TYPE_FIELD_PROPERTY, Constants.SENSOR_TYPE
+ , Constants.THREAT_SCORE_FIELD_PROPERTY, THREAT_FIELD_DEFAULT
+ )
+ ) {
+
+ @Override
+ protected String getDefaultThreatTriageField() {
+ return THREAT_FIELD_DEFAULT.replace(':', '.');
+ }
+
+ @Override
+ protected String getDefaultSourceTypeField() {
+ return Constants.SENSOR_TYPE;
+ }
+ };
+
SolrMetaAlertSearchDao searchDao = new SolrMetaAlertSearchDao(
solrDao.getSolrClient(solrDao.getZkHosts()),
- solrDao.getSolrSearchDao());
+ solrDao.getSolrSearchDao(), config);
SolrMetaAlertRetrieveLatestDao retrieveLatestDao = new SolrMetaAlertRetrieveLatestDao(solrDao);
SolrMetaAlertUpdateDao updateDao = new SolrMetaAlertUpdateDao(solrDao, searchDao,
retrieveLatestDao, config);