You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/20 15:40:30 UTC
[02/15] metron git commit: METRON-1660 On Solr,
sorting by threat score fails (justinleet) closes apache/metron#1102
METRON-1660 On Solr, sorting by threat score fails (justinleet) closes apache/metron#1102
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/28f4b570
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/28f4b570
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/28f4b570
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 28f4b570493eda0a23317f520b89cb370e606ca0
Parents: 7af11b6
Author: justinleet <ju...@gmail.com>
Authored: Wed Jul 11 15:48:08 2018 -0400
Committer: leet <le...@apache.org>
Committed: Wed Jul 11 15:48:08 2018 -0400
----------------------------------------------------------------------
.../dao/metaalert/MetaAlertIntegrationTest.java | 56 ++++++++++++++++++++
.../src/main/config/schema/metaalert/schema.xml | 6 ++-
2 files changed, 61 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/28f4b570/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
index 6f96fb5..f754b81 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java
@@ -51,6 +51,7 @@ import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.search.SortField;
+import org.apache.metron.indexing.dao.search.SortOrder;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -194,6 +195,60 @@ public abstract class MetaAlertIntegrationTest {
}
@Test
+ public void shouldSortByThreatTriageScore() throws Exception {
+ // Load alerts
+ List<Map<String, Object>> alerts = buildAlerts(2);
+ alerts.get(0).put(METAALERT_FIELD, "meta_active_0");
+ addRecords(alerts, getTestIndexFullName(), SENSOR_NAME);
+
+ // Load metaAlerts
+ List<Map<String, Object>> metaAlerts = buildMetaAlerts(1, MetaAlertStatus.ACTIVE,
+ Optional.of(Collections.singletonList(alerts.get(0))));
+ // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
+ addRecords(metaAlerts, getMetaAlertIndex(), METAALERT_TYPE);
+
+ // Verify load was successful
+ List<GetRequest> createdDocs = metaAlerts.stream().map(metaAlert ->
+ new GetRequest((String) metaAlert.get(Constants.GUID), METAALERT_TYPE))
+ .collect(Collectors.toList());
+ createdDocs.addAll(alerts.stream().map(alert ->
+ new GetRequest((String) alert.get(Constants.GUID), SENSOR_NAME))
+ .collect(Collectors.toList()));
+ findCreatedDocs(createdDocs);
+
+ // Test descending
+ SortField sf = new SortField();
+ sf.setField(getThreatTriageField());
+ sf.setSortOrder(SortOrder.DESC.getSortOrder());
+ SearchRequest sr = new SearchRequest();
+ sr.setQuery("*:*");
+ sr.setSize(5);
+ sr.setIndices(Arrays.asList(getTestIndexName(), METAALERT_TYPE));
+ sr.setSort(Collections.singletonList(sf));
+
+ SearchResponse result = metaDao.search(sr);
+ List<SearchResult> results = result.getResults();
+ Assert.assertEquals(2, results.size());
+ Assert.assertEquals("meta_active_0", results.get((0)).getId());
+ Assert.assertEquals("message_1", results.get((1)).getId());
+
+ // Test ascending
+ SortField sfAsc = new SortField();
+ sfAsc.setField(getThreatTriageField());
+ sfAsc.setSortOrder(SortOrder.ASC.getSortOrder());
+ SearchRequest srAsc = new SearchRequest();
+ srAsc.setQuery("*:*");
+ srAsc.setSize(2);
+ srAsc.setIndices(Arrays.asList(getTestIndexName(), METAALERT_TYPE));
+ srAsc.setSort(Collections.singletonList(sfAsc));
+ result = metaDao.search(srAsc);
+ results = result.getResults();
+ Assert.assertEquals("message_1", results.get((0)).getId());
+ Assert.assertEquals("meta_active_0", results.get((1)).getId());
+ Assert.assertEquals(2, results.size());
+ }
+
+ @Test
public void getAllMetaAlertsForAlertShouldThrowExceptionForEmptyGuid() throws Exception {
try {
metaDao.getAllMetaAlertsForAlert("");
@@ -960,6 +1015,7 @@ public abstract class MetaAlertIntegrationTest {
metaAlert.put(Constants.GUID, guid);
metaAlert.put(getSourceTypeField(), METAALERT_TYPE);
metaAlert.put(STATUS_FIELD, status.getStatusString());
+ metaAlert.put(getThreatTriageField(), 100.0d);
if (alerts.isPresent()) {
List<Map<String, Object>> alertsList = alerts.get();
metaAlert.put(ALERT_FIELD, alertsList);
http://git-wip-us.apache.org/repos/asf/metron/blob/28f4b570/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
index 63e729b..6555bf6 100644
--- a/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/metaalert/schema.xml
@@ -28,7 +28,6 @@
<field name="timestamp" type="plong" indexed="true" stored="true"/>
<field name="score" type="pdouble" indexed="true" stored="true"/>
<field name="status" type="string" indexed="true" stored="true"/>
- <field name="threat:triage:score" type="pdouble" indexed="true" stored="true"/>
<field name="average" type="pdouble" indexed="true" stored="true"/>
<field name="min" type="pdouble" indexed="true" stored="true"/>
<field name="median" type="pdouble" indexed="true" stored="true"/>
@@ -40,6 +39,11 @@
<!-- Ensure that metaalerts child field is multivalued -->
<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>
+ <!-- Threat Intel Scoring Field -->
+ <!-- This is a double from the method of calculation. It'll still sort alongside pfloat -->
+ <dynamicField name="*score" type="pdouble" multiValued="false" docValues="true"/>
+
+ <!-- Catch all, if we don't know about it, it gets dropped. -->
<dynamicField name="*" type="ignored" indexed="true" stored="true" multiValued="false" docValues="true"/>
<uniqueKey>guid</uniqueKey>