You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2018/05/02 14:11:11 UTC
[3/3] metron git commit: METRON-1540 Solr Integration tests should
use actual schemas (justinleet) closes apache/metron#1005
METRON-1540 Solr Integration tests should use actual schemas (justinleet) closes apache/metron#1005
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/832296f0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/832296f0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/832296f0
Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: 832296f0d4615bf95aed1c0742362c6127bd733a
Parents: d0a4e4c
Author: justinleet <ju...@gmail.com>
Authored: Wed May 2 10:10:27 2018 -0400
Committer: leet <le...@apache.org>
Committed: Wed May 2 10:10:27 2018 -0400
----------------------------------------------------------------------
.../ElasticsearchSearchIntegrationTest.java | 22 +-
.../indexing/dao/SearchIntegrationTest.java | 55 +-
.../metron/solr/dao/SolrColumnMetadataDao.java | 33 +-
.../apache/metron/solr/dao/SolrSearchDao.java | 3 +-
.../integration/SolrSearchIntegrationTest.java | 138 +-
.../resources/config/bro/conf/managed-schema | 50 -
.../resources/config/bro/conf/solrconfig.xml | 1601 ------------------
.../resources/config/snort/conf/managed-schema | 51 -
.../resources/config/snort/conf/solrconfig.xml | 1601 ------------------
9 files changed, 178 insertions(+), 3376 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 5569c54..bb28abb 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ExecutionException;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
@@ -109,7 +110,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
* "type": "text",
* "fielddata" : "true"
* },
- * "duplicate_name_field": {
+ * "ttl": {
* "type": "text",
* "fielddata" : "true"
* },
@@ -162,7 +163,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
* "snort_field": {
* "type": "integer"
* },
- * "duplicate_name_field": {
+ * "ttl": {
* "type": "integer"
* },
* "alert": {
@@ -275,7 +276,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
Assert.assertEquals(13, fieldTypes.size());
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
@@ -287,7 +288,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
}
// getColumnMetadata with only snort
@@ -295,7 +296,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
Assert.assertEquals(14, fieldTypes.size());
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
@@ -306,7 +307,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
}
}
@@ -328,7 +329,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
//NOTE: This is because the field is in both bro and snort and they have different types.
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
}
@@ -349,6 +350,11 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
Assert.assertEquals(1, response.getTotal());
List<SearchResult> results = response.getResults();
Assert.assertEquals("bro", results.get(0).getSource().get("source:type"));
- Assert.assertEquals("data 1", results.get(0).getSource().get("duplicate_name_field"));
+ Assert.assertEquals("data 1", results.get(0).getSource().get("ttl"));
+ }
+
+ @Override
+ protected String getSourceTypeField() {
+ return Constants.SENSOR_TYPE.replace('.', ':');
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 83046b8..56406f4 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -40,6 +40,7 @@ import org.apache.metron.integration.InMemoryComponent;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -47,11 +48,11 @@ import org.junit.rules.ExpectedException;
public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2", "guid":"bro_2"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3", "guid":"bro_3"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4", "guid":"bro_4"},
- * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5", "guid":"bro_5"}
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "ttl": "data 1", "guid":"bro_1"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "ttl": "data 2", "guid":"bro_2"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "ttl": "data 3", "guid":"bro_3"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "ttl": "data 4", "guid":"bro_4"},
+ * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "ttl": "data 5", "guid":"bro_5"}
* ]
*/
@Multiline
@@ -59,11 +60,11 @@ public abstract class SearchIntegrationTest {
/**
* [
- * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1, "guid":"snort_1", "threat:triage:score":10.0},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2, "guid":"snort_2", "threat:triage:score":20.0},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3, "guid":"snort_3"},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4, "guid":"snort_4"},
- * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5, "guid":"snort_5"}
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "ttl": 3, "guid":"snort_3"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "ttl": 4, "guid":"snort_4"},
+ * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "ttl": 5, "guid":"snort_5"}
* ]
*/
@Multiline
@@ -234,7 +235,7 @@ public abstract class SearchIntegrationTest {
* }
*/
@Multiline
- public static String facetQuery;
+ public static String facetQueryRaw;
/**
* {
@@ -291,7 +292,7 @@ public abstract class SearchIntegrationTest {
/**
* {
- * "facetFields": ["duplicate_name_field"],
+ * "facetFields": ["ttl"],
* "indices": ["bro", "snort"],
* "query": "*:*",
* "from": 0,
@@ -457,7 +458,7 @@ public abstract class SearchIntegrationTest {
/**
* {
* "indices": ["bro", "snort"],
- * "query": "duplicate_name_field:\"data 1\"",
+ * "query": "ttl:\"data 1\"",
* "from": 0,
* "size": 10,
* "sort": [
@@ -494,11 +495,11 @@ public abstract class SearchIntegrationTest {
List<SearchResult> results = response.getResults();
Assert.assertEquals(10, results.size());
for(int i = 0;i < 5;++i) {
- Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
+ Assert.assertEquals("snort", results.get(i).getSource().get(getSourceTypeField()));
Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
}
for (int i = 5; i < 10; ++i) {
- Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
+ Assert.assertEquals("bro", results.get(i).getSource().get(getSourceTypeField()));
Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
}
}
@@ -509,7 +510,7 @@ public abstract class SearchIntegrationTest {
Optional<Map<String, Object>> response = dao.getLatestResult(request);
Assert.assertTrue(response.isPresent());
Map<String, Object> doc = response.get();
- Assert.assertEquals("bro", doc.get("source:type"));
+ Assert.assertEquals("bro", doc.get(getSourceTypeField()));
Assert.assertEquals("3", doc.get("timestamp").toString());
}
@@ -524,8 +525,8 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(2, docs.size());
Assert.assertTrue(docs.keySet().contains("bro_1"));
Assert.assertTrue(docs.keySet().contains("snort_2"));
- Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type"));
- Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type"));
+ Assert.assertEquals("bro", docs.get("bro_1").getDocument().get(getSourceTypeField()));
+ Assert.assertEquals("snort", docs.get("snort_2").getDocument().get(getSourceTypeField()));
}
@Test
@@ -534,11 +535,11 @@ public abstract class SearchIntegrationTest {
SearchResponse response = dao.search(request);
Assert.assertEquals(3, response.getTotal());
List<SearchResult> results = response.getResults();
- Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+ Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
Assert.assertEquals("9", results.get(0).getSource().get("timestamp").toString());
- Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
+ Assert.assertEquals("snort", results.get(1).getSource().get(getSourceTypeField()));
Assert.assertEquals("7", results.get(1).getSource().get("timestamp").toString());
- Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+ Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField()));
Assert.assertEquals("1", results.get(2).getSource().get("timestamp").toString());
}
@@ -596,11 +597,11 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(10, response.getTotal());
List<SearchResult> results = response.getResults();
Assert.assertEquals(3, results.size());
- Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+ Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
Assert.assertEquals("6", results.get(0).getSource().get("timestamp").toString());
- Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
+ Assert.assertEquals("bro", results.get(1).getSource().get(getSourceTypeField()));
Assert.assertEquals("5", results.get(1).getSource().get("timestamp").toString());
- Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+ Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField()));
Assert.assertEquals("4", results.get(2).getSource().get("timestamp").toString());
}
@@ -611,19 +612,20 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(5, response.getTotal());
List<SearchResult> results = response.getResults();
for (int i = 5, j = 0; i > 0; i--, j++) {
- Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
+ Assert.assertEquals("bro", results.get(j).getSource().get(getSourceTypeField()));
Assert.assertEquals(i + "", results.get(j).getSource().get("timestamp").toString());
}
}
@Test
public void facet_query_yields_field_types() throws Exception {
+ String facetQuery = facetQueryRaw.replace("source:type", getSourceTypeField());
SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
SearchResponse response = dao.search(request);
Assert.assertEquals(10, response.getTotal());
Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
Assert.assertEquals(8, facetCounts.size());
- Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
+ Map<String, Long> sourceTypeCounts = facetCounts.get(getSourceTypeField());
Assert.assertEquals(2, sourceTypeCounts.size());
Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
@@ -951,4 +953,5 @@ public abstract class SearchIntegrationTest {
protected abstract IndexDao createDao() throws Exception;
protected abstract InMemoryComponent startIndex() throws Exception;
protected abstract void loadTestData() throws Exception;
+ protected abstract String getSourceTypeField();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
index 61f0209..1e074a9 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
@@ -20,17 +20,25 @@ package org.apache.metron.solr.dao;
import com.google.common.collect.Sets;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest.DynamicFields;
+import org.apache.solr.client.solrj.response.LukeResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.schema.SchemaRepresentation;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse.DynamicFieldsResponse;
import org.apache.solr.common.SolrException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,6 +57,7 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao {
fieldTypeMap.put("pfloat", FieldType.FLOAT);
fieldTypeMap.put("pdouble", FieldType.DOUBLE);
fieldTypeMap.put("boolean", FieldType.BOOLEAN);
+ fieldTypeMap.put("ip", FieldType.IP);
solrTypeMap = Collections.unmodifiableMap(fieldTypeMap);
}
@@ -104,12 +113,30 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao {
return indexColumnMetadata;
}
- protected List<Map<String, Object>> getIndexFields(String index) throws IOException, SolrServerException {
+ protected List<Map<String, Object>> getIndexFields(String index)
+ throws IOException, SolrServerException {
CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build();
client.setDefaultCollection(index);
+
+ List<Map<String, Object>> indexFields = new ArrayList<>();
+
+ // Get all the fields in use, including dynamic fields
+ LukeRequest lukeRequest = new LukeRequest();
+ LukeResponse lukeResponse = lukeRequest.process(client);
+ for (Entry<String, LukeResponse.FieldInfo> field : lukeResponse.getFieldInfo().entrySet()) {
+ Map<String, Object> fieldData = new HashMap<>();
+ fieldData.put("name", field.getValue().getName());
+ fieldData.put("type", field.getValue().getType());
+ indexFields.add(fieldData);
+
+ }
+
+ // Get all the schema fields
SchemaRepresentation schemaRepresentation = new SchemaRequest().process(client)
- .getSchemaRepresentation();
- return schemaRepresentation.getFields();
+ .getSchemaRepresentation();
+ indexFields.addAll(schemaRepresentation.getFields());
+
+ return indexFields;
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index 272b96a..f1b7102 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -47,6 +47,7 @@ import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,7 +92,7 @@ public class SolrSearchDao implements SearchDao {
SolrQuery query = buildSearchRequest(searchRequest);
QueryResponse response = client.query(query);
return buildSearchResponse(searchRequest, response);
- } catch (IOException | SolrServerException e) {
+ } catch (SolrException | IOException | SolrServerException e) {
String msg = e.getMessage();
LOG.error(msg, e);
throw new InvalidSearchException(msg, e);
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index a9ce650..92941cd 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -21,6 +21,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
@@ -36,12 +37,16 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.json.simple.JSONArray;
import org.json.simple.parser.JSONParser;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
public class SolrSearchIntegrationTest extends SearchIntegrationTest {
private SolrComponent solrComponent;
+ private String broData = SearchIntegrationTest.broData.replace("source:type", "source.type");
+ private String snortData = SearchIntegrationTest.snortData.replace("source:type", "source.type");
+
@Override
protected IndexDao createDao() throws Exception {
AccessConfig config = new AccessConfig();
@@ -60,9 +65,10 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
@Override
protected InMemoryComponent startIndex() throws Exception {
+
solrComponent = new SolrComponent.Builder()
- .addCollection("bro", "../metron-solr/src/test/resources/config/bro/conf")
- .addCollection("snort", "../metron-solr/src/test/resources/config/snort/conf")
+ .addCollection("bro", "../metron-solr/src/main/config/schema/bro")
+ .addCollection("snort", "../metron-solr/src/main/config/schema/snort")
.build();
solrComponent.start();
return solrComponent;
@@ -83,65 +89,122 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
}
@Override
+ @Test
public void returns_column_metadata_for_specified_indices() throws Exception {
// getColumnMetadata with only bro
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
- Assert.assertEquals(12, fieldTypes.size());
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+ // Don't test all 256, just test a sample of different fields
+ Assert.assertEquals(261, fieldTypes.size());
+
+ // Fields present in both with same type
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
+ Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
- Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
- Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp"));
- Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
- Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+
+ // Bro only field
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username"));
+
+ // A dynamic field present in both with same type
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));
+
+ // Dyanamic field present in both with nonstandard types.
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+
+ // Field with nonstandard type
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
+
+ // Bro only field in the dynamic catch all
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field"));
+
+ // A field is in both bro and snort and they have different types.
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
+
+ // Field only present in Snort
+ Assert.assertEquals(null, fieldTypes.get("dgmlen"));
+
+ // Field that doesn't exist
+ Assert.assertEquals(null, fieldTypes.get("fake.field"));
}
// getColumnMetadata with only snort
{
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
- Assert.assertEquals(13, fieldTypes.size());
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(31, fieldTypes.size());
+
+ // Fields present in both with same type
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
+ Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
- Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
- Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp"));
- Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
- Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+
+ // Snort only field
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen"));
+
+ // A dynamic field present in both with same type
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));
+
+ // Dyanamic field present in both with nonstandard types.
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+
+ // Field with nonstandard type
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
+
+ // Snort only field in the dynamic catch all
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field"));
+
+ // A field is in both bro and snort and they have different types.
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
+
+ // Field only present in Bro
+ Assert.assertEquals(null, fieldTypes.get("username"));
+
+ // Field that doesn't exist
+ Assert.assertEquals(null, fieldTypes.get("fake.field"));
}
}
@Override
+ @Test
public void returns_column_data_for_multiple_indices() throws Exception {
Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
- Assert.assertEquals(14, fieldTypes.size());
+ // Don't test everything, just test a variety of fields, including fields across collections.
+
+ // Fields present in both with same type
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
+ Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
- Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
- Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp"));
- Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
- Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+
+ // Bro only field
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username"));
+
+ // Snort only field
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen"));
+
+ // A dynamic field present in both with same type
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));
+
+ // Dyanamic field present in both with nonstandard types.
Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
- Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
- Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
- //NOTE: This is because the field is in both bro and snort and they have different types.
- Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field"));
- Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
+
+ // Field present in both with nonstandard type
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));
+
+ // Bro only field in the dynamic catch all
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field"));
+
+ // Snort only field in the dynamic catch all
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field"));
+
+ // A field is in both bro and snort and they have different types.
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));
+
+ // Field that doesn't exist
+ Assert.assertEquals(null, fieldTypes.get("fake.field"));
}
@Test
@@ -150,4 +213,9 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);
SearchResponse response = dao.search(request);
}
+
+ @Override
+ protected String getSourceTypeField() {
+ return Constants.SENSOR_TYPE;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
deleted file mode 100644
index 3bcb2bc..0000000
--- a/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<schema name="example" version="1.6">
- <field name="_version_" type="plong" indexed="false" stored="false"/>
- <field name="_root_" type="string" indexed="true" stored="false" docValues="false" />
-
- <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" />
-
- <field name="source:type" type="string" indexed="true" stored="true"/>
- <field name="ip_src_addr" type="string" indexed="true" stored="true" />
- <field name="ip_src_port" type="pint" indexed="true" stored="true"/>
- <field name="long_field" type="plong" indexed="true" stored="true" />
- <field name="timestamp" type="plong" indexed="true" stored="true" />
- <field name="latitude" type="pfloat" indexed="true" stored="true" />
- <field name="score" type="pdouble" indexed="true" stored="true" />
- <field name="is_alert" type="boolean" indexed="true" stored="true"/>
- <field name="location_point" type="location" indexed="true" stored="true"/>
- <field name="bro_field" type="string" indexed="true" stored="true" />
- <field name="duplicate_name_field" type="string" indexed="true" stored="true" />
-
- <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
-
- <uniqueKey>guid</uniqueKey>
-
-
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
- <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
- <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
- <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
- <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
- <fieldType name="location" class="solr.LatLonPointSpatialField" docValues="true"/>
- <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
-
-</schema>