You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2017/08/24 20:32:35 UTC
metron git commit: METRON-1073 Add option to limit fields returned
from search query (merrimanr) closes apache/metron#677
Repository: metron
Updated Branches:
refs/heads/master 8059c9fd4 -> f34affdb2
METRON-1073 Add option to limit fields returned from search query (merrimanr) closes apache/metron#677
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f34affdb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f34affdb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f34affdb
Branch: refs/heads/master
Commit: f34affdb25422f19f8e70a91b7a796a15a7745a5
Parents: 8059c9f
Author: merrimanr <me...@gmail.com>
Authored: Thu Aug 24 15:32:23 2017 -0500
Committer: merrimanr <me...@apache.org>
Committed: Thu Aug 24 15:32:23 2017 -0500
----------------------------------------------------------------------
.../elasticsearch/dao/ElasticsearchDao.java | 20 ++++++-
.../indexing/dao/search/SearchRequest.java | 9 +++
.../indexing/dao/SearchIntegrationTest.java | 63 ++++++++++++++++++++
3 files changed, 90 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f34affdb/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 217da84..01c113c 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -69,6 +69,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -114,8 +115,14 @@ public class ElasticsearchDao implements IndexDao {
.size(searchRequest.getSize())
.from(searchRequest.getFrom())
.query(new QueryStringQueryBuilder(searchRequest.getQuery()))
- .fetchSource(true)
+
.trackScores(true);
+ Optional<List<String>> fields = searchRequest.getFields();
+ if (fields.isPresent()) {
+ searchSourceBuilder.fields(fields.get());
+ } else {
+ searchSourceBuilder.fetchSource(true);
+ }
for (SortField sortField : searchRequest.getSort()) {
FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(sortField.getField());
if (sortField.getSortOrder() == org.apache.metron.indexing.dao.search.SortOrder.DESC) {
@@ -142,7 +149,16 @@ public class ElasticsearchDao implements IndexDao {
searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> {
SearchResult searchResult = new SearchResult();
searchResult.setId(searchHit.getId());
- searchResult.setSource(searchHit.getSource());
+ Map<String, Object> source;
+ if (fields.isPresent()) {
+ source = new HashMap<>();
+ searchHit.getFields().forEach((key, value) -> {
+ source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues());
+ });
+ } else {
+ source = searchHit.getSource();
+ }
+ searchResult.setSource(source);
searchResult.setScore(searchHit.getScore());
searchResult.setIndex(searchHit.getIndex());
return searchResult;
http://git-wip-us.apache.org/repos/asf/metron/blob/f34affdb/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java
index 897f918..3a72002 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java
@@ -28,6 +28,7 @@ public class SearchRequest {
private int size;
private int from;
private List<SortField> sort;
+ private List<String> fields;
private List<String> facetFields;
public SearchRequest() {
@@ -99,6 +100,14 @@ public class SearchRequest {
this.sort = sort;
}
+ public Optional<List<String>> getFields() {
+ return fields == null || fields.size() == 0 ? Optional.empty() : Optional.of(fields);
+ }
+
+ public void setFields(List<String> fields) {
+ this.fields = fields;
+ }
+
public Optional<List<String>> getFacetFields() {
return facetFields == null || facetFields.size() == 0 ? Optional.empty() : Optional.of(facetFields);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/f34affdb/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index e262269..2645df2 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -25,6 +25,10 @@ import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.integration.InMemoryComponent;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import org.junit.*;
import java.util.ArrayList;
@@ -213,6 +217,42 @@ public abstract class SearchIntegrationTest {
@Multiline
public static String exceededMaxResultsQuery;
+ /**
+ * {
+ * "fields": ["ip_src_addr"],
+ * "indices": ["bro", "snort"],
+ * "query": "*",
+ * "from": 0,
+ * "size": 10,
+ * "sort": [
+ * {
+ * "field": "timestamp",
+ * "sortOrder": "desc"
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ public static String fieldsQuery;
+
+ /**
+ * {
+ * "fields": ["ip_src_addr"],
+ * "indices": ["bro", "snort"],
+ * "query": "ip_src_addr:192.168.1.9",
+ * "from": 0,
+ * "size": 10,
+ * "sort": [
+ * {
+ * "field": "timestamp",
+ * "sortOrder": "desc"
+ * }
+ * ]
+ * }
+ */
+ @Multiline
+ public static String noResultsFieldsQuery;
+
protected static IndexDao dao;
protected static InMemoryComponent indexComponent;
@@ -464,6 +504,29 @@ public abstract class SearchIntegrationTest {
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
}
+ //Fields query
+ {
+ SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ for(int i = 0;i < 5;++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertNotNull(source.get("ip_src_addr"));
+ }
+ for(int i = 5;i < 10;++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertNotNull(source.get("ip_src_addr"));
+ }
+ }
+ //No results fields query
+ {
+ SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(0, response.getTotal());
+ }
}
@AfterClass