You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2020/05/14 06:03:18 UTC
[atlas] branch master updated: ATLAS-3758: support sort params in
FreeTextSearchProcessor
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 103e867 ATLAS-3758: support sort params in FreeTextSearchProcessor
103e867 is described below
commit 103e867cc126ddb84e64bf262791a01a55bee6e5
Author: Damian Warszawski <da...@ing.com>
AuthorDate: Thu May 14 00:47:29 2020 +0200
ATLAS-3758: support sort params in FreeTextSearchProcessor
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../atlas/discovery/EntitySearchProcessor.java | 9 +-
.../atlas/discovery/FreeTextSearchProcessor.java | 2 +-
.../apache/atlas/discovery/SearchProcessor.java | 28 +++++
.../discovery/FreeTextSearchProcessorTest.java | 138 +++++++++++++++++++++
.../resources/solr/core-template/solrconfig.xml | 17 +++
5 files changed, 185 insertions(+), 9 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index fb12244..56956e6 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -287,14 +287,7 @@ public class EntitySearchProcessor extends SearchProcessor {
final boolean isLastResultPage;
if (indexQuery != null) {
- Iterator<AtlasIndexQuery.Result> idxQueryResult;
-
- if (StringUtils.isEmpty(sortBy)) {
- idxQueryResult = indexQuery.vertices(qryOffset, limit);
- } else {
- Order qrySortOrder = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
- idxQueryResult = indexQuery.vertices(qryOffset, limit, sortBy, qrySortOrder);
- }
+ Iterator<AtlasIndexQuery.Result> idxQueryResult = executeIndexQuery(context, indexQuery, qryOffset, limit);
getVerticesFromIndexQueryResult(idxQueryResult, entityVertices);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
index 9850d8e..6e3a760 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -113,7 +113,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
break;
}
- Iterator<AtlasIndexQuery.Result> idxQueryResult = indexQuery.vertices(qryOffset, limit);
+ Iterator<AtlasIndexQuery.Result> idxQueryResult = executeIndexQuery(context, indexQuery, qryOffset, limit);
final boolean isLastResultPage;
int resultCount = 0;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 11eb7ca..804c694 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -19,6 +19,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
@@ -39,6 +40,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,6 +49,7 @@ import java.math.BigInteger;
import java.util.*;
import java.util.regex.Pattern;
+import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
@@ -986,4 +989,29 @@ public abstract class SearchProcessor {
return defaultValue;
}
+ private static String getSortByAttribute(SearchContext context) {
+ final AtlasEntityType entityType = context.getEntityType();
+ String sortBy = context.getSearchParameters().getSortBy();
+ AtlasStructType.AtlasAttribute sortByAttribute = entityType != null ? entityType.getAttribute(sortBy) : null;
+ if (sortByAttribute != null) {
+ return sortByAttribute.getVertexPropertyName();
+ }
+ return null;
+ }
+
+ private static Order getSortOrderAttribute(SearchContext context) {
+ SortOrder sortOrder = context.getSearchParameters().getSortOrder();
+ if (sortOrder == null) sortOrder = ASCENDING;
+
+ return sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
+ }
+
+ protected static Iterator<AtlasIndexQuery.Result> executeIndexQuery(SearchContext context, AtlasIndexQuery indexQuery, int qryOffset, int limit) {
+ String sortBy = getSortByAttribute(context);
+ if (sortBy != null && !sortBy.isEmpty()) {
+ Order sortOrder = getSortOrderAttribute(context);
+ return indexQuery.vertices(qryOffset, limit, sortBy, sortOrder);
+ }
+ return indexQuery.vertices(qryOffset, limit);
+ }
}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java
new file mode 100644
index 0000000..464b281
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import com.google.common.collect.Sets;
+import org.apache.atlas.BasicTestSetup;
+import org.apache.atlas.SortOrder;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.Collections;
+import java.util.List;
+
+import static org.testng.Assert.assertEquals;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class FreeTextSearchProcessorTest extends BasicTestSetup {
+
+ @Inject
+ private AtlasGraph graph;
+
+ @Inject
+ private AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private EntityGraphRetriever entityRetriever;
+
+ @BeforeClass
+ public void setup() {
+ setupTestData();
+ }
+
+ @Test
+ public void searchTablesByName() throws AtlasBaseException, InterruptedException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName("hive_table");
+ params.setQuery("sales");
+ params.setExcludeDeletedEntities(true);
+ params.setLimit(3);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ FreeTextSearchProcessor processor = new FreeTextSearchProcessor(context);
+
+ assertEquals(processor.getResultCount(), 3);
+ assertEquals(processor.execute().size(), 3);
+ }
+
+ @Test
+ public void searchByNameSortBy() throws AtlasBaseException, InterruptedException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName("hive_table");
+ params.setQuery("sales");
+ params.setExcludeDeletedEntities(true);
+ params.setLimit(3);
+ params.setSortBy("owner");
+ params.setSortOrder(SortOrder.ASCENDING);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ FreeTextSearchProcessor processor = new FreeTextSearchProcessor(context);
+
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(processor.getResultCount(), 3);
+ assertEquals(vertices.size(), 3);
+
+ AtlasVertex firstVertex = vertices.get(0);
+
+ String firstOwner = (String) entityRetriever.toAtlasEntityHeader(firstVertex, Sets.newHashSet(params.getSortBy())).getAttribute(params.getSortBy());
+
+ AtlasVertex secondVertex = vertices.get(1);
+ String secondOwner = (String) entityRetriever.toAtlasEntityHeader(secondVertex, Sets.newHashSet(params.getSortBy())).getAttribute(params.getSortBy());
+
+ assertEquals(firstOwner, "Jane BI");
+ assertEquals(secondOwner, "Joe");
+ }
+
+ @Test
+ public void emptySearch() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName("hive_table");
+ params.setQuery("not_exists");
+ params.setExcludeDeletedEntities(true);
+ params.setLimit(3);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ FreeTextSearchProcessor processor = new FreeTextSearchProcessor(context);
+
+ assertEquals(processor.getResultCount(), 0);
+ assertEquals(processor.execute().size(), 0);
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "not_exists_type: Unknown/invalid typename")
+ public void searchByNonExistingClassification() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName("not_exists_type");
+ params.setQuery("aaa");
+ params.setExcludeDeletedEntities(true);
+ params.setLimit(3);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+ new FreeTextSearchProcessor(context);
+ }
+
+ @AfterClass
+ public void teardown() {
+ AtlasGraphProvider.cleanup();
+ }
+
+}
\ No newline at end of file
diff --git a/test-tools/src/main/resources/solr/core-template/solrconfig.xml b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
index 9264f99..ef74a0a 100644
--- a/test-tools/src/main/resources/solr/core-template/solrconfig.xml
+++ b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
@@ -428,6 +428,23 @@
</requestHandler>
+ <requestHandler name="/freetext" class="solr.SearchHandler">
+ <!-- default values for query parameters can be specified, these
+ will be overridden by parameters in the request
+ -->
+ <lst name="defaults">
+ <str name="defType">edismax</str>
+ <str name="qf">7shx_t^3 7qx1_t^3 3pj9_t^3 7cp1_t^3 43r9_t^3 cn9_t^1 79j9_t^3 3vut_t^3 3thh_t^3 3qbp_s^3 7wg5_t^3 7b45_t^3 3nyd_s^3 7vnp_t^3 7rph_t^3 8ydh_t^3 x6t_t^3 7tad_t^3 7jt1_t^3 3mdh_s^3 7e9x_t^3 3rwl_s^3 93wl_t^3 yrp_t^10 7pc5_t^3 7u2t_t^3 3xfp_t^3 426d_t^3 iyt_t^10 3g1x_t^3 40lh_t^3</str>
+ <str name="hl.fl">*</str>
+ <bool name="hl.requireFieldMatch">true</bool>
+ <bool name="lowercaseOperators">true</bool>
+ <str name="echoParams">explicit</str>
+ <int name="rows">10</int>
+
+ </lst>
+
+ </requestHandler>
+
<!-- A request handler that returns indented JSON by default -->
<requestHandler name="/query" class="solr.SearchHandler">
<lst name="defaults">