You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/07/05 23:27:53 UTC
[atlas] branch master updated: ATLAS-3308: enhanced Quicksearch API
to support parameters via POST method
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new a7d8044 ATLAS-3308: enhanced Quicksearch API to support parameters via POST method
a7d8044 is described below
commit a7d8044920a81f3dbb0b8e688ddecf68505998f8
Author: skoritala <sk...@cloudera.com>
AuthorDate: Wed Jun 19 16:46:42 2019 -0700
ATLAS-3308: enhanced Quicksearch API to support parameters via POST method
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../org/apache/atlas/repository/Constants.java | 4 +-
.../repository/graphdb/AggregationContext.java | 93 ++++++
.../atlas/repository/graphdb/AtlasGraph.java | 7 -
.../repository/graphdb/AtlasGraphIndexClient.java | 10 +-
.../repository/graphdb/AtlasGraphManagement.java | 3 +-
graphdb/janus/pom.xml | 6 +
.../repository/graphdb/janus/AtlasJanusGraph.java | 19 +-
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 156 +++++----
.../graphdb/janus/AtlasJanusGraphManagement.java | 24 +-
.../graphdb/janus/AtlasSolrQueryBuilder.java | 347 +++++++++++++++++++++
.../janus/AtlasJanusGraphIndexClientTest.java | 57 ++--
.../graphdb/janus/AtlasSolrQueryBuilderTest.java | 254 +++++++++++++++
.../src/test/resources/searchParameters3.json | 36 +++
.../src/test/resources/searchParametersGT.json | 26 ++
.../src/test/resources/searchParametersGTE.json | 31 ++
.../src/test/resources/searchParametersLT.json | 26 ++
.../src/test/resources/searchParametersLTE.json | 31 ++
.../test/resources/searchParametersStartsWith.json | 22 ++
.../src/test/resources/searchparameters0.json | 22 ++
.../src/test/resources/searchparameters1AND.json | 27 ++
.../src/test/resources/searchparameters1OR.json | 27 ++
.../src/test/resources/searchparameters2AND.json | 32 ++
.../src/test/resources/searchparameters2OR.json | 32 ++
.../atlas/listener/TypeDefChangeListener.java | 1 +
.../model/discovery/QuickSearchParameters.java | 138 ++++++++
.../org/apache/atlas/store/AtlasTypeDefStore.java | 2 +
.../org/apache/atlas/type/AtlasStructType.java | 14 +-
.../org/apache/atlas/type/AtlasTypeRegistry.java | 37 ++-
.../atlas/discovery/AtlasDiscoveryService.java | 14 +-
.../atlas/discovery/EntityDiscoveryService.java | 84 +++--
.../apache/atlas/discovery/SearchAggregator.java | 14 +-
.../atlas/discovery/SearchAggregatorImpl.java | 114 +++++--
.../repository/graph/GraphBackedSearchIndexer.java | 178 ++++++++---
.../atlas/repository/graph/SolrIndexHelper.java | 65 ++--
.../bootstrap/AtlasTypeDefStoreInitializer.java | 2 +-
.../store/graph/AtlasTypeDefGraphStore.java | 11 +
.../org/apache/atlas/web/rest/DiscoveryREST.java | 117 +++++--
37 files changed, 1793 insertions(+), 290 deletions(-)
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java
index 9f7364c..eb536cd 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -47,8 +47,8 @@ public final class Constants {
*/
public static final String ENTITY_TYPE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName");
public static final String TYPE_NAME_INTERNAL = INTERNAL_PROPERTY_KEY_PREFIX + "internal";
- public static final String ASSET_OWNER_PROPERTY_KEY = "Asset.owner";
-
+ public static final String ASSET_ENTITY_TYPE = "Asset";
+ public static final String OWNER_ATTRIBUTE = "owner";
/**
* Entity type's super types property key.
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
new file mode 100644
index 0000000..6006fef
--- /dev/null
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb;
+
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+
+import java.util.Map;
+import java.util.Set;
+
+public class AggregationContext {
+ private final String queryString;
+ private final FilterCriteria filterCriteria;
+ private final AtlasEntityType searchForEntityType;
+ private final Set<String> aggregationFieldNames;
+ private final Set<AtlasAttribute> aggregationAttributes;
+ private final Map<String, String> indexFieldNameCache;
+ private final boolean excludeDeletedEntities;
+ private final boolean includeSubTypes;
+
+ /**
+ * @param queryString the query string whose aggregation metrics need to be retrieved.
+ * @param searchForEntityType
+ * @param aggregationFieldNames the set of aggregation fields.
+ * @param indexFieldNameCache
+ * @param excludeDeletedEntities a boolean flag to indicate if the deleted entities need to be excluded in search
+ */
+ public AggregationContext(String queryString,
+ FilterCriteria filterCriteria,
+ AtlasEntityType searchForEntityType,
+ Set<String> aggregationFieldNames,
+ Set<AtlasAttribute> aggregationAttributes,
+ Map<String, String> indexFieldNameCache,
+ boolean excludeDeletedEntities,
+ boolean includeSubTypes) {
+ this.queryString = queryString;
+ this.filterCriteria = filterCriteria;
+ this.searchForEntityType = searchForEntityType;
+ this.aggregationFieldNames = aggregationFieldNames;
+ this.aggregationAttributes = aggregationAttributes;
+ this.indexFieldNameCache = indexFieldNameCache;
+ this.excludeDeletedEntities = excludeDeletedEntities;
+ this.includeSubTypes = includeSubTypes;
+ }
+
+ public String getQueryString() {
+ return queryString;
+ }
+
+ public FilterCriteria getFilterCriteria() {
+ return filterCriteria;
+ }
+
+ public AtlasEntityType getSearchForEntityType() {
+ return searchForEntityType;
+ }
+
+ public Set<String> getAggregationFieldNames() {
+ return aggregationFieldNames;
+ }
+
+ public Set<AtlasAttribute> getAggregationAttributes() {
+ return aggregationAttributes;
+ }
+
+ public Map<String, String> getIndexFieldNameCache() {
+ return indexFieldNameCache;
+ }
+
+ public boolean isExcludeDeletedEntities() {
+ return excludeDeletedEntities;
+ }
+
+ public boolean isIncludeSubTypes() {
+ return includeSubTypes;
+ }
+}
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
index 53a5a7a..7bd5f2d 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
@@ -340,13 +340,6 @@ public interface AtlasGraph<V, E> {
boolean isMultiProperty(String name);
/**
- * return the encoded name used for the attribute identified by property key and index name.
- * @param propertyKey the property key of attributed
- * @param indexName the name of the index containing the property.
- * @return the encoded name of the property.
- */
- String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName);
- /**
* Create Index query parameter for use with atlas graph.
* @param parameterName the name of the parameter that needs to be passed to index layer.
* @param parameterValue the value of the paratemeter that needs to be passed to the index layer.
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
index 53c7cb1..09a88d9 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
@@ -18,6 +18,8 @@
package org.apache.atlas.repository.graphdb;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.type.AtlasEntityType;
import java.util.List;
import java.util.Map;
@@ -30,11 +32,9 @@ public interface AtlasGraphIndexClient {
/**
* Gets aggregated metrics for the given query string and aggregation field names.
- * @param queryString the query string whose aggregation metrics need to be retrieved.
- * @param propertyKeyNames the set of aggregation fields.
* @return A map of aggregation field to value-count pairs.
*/
- Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(String queryString, Set<String> propertyKeyNames);
+ Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(AggregationContext aggregationContext);
/**
* Returns top 5 suggestions for the given prefix string.
@@ -46,9 +46,9 @@ public interface AtlasGraphIndexClient {
/**
* The implementers should apply the search weights for the passed in properties.
* @param collectionName the name of the collection for which the search weight needs to be applied
- * @param propertyName2SearchWeightMap the map containing search weights from property name to search weights.
+ * @param indexFieldName2SearchWeightMap the map containing search weights from index field name to search weights.
*/
- void applySearchWeight(String collectionName, Map<String, Integer> propertyName2SearchWeightMap);
+ void applySearchWeight(String collectionName, Map<String, Integer> indexFieldName2SearchWeightMap);
/**
* The implementors should take the passed in list of suggestion properties for suggestions functionality.
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index 6fe3460..08923f8 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -155,8 +155,9 @@ public interface AtlasGraphManagement {
*
* @param vertexIndex
* @param propertyKey
+ * @return the index field name used for the given property
*/
- void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
+ String addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
/**
* Gets the index field name for the vertex property.
diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml
index 48e730f..adc9a9e 100644
--- a/graphdb/janus/pom.xml
+++ b/graphdb/janus/pom.xml
@@ -236,6 +236,12 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index 499e8d1..613a714 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -75,12 +75,13 @@ import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.
*/
public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraph.class);
- private static Configuration APPLICATION_PROPERTIES = null;
+
+ private static final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
+ private static Configuration APPLICATION_PROPERTIES = null;
private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
private final Set<String> multiProperties = new HashSet<>();
private final StandardJanusGraph janusGraph;
- private final Parameter[] EMPTY_PARAMETER_ARRAY = new Parameter[0];
public AtlasJanusGraph() {
this(getGraphInstance());
@@ -198,7 +199,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
try {
initApplicationProperties();
- return new AtlasJanusGraphIndexClient(this, APPLICATION_PROPERTIES);
+ return new AtlasJanusGraphIndexClient(APPLICATION_PROPERTIES);
} catch (Exception e) {
LOG.error("Error encountered in creating Graph Index Client.", e);
throw new AtlasException(e);
@@ -417,6 +418,13 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
+ String getIndexFieldName(AtlasPropertyKey propertyKey, JanusGraphIndex graphIndex) {
+ PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
+
+ return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, EMPTY_PARAMETER_ARRAY, graphIndex.getBackingIndex());
+ }
+
+
private String getIndexQueryPrefix() {
final String ret;
@@ -524,9 +532,4 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return convertGremlinValue(input);
}
}
-
- public String getIndexFieldName(AtlasPropertyKey propertyKey, String indexName) {
- PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
- return janusGraph.getIndexSerializer().getDefaultFieldName(janusKey, EMPTY_PARAMETER_ARRAY, indexName);
- }
}
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index 3a64d31..113ea6c 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -21,10 +21,10 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AggregationContext;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
-import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.configuration.Configuration;
@@ -54,18 +54,17 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
private static final FreqComparator FREQ_COMPARATOR = new FreqComparator();
private static final int DEFAULT_SUGGESTION_COUNT = 5;
+ private static final int MIN_FACET_COUNT_REQUIRED = 1;
- private final AtlasGraph graph;
private final Configuration configuration;
- public AtlasJanusGraphIndexClient(AtlasGraph graph, Configuration configuration) {
- this.graph = graph;
+ public AtlasJanusGraphIndexClient(Configuration configuration) {
this.configuration = configuration;
}
@Override
- public void applySearchWeight(String collectionName, Map<String, Integer> propertyName2SearchWeightMap) {
+ public void applySearchWeight(String collectionName, Map<String, Integer> indexFieldName2SearchWeightMap) {
SolrClient solrClient = null;
try {
@@ -98,7 +97,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
try {
LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- updateFreeTextRequestHandler(solrClient, collectionName, propertyName2SearchWeightMap);
+ updateFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap);
LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
@@ -112,7 +111,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
try {
LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- createFreeTextRequestHandler(solrClient, collectionName, propertyName2SearchWeightMap);
+ createFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap);
LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
return;
@@ -133,10 +132,10 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
}
+
@Override
- public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(String queryString, Set<String> propertyKeyNames) {
- SolrClient solrClient = null;
- AtlasGraphManagement management = graph.getManagementSystem();
+ public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(AggregationContext aggregationContext) {
+ SolrClient solrClient = null;
try {
solrClient = Solr6Index.getSolrClient(); // get solr client using same settings as that of Janus Graph
@@ -147,28 +146,69 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return Collections.EMPTY_MAP;
}
- if (propertyKeyNames.size() <= 0) {
- LOG.warn("There no fields provided for aggregation purpose.");
+ Set<String> aggregationCommonFields = aggregationContext.getAggregationFieldNames();
+ Set<AtlasAttribute> aggregationAttributes = aggregationContext.getAggregationAttributes();
+ Map<String, String> indexFieldNameCache = aggregationContext.getIndexFieldNameCache();
- return Collections.EMPTY_MAP;
+ if (CollectionUtils.isEmpty(aggregationCommonFields)) {
+ LOG.warn("There are no fields provided for aggregation purpose.");
+
+ if (CollectionUtils.isEmpty(aggregationAttributes)) {
+ LOG.warn("There are no aggregation fields or attributes are provided. Will return empty metrics.");
+
+ return Collections.EMPTY_MAP;
+ }
+ }
+
+ if (CollectionUtils.isEmpty(aggregationAttributes)) {
+ LOG.warn("There no attributes provided for aggregation purpose.");
}
- SolrQuery solrQuery = new SolrQuery();
- Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>();
+ Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>();
+ AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder();
- solrQuery.setQuery(queryString);
+ solrQueryBuilder.withEntityType(aggregationContext.getSearchForEntityType())
+ .withQueryString(aggregationContext.getQueryString())
+ .withCriteria(aggregationContext.getFilterCriteria())
+ .withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities())
+ .withIncludeSubTypes(aggregationContext.isIncludeSubTypes())
+ .withCommonIndexFieldNames(indexFieldNameCache);
+
+
+ SolrQuery solrQuery = new SolrQuery();
+ String finalSolrQuery = solrQueryBuilder.build();
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Final query string prepared is {}", finalSolrQuery);
+ }
+
+ solrQuery.setQuery(finalSolrQuery);
solrQuery.setRequestHandler(FREETEXT_REQUEST_HANDLER);
- for (String propertyName : propertyKeyNames) {
- AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
- String indexFieldName = management.getIndexFieldName(VERTEX_INDEX, propertyKey);
+ if (CollectionUtils.isNotEmpty(aggregationCommonFields)) {
+ for (String propertyName : aggregationCommonFields) {
+ // resolve index field names for aggregation fields.
+ String indexFieldName = indexFieldNameCache.get(propertyName);
+
+ indexFieldName2PropertyKeyNameMap.put(indexFieldName, propertyName);
- indexFieldName2PropertyKeyNameMap.put(indexFieldName, propertyName);
+ solrQuery.addFacetField(indexFieldName);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(aggregationAttributes)) {
+ for (AtlasAttribute attribute : aggregationAttributes) {
+ String indexFieldName = attribute.getIndexFieldName();
+
+ indexFieldName2PropertyKeyNameMap.put(indexFieldName, attribute.getQualifiedName());
- solrQuery.addFacetField(indexFieldName);
+ solrQuery.addFacetField(indexFieldName);
+ }
}
- QueryResponse queryResponse = solrClient.query(VERTEX_INDEX, solrQuery);
+ solrQuery.setFacetMinCount(MIN_FACET_COUNT_REQUIRED);
+
+ QueryResponse queryResponse = solrClient.query(VERTEX_INDEX, solrQuery, SolrRequest.METHOD.POST);
List<FacetField> facetFields = queryResponse == null ? null : queryResponse.getFacetFields();
if (CollectionUtils.isNotEmpty(facetFields)) {
@@ -183,6 +223,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
entries.add(new AtlasAggregationEntry(count.getName(), count.getCount()));
}
+ //get the original propertyName from the index field name.
String propertyKeyName = indexFieldName2PropertyKeyNameMap.get(indexFieldName);
ret.put(propertyKeyName, entries);
@@ -191,10 +232,8 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return ret;
}
} catch (Exception e) {
- LOG.error("Error enocunted in getting the aggregation metrics. Will return empty agregation.", e);
+ LOG.error("Error encountered in getting the aggregation metrics. Will return empty aggregation.", e);
} finally {
- graphManagementCommit(management);
-
Solr6Index.releaseSolrClient(solrClient);
}
@@ -215,8 +254,9 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
//update the request handler
- performRequestHandlerAction(collectionName, solrClient,
- generatePayLoadForSuggestions(generateSuggestionsString(collectionName, suggestionProperties)));
+ performRequestHandlerAction(collectionName,
+ solrClient,
+ generatePayLoadForSuggestions(generateSuggestionsString(suggestionProperties)));
} catch (Throwable t) {
String msg = String.format("Error encountered in creating the request handler '%s' for collection '%s'", Constants.TERMS_REQUEST_HANDLER, collectionName);
@@ -373,61 +413,55 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
- private String generateSearchWeightString(String indexName, Map<String, Integer> propertyName2SearchWeightMap) {
- StringBuilder searchWeightBuilder = new StringBuilder();
- AtlasGraphManagement management = graph.getManagementSystem();
-
- try {
- for (Map.Entry<String, Integer> entry : propertyName2SearchWeightMap.entrySet()) {
- AtlasPropertyKey propertyKey = management.getPropertyKey(entry.getKey());
- String indexFieldName = management.getIndexFieldName(indexName, propertyKey);
-
- searchWeightBuilder.append(" ")
- .append(indexFieldName)
- .append("^")
- .append(entry.getValue().intValue());
- }
- } finally {
- graphManagementCommit(management);
+ @VisibleForTesting
+ protected static String generateSearchWeightString(Map<String, Integer> indexFieldName2SearchWeightMap) {
+ StringBuilder searchWeightBuilder = new StringBuilder();
+
+ for (Map.Entry<String, Integer> entry : indexFieldName2SearchWeightMap.entrySet()) {
+ searchWeightBuilder.append(" ")
+ .append(entry.getKey())
+ .append("^")
+ .append(entry.getValue().intValue());
}
return searchWeightBuilder.toString();
}
- private String generateSuggestionsString(String collectionName, List<String> suggestionProperties) {
- StringBuilder ret = new StringBuilder();
- AtlasGraphManagement management = graph.getManagementSystem();
+ @VisibleForTesting
+ protected static String generateSuggestionsString(List<String> suggestionIndexFieldNames) {
+ StringBuilder ret = new StringBuilder();
+ Iterator<String> iterator = suggestionIndexFieldNames.iterator();
- try {
- for (String propertyName : suggestionProperties) {
- AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
- String indexFieldName = management.getIndexFieldName(collectionName, propertyKey);
+ while(iterator.hasNext()) {
+ ret.append("'").append(iterator.next()).append("'");
- ret.append("'").append(indexFieldName).append("', ");
+ if(iterator.hasNext()) {
+ ret.append(", ");
}
- } finally {
- graphManagementCommit(management);
}
return ret.toString();
}
- private V2Response updateFreeTextRequestHandler(SolrClient solrClient, String collectionName, Map<String, Integer> propertyName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
- String searchWeightString = generateSearchWeightString(collectionName, propertyName2SearchWeightMap);
+ private V2Response updateFreeTextRequestHandler(SolrClient solrClient, String collectionName,
+ Map<String, Integer> indexFieldName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
+ String searchWeightString = generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("update-requesthandler", searchWeightString);
return performRequestHandlerAction(collectionName, solrClient, payLoadString);
}
- private V2Response createFreeTextRequestHandler(SolrClient solrClient, String collectionName, Map<String, Integer> propertyName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
- String searchWeightString = generateSearchWeightString(collectionName, propertyName2SearchWeightMap);
+ private V2Response createFreeTextRequestHandler(SolrClient solrClient, String collectionName,
+ Map<String, Integer> indexFieldName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
+ String searchWeightString = generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("create-requesthandler", searchWeightString);
return performRequestHandlerAction(collectionName, solrClient, payLoadString);
}
- private V2Response performRequestHandlerAction(String collectionName, SolrClient solrClient, String actionPayLoad)
- throws IOException, SolrServerException, AtlasBaseException {
+ private V2Response performRequestHandlerAction(String collectionName,
+ SolrClient solrClient,
+ String actionPayLoad) throws IOException, SolrServerException, AtlasBaseException {
V2Request v2Request = new V2Request.Builder(String.format("/collections/%s/config", collectionName))
.withMethod(SolrRequest.METHOD.POST)
.withPayload(actionPayLoad)
@@ -487,7 +521,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return v2Response;
}
- static final class TermFreq {
+ static final class TermFreq {
private final String term;
private long freq;
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index feee036..b6889c8 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -23,10 +23,7 @@ import org.apache.tinkerpop.gremlin.structure.Direction;
import org.janusgraph.core.Cardinality;
import org.janusgraph.core.EdgeLabel;
import org.janusgraph.core.PropertyKey;
-import org.janusgraph.core.schema.Mapping;
-import org.janusgraph.core.schema.PropertyKeyMaker;
-import org.janusgraph.core.schema.JanusGraphIndex;
-import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.*;
import org.janusgraph.core.schema.JanusGraphManagement.IndexBuilder;
import org.janusgraph.graphdb.internal.Token;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
@@ -195,19 +192,24 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
}
@Override
- public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
- PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
- JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
+ public String addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
+ PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
+ JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
+
+ management.addIndexKey(janusGraphIndex, janusKey);
+
+ String encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex);
- management.addIndexKey(vertexIndex, janusKey);
- String encodedName = graph.getIndexFieldName(propertyKey, vertexIndex.getBackingIndex());
LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(), encodedName);
+
+ return encodedName;
}
@Override
public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey) {
- JanusGraphIndex index = management.getGraphIndex(indexName);
- return graph.getIndexFieldName(propertyKey, index.getBackingIndex());
+ JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
+
+ return graph.getIndexFieldName(propertyKey, janusGraphIndex);
}
@Override
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
new file mode 100644
index 0000000..05db148
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
@@ -0,0 +1,347 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.Operator;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class AtlasSolrQueryBuilder {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
+
+ private AtlasEntityType entityType;
+ private String queryString;
+ private FilterCriteria criteria;
+ private boolean excludeDeletedEntities;
+ private boolean includeSubtypes;
+ private Map<String, String> indexFieldNameCache;
+
+
+ public AtlasSolrQueryBuilder() {
+ }
+
+ public AtlasSolrQueryBuilder withEntityType(AtlasEntityType searchForEntityType) {
+ this.entityType = searchForEntityType;
+
+ return this;
+ }
+
+ public AtlasSolrQueryBuilder withQueryString(String queryString) {
+ this.queryString = queryString;
+
+ return this;
+ }
+
+ public AtlasSolrQueryBuilder withCriteria(FilterCriteria criteria) {
+ this.criteria = criteria;
+
+ return this;
+ }
+
+ public AtlasSolrQueryBuilder withExcludedDeletedEntities(boolean excludeDeletedEntities) {
+ this.excludeDeletedEntities = excludeDeletedEntities;
+
+ return this;
+ }
+
+ public AtlasSolrQueryBuilder withIncludeSubTypes(boolean includeSubTypes) {
+ this.includeSubtypes = includeSubTypes;
+
+ return this;
+ }
+
+ public AtlasSolrQueryBuilder withCommonIndexFieldNames(Map<String, String> indexFieldNameCache) {
+ this.indexFieldNameCache = indexFieldNameCache;
+
+ return this;
+ }
+
+ public String build() throws AtlasBaseException {
+ StringBuilder queryBuilder = new StringBuilder();
+ boolean isAndNeeded = false;
+
+ if (queryString != null ) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initial query string is {}.", queryString);
+ }
+
+ queryBuilder.append("+").append(queryString.trim()).append(" ");
+
+ isAndNeeded = true;
+ }
+
+ if (excludeDeletedEntities) {
+ if (isAndNeeded) {
+ queryBuilder.append(" AND ");
+ }
+
+ dropDeletedEntities(queryBuilder);
+
+ isAndNeeded = true;
+ }
+
+ if (entityType != null) {
+ if (isAndNeeded) {
+ queryBuilder.append(" AND ");
+ }
+
+ buildForEntityType(queryBuilder);
+
+ isAndNeeded = true;
+ }
+
+ if (criteria != null) {
+ StringBuilder attrFilterQueryBuilder = new StringBuilder();
+
+ withCriteria(attrFilterQueryBuilder, criteria);
+
+ if (attrFilterQueryBuilder.length() != 0) {
+ if (isAndNeeded) {
+ queryBuilder.append(" AND ");
+ }
+
+ queryBuilder.append(" ").append(attrFilterQueryBuilder.toString());
+ }
+ }
+
+ return queryBuilder.toString();
+ }
+
+ private void buildForEntityType(StringBuilder queryBuilder) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Search is being done for entities of type {}", entityType.getTypeName());
+ }
+
+ String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY);
+
+ queryBuilder.append(" +")
+ .append(typeIndexFieldName)
+ .append(":(")
+ .append(entityType.getTypeName())
+ .append(" ");
+
+ if (includeSubtypes) {
+ Set<String> allSubTypes = entityType.getAllSubTypes();
+
+ if(allSubTypes.size() != 0 ) {
+ for(String subTypeName: allSubTypes) {
+ queryBuilder.append(subTypeName).append(" ");
+ }
+ }
+ }
+
+ queryBuilder.append(" ) ");
+ }
+
+ private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("excluding the deleted entities.");
+ }
+
+ String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY);
+
+ if (indexFieldName == null) {
+ String msg = String.format("There is no index field name defined for attribute '%s' for entity '%s'",
+ Constants.STATE_PROPERTY_KEY,
+ entityType.getTypeName());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+
+ queryBuilder.append(" -").append(indexFieldName).append(":").append(AtlasEntity.Status.DELETED.name());
+ }
+
+ private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException {
+ List<FilterCriteria> criterion = criteria.getCriterion();
+
+ if(criterion == null || CollectionUtils.isEmpty(criteria.getCriterion())) { // no child criterion
+ withPropertyCondition(queryBuilder, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
+ } else {
+ beginCriteria(queryBuilder);
+
+ for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) {
+ FilterCriteria childCriteria = iterator.next();
+
+ withCriteria(queryBuilder, childCriteria);
+
+ if (iterator.hasNext()) {
+ withCondition(queryBuilder, criteria.getCondition().name());
+ }
+ }
+
+ endCriteria(queryBuilder);
+ }
+
+ return this;
+ }
+
+ private void withPropertyCondition(StringBuilder queryBuilder, String attributeName, Operator operator, String attributeValue) throws AtlasBaseException {
+ if (StringUtils.isNotEmpty(attributeName) && operator != null) {
+ if (attributeValue != null) {
+ attributeValue = attributeValue.trim();
+ }
+
+ AtlasAttribute attribute = entityType.getAttribute(attributeName);
+
+ if (attribute == null) {
+ String msg = String.format("Received unknown attribute '%s' for type '%s'.", attributeName, entityType.getTypeName());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+
+ String indexFieldName = attribute.getIndexFieldName();
+
+ if (indexFieldName == null) {
+ String msg = String.format("Received non-index attribute %s for type %s.", attributeName, entityType.getTypeName());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+
+ switch (operator) {
+ case EQ:
+ withEqual(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case NEQ:
+ withNotEqual(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case STARTS_WITH:
+ withStartsWith(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case ENDS_WITH:
+ withEndsWith(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case CONTAINS:
+ withContains(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case IS_NULL:
+ withIsNull(queryBuilder, indexFieldName);
+ break;
+ case NOT_NULL:
+ withIsNotNull(queryBuilder, indexFieldName);
+ break;
+ case LT:
+ withLessthan(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case GT:
+ withGreaterThan(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case LTE:
+ withLessthanOrEqual(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case GTE:
+ withGreaterThanOrEqual(queryBuilder, indexFieldName, attributeValue);
+ break;
+ case IN:
+ case LIKE:
+ case CONTAINS_ANY:
+ case CONTAINS_ALL:
+ default:
+ String msg = String.format("%s is not supported operation.", operator.getSymbol());
+ LOG.error(msg);
+ throw new AtlasBaseException(msg);
+ }
+ }
+ }
+
+ private void beginCriteria(StringBuilder queryBuilder) {
+ queryBuilder.append("( ");
+ }
+
+ private void endCriteria(StringBuilder queryBuilder) {
+ queryBuilder.append(" )");
+ }
+
+ private void withEndsWith(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ queryBuilder.append("+").append(indexFieldName).append(":*").append(attributeValue).append(" ");
+ }
+
+ private void withStartsWith(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ queryBuilder.append("+").append(indexFieldName).append(":").append(attributeValue).append("* ");
+ }
+
+ private void withNotEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ queryBuilder.append("-").append(indexFieldName).append(":").append(attributeValue).append(" ");
+ }
+
+ private void withEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ queryBuilder.append("+").append(indexFieldName).append(":").append(attributeValue).append(" ");
+ }
+
+ private void withGreaterThan(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ //{ == exclusive
+ //] == inclusive
+ //+__timestamp_l:{<attributeValue> TO *]
+ queryBuilder.append("+").append(indexFieldName).append(":{ ").append(attributeValue).append(" TO * ] ");
+ }
+
+ private void withGreaterThanOrEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ //[ == inclusive
+ //] == inclusive
+ //+__timestamp_l:[<attributeValue> TO *]
+ queryBuilder.append("+").append(indexFieldName).append(":[ ").append(attributeValue).append(" TO * ] ");
+ }
+
+ private void withLessthan(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ //[ == inclusive
+ //} == exclusive
+ //+__timestamp_l:[* TO <attributeValue>}
+ queryBuilder.append("+").append(indexFieldName).append(":[ * TO").append(attributeValue).append("} ");
+ }
+
+ private void withLessthanOrEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ //[ == inclusive
+ //[ == inclusive
+ //+__timestamp_l:[* TO <attributeValue>]
+ queryBuilder.append("+").append(indexFieldName).append(":[ * TO ").append(attributeValue).append(" ] ");
+ }
+
+ private void withContains(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ queryBuilder.append("+").append(indexFieldName).append(":*").append(attributeValue).append("* ");
+ }
+
+ private void withIsNull(StringBuilder queryBuilder, String indexFieldName) {
+ queryBuilder.append("-").append(indexFieldName).append(":*").append(" ");
+ }
+
+ private void withIsNotNull(StringBuilder queryBuilder, String indexFieldName) {
+ queryBuilder.append("+").append(indexFieldName).append(":*").append(" ");
+ }
+
+ private void withCondition(StringBuilder queryBuilder, String condition) {
+ queryBuilder.append(" ").append(condition).append(" ");
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java
index 3796c68..7fdb50f 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java
@@ -20,6 +20,7 @@ package org.apache.atlas.repository.graphdb.janus;
import org.testng.Assert;
import org.testng.annotations.Test;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -27,81 +28,99 @@ import java.util.Map;
public class AtlasJanusGraphIndexClientTest {
@Test
- public void testGetTop5TermsAsendingInput() {
+ public void testGetTopTermsAsendingInput() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 12, 15);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
- validateOrder(top5Terms, 2,1,0);
+ assertOrder(top5Terms, 2,1,0);
}
@Test
- public void testGetTop5TermsAsendingInput2() {
+ public void testGetTopTermsAsendingInput2() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 12, 15, 20, 25, 26, 30, 40);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
- validateOrder(top5Terms, 7, 6, 5, 4, 3);
+ assertOrder(top5Terms, 7, 6, 5, 4, 3);
}
@Test
- public void testGetTop5TermsDescendingInput() {
+ public void testGetTopTermsDescendingInput() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 9, 8);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
- validateOrder(top5Terms, 0, 1, 2);
+ assertOrder(top5Terms, 0, 1, 2);
}
@Test
- public void testGetTop5TermsDescendingInput2() {
+ public void testGetTopTermsDescendingInput2() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 9, 8, 7, 6, 5, 4, 3, 2);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
- validateOrder(top5Terms, 0, 1, 2, 3, 4);
+ assertOrder(top5Terms, 0, 1, 2, 3, 4);
}
@Test
- public void testGetTop5TermsRandom() {
+ public void testGetTopTermsRandom() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 19, 28, 27, 16, 1, 30, 3, 36);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//10, 19, 28, 27, 16, 1, 30, 3, 36
//0, 1, 2, 3, 4, 5, 6, 7, 8
- validateOrder(top5Terms, 8, 6, 2, 3, 1);
+ assertOrder(top5Terms, 8, 6, 2, 3, 1);
}
@Test
- public void testGetTop5TermsRandom2() {
+ public void testGetTopTermsRandom2() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 36, 19, 28, 27, 16, 1, 30, 3, 10);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//36, 19, 28, 27, 16, 1, 30, 3, 10
//0, 1, 2, 3, 4, 5, 6, 7, 8
- validateOrder(top5Terms, 0, 6, 2, 3, 1);
+ assertOrder(top5Terms, 0, 6, 2, 3, 1);
}
@Test
- public void testGetTop5TermsRandom3() {
+ public void testGetTopTermsRandom3() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 36, 36, 28, 27, 16, 1, 30, 3, 10);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//36, 36, 28, 27, 16, 1, 30, 3, 10
//0, 1, 2, 3, 4, 5, 6, 7, 8
- validateOrder(top5Terms, 0, 1, 6, 2, 3);
+ assertOrder(top5Terms, 0, 1, 6, 2, 3);
}
@Test
- public void testGetTop5TermsRandom4() {
+ public void testGetTopTermsRandom4() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 10, 10, 28, 27, 16, 1, 30, 36, 36);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//10, 10, 28, 27, 16, 1, 30, 36, 36
//0, 1, 2, 3, 4, 5, 6, 7, 8
- validateOrder(top5Terms, 7, 8, 6, 2, 3);
+ assertOrder(top5Terms, 7, 8, 6, 2, 3);
}
@Test
- public void testGetTop5TermsRandom5() {
+ public void testGetTopTermsRandom5() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms( 36, 10, 28, 27, 16, 1, 30, 36, 36);
List<String> top5Terms = AtlasJanusGraphIndexClient.getTopTerms(terms);
//36, 10, 28, 27, 16, 1, 30, 36, 36
//0, 1, 2, 3, 4, 5, 6, 7, 8
- validateOrder(top5Terms, 0, 7, 8, 6, 2);
+ assertOrder(top5Terms, 0, 7, 8, 6, 2);
}
+ @Test
+ public void testGenerateSuggestionString() {
+ List<String> fields = new ArrayList<>();
+ fields.add("one");
+ fields.add("two");
+ fields.add("three");
+ String generatedString = AtlasJanusGraphIndexClient.generateSuggestionsString(fields);
+ Assert.assertEquals(generatedString, "'one', 'two', 'three'");
+ }
+ @Test
+ public void testGenerateSearchWeightString() {
+ Map<String, Integer> fields = new HashMap<>();
+ fields.put("one", 10);
+ fields.put("two", 1);
+ fields.put("three", 15);
+ String generatedString = AtlasJanusGraphIndexClient.generateSearchWeightString(fields);
+ Assert.assertEquals(generatedString, " one^10 two^1 three^15");
+ }
- private void validateOrder(List<String> topTerms, int ... indices) {
+ private void assertOrder(List<String> topTerms, int ... indices) {
Assert.assertEquals(topTerms.size(), indices.length);
int i = 0;
for(String term: topTerms) {
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
new file mode 100644
index 0000000..2af1818
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.Assert;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.when;
+
+
+public class AtlasSolrQueryBuilderTest {
+
+ @Mock
+ private AtlasEntityType hiveTableEntityTypeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute nameAttributeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute commentAttributeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute stateAttributeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute descrptionAttributeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute createdAttributeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute startedAttributeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute entitypeAttributeMock;
+
+ @Mock
+ private AtlasStructType.AtlasAttribute qualifiedNameAttributeMock;
+
+ private Map<String, String> indexFieldNamesMap = new HashMap<>();
+
+
+ @BeforeTest
+ public void setup() {
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ MockitoAnnotations.initMocks(this);
+ when(hiveTableEntityTypeMock.getAttribute("name")).thenReturn(nameAttributeMock);
+ when(hiveTableEntityTypeMock.getAttribute("comment")).thenReturn(commentAttributeMock);
+ when(hiveTableEntityTypeMock.getAttribute("__state")).thenReturn(stateAttributeMock);
+ when(hiveTableEntityTypeMock.getAttribute("description")).thenReturn(descrptionAttributeMock);
+ when(hiveTableEntityTypeMock.getAttribute("created")).thenReturn(createdAttributeMock);
+ when(hiveTableEntityTypeMock.getAttribute("started")).thenReturn(startedAttributeMock);
+ when(hiveTableEntityTypeMock.getAttribute("Constants.ENTITY_TYPE_PROPERTY_KEY")).thenReturn(entitypeAttributeMock);
+ when(hiveTableEntityTypeMock.getAttribute("qualifiedName")).thenReturn(qualifiedNameAttributeMock);
+
+
+ indexFieldNamesMap.put("name", "name_index");
+ indexFieldNamesMap.put("comment", "comment_index");
+ indexFieldNamesMap.put("__state", "__state_index");
+ indexFieldNamesMap.put("description", "descrption__index");
+ indexFieldNamesMap.put("created", "created__index");
+ indexFieldNamesMap.put("started", "started__index");
+ indexFieldNamesMap.put(Constants.ENTITY_TYPE_PROPERTY_KEY, Constants.ENTITY_TYPE_PROPERTY_KEY + "__index");
+
+
+ when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table");
+
+ when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index");
+ when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index");
+ when(stateAttributeMock.getIndexFieldName()).thenReturn("__state_index");
+ when(descrptionAttributeMock.getIndexFieldName()).thenReturn("descrption__index");
+ when(createdAttributeMock.getIndexFieldName()).thenReturn("created__index");
+ when(startedAttributeMock.getIndexFieldName()).thenReturn("started__index");
+ when(entitypeAttributeMock.getIndexFieldName()).thenReturn(Constants.ENTITY_TYPE_PROPERTY_KEY + "__index");
+ when(qualifiedNameAttributeMock.getIndexFieldName()).thenReturn("qualifiedName" + "__index");
+
+ }
+
+ @Test
+ public void testGenerateSolrQueryString() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparameters2OR.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 OR +comment_index:*t10* )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryString2() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparameters1OR.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryString3() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparameters2AND.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 AND +comment_index:*t10* )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryString4() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparameters1AND.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryString5() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparameters0.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND +name_index:t10 ");
+ }
+
+ @Test
+ public void testGenerateSolrQueryString6() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparameters3.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +comment_index:*United States* AND +descrption__index:*nothing* AND +name_index:*t100* )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryStringGT() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparametersGT.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:{ 100 TO * ] )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryStringGTE() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparametersGTE.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ 100 TO * ] AND +started__index:[ 100 TO * ] )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryStringLT() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparametersLT.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ * TO100} )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryStringLE() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparametersLTE.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ * TO 100 ] AND +started__index:[ * TO 100 ] )");
+ }
+
+ @Test
+ public void testGenerateSolrQueryStartsWith() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparametersStartsWith.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParameters(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +qualifiedName__index:testdb.t1* )");
+ }
+
+
+
+
+ private void validateOrder(List<String> topTerms, int ... indices) {
+ Assert.assertEquals(topTerms.size(), indices.length);
+ int i = 0;
+ for(String term: topTerms) {
+ Assert.assertEquals(Integer.toString(indices[i++]), term);
+ }
+ Assert.assertEquals(topTerms.size(), indices.length);
+ }
+
+ private Map<String, AtlasJanusGraphIndexClient.TermFreq> generateTerms(int ... termFreqs) {
+ int i =0;
+ Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = new HashMap<>();
+ for(int count: termFreqs) {
+ AtlasJanusGraphIndexClient.TermFreq termFreq1 = new AtlasJanusGraphIndexClient.TermFreq(Integer.toString(i++), count);
+ terms.put(termFreq1.getTerm(), termFreq1);
+ }
+ return terms;
+ }
+
+ private void processSearchParameters(String fileName, AtlasSolrQueryBuilder underTest) throws IOException, AtlasBaseException {
+ ObjectMapper mapper = new ObjectMapper();
+ SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
+
+
+ underTest.withEntityType(hiveTableEntityTypeMock)
+ .withQueryString(searchParameters.getQuery())
+ .withCriteria(searchParameters.getEntityFilters())
+ .withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
+ .withCommonIndexFieldNames(indexFieldNamesMap);
+ }
+
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchParameters3.json b/graphdb/janus/src/test/resources/searchParameters3.json
new file mode 100644
index 0000000..9175db4
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchParameters3.json
@@ -0,0 +1,36 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"AND",
+ "criterion":[
+ {
+ "attributeName":"comment",
+ "operator":"contains",
+ "attributeValue":"United States"
+ },
+ {
+ "attributeName":"description",
+ "operator":"contains",
+ "attributeValue":"nothing"
+ },
+ {
+ "attributeName":"name",
+ "operator":"contains",
+ "attributeValue":"t100"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t10",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchParametersGT.json b/graphdb/janus/src/test/resources/searchParametersGT.json
new file mode 100644
index 0000000..bff379b
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchParametersGT.json
@@ -0,0 +1,26 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"AND",
+ "criterion":[
+ {
+ "attributeName":"created",
+ "operator":"gt",
+ "attributeValue":"100"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t10",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchParametersGTE.json b/graphdb/janus/src/test/resources/searchParametersGTE.json
new file mode 100644
index 0000000..65c953e
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchParametersGTE.json
@@ -0,0 +1,31 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"AND",
+ "criterion":[
+ {
+ "attributeName":"created",
+ "operator":"gte",
+ "attributeValue":100
+ },
+ {
+ "attributeName":"started",
+ "operator":"gte",
+ "attributeValue": 100
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t10",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchParametersLT.json b/graphdb/janus/src/test/resources/searchParametersLT.json
new file mode 100644
index 0000000..b221e83
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchParametersLT.json
@@ -0,0 +1,26 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"AND",
+ "criterion":[
+ {
+ "attributeName":"created",
+ "operator":"lt",
+ "attributeValue":"100"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t10",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchParametersLTE.json b/graphdb/janus/src/test/resources/searchParametersLTE.json
new file mode 100644
index 0000000..c8ede38
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchParametersLTE.json
@@ -0,0 +1,31 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"AND",
+ "criterion":[
+ {
+ "attributeName":"created",
+ "operator":"lte",
+ "attributeValue":"100"
+ },
+ {
+ "attributeName":"started",
+ "operator":"lte",
+ "attributeValue": 100
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t10",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchParametersStartsWith.json b/graphdb/janus/src/test/resources/searchParametersStartsWith.json
new file mode 100644
index 0000000..271dfe7
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchParametersStartsWith.json
@@ -0,0 +1,22 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":
+ {
+ "condition":"AND",
+ "criterion":
+ [
+ {
+ "attributeName":"qualifiedName",
+ "operator":"startsWith",
+ "attributeValue":"testdb.t1"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table"
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchparameters0.json b/graphdb/janus/src/test/resources/searchparameters0.json
new file mode 100644
index 0000000..bd04371
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchparameters0.json
@@ -0,0 +1,22 @@
+
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "attributeName":"name",
+ "operator":"eq",
+ "attributeValue":"t10"
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchparameters1AND.json b/graphdb/janus/src/test/resources/searchparameters1AND.json
new file mode 100644
index 0000000..7c7267f
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchparameters1AND.json
@@ -0,0 +1,27 @@
+
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"AND",
+ "criterion":[
+ {
+ "attributeName":"name",
+ "operator":"eq",
+ "attributeValue":"t10"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchparameters1OR.json b/graphdb/janus/src/test/resources/searchparameters1OR.json
new file mode 100644
index 0000000..91e0503
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchparameters1OR.json
@@ -0,0 +1,27 @@
+
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"OR",
+ "criterion":[
+ {
+ "attributeName":"name",
+ "operator":"eq",
+ "attributeValue":"t10"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
\ No newline at end of file
diff --git a/graphdb/janus/src/test/resources/searchparameters2AND.json b/graphdb/janus/src/test/resources/searchparameters2AND.json
new file mode 100644
index 0000000..4a8bdb3
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchparameters2AND.json
@@ -0,0 +1,32 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"AND",
+ "criterion":[
+ {
+ "attributeName":"name",
+ "operator":"eq",
+ "attributeValue":"t10"
+ },
+ {
+ "attributeName":"comment",
+ "operator":"contains",
+ "attributeValue":"t10"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
+
diff --git a/graphdb/janus/src/test/resources/searchparameters2OR.json b/graphdb/janus/src/test/resources/searchparameters2OR.json
new file mode 100644
index 0000000..7839e60
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchparameters2OR.json
@@ -0,0 +1,32 @@
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ "condition":"OR",
+ "criterion":[
+ {
+ "attributeName":"name",
+ "operator":"eq",
+ "attributeValue":"t10"
+ },
+ {
+ "attributeName":"comment",
+ "operator":"contains",
+ "attributeValue":"t10"
+ }
+ ]
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table",
+ "classification":null,
+ "termName":null
+}
+
diff --git a/intg/src/main/java/org/apache/atlas/listener/TypeDefChangeListener.java b/intg/src/main/java/org/apache/atlas/listener/TypeDefChangeListener.java
index e8ac8f4..02264c5 100644
--- a/intg/src/main/java/org/apache/atlas/listener/TypeDefChangeListener.java
+++ b/intg/src/main/java/org/apache/atlas/listener/TypeDefChangeListener.java
@@ -21,4 +21,5 @@ import org.apache.atlas.exception.AtlasBaseException;
public interface TypeDefChangeListener {
void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException;
+ void onLoadCompletion() throws AtlasBaseException;
}
diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
new file mode 100644
index 0000000..9e5e977
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.model.discovery;
+
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+
+import java.io.Serializable;
+import java.util.Set;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+/**
+ * This is the root class representing the input for quick search puroposes.
+ */
+public class QuickSearchParameters implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private String query;
+ private String typeName;
+ private FilterCriteria entityFilters;
+ private boolean includeSubTypes;
+ private boolean excludeDeletedEntities;
+ private int offset;
+ private int limit;
+ private Set<String> attributes;
+
+ /**
+ * for framework use.
+ */
+ public QuickSearchParameters() {
+ }
+
+ public QuickSearchParameters(String query,
+ String typeName,
+ FilterCriteria entityFilters,
+ boolean includeSubTypes,
+ boolean excludeDeletedEntities,
+ int offset,
+ int limit,
+ Set<String> attributes) {
+ this.query = query;
+ this.typeName = typeName;
+ this.entityFilters = entityFilters;
+ this.includeSubTypes = includeSubTypes;
+ this.excludeDeletedEntities = excludeDeletedEntities;
+ this.offset = offset;
+ this.limit = limit;
+ this.attributes = attributes;
+ }
+
+ public String getQuery() {
+ return query;
+ }
+
+ public void setQuery(String query) {
+ this.query = query;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+
+ public FilterCriteria getEntityFilters() {
+ return entityFilters;
+ }
+
+ public void setEntityFilters(FilterCriteria entityFilters) {
+ this.entityFilters = entityFilters;
+ }
+
+ public boolean getIncludeSubTypes() {
+ return includeSubTypes;
+ }
+
+ public void setIncludeSubTypes(boolean includeSubTypes) {
+ this.includeSubTypes = includeSubTypes;
+ }
+
+ public boolean getExcludeDeletedEntities() {
+ return excludeDeletedEntities;
+ }
+
+ public void setExcludeDeletedEntities(boolean excludeDeletedEntities) {
+ this.excludeDeletedEntities = excludeDeletedEntities;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public void setOffset(int offset) {
+ this.offset = offset;
+ }
+
+ public int getLimit() {
+ return limit;
+ }
+
+ public void setLimit(int limit) {
+ this.limit = limit;
+ }
+
+ public Set<String> getAttributes() {
+ return attributes;
+ }
+
+ public void setAttributes(Set<String> attributes) {
+ this.attributes = attributes;
+ }
+}
diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
index 9a74627..4ee68a9 100644
--- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
+++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
@@ -106,4 +106,6 @@ public interface AtlasTypeDefStore {
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
void deleteTypeByName(String typeName) throws AtlasBaseException;
+
+ void notifyLoadCompletion();
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index 254eee4..0fe47bd 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -33,10 +33,8 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-import static org.apache.atlas.model.TypeCategory.*;
-import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE;
-import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF;
-import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF;
+import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.*;
/**
* class that implements behaviour of a struct-type.
@@ -710,6 +708,7 @@ public class AtlasStructType extends AtlasType {
private String relationshipEdgeLabel;
private AtlasRelationshipEdgeDirection relationshipEdgeDirection;
private boolean isLegacyAttribute;
+ private String indexFieldName;
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType, String relationshipName, String relationshipLabel) {
this.definedInType = definedInType;
@@ -821,6 +820,13 @@ public class AtlasStructType extends AtlasType {
public void setLegacyAttribute(boolean legacyAttribute) { isLegacyAttribute = legacyAttribute; }
+ public String getIndexFieldName() { return indexFieldName; }
+
+ public void setIndexFieldName(String indexFieldName) { this.indexFieldName = indexFieldName; }
+
+ public int getSearchWeight() { return attributeDef.getSearchWeight(); }
+
+
public static String getEdgeLabel(String property) {
return "__" + property;
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index c0ac7ad..5c94c33 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -33,13 +33,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.inject.Singleton;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
@@ -57,19 +51,22 @@ public class AtlasTypeRegistry {
protected RegistryData registryData;
private final TypeRegistryUpdateSynchronizer updateSynchronizer;
private final Set<String> missingRelationshipDefs;
+ private final Map<String, String> commonIndexFieldNameCache;
public AtlasTypeRegistry() {
- registryData = new RegistryData();
- updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
- missingRelationshipDefs = new HashSet<>();
+ registryData = new RegistryData();
+ updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
+ missingRelationshipDefs = new HashSet<>();
+ commonIndexFieldNameCache = new HashMap<>();
}
// used only by AtlasTransientTypeRegistry
protected AtlasTypeRegistry(AtlasTypeRegistry other) {
- registryData = new RegistryData();
- updateSynchronizer = other.updateSynchronizer;
- missingRelationshipDefs = other.missingRelationshipDefs;
+ registryData = new RegistryData();
+ updateSynchronizer = other.updateSynchronizer;
+ missingRelationshipDefs = other.missingRelationshipDefs;
+ commonIndexFieldNameCache = other.commonIndexFieldNameCache;
}
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
@@ -240,6 +237,19 @@ public class AtlasTypeRegistry {
}
}
+ public void addIndexFieldName(String propertyName, String indexFieldName) {
+ commonIndexFieldNameCache.put(propertyName, indexFieldName);
+ }
+
+ /**
+ * retrieves the index field name for the common field passed in.
+ * @param propertyName the name of the common field.
+ * @return the index name for the common field passed in.
+ */
+ public String getIndexFieldName(String propertyName) {
+ return commonIndexFieldNameCache.get(propertyName);
+ }
+
static class RegistryData {
final TypeCache allTypes;
final TypeDefCache<AtlasEnumDef, AtlasEnumType> enumDefs;
@@ -1163,4 +1173,5 @@ class TypeDefCache<T1 extends AtlasBaseTypeDef, T2 extends AtlasType> {
typeDefNameMap.clear();
typeNameMap.clear();
}
+
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index b17e866..3feae27 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -19,12 +19,10 @@
package org.apache.atlas.discovery;
+import com.sun.xml.bind.v2.model.annotation.Quick;
import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.AtlasQuickSearchResult;
-import org.apache.atlas.model.discovery.AtlasSearchResult;
-import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
-import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.*;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import java.util.List;
@@ -142,7 +140,13 @@ public interface AtlasDiscoveryService {
*/
void deleteSavedSearch(String currentUser, String guid) throws AtlasBaseException;
- AtlasQuickSearchResult quickSearchWithParameters(SearchParameters searchParameters) throws AtlasBaseException;
+ /**
+ * Search for entities matching the search criteria
+ * @param searchParameters Search criteria
+ * @return Matching entities
+ * @throws AtlasBaseException
+ */
+ AtlasQuickSearchResult quickSearch(QuickSearchParameters searchParameters) throws AtlasBaseException;
/**
* Should return top 5 suggestion strings for the given prefix.
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index fe144f8..c67e347 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -26,14 +26,10 @@ import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.authorize.AtlasSearchResultScrubRequest;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.AtlasQuickSearchResult;
-import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.discovery.*;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
-import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
-import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
@@ -43,20 +39,14 @@ import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.repository.userprofile.UserProfileService;
-import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
-import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
-import org.apache.atlas.type.AtlasType;
-import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.atlas.util.SearchTracker;
@@ -80,6 +70,7 @@ import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.SortOrder.DESCENDING;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
+import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.*;
@Component
@@ -102,7 +93,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Inject
EntityDiscoveryService(AtlasTypeRegistry typeRegistry,
- AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker,
+ AtlasGraph graph,
+ GraphBackedSearchIndexer indexer,
+ SearchTracker searchTracker,
UserProfileService userProfileService) throws AtlasException {
this.graph = graph;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
@@ -421,8 +414,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
@GraphTransaction
- public AtlasQuickSearchResult quickSearchWithParameters(SearchParameters searchParameters) throws AtlasBaseException {
- SearchContext searchContext = new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys());
+ public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
+ SearchContext searchContext = new SearchContext(createSearchParameters(quickSearchParameters),
+ typeRegistry,
+ graph,
+ indexer.getVertexIndexKeys());
if(LOG.isDebugEnabled()) {
LOG.debug("Generating the search results for the query {} .", searchContext.getSearchParameters().getQuery());
@@ -434,9 +430,12 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
LOG.debug("Generating the aggregated metrics for the query {} .", searchContext.getSearchParameters().getQuery());
}
- SearchAggregator searchAggregator = new SearchAggregatorImpl(searchContext);
- Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = searchAggregator.getAggregatedMetrics();
- AtlasQuickSearchResult ret = new AtlasQuickSearchResult(searchResult, aggregatedMetrics);
+ // load the facet fields and attributes.
+ Set<String> aggregationFields = getAggregationFields();
+ Set<AtlasAttribute> aggregationAttributes = getAggregationAtlasAttributes();
+ SearchAggregator searchAggregator = new SearchAggregatorImpl(searchContext);
+ Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = searchAggregator.getAggregatedMetrics(aggregationFields, aggregationAttributes);
+ AtlasQuickSearchResult ret = new AtlasQuickSearchResult(searchResult, aggregatedMetrics);
return ret;
}
@@ -946,6 +945,21 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return queryStr;
}
+ public static SearchParameters createSearchParameters(QuickSearchParameters quickSearchParameters) {
+ SearchParameters searchParameters = new SearchParameters();
+
+ searchParameters.setQuery(quickSearchParameters.getQuery());
+ searchParameters.setTypeName(quickSearchParameters.getTypeName());
+ searchParameters.setExcludeDeletedEntities(quickSearchParameters.getExcludeDeletedEntities());
+ searchParameters.setIncludeSubTypes(quickSearchParameters.getIncludeSubTypes());
+ searchParameters.setLimit(quickSearchParameters.getLimit());
+ searchParameters.setOffset(quickSearchParameters.getOffset());
+ searchParameters.setEntityFilters(quickSearchParameters.getEntityFilters());
+ searchParameters.setAttributes(quickSearchParameters.getAttributes());
+
+ return searchParameters;
+ }
+
private String escapeTypeName(String typeName) {
String ret;
@@ -968,4 +982,36 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private void scrubSearchResults(AtlasSearchResult result) throws AtlasBaseException {
AtlasAuthorizationUtils.scrubSearchResults(new AtlasSearchResultScrubRequest(typeRegistry, result));
}
+
+ private Set<String> getAggregationFields() {
+ Set<String> ret = new HashSet<>(); // for non-modeled attributes.
+
+ ret.add(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ ret.add(Constants.STATE_PROPERTY_KEY);
+
+ return ret;
+ }
+
+ private Set<AtlasAttribute> getAggregationAtlasAttributes() {
+ Set<AtlasAttribute> ret = new HashSet<>(); // for modeled attributes, like Asset.owner
+
+ ret.add(getAtlasAttributeForAssetOwner());
+
+ return ret;
+ }
+
+ private AtlasAttribute getAtlasAttributeForAssetOwner() {
+ AtlasEntityType typeAsset = typeRegistry.getEntityTypeByName(ASSET_ENTITY_TYPE);
+ AtlasAttribute atttOwner = typeAsset != null ? typeAsset.getAttribute(OWNER_ATTRIBUTE) : null;
+
+ if(atttOwner == null) {
+ String msg = String.format("Unable to resolve the attribute %s.%s", ASSET_ENTITY_TYPE, OWNER_ATTRIBUTE);
+
+ LOG.error(msg);
+
+ throw new RuntimeException(msg);
+ }
+
+ return atttOwner;
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java
index d5c3da8..b7c27e5 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java
@@ -18,10 +18,22 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
+import org.apache.atlas.type.AtlasStructType;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+/**
+ * This is an interface to search aggregation mwntrics providers.
+ */
public interface SearchAggregator {
- Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics();
+ /**
+ * returns aggregation metrics for passed in aggregation fields.
+ * @param aggregationFields the set of aggregation attribute names.
+ * @param aggregationAttrbutes the set of aggregationAttributes
+ * @return the result of aggreggations by aggregation fields.
+ */
+ Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields,
+ Set<AtlasStructType.AtlasAttribute> aggregationAttrbutes);
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
index fcd6ff0..e8f7dbc 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
@@ -19,10 +19,15 @@ package org.apache.atlas.discovery;
import org.apache.atlas.AtlasException;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
+import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AggregationContext;
import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,31 +39,66 @@ public class SearchAggregatorImpl implements SearchAggregator {
private final SearchContext searchContext;
+
public SearchAggregatorImpl(SearchContext searchContext) {
this.searchContext = searchContext;
}
- public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics() {
- String queryString = searchContext.getSearchParameters().getQuery();
- AtlasGraph atlasGraph = searchContext.getGraph();
- Set<String> aggregationFields = new HashSet<>();
- List<PostProcessor> postProcessors = new ArrayList<>();
-
- aggregationFields.add(Constants.ENTITY_TYPE_PROPERTY_KEY);
- aggregationFields.add(Constants.ASSET_OWNER_PROPERTY_KEY);
+ public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields,
+ Set<AtlasAttribute> aggregationAttributes) {
+ SearchParameters searchParameters = searchContext.getSearchParameters();
+ AtlasGraph graph = searchContext.getGraph();
+ AtlasTypeRegistry typeRegistry = searchContext.getTypeRegistry();
+ String queryString = searchParameters.getQuery();
+ List<PostProcessor> postProcessors = new ArrayList<>();
postProcessors.add(new ServiceTypeAggregator(searchContext.getTypeRegistry()));
try {
- Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = atlasGraph.getGraphIndexClient().getAggregatedMetrics(queryString, aggregationFields);
- Set<String> aggregationMetricNames = aggregatedMetrics.keySet();
+ AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
+ String searchedOnTypeName = searchParameters.getTypeName();
+ AtlasEntityType searchForEntityType = null;
+
+ if (searchedOnTypeName != null) {
+ searchForEntityType = typeRegistry.getEntityTypeByName(searchedOnTypeName);
+ }
+
+ Map<String, String> indexFieldNameCache = new HashMap<>();
+
+ for (String fieldName: aggregationFields) {
+ String indexFieldName = getIndexFieldNameForCommonFieldName(typeRegistry, fieldName);
+
+ indexFieldNameCache.put(fieldName, indexFieldName);
+ }
+
+ for (AtlasAttribute attribute: aggregationAttributes) {
+ String indexFieldName = attribute.getIndexFieldName();
- for(String aggregationMetricName: aggregationMetricNames) {
- for(PostProcessor postProcessor: postProcessors) {
- if(postProcessor.needsProcessing(aggregationMetricName)) {
+ if (indexFieldName == null) {
+ //there is no index field name.
+ indexFieldName = attribute.getQualifiedName();
+ }
+
+ indexFieldNameCache.put(attribute.getQualifiedName(), indexFieldName);
+ }
+
+ AggregationContext aggregatorContext = new AggregationContext(queryString,
+ searchParameters.getEntityFilters(),
+ searchForEntityType,
+ aggregationFields,
+ aggregationAttributes,
+ indexFieldNameCache,
+ searchParameters.getExcludeDeletedEntities(),
+ searchParameters.getIncludeSubTypes());
+
+ Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = graphIndexClient.getAggregatedMetrics(aggregatorContext);
+
+ for (String aggregationMetricName: aggregatedMetrics.keySet()) {
+ for (PostProcessor postProcessor: postProcessors) {
+ if (postProcessor.needsProcessing(aggregationMetricName)) {
postProcessor.prepareForMetric(aggregationMetricName);
- for(AtlasAggregationEntry aggregationEntry: aggregatedMetrics.get(aggregationMetricName)) {
+ for (AtlasAggregationEntry aggregationEntry: aggregatedMetrics.get(aggregationMetricName)) {
postProcessor.process(aggregationEntry);
}
@@ -67,21 +107,10 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
}
- for(PostProcessor postProcessor: postProcessors) {
+ for (PostProcessor postProcessor: postProcessors) {
postProcessor.handleCompletion(aggregatedMetrics);
}
- // remove entries with 0 counts
- for (List<AtlasAggregationEntry> entries : aggregatedMetrics.values()) {
- for (ListIterator<AtlasAggregationEntry> iter = entries.listIterator(); iter.hasNext(); ) {
- AtlasAggregationEntry entry = iter.next();
-
- if (entry.getCount() <= 0) {
- iter.remove();
- }
- }
- }
-
return aggregatedMetrics;
} catch (AtlasException e) {
LOG.error("Error encountered in post processing stage of aggrgation metrics collection. Empty metrics will be returned.", e);
@@ -90,6 +119,20 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
}
+ private String getIndexFieldNameForCommonFieldName(AtlasTypeRegistry typeRegistry, String fieldName) {
+ String indexFieldName = typeRegistry.getIndexFieldName(fieldName);
+
+ if(indexFieldName != null) {
+ return indexFieldName;
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Could not find index field name from type registry for attribute {}. Will use use the field name as is.", fieldName);
+ }
+
+ return fieldName;
+ }
+
static interface PostProcessor {
boolean needsProcessing(String metricName);
void prepareForMetric(String metricName);
@@ -102,8 +145,8 @@ public class SearchAggregatorImpl implements SearchAggregator {
private static final String SERVICE_TYPE = "ServiceType";
private final AtlasTypeRegistry typeRegistry;
- private List<AtlasAggregationEntry> entries;
- private Map<String, AtlasAggregationEntry> entityType2MetricsMap;
+ private final List<AtlasAggregationEntry> entries = new ArrayList<>();;
+ private final Map<String, AtlasAggregationEntry> entityType2MetricsMap = new HashMap<>();
public ServiceTypeAggregator(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
@@ -118,8 +161,6 @@ public class SearchAggregatorImpl implements SearchAggregator {
public void prepareForMetric(String metricName) {
Map<String, AtlasAggregationEntry> serviceName2MetricsMap = new HashMap<>();
- entries = new ArrayList<>();
-
//prepare the service map to aggregations
for(String serviceName: typeRegistry.getAllServiceTypes()) {
AtlasAggregationEntry serviceMetrics = new AtlasAggregationEntry(serviceName, 0);
@@ -130,8 +171,6 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
//prepare the map from entity type to aggregations
- entityType2MetricsMap = new HashMap<>();
-
for(AtlasEntityType entityType: typeRegistry.getAllEntityTypes()) {
String serviceName = entityType.getServiceType();
@@ -157,7 +196,16 @@ public class SearchAggregatorImpl implements SearchAggregator {
@Override
public void handleCompletion(Map<String, List<AtlasAggregationEntry>> aggregatedMetrics) {
- aggregatedMetrics.put(SERVICE_TYPE, entries);
+ //remove all zero count entries.
+ for (int i = entries.size() - 1; i >= 0; i--) {
+ if (entries.get(i).getCount() == 0) {
+ entries.remove(i);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(entries)) {
+ aggregatedMetrics.put(SERVICE_TYPE, entries);
+ }
}
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 408dac4..95ae504 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -29,7 +29,9 @@ import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.ActiveStateChangeHandler;
import org.apache.atlas.listener.ChangedTypeDefs;
import org.apache.atlas.listener.TypeDefChangeListener;
+import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
@@ -39,6 +41,7 @@ import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.*;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
@@ -49,13 +52,7 @@ import org.springframework.core.annotation.Order;
import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
import static org.apache.atlas.repository.Constants.*;
@@ -106,8 +103,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
return true;
}
- public static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) {
- return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName());
+ public static boolean isStringAttribute(AtlasAttribute attribute) {
+ return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attribute.getTypeName());
}
public enum UniqueKind { NONE, GLOBAL_UNIQUE, PER_TYPE_UNIQUE }
@@ -120,14 +117,15 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
@VisibleForTesting
GraphBackedSearchIndexer(IAtlasGraphProvider provider, Configuration configuration, AtlasTypeRegistry typeRegistry)
throws IndexException, RepositoryException {
- this.provider = provider;
+ this.provider = provider;
this.typeRegistry = typeRegistry;
+
//make sure solr index follows graph backed index listener
addIndexListener(new SolrIndexHelper(typeRegistry));
+
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize(provider.get());
}
-
}
public void addIndexListener(IndexChangeListener listener) {
@@ -192,6 +190,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ //resolve index fields names for the new entity attributes.
+ resolveIndexFieldNames(management, changedTypeDefs);
//Commit indexes
commit(management);
} catch (RepositoryException | IndexException e) {
@@ -202,6 +202,32 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
notifyChangeListeners(changedTypeDefs);
}
+ @Override
+ public void onLoadCompletion() throws AtlasBaseException {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Type definition load completed. Informing the completion to IndexChangeListeners.");
+ }
+
+ Collection<AtlasEntityDef> entityDefs = typeRegistry.getAllEntityDefs();
+ ChangedTypeDefs changedTypeDefs = new ChangedTypeDefs(null, new ArrayList<>(entityDefs), null);
+ AtlasGraphManagement management = null;
+
+ try {
+ management = provider.get().getManagementSystem();
+
+ //resolve index fields names for the new entity attributes.
+ resolveIndexFieldNames(management, changedTypeDefs);
+
+ //Commit indexes
+ commit(management);
+
+ notifyChangeListeners(changedTypeDefs);
+ } catch (RepositoryException | IndexException e) {
+ LOG.error("Failed to update indexes for changed typedefs", e);
+ attemptRollback(changedTypeDefs, management);
+ }
+ }
+
public Set<String> getVertexIndexKeys() {
if (recomputeIndexedKeys) {
AtlasGraphManagement management = null;
@@ -277,30 +303,30 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
// create vertex indexes
- createVertexIndex(management, GUID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
- createVertexIndex(management, HISTORICAL_GUID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
-
- createVertexIndex(management, TYPENAME_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
- createVertexIndex(management, TYPESERVICETYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
- createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
- createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, UniqueKind.NONE, Long.class, SINGLE, true, false);
-
- createVertexIndex(management, ENTITY_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
- createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
- createVertexIndex(management, TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
- createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
- createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
- createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
- createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
- createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
- createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
- createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
-
- createVertexIndex(management, PATCH_ID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
- createVertexIndex(management, PATCH_DESCRIPTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
- createVertexIndex(management, PATCH_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
- createVertexIndex(management, PATCH_ACTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
- createVertexIndex(management, PATCH_STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, GUID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, HISTORICAL_GUID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
+
+ createCommonVertexIndex(management, TYPENAME_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, TYPESERVICETYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, UniqueKind.NONE, Long.class, SINGLE, true, false);
+
+ createCommonVertexIndex(management, ENTITY_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
+ createCommonVertexIndex(management, TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
+ createCommonVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
+ createCommonVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
+ createCommonVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
+ createCommonVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
+ createCommonVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
+ createCommonVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
+ createCommonVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
+
+ createCommonVertexIndex(management, PATCH_ID_PROPERTY_KEY, UniqueKind.GLOBAL_UNIQUE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, PATCH_DESCRIPTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, PATCH_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, PATCH_ACTION_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
+ createCommonVertexIndex(management, PATCH_STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
// create vertex-centric index
createVertexCentricIndex(management, CLASSIFICATION_LABEL, AtlasEdgeDirection.BOTH, CLASSIFICATION_EDGE_NAME_PROPERTY_KEY, String.class, SINGLE);
@@ -325,6 +351,74 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ private void resolveIndexFieldNames(AtlasGraphManagement managementSystem, ChangedTypeDefs changedTypeDefs) {
+ List<? extends AtlasBaseTypeDef> createdTypeDefs = changedTypeDefs.getCreatedTypeDefs();
+
+ if(createdTypeDefs != null) {
+ resolveIndexFieldNames(managementSystem, createdTypeDefs);
+ }
+
+ List<? extends AtlasBaseTypeDef> updatedTypeDefs = changedTypeDefs.getUpdatedTypeDefs();
+
+ if(updatedTypeDefs != null) {
+ resolveIndexFieldNames(managementSystem, updatedTypeDefs);
+ }
+ }
+
+ private void resolveIndexFieldNames(AtlasGraphManagement managementSystem, List<? extends AtlasBaseTypeDef> typeDefs) {
+ for(AtlasBaseTypeDef baseTypeDef: typeDefs) {
+ if(TypeCategory.ENTITY.equals(baseTypeDef.getCategory())) {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(baseTypeDef.getName());
+
+ resolveIndexFieldNames(managementSystem, entityType);
+ } else {
+ LOG.debug("Ignoring the non-entity type definition {}", baseTypeDef.getName());
+ }
+ }
+ }
+
+ private void resolveIndexFieldNames(AtlasGraphManagement managementSystem, AtlasEntityType entityType) {
+ for(AtlasAttribute attribute: entityType.getAllAttributes().values()) {
+ if(needsIndexFieldNameResolution(attribute)) {
+ resolveIndexFieldName(managementSystem, attribute);
+ }
+ }
+ }
+
+ private void resolveIndexFieldName(AtlasGraphManagement managementSystem,
+ AtlasAttribute attribute) {
+ AtlasPropertyKey propertyKey = managementSystem.getPropertyKey(attribute.getQualifiedName());
+ String indexFieldName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, propertyKey);
+
+ attribute.setIndexFieldName(indexFieldName);
+
+ LOG.info("Property {} is mapped to index field name {}", attribute.getQualifiedName(), attribute.getIndexFieldName());
+ }
+
+ private boolean needsIndexFieldNameResolution(AtlasAttribute attribute) {
+ return attribute.getIndexFieldName() == null &&
+ TypeCategory.PRIMITIVE.equals(attribute.getAttributeType().getTypeCategory());
+ }
+
+ private void createCommonVertexIndex(AtlasGraphManagement management,
+ String propertyName,
+ UniqueKind uniqueKind,
+ Class propertyClass,
+ AtlasCardinality cardinality,
+ boolean createCompositeIndex,
+ boolean createCompositeIndexWithTypeAndSuperTypes) {
+ final String indexFieldName = createVertexIndex(management,
+ propertyName,
+ uniqueKind,
+ propertyClass,
+ cardinality,
+ createCompositeIndex,
+ createCompositeIndexWithTypeAndSuperTypes);
+ if(indexFieldName != null) {
+ typeRegistry.addIndexFieldName(propertyName, indexFieldName);
+ }
+ }
+
private void addIndexForType(AtlasGraphManagement management, AtlasBaseTypeDef typeDef) {
if (typeDef instanceof AtlasEnumDef) {
// Only handle complex types like Struct, Classification and Entity
@@ -560,8 +654,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
return propertyKey;
}
- public void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
+ public String createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes) {
+ String indexFieldName = null;
+
if (propertyName != null) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
@@ -573,12 +669,15 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
- management.addMixedIndex(VERTEX_INDEX, propertyKey);
+ indexFieldName = management.addMixedIndex(VERTEX_INDEX, propertyKey);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
-
}
}
+ if(indexFieldName == null) {
+ indexFieldName = management.getIndexFieldName(VERTEX_INDEX, propertyKey);
+ }
+
if (propertyKey != null) {
if (createCompositeIndex || uniqueKind == UniqueKind.GLOBAL_UNIQUE || uniqueKind == UniqueKind.PER_TYPE_UNIQUE) {
createVertexCompositeIndex(management, propertyClass, propertyKey, uniqueKind == UniqueKind.GLOBAL_UNIQUE);
@@ -592,6 +691,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.warn("Index not created for {}: propertyKey is null", propertyName);
}
}
+
+ return indexFieldName;
}
private void createVertexCentricIndex(AtlasGraphManagement management, String edgeLabel, AtlasEdgeDirection edgeDirection,
@@ -819,4 +920,5 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
}
+
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
index aa012fb..f337fb3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
@@ -19,14 +19,15 @@ package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.ChangedTypeDefs;
-import org.apache.atlas.model.typedef.AtlasEntityDef;
-import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,13 +66,12 @@ public class SolrIndexHelper implements IndexChangeListener {
}
try {
- AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance();
- AtlasGraphIndexClient atlasGraphIndexClient = atlasGraph.getGraphIndexClient();
+ AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
+ AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
Map<String, Integer> propertyName2SearchWeightMap = gePropertiesWithSearchWeights();
- atlasGraphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, propertyName2SearchWeightMap);
- atlasGraphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getPropertiesForSuggestions(propertyName2SearchWeightMap));
-
+ graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, propertyName2SearchWeightMap);
+ graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getPropertiesForSuggestions(propertyName2SearchWeightMap));
} catch (AtlasException e) {
LOG.error("Error encountered in handling type system change notification.", e);
throw new RuntimeException("Error encountered in handling type system change notification.", e);
@@ -79,7 +79,7 @@ public class SolrIndexHelper implements IndexChangeListener {
}
private List<String> getPropertiesForSuggestions(Map<String, Integer> propertyName2SearchWeightMap) {
- List<String> propertiesForSuggestions = new ArrayList<>();
+ List<String> ret = new ArrayList<>();
for(Map.Entry<String, Integer> entry: propertyName2SearchWeightMap.entrySet()) {
if(entry.getValue().intValue() >= MIN_SEARCH_WEIGHT_FOR_SUGGESTIONS) {
@@ -89,55 +89,64 @@ public class SolrIndexHelper implements IndexChangeListener {
LOG.debug("Adding the property {} for suggestions.", propertyName);
}
- propertiesForSuggestions.add(propertyName);
+ ret.add(propertyName);
}
}
- return propertiesForSuggestions;
+ return ret;
}
private Map<String, Integer> gePropertiesWithSearchWeights() {
- Map<String, Integer> propertiesWithSearchWeights = new HashMap<>();
- Collection<AtlasEntityDef> allEntityDefs = typeRegistry.getAllEntityDefs();
+ Map<String, Integer> ret = new HashMap<>();
+ Collection<AtlasEntityType> entityTypes = typeRegistry.getAllEntityTypes();
+
+ //the following two properties are specially added manually.
+ //as, they don't come in the entity definitions as attributes.
- propertiesWithSearchWeights.put(CLASSIFICATION_TEXT_KEY, SEARCHWEIGHT_FOR_CLASSIFICATIONS);
- propertiesWithSearchWeights.put(TYPE_NAME_PROPERTY_KEY, SEARCHWEIGHT_FOR_TYPENAME);
+ ret.put(typeRegistry.getIndexFieldName(CLASSIFICATION_TEXT_KEY), SEARCHWEIGHT_FOR_CLASSIFICATIONS);
+ ret.put(typeRegistry.getIndexFieldName(TYPE_NAME_PROPERTY_KEY), SEARCHWEIGHT_FOR_TYPENAME);
- if (CollectionUtils.isNotEmpty(allEntityDefs)) {
- for (AtlasEntityDef entityDef : allEntityDefs) {
- processEntity(propertiesWithSearchWeights, entityDef);
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ for (AtlasEntityType entityType : entityTypes) {
+ processEntityType(ret, entityType);
}
}
- return propertiesWithSearchWeights;
+ return ret;
}
- private void processEntity(Map<String, Integer> propertiesWithSearchWeights, AtlasEntityDef entityDef) {
- for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) {
- processAttributeDefinition(propertiesWithSearchWeights, entityDef, attributeDef);
+ private void processEntityType(Map<String, Integer> indexFieldNameWithSearchWeights, AtlasEntityType entityType) {
+ Map<String, AtlasAttribute> attributes = entityType.getAllAttributes();
+
+ if(MapUtils.isNotEmpty(attributes)) {
+ for (AtlasAttribute attribute : attributes.values()) {
+ processAttribute(indexFieldNameWithSearchWeights, attribute);
+ }
+ } else {
+ LOG.debug("No attributes are defined for entity {}", entityType.getTypeName());
}
}
- private void processAttributeDefinition(Map<String, Integer> propertiesWithSearchWeights, AtlasEntityDef entityDef, AtlasAttributeDef attributeDef) {
- if (GraphBackedSearchIndexer.isStringAttribute(attributeDef)) {
- final String propertyName = GraphBackedSearchIndexer.getEncodedPropertyName(entityDef.getName(), attributeDef);
- int searchWeight = attributeDef.getSearchWeight();
+ private void processAttribute(Map<String, Integer> indexFieldNameWithSearchWeights, AtlasAttribute attribute) {
+ if (GraphBackedSearchIndexer.isStringAttribute(attribute)) {
+ int searchWeight = attribute.getSearchWeight();
if (searchWeight == DEFAULT_SEARCHWEIGHT) {
//We will use default search weight of 3 for string attributes.
//this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
} else if (!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) { //validate the value provided in the model.
- LOG.warn("Invalid search weight {} for attribute {}.{}. Will use default {}", searchWeight, entityDef.getName(), propertyName, DEFAULT_SEARCHWEIGHT_FOR_STRINGS);
+ LOG.warn("Invalid search weight {} for attribute {}. Will use default {}",
+ searchWeight, attribute.getQualifiedName(), DEFAULT_SEARCHWEIGHT_FOR_STRINGS);
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Applying search weight {} for attribute {}.{}", searchWeight, entityDef.getName(), propertyName);
+ LOG.debug("Applying search weight {} for attribute {}", searchWeight, attribute.getQualifiedName());
}
- propertiesWithSearchWeights.put(propertyName, searchWeight);
+ indexFieldNameWithSearchWeights.put(attribute.getIndexFieldName(), searchWeight);
}
}
}
\ No newline at end of file
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index adf1972..de79cbe 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -352,7 +352,7 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
try {
typeDefStore.init();
loadBootstrapTypeDefs();
-
+ typeDefStore.notifyLoadCompletion();
try {
AtlasAuthorizerFactory.getAtlasAuthorizer();
} catch (Throwable t) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
index 2dac776..2e2ab1a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
@@ -1006,6 +1006,17 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
}
+ @Override
+ public void notifyLoadCompletion(){
+ for (TypeDefChangeListener changeListener : typeDefChangeListeners) {
+ try {
+ changeListener.onLoadCompletion();
+ } catch (Throwable t) {
+ LOG.error("OnLoadCompletion failed for listener {}", changeListener.getClass().getName(), t);
+ }
+ }
+ }
+
private void tryUpdateByName(String name, AtlasBaseTypeDef typeDef, AtlasTransientTypeRegistry ttr) throws AtlasBaseException {
try {
ttr.updateTypeByName(name, typeDef);
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index f70938e..8ec3427 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -21,11 +21,9 @@ import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.SortOrder;
import org.apache.atlas.discovery.AtlasDiscoveryService;
+import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.AtlasQuickSearchResult;
-import org.apache.atlas.model.discovery.AtlasSearchResult;
-import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
-import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.*;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import org.apache.atlas.repository.Constants;
@@ -74,12 +72,12 @@ public class DiscoveryREST {
private final int maxDslQueryLength;
private final AtlasTypeRegistry typeRegistry;
- private final AtlasDiscoveryService atlasDiscoveryService;
+ private final AtlasDiscoveryService discoveryService;
@Inject
- public DiscoveryREST(AtlasTypeRegistry typeRegistry, AtlasDiscoveryService atlasDiscoveryService, Configuration configuration) {
+ public DiscoveryREST(AtlasTypeRegistry typeRegistry, AtlasDiscoveryService discoveryService, Configuration configuration) {
this.typeRegistry = typeRegistry;
- this.atlasDiscoveryService = atlasDiscoveryService;
+ this.discoveryService = discoveryService;
this.maxFullTextQueryLength = configuration.getInt(Constants.MAX_FULLTEXT_QUERY_STR_LENGTH, 4096);
this.maxDslQueryLength = configuration.getInt(Constants.MAX_DSL_QUERY_STR_LENGTH, 4096);
}
@@ -123,9 +121,9 @@ public class DiscoveryREST {
+ "," + classification + "," + limit + "," + offset + ")");
}
- String queryStr = atlasDiscoveryService.getDslQueryUsingTypeNameClassification(query, typeName, classification);
+ String queryStr = discoveryService.getDslQueryUsingTypeNameClassification(query, typeName, classification);
- return atlasDiscoveryService.searchUsingDslQuery(queryStr, limit, offset);
+ return discoveryService.searchUsingDslQuery(queryStr, limit, offset);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -164,7 +162,7 @@ public class DiscoveryREST {
limit + "," + offset + ")");
}
- return atlasDiscoveryService.searchUsingFullTextQuery(query, excludeDeletedEntities, limit, offset);
+ return discoveryService.searchUsingFullTextQuery(query, excludeDeletedEntities, limit, offset);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -214,7 +212,7 @@ public class DiscoveryREST {
searchParameters.setLimit(limit);
searchParameters.setOffset(offset);
- return atlasDiscoveryService.searchWithParameters(searchParameters);
+ return discoveryService.searchWithParameters(searchParameters);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -335,7 +333,7 @@ public class DiscoveryREST {
validateSearchParameters(parameters);
- return atlasDiscoveryService.searchWithParameters(parameters);
+ return discoveryService.searchWithParameters(parameters);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -376,7 +374,7 @@ public class DiscoveryREST {
", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + excludeDeletedEntities + ", " + ", " + limit + ", " + offset + ")");
}
- return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, excludeDeletedEntities, limit, offset);
+ return discoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, excludeDeletedEntities, limit, offset);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -400,7 +398,7 @@ public class DiscoveryREST {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.addSavedSearch(userName=" + savedSearch.getOwnerName() + ", name=" + savedSearch.getName() + ", searchType=" + savedSearch.getSearchType() + ")");
}
- return atlasDiscoveryService.addSavedSearch(Servlets.getUserName(httpServletRequest), savedSearch);
+ return discoveryService.addSavedSearch(Servlets.getUserName(httpServletRequest), savedSearch);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -424,7 +422,7 @@ public class DiscoveryREST {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.updateSavedSearch(userName=" + savedSearch.getOwnerName() + ", name=" + savedSearch.getName() + ", searchType=" + savedSearch.getSearchType() + ")");
}
- return atlasDiscoveryService.updateSavedSearch(Servlets.getUserName(httpServletRequest), savedSearch);
+ return discoveryService.updateSavedSearch(Servlets.getUserName(httpServletRequest), savedSearch);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -451,7 +449,7 @@ public class DiscoveryREST {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.getSavedSearch(userName=" + userName + ", name=" + searchName + ")");
}
- return atlasDiscoveryService.getSavedSearchByName(Servlets.getUserName(httpServletRequest), userName, searchName);
+ return discoveryService.getSavedSearchByName(Servlets.getUserName(httpServletRequest), userName, searchName);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -475,7 +473,7 @@ public class DiscoveryREST {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.getSavedSearches(userName=" + userName + ")");
}
- return atlasDiscoveryService.getSavedSearches(Servlets.getUserName(httpServletRequest), userName);
+ return discoveryService.getSavedSearches(Servlets.getUserName(httpServletRequest), userName);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -496,7 +494,7 @@ public class DiscoveryREST {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.deleteSavedSearch(guid=" + guid + ")");
}
- atlasDiscoveryService.deleteSavedSearch(Servlets.getUserName(httpServletRequest), guid);
+ discoveryService.deleteSavedSearch(Servlets.getUserName(httpServletRequest), guid);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -526,7 +524,7 @@ public class DiscoveryREST {
"DiscoveryREST.executeSavedSearchByName(userName=" + userName + ", " + "name=" + searchName + ")");
}
- AtlasUserSavedSearch savedSearch = atlasDiscoveryService.getSavedSearchByName(Servlets.getUserName(httpServletRequest), userName, searchName);
+ AtlasUserSavedSearch savedSearch = discoveryService.getSavedSearchByName(Servlets.getUserName(httpServletRequest), userName, searchName);
return executeSavedSearch(savedSearch);
} finally {
@@ -553,7 +551,7 @@ public class DiscoveryREST {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.executeSavedSearchByGuid(" + searchGuid + ")");
}
- AtlasUserSavedSearch savedSearch = atlasDiscoveryService.getSavedSearchByGuid(Servlets.getUserName(httpServletRequest), searchGuid);
+ AtlasUserSavedSearch savedSearch = discoveryService.getSavedSearchByGuid(Servlets.getUserName(httpServletRequest), searchGuid);
return executeSavedSearch(savedSearch);
} finally {
@@ -570,10 +568,14 @@ public class DiscoveryREST {
*/
@Path("/quick")
@GET
- public AtlasQuickSearchResult searchUsingFreeText(@QueryParam("query") String query,
- @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
- @QueryParam("limit") int limit,
- @QueryParam("offset") int offset) throws AtlasBaseException {
+ public AtlasQuickSearchResult quickSearch(@QueryParam("query") String query,
+ @QueryParam("typeName") String typeName,
+ @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
+ @QueryParam("offset") int offset,
+ @QueryParam("limit") int limit) throws AtlasBaseException {
+
+
+
if (StringUtils.isNotEmpty(query) && query.length() > maxFullTextQueryLength) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_QUERY_LENGTH, Constants.MAX_FULLTEXT_QUERY_STR_LENGTH);
}
@@ -586,14 +588,55 @@ public class DiscoveryREST {
"excludeDeletedEntities:" + excludeDeletedEntities + "," + limit + "," + offset + ")");
}
- SearchParameters searchParameters = new SearchParameters();
+ QuickSearchParameters quickSearchParameters = new QuickSearchParameters(query,
+ typeName,
+ null, // entityFilters
+ false, // includeSubTypes
+ excludeDeletedEntities,
+ offset,
+ limit,
+ null); // attributes
- searchParameters.setQuery(query);
- searchParameters.setExcludeDeletedEntities(excludeDeletedEntities);
- searchParameters.setLimit(limit);
- searchParameters.setOffset(offset);
+ return discoveryService.quickSearch(quickSearchParameters);
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+ }
+
+ /**
+ * Attribute based search for entities satisfying the search parameters
+ *@return Atlas search result
+ * @throws AtlasBaseException
+ * @HTTP 200 On successful search
+ * @HTTP 400 Entity/attribute doesn't exist or entity filter is present without type name
+ */
+ @Path("/quick")
+ @POST
+ public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
+ AtlasPerfTracer perf = null;
- return atlasDiscoveryService.quickSearchWithParameters(searchParameters);
+ try {
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.searchWithParameters(" + quickSearchParameters + ")");
+ }
+
+ if (quickSearchParameters.getLimit() < 0 || quickSearchParameters.getOffset() < 0) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Limit/offset should be non-negative");
+ }
+
+ if (StringUtils.isEmpty(quickSearchParameters.getTypeName()) &&
+ !isEmpty(quickSearchParameters.getEntityFilters())) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "EntityFilters specified without Type name");
+ }
+
+ if (StringUtils.isEmpty(quickSearchParameters.getTypeName()) &&
+ StringUtils.isEmpty(quickSearchParameters.getQuery())){
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_SEARCH_PARAMS);
+ }
+
+ validateSearchParameters(quickSearchParameters);
+
+ return discoveryService.quickSearch(quickSearchParameters);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -609,7 +652,7 @@ public class DiscoveryREST {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.getSuggestions(" + prefixString + ")");
}
- return atlasDiscoveryService.getSuggestions(prefixString);
+ return discoveryService.getSuggestions(prefixString);
} finally {
AtlasPerfTracer.log(perf);
}
@@ -624,11 +667,11 @@ public class DiscoveryREST {
SearchParameters sp = savedSearch.getSearchParameters();
if(savedSearch.getSearchType() == AtlasUserSavedSearch.SavedSearchType.ADVANCED) {
- String dslQuery = atlasDiscoveryService.getDslQueryUsingTypeNameClassification(sp.getQuery(), sp.getTypeName(), sp.getClassification());
+ String dslQuery = discoveryService.getDslQueryUsingTypeNameClassification(sp.getQuery(), sp.getTypeName(), sp.getClassification());
- return atlasDiscoveryService.searchUsingDslQuery(dslQuery, sp.getLimit(), sp.getOffset());
+ return discoveryService.searchUsingDslQuery(dslQuery, sp.getLimit(), sp.getOffset());
} else {
- return atlasDiscoveryService.searchWithParameters(sp);
+ return discoveryService.searchWithParameters(sp);
}
}
@@ -652,4 +695,10 @@ public class DiscoveryREST {
}
}
+
+ private void validateSearchParameters(QuickSearchParameters parameters) throws AtlasBaseException {
+ if (parameters != null) {
+ validateSearchParameters(EntityDiscoveryService.createSearchParameters(parameters));
+ }
+ }
}