You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/08/15 16:37:20 UTC
[atlas] branch master updated: ATLAS-3374: fix for incorrect
basic/quick search results
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 4b7b1a9 ATLAS-3374: fix for incorrect basic/quick search results
4b7b1a9 is described below
commit 4b7b1a998085303a98cde198476679f5b27b0917
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Thu Aug 15 01:03:57 2019 -0700
ATLAS-3374: fix for incorrect basic/quick search results
---
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 8 +++++
.../org/apache/atlas/type/AtlasTypeRegistry.java | 18 +++++++++++
.../repository/graph/GraphBackedSearchIndexer.java | 37 ++++++++++++++--------
.../atlas/repository/graph/SolrIndexHelper.java | 37 +++++++++++-----------
4 files changed, 68 insertions(+), 32 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index 113ea6c..278ec5d 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -424,6 +424,10 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
.append(entry.getValue().intValue());
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("generateSearchWeightString(fieldsCount={}): ret={}", indexFieldName2SearchWeightMap.size(), searchWeightBuilder.toString());
+ }
+
return searchWeightBuilder.toString();
}
@@ -440,6 +444,10 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("generateSuggestionsString(fieldsCount={}): ret={}", suggestionIndexFieldNames.size(), ret.toString());
+ }
+
return ret.toString();
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 5c94c33..8b4fd1c 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -26,6 +26,7 @@ import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -918,6 +919,23 @@ public class AtlasTypeRegistry {
new Exception().fillInStackTrace());
} else if (typeRegistryUpdateLock.getHoldCount() == 1) {
if (ttr != null && commitUpdates) {
+ // copy indexName for entity attributes from current typeRegistry to new one
+ for (AtlasEntityType ttrEntityType : ttr.getAllEntityTypes()) {
+ AtlasEntityType currEntityType = typeRegistry.getEntityTypeByName(ttrEntityType.getTypeName());
+
+ if (currEntityType != null) { // ttrEntityType could be a new type introduced
+ for (AtlasAttribute attribute : ttrEntityType.getAllAttributes().values()) {
+ if (StringUtils.isEmpty(attribute.getIndexFieldName())) {
+ AtlasAttribute currAttribute = currEntityType.getAttribute(attribute.getName());
+
+ if (currAttribute != null) {
+ attribute.setIndexFieldName(currAttribute.getIndexFieldName());
+ }
+ }
+ }
+ }
+ }
+
typeRegistry.registryData = ttr.registryData;
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 8d8e4ad..255050d 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -386,26 +386,35 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
private void resolveIndexFieldName(AtlasGraphManagement managementSystem, AtlasAttribute attribute) {
- if (attribute.getIndexFieldName() == null && TypeCategory.PRIMITIVE.equals(attribute.getAttributeType().getTypeCategory())) {
- AtlasStructType definedInType = attribute.getDefinedInType();
- AtlasAttribute baseInstance = definedInType != null ? definedInType.getAttribute(attribute.getName()) : null;
+ try {
+ if (attribute.getIndexFieldName() == null && TypeCategory.PRIMITIVE.equals(attribute.getAttributeType().getTypeCategory())) {
+ AtlasStructType definedInType = attribute.getDefinedInType();
+ AtlasAttribute baseInstance = definedInType != null ? definedInType.getAttribute(attribute.getName()) : null;
- if (baseInstance != null && baseInstance.getIndexFieldName() != null) {
- attribute.setIndexFieldName(baseInstance.getIndexFieldName());
- } else {
- AtlasPropertyKey propertyKey = managementSystem.getPropertyKey(attribute.getVertexPropertyName());
- String indexFieldName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, propertyKey);
+ if (baseInstance != null && baseInstance.getIndexFieldName() != null) {
+ attribute.setIndexFieldName(baseInstance.getIndexFieldName());
+ } else if (isIndexApplicable(getPrimitiveClass(attribute.getTypeName()), toAtlasCardinality(attribute.getAttributeDef().getCardinality()))) {
+ AtlasPropertyKey propertyKey = managementSystem.getPropertyKey(attribute.getVertexPropertyName());
- attribute.setIndexFieldName(indexFieldName);
+ if (propertyKey != null) {
+ String indexFieldName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, propertyKey);
- if (baseInstance != null) {
- baseInstance.setIndexFieldName(indexFieldName);
- }
+ attribute.setIndexFieldName(indexFieldName);
+
+ if (baseInstance != null) {
+ baseInstance.setIndexFieldName(indexFieldName);
+ }
- typeRegistry.addIndexFieldName(attribute.getVertexPropertyName(), indexFieldName);
+ typeRegistry.addIndexFieldName(attribute.getVertexPropertyName(), indexFieldName);
- LOG.info("Property {} is mapped to index field name {}", attribute.getQualifiedName(), attribute.getIndexFieldName());
+ LOG.info("Property {} is mapped to index field name {}", attribute.getQualifiedName(), attribute.getIndexFieldName());
+ } else {
+ LOG.warn("resolveIndexFieldName(attribute={}): propertyKey is null for vertextPropertyName={}", attribute.getQualifiedName(), attribute.getVertexPropertyName());
+ }
+ }
}
+ } catch (Exception excp) {
+ LOG.warn("resolveIndexFieldName(attribute={}) failed.", attribute.getQualifiedName(), excp);
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
index f337fb3..4380a7e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
@@ -19,6 +19,7 @@ package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.ChangedTypeDefs;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
@@ -27,7 +28,7 @@ import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,37 +67,37 @@ public class SolrIndexHelper implements IndexChangeListener {
}
try {
- AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
- AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
- Map<String, Integer> propertyName2SearchWeightMap = gePropertiesWithSearchWeights();
+ AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
+ AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
+ Map<String, Integer> indexFieldName2SearchWeightMap = geIndexFieldNamesWithSearchWeights();
- graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, propertyName2SearchWeightMap);
- graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getPropertiesForSuggestions(propertyName2SearchWeightMap));
+ graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, indexFieldName2SearchWeightMap);
+ graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getIndexFieldNamesForSuggestions(indexFieldName2SearchWeightMap));
} catch (AtlasException e) {
LOG.error("Error encountered in handling type system change notification.", e);
throw new RuntimeException("Error encountered in handling type system change notification.", e);
}
}
- private List<String> getPropertiesForSuggestions(Map<String, Integer> propertyName2SearchWeightMap) {
+ private List<String> getIndexFieldNamesForSuggestions(Map<String, Integer> indexFieldName2SearchWeightMap) {
List<String> ret = new ArrayList<>();
- for(Map.Entry<String, Integer> entry: propertyName2SearchWeightMap.entrySet()) {
+ for(Map.Entry<String, Integer> entry: indexFieldName2SearchWeightMap.entrySet()) {
if(entry.getValue().intValue() >= MIN_SEARCH_WEIGHT_FOR_SUGGESTIONS) {
- String propertyName = entry.getKey();
+ String indexFieldName = entry.getKey();
if (LOG.isDebugEnabled()) {
- LOG.debug("Adding the property {} for suggestions.", propertyName);
+ LOG.debug("Adding indexFieldName {} for suggestions.", indexFieldName);
}
- ret.add(propertyName);
+ ret.add(indexFieldName);
}
}
return ret;
}
- private Map<String, Integer> gePropertiesWithSearchWeights() {
+ private Map<String, Integer> geIndexFieldNamesWithSearchWeights() {
Map<String, Integer> ret = new HashMap<>();
Collection<AtlasEntityType> entityTypes = typeRegistry.getAllEntityTypes();
@@ -116,11 +117,11 @@ public class SolrIndexHelper implements IndexChangeListener {
}
private void processEntityType(Map<String, Integer> indexFieldNameWithSearchWeights, AtlasEntityType entityType) {
- Map<String, AtlasAttribute> attributes = entityType.getAllAttributes();
+ List<AtlasAttributeDef> attributes = entityType.getEntityDef().getAttributeDefs();
- if(MapUtils.isNotEmpty(attributes)) {
- for (AtlasAttribute attribute : attributes.values()) {
- processAttribute(indexFieldNameWithSearchWeights, attribute);
+ if(CollectionUtils.isNotEmpty(attributes)) {
+ for (AtlasAttributeDef attribute : attributes) {
+ processAttribute(indexFieldNameWithSearchWeights, entityType.getAttribute(attribute.getName()));
}
} else {
LOG.debug("No attributes are defined for entity {}", entityType.getTypeName());
@@ -128,7 +129,7 @@ public class SolrIndexHelper implements IndexChangeListener {
}
private void processAttribute(Map<String, Integer> indexFieldNameWithSearchWeights, AtlasAttribute attribute) {
- if (GraphBackedSearchIndexer.isStringAttribute(attribute)) {
+ if (attribute != null && GraphBackedSearchIndexer.isStringAttribute(attribute) && StringUtils.isNotEmpty(attribute.getIndexFieldName())) {
int searchWeight = attribute.getSearchWeight();
if (searchWeight == DEFAULT_SEARCHWEIGHT) {
@@ -143,7 +144,7 @@ public class SolrIndexHelper implements IndexChangeListener {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Applying search weight {} for attribute {}", searchWeight, attribute.getQualifiedName());
+ LOG.debug("Applying search weight {} for attribute={}: indexFieldName={}", searchWeight, attribute.getQualifiedName(), attribute.getIndexFieldName());
}
indexFieldNameWithSearchWeights.put(attribute.getIndexFieldName(), searchWeight);