You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/04/03 17:13:45 UTC
[atlas] branch master updated: ATLAS-3112: Allow Indexing of array
attributes (LIST or SET) in indexing store
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new d234de2 ATLAS-3112: Allow Indexing of array attributes (LIST or SET) in indexing store
d234de2 is described below
commit d234de2d7ef4a935db313a852ce80b8aab126c3b
Author: Sarath Subramanian <ss...@hortonworks.com>
AuthorDate: Wed Apr 3 10:11:38 2019 -0700
ATLAS-3112: Allow Indexing of array attributes (LIST or SET) in indexing store
---
.../repository/graphdb/AtlasGraphManagement.java | 3 +-
.../graphdb/janus/AtlasJanusGraphManagement.java | 8 ++++--
.../graphdb/janus/AbstractGraphDatabaseTest.java | 2 +-
.../repository/graph/GraphBackedSearchIndexer.java | 24 ++++++++--------
.../apache/atlas/repository/graph/GraphHelper.java | 4 +--
.../store/graph/v2/AtlasGraphUtilsV2.java | 27 ++++++++++++++----
.../store/graph/v2/EntityGraphMapper.java | 32 ++++++++++++++++------
.../store/graph/v2/EntityStateChecker.java | 4 +--
8 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index b1e2c5d..cfa1cb4 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -155,6 +155,7 @@ public interface AtlasGraphManagement {
*
* @param vertexIndex
* @param propertyKey
+ * @param propertyClass
*/
- void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey);
+ void addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, Class propertyClass);
}
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index 4e18432..7b034ba 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -195,11 +195,15 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
}
@Override
- public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey) {
+ public void addMixedIndex(String indexName, AtlasPropertyKey propertyKey, Class propertyClass) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex vertexIndex = management.getGraphIndex(indexName);
- management.addIndexKey(vertexIndex, janusKey);
+ if (propertyClass == String.class) {
+ management.addIndexKey(vertexIndex, janusKey, Mapping.STRING.asParameter());
+ } else {
+ management.addIndexKey(vertexIndex, janusKey);
+ }
}
@Override
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
index f72b412..a139d08 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
@@ -99,7 +99,7 @@ public abstract class AbstractGraphDatabaseTest {
AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
try {
if (propertyClass != Integer.class) {
- management.addMixedIndex(BACKING_INDEX_NAME, key);
+ management.addMixedIndex(BACKING_INDEX_NAME, key, propertyClass);
}
} catch(Throwable t) {
//ok
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index c57f8e3..9f75d2a 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -271,12 +271,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, UniqueKind.NONE, Long.class, SINGLE, true, false);
createVertexIndex(management, ENTITY_TYPE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
- createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
createVertexIndex(management, TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
+
+ createVertexIndex(management, SUPER_TYPES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, false);
createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
@@ -351,10 +352,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createLabelIfNeeded(management, propertyName, attribTypeName);
AtlasArrayType arrayType = (AtlasArrayType) attributeType;
- boolean isReference = isReference(arrayType.getElementType());
+ AtlasType elementType = arrayType.getElementType();
+ boolean isReference = isReference(elementType);
if (!isReference) {
- createPropertyKey(management, propertyName, ArrayList.class, SINGLE);
+ createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(elementType.getTypeName()), cardinality, isIndexable, false);
}
}
@@ -508,12 +510,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
- if (isIndexApplicable(propertyClass, cardinality)) {
+ if (isIndexApplicable(propertyClass)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
- management.addMixedIndex(VERTEX_INDEX, propertyKey);
+ management.addMixedIndex(VERTEX_INDEX, propertyKey, propertyClass);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
@@ -591,12 +593,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
- if (isIndexApplicable(propertyClass, cardinality)) {
+ if (isIndexApplicable(propertyClass)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
}
- management.addMixedIndex(EDGE_INDEX, propertyKey);
+ management.addMixedIndex(EDGE_INDEX, propertyKey, propertyClass);
LOG.info("Created backing index for edge property {} of type {} ", propertyName, propertyClass.getName());
}
@@ -619,12 +621,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
- if (isIndexApplicable(propertyClass, cardinality)) {
+ if (isIndexApplicable(propertyClass)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
- management.addMixedIndex(FULLTEXT_INDEX, propertyKey);
+ management.addMixedIndex(FULLTEXT_INDEX, propertyKey, propertyClass);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
@@ -700,8 +702,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
- private boolean isIndexApplicable(Class propertyClass, AtlasCardinality cardinality) {
- return !(INDEX_EXCLUSION_CLASSES.contains(propertyClass) || cardinality.isMany());
+ private boolean isIndexApplicable(Class propertyClass) {
+ return !INDEX_EXCLUSION_CLASSES.contains(propertyClass);
}
public void commit(AtlasGraphManagement management) throws IndexException {
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 449f382..237389e 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -199,7 +199,7 @@ public final class GraphHelper {
AtlasGraphUtilsV2.setEncodedProperty(ret, MODIFIED_BY_KEY, RequestContext.get().getUser());
for (String superTypeName : superTypeNames) {
- AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
+ AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
return ret;
@@ -1545,7 +1545,7 @@ public final class GraphHelper {
if (isReference(elementType)) {
return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
} else {
- return (List) instanceVertex.getListProperty(propertyName);
+ return (List) instanceVertex.getPropertyValues(propertyName, List.class);
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
index 80141b4..084958f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
@@ -32,6 +32,7 @@ import org.apache.atlas.model.patches.AtlasPatch;
import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches;
import org.apache.atlas.model.patches.AtlasPatch.PatchStatus;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
@@ -64,6 +65,8 @@ import java.util.Map;
import java.util.Set;
import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.LIST;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
@@ -187,15 +190,23 @@ public class AtlasGraphUtilsV2 {
* @param propertyName
* @param value
*/
- public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Object value) {
- return addProperty(vertex, propertyName, value, false);
+ public static AtlasVertex addToListProperty(AtlasVertex vertex, String propertyName, Object value) {
+ return addProperty(vertex, propertyName, LIST, false, value);
}
- public static AtlasVertex addEncodedProperty(AtlasVertex vertex, String propertyName, Object value) {
- return addProperty(vertex, propertyName, value, true);
+ public static AtlasVertex addToSetProperty(AtlasVertex vertex, String propertyName, Object value) {
+ return addProperty(vertex, propertyName, SET, false, value);
}
- public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Object value, boolean isEncoded) {
+ public static AtlasVertex addToEncodedListProperty(AtlasVertex vertex, String propertyName, Object value) {
+ return addProperty(vertex, propertyName, LIST, true, value);
+ }
+
+ public static AtlasVertex addToEncodedSetProperty(AtlasVertex vertex, String propertyName, Object value) {
+ return addProperty(vertex, propertyName, SET, true, value);
+ }
+
+ public static AtlasVertex addProperty(AtlasVertex vertex, String propertyName, Cardinality cardinality, boolean isEncoded, Object value) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> addProperty({}, {}, {})", toString(vertex), propertyName, value);
}
@@ -204,7 +215,11 @@ public class AtlasGraphUtilsV2 {
propertyName = encodePropertyKey(propertyName);
}
- vertex.addProperty(propertyName, value);
+ if (cardinality == LIST) {
+ vertex.addListProperty(propertyName, value);
+ } else {
+ vertex.addProperty(propertyName, value);
+ }
return vertex;
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 4f93c8f..14f67c9 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -78,6 +78,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.CR
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.LIST;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
@@ -116,6 +117,8 @@ import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled
import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
+import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.addToListProperty;
+import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.addToSetProperty;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference;
import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
@@ -165,7 +168,7 @@ public class EntityGraphMapper {
AtlasVertex ret = createStructVertex(entity);
for (String superTypeName : entityType.getAllSuperTypes()) {
- AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
+ AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
AtlasGraphUtilsV2.setEncodedProperty(ret, GUID_PROPERTY_KEY, guid);
@@ -310,7 +313,10 @@ public class EntityGraphMapper {
AtlasVertex ret = createStructVertex(classification);
- AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, classificationType.getAllSuperTypes());
+ for (String superTypeName : classificationType.getAllSuperTypes()) {
+ AtlasGraphUtilsV2.addToEncodedSetProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
+ }
+
AtlasGraphUtilsV2.setEncodedProperty(ret, CLASSIFICATION_ENTITY_GUID, classification.getEntityGuid());
AtlasGraphUtilsV2.setEncodedProperty(ret, CLASSIFICATION_ENTITY_STATUS, classification.getEntityStatus().name());
@@ -1080,9 +1086,9 @@ public class EntityGraphMapper {
}
if (isNewElementsNull) {
- setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), null);
+ setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), null, cardinality);
} else {
- setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated);
+ setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated, cardinality);
}
if (LOG.isDebugEnabled()) {
@@ -1392,7 +1398,7 @@ public class EntityGraphMapper {
return (List)vertex.getListProperty(vertexPropertyName, AtlasEdge.class);
}
else {
- return (List)vertex.getListProperty(vertexPropertyName);
+ return (List) vertex.getPropertyValues(vertexPropertyName, List.class);
}
}
@@ -1436,9 +1442,17 @@ public class EntityGraphMapper {
return Collections.emptyList();
}
- private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String vertexPropertyName, List<Object> values) {
+
+ private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String propertyName, List<Object> values, Cardinality cardinality) {
if (!isReference(elementType) || isSoftReference) {
- AtlasGraphUtilsV2.setEncodedProperty(vertex, vertexPropertyName, values);
+ //remove existing array values before setting new values
+ vertex.removeProperty(propertyName);
+
+ if (cardinality == LIST) {
+ values.forEach(value -> addToListProperty(vertex, propertyName, value));
+ } else {
+ values.forEach(value -> addToSetProperty(vertex, propertyName, value));
+ }
}
}
@@ -1523,7 +1537,7 @@ public class EntityGraphMapper {
LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
}
- AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
+ AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
// add a new AtlasVertex for the struct or trait instance
AtlasVertex classificationVertex = createClassificationVertex(classification);
@@ -1924,7 +1938,7 @@ public class EntityGraphMapper {
entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
for (String traitName : traitNames) {
- AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
+ AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
}
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
index eb594f2..07df9de 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
@@ -268,7 +268,7 @@ public final class EntityStateChecker {
entityVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
for (String classificationName : traitVertexNames) {
- AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
+ AtlasGraphUtilsV2.addToEncodedSetProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
}
}
@@ -284,7 +284,7 @@ public final class EntityStateChecker {
entityVertex.removeProperty(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
for (String classificationName : propagatedTraitVertexNames) {
- AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
+ AtlasGraphUtilsV2.addToEncodedListProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
}
}