You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2015/08/03 12:19:27 UTC
incubator-atlas git commit: ATLAS-79 Unique constraint is not
honoured (shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 96059e0a9 -> 39c52b2b7
ATLAS-79 Unique constraint is not honoured (shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/39c52b2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/39c52b2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/39c52b2b
Branch: refs/heads/master
Commit: 39c52b2b7b1b27cd52dbba00599f74b7cf8c635e
Parents: 96059e0
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Mon Aug 3 15:49:18 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Mon Aug 3 15:49:18 2015 +0530
----------------------------------------------------------------------
release-log.txt | 1 +
.../org/apache/atlas/repository/Constants.java | 4 -
.../graph/GraphBackedSearchIndexer.java | 105 ++++++++-----------
.../web/resources/EntityJerseyResourceIT.java | 43 ++++++++
4 files changed, 89 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 3642095..0965bf6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -8,6 +8,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES:
+ATLAS-79 Unique constraint is not honoured (shwethags)
ATLAS-25 Fix Atlas on Java 8 (sandeep.samudrala via shwethags)
ATLAS-86 Jenkins build failing as of build #41 (shwethags)
ATLAS-80 Support for variables in application properties (shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/repository/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java
index fe43ad0..d5d3b3c 100755
--- a/repository/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java
@@ -26,19 +26,16 @@ public final class Constants {
public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid";
- public static final String GUID_INDEX = "guid_index";
/**
* Entity type name property key.
*/
public static final String ENTITY_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
- public static final String ENTITY_TYPE_INDEX = "type_index";
/**
* Entity type's super types property key.
*/
public static final String SUPER_TYPES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames";
- public static final String SUPER_TYPES_INDEX = "super_types_index";
/**
* Full-text for the entity for enabling full-text search.
@@ -57,7 +54,6 @@ public final class Constants {
* Trait names property key and index name.
*/
public static final String TRAIT_NAMES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "traitNames";
- public static final String TRAIT_NAMES_INDEX = "trait_names_index";
public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index c43c64f..10babed 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -36,6 +36,7 @@ import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TraitType;
import org.slf4j.Logger;
@@ -44,8 +45,9 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.util.ArrayList;
import java.util.Collection;
-import java.util.Date;
+import java.util.List;
import java.util.Map;
/**
@@ -59,6 +61,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
private TitanManagement management;
+ List<Class> MIXED_INDEX_EXCLUSIONS = new ArrayList() {{
+ add(Boolean.class);
+ add(BigDecimal.class);
+ add(BigInteger.class);
+ }};
+
@Inject
public GraphBackedSearchIndexer(GraphProvider<TitanGraph> graphProvider) throws RepositoryException {
@@ -74,7 +82,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
* Initializes the indices for the graph - create indices for Global Vertex Keys
*/
private void initialize() {
- if (management.containsPropertyKey(Constants.GUID_PROPERTY_KEY)) {
+ if (management.containsPropertyKey(Constants.VERTEX_TYPE_PROPERTY_KEY)) {
LOG.info("Global indexes already exist for graph");
return;
}
@@ -85,20 +93,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
management.buildIndex(Constants.EDGE_INDEX, Edge.class).buildMixedIndex(Constants.BACKING_INDEX);
// create a composite index for guid as its unique
- createCompositeIndex(Constants.GUID_INDEX, Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE);
+ createCompositeAndMixedIndex(Constants.GUID_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, true);
// create a composite and mixed index for type since it can be combined with other keys
- createCompositeAndMixedIndex(Constants.ENTITY_TYPE_INDEX, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class,
- false, Cardinality.SINGLE);
+ createCompositeAndMixedIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true);
// create a composite and mixed index for type since it can be combined with other keys
- createCompositeAndMixedIndex(Constants.SUPER_TYPES_INDEX, Constants.SUPER_TYPES_PROPERTY_KEY, String.class,
- false, Cardinality.SET);
+ createCompositeAndMixedIndex(Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET, true);
// create a composite and mixed index for traitNames since it can be combined with other
// keys. Traits must be a set and not a list.
- createCompositeAndMixedIndex(Constants.TRAIT_NAMES_INDEX, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class,
- false, Cardinality.SET);
+ createCompositeAndMixedIndex(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET, true);
// Index for full text search
createFullTextIndex();
@@ -125,13 +130,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
private void createTypeStoreIndexes() {
//Create unique index on typeName
- createCompositeIndex(Constants.TYPENAME_PROPERTY_KEY, Constants.TYPENAME_PROPERTY_KEY, String.class, true,
- Cardinality.SINGLE);
+ createCompositeAndMixedIndex(Constants.TYPENAME_PROPERTY_KEY, String.class, true, Cardinality.SINGLE, true);
//create index on vertex type
- createCompositeIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class,
- false, Cardinality.SINGLE);
-
+ createCompositeAndMixedIndex(Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, true);
}
/**
@@ -201,11 +203,14 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
final String propertyName = typeName + "." + field.name;
switch (field.dataType().getTypeCategory()) {
case PRIMITIVE:
- createVertexMixedIndex(propertyName, getPrimitiveClass(field.dataType()));
+ Cardinality cardinality = getCardinality(field.multiplicity);
+ createCompositeAndMixedIndex(propertyName, getPrimitiveClass(field.dataType()), field.isUnique,
+ cardinality, false);
break;
case ENUM:
- createVertexMixedIndex(propertyName, String.class);
+ cardinality = getCardinality(field.multiplicity);
+ createCompositeAndMixedIndex(propertyName, String.class, field.isUnique, cardinality, false);
break;
case ARRAY:
@@ -264,7 +269,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
throw new IllegalArgumentException("unknown data type " + dataType);
}
-/*
+
private Cardinality getCardinality(Multiplicity multiplicity) {
if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
return Cardinality.SINGLE;
@@ -277,66 +282,46 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
// todo - default to LIST as this is the most forgiving
return Cardinality.LIST;
}
-*/
-
- private void createCompositeAndMixedIndex(String indexName, String propertyName, Class propertyClass,
- boolean isUnique, Cardinality cardinality) {
- createCompositeIndex(indexName, propertyName, propertyClass, isUnique, cardinality);
- createVertexMixedIndex(propertyName, propertyClass);
- }
-
- private PropertyKey createCompositeIndex(String indexName, String propertyName, Class propertyClass,
- boolean isUnique, Cardinality cardinality) {
- PropertyKey propertyKey = management.getPropertyKey(propertyName);
- if (propertyKey == null) {
- propertyKey =
- management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality).make();
-
- TitanManagement.IndexBuilder indexBuilder =
- management.buildIndex(indexName, Vertex.class).addKey(propertyKey);
-
- if (isUnique) {
- indexBuilder = indexBuilder.unique();
- }
-
- indexBuilder.buildCompositeIndex();
- LOG.info("Created index for property {} in composite index {}", propertyName, indexName);
- }
- return propertyKey;
- }
- private PropertyKey createVertexMixedIndex(String propertyName, Class propertyClass) {
+ private PropertyKey createCompositeAndMixedIndex(String propertyName, Class propertyClass,
+ boolean isUnique, Cardinality cardinality, boolean force) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
- // ignored cardinality as Can only index single-valued property keys on vertices
- propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).make();
+ propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality)
+ .make();
- if (!checkIfMixedIndexApplicable(propertyClass)) {
- LOG.debug("Creating composite index for property {} of type {} ", propertyName,
- propertyClass.getName());
- //Use standard index as backing index only supports string, int and geo types
- management.buildIndex(propertyName, Vertex.class).addKey(propertyKey).buildCompositeIndex();
- LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
- } else {
+ if (checkIfMixedIndexApplicable(propertyClass, cardinality)) {
//Use backing index
LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
LOG.debug("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
}
- LOG.info("Created mixed vertex index for property {}", propertyName);
+
+ //Create mixed index only for meta properties and unique constraints:
+ //Unique can't be achieved with backing/mixed index
+ //Creating composite index for every attribute will bloat up the index
+ if (force || isUnique) {
+ LOG.debug("Creating composite index for property {} of type {} ", propertyName,
+ propertyClass.getName());
+ TitanManagement.IndexBuilder indexBuilder =
+ management.buildIndex(propertyName, Vertex.class).addKey(propertyKey);
+ if (isUnique) {
+ indexBuilder.unique();
+ }
+ indexBuilder.buildCompositeIndex();
+ LOG.debug("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
+ }
}
return propertyKey;
}
- private boolean checkIfMixedIndexApplicable(Class propertyClass) {
- if (propertyClass == Boolean.class || propertyClass == BigDecimal.class || propertyClass == BigInteger.class) {
- return false;
- }
- return true;
+ private boolean checkIfMixedIndexApplicable(Class propertyClass, Cardinality cardinality) {
+ return !(MIXED_INDEX_EXCLUSIONS.contains(propertyClass) || cardinality == Cardinality.LIST || cardinality ==
+ Cardinality.SET);
}
public void commit() throws IndexException {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/39c52b2b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
index 7361173..e6a7325 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
@@ -116,6 +116,49 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
}
@Test
+ public void testUniqueAttribute() throws Exception {
+ //create type
+ String typeName = "type" + randomString();
+ HierarchicalTypeDefinition<ClassType> typeDefinition = TypesUtil
+ .createClassTypeDef(typeName, ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
+ TypesDef typesDef = TypeUtils
+ .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(typeDefinition));
+ createType(typesDef);
+
+ //create entity
+ String name = "name" + randomString();
+ Referenceable referenceable = new Referenceable(typeName);
+ referenceable.set("name", name);
+ createInstance(referenceable);
+
+ //create entity with same name again - should fail
+ try {
+ createInstance(referenceable);
+ Assert.fail("Expected exception");
+ } catch(Exception e) {
+ //expected exception
+ }
+
+ //create another type with same attribute - should allow
+ typeName = "type" + randomString();
+ typeDefinition = TypesUtil
+ .createClassTypeDef(typeName, ImmutableList.<String>of(),
+ TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
+ typesDef = TypeUtils
+ .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(typeDefinition));
+ createType(typesDef);
+
+ referenceable = new Referenceable(typeName);
+ referenceable.set("name", name);
+ createInstance(referenceable);
+ }
+
+ @Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
try {