You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/08/05 13:09:15 UTC
[1/2] incubator-atlas git commit: ATLAS-1060 Add composite indexes
for exact match performance improvements for all attributes (sumasai via
shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master d671b1276 -> 91072c106
ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/91072c10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/91072c10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/91072c10
Branch: refs/heads/master
Commit: 91072c10669c390338e39a9b705ffdd3bb5a3868
Parents: 89f25d4
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Fri Aug 5 18:38:29 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Fri Aug 5 18:39:07 2016 +0530
----------------------------------------------------------------------
.../falcon/model/FalconDataModelGenerator.java | 10 +-
.../org/apache/atlas/fs/model/FSDataModel.scala | 6 +-
.../hive/model/HiveDataModelGenerator.java | 10 +-
.../hive/bridge/HiveMetastoreBridgeIT.java | 2 +-
.../org/apache/atlas/hive/hook/HiveHookIT.java | 2 +-
.../sqoop/model/SqoopDataModelGenerator.java | 8 +-
release-log.txt | 2 +-
.../graph/GraphBackedSearchIndexer.java | 112 ++++++++++++-------
.../atlas/services/DefaultMetadataService.java | 7 +-
.../GraphBackedDiscoveryServiceTest.java | 4 +
.../graph/GraphBackedSearchIndexerTest.java | 11 +-
.../org/apache/atlas/query/GremlinTest.scala | 78 ++++++-------
.../apache/atlas/query/LineageQueryTest.scala | 28 ++---
.../typesystem/types/AttributeDefinition.java | 2 +-
14 files changed, 158 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
index 3250a23..fc0a9b2 100644
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
+++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
@@ -97,7 +97,7 @@ public class FalconDataModelGenerator {
private void createClusterEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
+ false, true, null),
// map of tags
new AttributeDefinition(TAGS,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
@@ -113,7 +113,7 @@ public class FalconDataModelGenerator {
private void createFeedCreationEntityClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, null)
+ false, false, true, null)
};
HierarchicalTypeDefinition<ClassType> definition =
@@ -127,7 +127,7 @@ public class FalconDataModelGenerator {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, null),
+ false, false, true, null),
new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
// map of tags
new AttributeDefinition(TAGS,
@@ -156,12 +156,12 @@ public class FalconDataModelGenerator {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, null),
+ false, false, true, null),
// map of tags
new AttributeDefinition(TAGS,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
// wf properties
new AttributeDefinition(WFPROPERTIES,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
index 4941a5f..ad40340 100644
--- a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
+++ b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
@@ -45,9 +45,9 @@ object FSDataModel extends App {
//Is a regular file or a directory. If true, it is a file else a directory
"isFile" ~ (boolean, optional, indexed)
//Is a symlink or not
- "isSymlink" ~ (boolean, optional, indexed)
+ "isSymlink" ~ (boolean, optional)
//Optional and may not be set for a directory
- "fileSize" ~ (long, optional, indexed)
+ "fileSize" ~ (long, optional)
"group" ~ (string, optional, indexed)
"posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed)
}
@@ -66,7 +66,7 @@ object FSDataModel extends App {
//Making cluster optional since path is already unique containing the namenode URI
AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed)
"numberOfReplicas" ~ (int, optional, indexed)
- "extendedAttributes" ~ (map(string, string), optional, indexed)
+ "extendedAttributes" ~ (map(string, string), optional)
}
//TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
index a3d97eb..aac3924 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
@@ -204,7 +204,7 @@ public class HiveDataModelGenerator {
private void createDBClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
+ false, true, null),
new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
@@ -220,7 +220,7 @@ public class HiveDataModelGenerator {
private void createColumnClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
//Making this optional since this is an incompatible change
//Reverse attribute to 'columns' in Table
@@ -258,7 +258,7 @@ public class HiveDataModelGenerator {
new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),};
+ false, true, null),};
HierarchicalTypeDefinition<ClassType> definition =
new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
@@ -271,9 +271,9 @@ public class HiveDataModelGenerator {
new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
+ false, true, null),
new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
+ false, true, null),
new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java
index 57f93a8..d09db1b 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetastoreBridgeIT.java
@@ -54,7 +54,7 @@ public class HiveMetastoreBridgeIT extends HiveITBase {
int tableCount = atlasClient.listEntities(HiveDataTypes.HIVE_TABLE.getName()).size();
- //Now import using import tool - should be no-op
+ //Now import using import tool - should be no-op. This also tests update since table exists
hiveMetaStoreBridge.importTable(atlasClient.getEntity(dbId), DEFAULT_DB, tableName, true);
String tableId2 = assertTableIsRegistered(DEFAULT_DB, tableName);
assertEquals(tableId2, tableId);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 9258b3e..a5838b4 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -1502,7 +1502,7 @@ public class HiveHookIT extends HiveITBase {
final String fmtQuery = "alter database %s set OWNER %s %s";
String query = String.format(fmtQuery, dbName, "USER", owner);
- runCommand(query);
+ runCommandWithDelay(query, 1000);
assertDatabaseIsRegistered(dbName, new AssertPredicate() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
index 0ea4d7f..3373246 100644
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
+++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
@@ -119,13 +119,13 @@ public class SqoopDataModelGenerator {
private void createSqoopDbStoreClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(DB_STORE_TYPE,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(DB_STORE_USAGE,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(STORE_URI,
DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(SOURCE,
- DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null)
+ DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)
};
HierarchicalTypeDefinition<ClassType> definition =
@@ -139,12 +139,12 @@ public class SqoopDataModelGenerator {
private void createSqoopProcessClass() throws AtlasException {
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition(OPERATION,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
+ DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
new AttributeDefinition(USER,
- DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
+ DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
};
HierarchicalTypeDefinition<ClassType> definition =
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f6daa1c..294e6b3 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -3,7 +3,7 @@ Apache Atlas Release Notes
--trunk - unreleased
INCOMPATIBLE CHANGES:
-
+ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags)
ALL CHANGES:
ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index c77004d..f2e40f9 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -101,32 +101,32 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
// create a composite index for guid as its unique
createIndexes(management, Constants.GUID_PROPERTY_KEY, String.class, true,
- Cardinality.SINGLE, true);
+ Cardinality.SINGLE, true, true);
// create a composite index for entity state
- createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true);
+ createIndexes(management, Constants.TIMESTAMP_PROPERTY_KEY, Long.class, false, Cardinality.SINGLE, true, true);
// create a mixed index for entity state. Set systemProperty flag deliberately to false
// so that it doesnt create a composite index which has issues with
// titan 0.5.4 - Refer https://groups.google.com/forum/#!searchin/aureliusgraphs/hemanth/aureliusgraphs/bx7T843mzXU/fjAsclx7GAAJ
- createIndexes(management, Constants.STATE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE, false);
+ createStateMixedIndex(management);
// create a composite index for entity state
createIndexes(management, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class, false,
- Cardinality.SINGLE, true);
+ Cardinality.SINGLE, false, false);
// create a composite and mixed index for type since it can be combined with other keys
createIndexes(management, Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE,
- true);
+ true, true);
// create a composite and mixed index for type since it can be combined with other keys
createIndexes(management, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Cardinality.SET,
- true);
+ true, true);
// create a composite and mixed index for traitNames since it can be combined with other
// keys. Traits must be a set and not a list.
createIndexes(management, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Cardinality.SET,
- true);
+ true, true);
// Index for full text search
createFullTextIndex(management);
@@ -142,6 +142,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ private void createStateMixedIndex(TitanManagement management) {
+ PropertyKey propertyKey = management.getPropertyKey(Constants.STATE_PROPERTY_KEY);
+ if (propertyKey == null) {
+ propertyKey = management.makePropertyKey(Constants.STATE_PROPERTY_KEY).dataType(String.class).cardinality(Cardinality.SINGLE)
+ .make();
+ }
+ enhanceMixedIndex(management, Constants.STATE_PROPERTY_KEY, String.class, Cardinality.SINGLE, propertyKey);
+ }
+
+
private void createFullTextIndex(TitanManagement management) {
PropertyKey fullText =
management.makePropertyKey(Constants.ENTITY_TEXT_PROPERTY_KEY).dataType(String.class).make();
@@ -155,11 +165,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private void createTypeStoreIndexes(TitanManagement management) {
//Create unique index on typeName
createIndexes(management, Constants.TYPENAME_PROPERTY_KEY, String.class, true, Cardinality.SINGLE,
- true);
+ true, true);
//create index on vertex type
createIndexes(management, Constants.VERTEX_TYPE_PROPERTY_KEY, String.class, false, Cardinality.SINGLE,
- true);
+ true, true);
}
/**
@@ -172,7 +182,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
public void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException {
TitanManagement management = titanGraph.getManagementSystem();
for (IDataType dataType : dataTypes) {
- LOG.info("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass());
+ LOG.debug("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass());
try {
addIndexForType(management, dataType);
LOG.info("Index creation for type {} complete", dataType.getName());
@@ -225,9 +235,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private void createIndexForFields(TitanManagement management, IDataType dataType, Map<String, AttributeInfo> fields) {
for (AttributeInfo field : fields.values()) {
- if (field.isIndexable) {
- createIndexForAttribute(management, dataType.getName(), field);
- }
+ createIndexForAttribute(management, dataType.getName(), field);
}
}
@@ -237,12 +245,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
case PRIMITIVE:
Cardinality cardinality = getCardinality(field.multiplicity);
createIndexes(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique,
- cardinality, false);
+ cardinality, false, field.isIndexable);
break;
case ENUM:
cardinality = getCardinality(field.multiplicity);
- createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false);
+ createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false, field.isIndexable);
break;
case ARRAY:
@@ -317,7 +325,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private PropertyKey createIndexes(TitanManagement management, String propertyName,
Class propertyClass, boolean isUnique, Cardinality cardinality,
- boolean isSystemProperty) {
+ boolean createCompositeForAttribute, boolean createCompositeWithTypeandSuperTypes) {
PropertyKey propertyKey = management.getPropertyKey(propertyName);
if (propertyKey == null) {
propertyKey = management.makePropertyKey(propertyName).dataType(propertyClass).cardinality(cardinality)
@@ -325,52 +333,74 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
enhanceMixedIndex(management, propertyName, propertyClass, cardinality, propertyKey);
- if (isSystemProperty) {
- createCompositeIndex(management, propertyName, propertyClass, propertyKey, isUnique);
- } else if (isUnique) {
- // send uniqueness as false because there can be many vertexes with the same property value
- // but state can be active / deleted.
- createCompositeIndexWithTypeName(management, propertyName, propertyClass, propertyKey);
- }
+ }
+
+ if (createCompositeForAttribute) {
+ createCompositeIndex(management, propertyName, propertyClass, propertyKey, isUnique);
+ } else if (createCompositeWithTypeandSuperTypes) {
+ //Index with typename since typename+property key queries need to speed up
+ createCompositeIndexWithTypeName(management, propertyName, propertyClass, propertyKey);
+ createCompositeIndexWithSuperTypeName(management, propertyName, propertyClass, propertyKey);
}
return propertyKey;
}
private void createCompositeIndex(TitanManagement management, String propertyName, Class propertyClass,
- PropertyKey propertyKey, boolean enforceUniqueness) {
- LOG.info("Creating composite index for property {} of type {} ", propertyName,
- propertyClass.getName());
- TitanManagement.IndexBuilder indexBuilder =
+ PropertyKey propertyKey, boolean enforceUniqueness) {
+ LOG.debug("Creating composite index for property {} of type {} ", propertyName,
+ propertyClass.getName());
+
+ TitanGraphIndex existingIndex = management.getGraphIndex(propertyName);
+ if ( existingIndex == null) {
+ TitanManagement.IndexBuilder indexBuilder =
management.buildIndex(propertyName, Vertex.class).addKey(propertyKey);
- if (enforceUniqueness) {
- indexBuilder.unique();
+ if (enforceUniqueness) {
+ LOG.debug("Enabling unique index for property {} of type {} ", propertyName,
+ propertyClass.getName());
+ indexBuilder.unique();
+ }
+ indexBuilder.buildCompositeIndex();
}
- indexBuilder.buildCompositeIndex();
LOG.info("Created composite index for property {} of type {} ", propertyName, propertyClass.getName());
}
private void createCompositeIndexWithTypeName(TitanManagement management, String propertyName, Class propertyClass,
PropertyKey propertyKey) {
- LOG.info("Creating composite index for property {} of type {} and {}", propertyName,
- propertyClass.getName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
- PropertyKey typePropertyKey = management.getPropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ createCompositeIndexWithSystemProperty(management, propertyName, propertyClass, propertyKey, Constants.ENTITY_TYPE_PROPERTY_KEY, Cardinality.SINGLE);
+ }
+
+ private void createCompositeIndexWithSuperTypeName(TitanManagement management, String propertyName, Class propertyClass,
+ PropertyKey propertyKey) {
+ createCompositeIndexWithSystemProperty(management, propertyName, propertyClass, propertyKey, Constants.SUPER_TYPES_PROPERTY_KEY, Cardinality.SET);
+ }
+
+ private void createCompositeIndexWithSystemProperty(TitanManagement management, String propertyName, Class propertyClass,
+ PropertyKey propertyKey, final String systemPropertyKey, Cardinality cardinality) {
+ LOG.debug("Creating composite index for property {} of type {} and {}", propertyName,
+ propertyClass.getName(), systemPropertyKey);
+ PropertyKey typePropertyKey = management.getPropertyKey(systemPropertyKey);
if (typePropertyKey == null) {
- typePropertyKey = management.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY).
- dataType(String.class).cardinality(Cardinality.SINGLE)
+ typePropertyKey = management.makePropertyKey(systemPropertyKey).
+ dataType(String.class).cardinality(cardinality)
.make();
}
- TitanManagement.IndexBuilder indexBuilder =
- management.buildIndex(propertyName + Constants.ENTITY_TYPE_PROPERTY_KEY, Vertex.class).
- addKey(propertyKey).addKey(typePropertyKey);
- indexBuilder.buildCompositeIndex();
- LOG.info("Created composite index for property {} of type {} and {}", propertyName, propertyClass.getName(), Constants.ENTITY_TYPE_PROPERTY_KEY);
+ final String indexName = propertyName + systemPropertyKey;
+ TitanGraphIndex existingIndex = management.getGraphIndex(indexName);
+
+ if ( existingIndex == null) {
+ TitanManagement.IndexBuilder indexBuilder =
+ management.buildIndex(indexName, Vertex.class).
+ addKey(propertyKey).addKey(typePropertyKey);
+ indexBuilder.buildCompositeIndex();
+ LOG.info("Created composite index for property {} of type {} and {}", propertyName, propertyClass.getName(), systemPropertyKey);
+ }
}
private void enhanceMixedIndex(TitanManagement management, String propertyName, Class propertyClass,
Cardinality cardinality, PropertyKey propertyKey) {
if (checkIfMixedIndexApplicable(propertyClass, cardinality)) {
//Use backing index
- LOG.info("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
+ LOG.debug("Creating backing index for property {} of type {} ", propertyName, propertyClass.getName());
TitanGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
management.addIndexKey(vertexIndex, propertyKey);
LOG.info("Created backing index for property {} of type {} ", propertyName, propertyClass.getName());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index e6b75f4..4d05d49 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -185,15 +185,14 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private void createSuperTypes() throws AtlasException {
HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
.createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
- DataTypes.STRING_TYPE));
+ new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
createType(referenceableType);
HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
.createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
- TypesUtil.createRequiredAttrDef(AtlasClient.NAME, DataTypes.STRING_TYPE),
+ new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
- TypesUtil.createOptionalAttrDef(AtlasClient.OWNER, DataTypes.STRING_TYPE));
+ new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
createType(assetType);
HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index df3fe87..74438cd 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -385,6 +385,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_table where name='sales_fact', db where name='Reporting'", 0},
{"hive_partition as p where values = ['2015-01-01']", 1},
// {"StorageDesc select cols", 6} //Not working since loading of lists needs to be fixed yet
+
+ //check supertypeNames
+ {"DataSet where name='sales_fact'", 1},
+ {"Asset where name='sales_fact'", 1}
};
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
index 1fa0619..3291e72 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexerTest.java
@@ -135,6 +135,7 @@ public class GraphBackedSearchIndexerTest {
verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY, false);
verifyVertexIndexContains(managementSystem, "Database.name" + Constants.ENTITY_TYPE_PROPERTY_KEY);
+ verifySystemCompositeIndex(managementSystem, "Database.name" + Constants.SUPER_TYPES_PROPERTY_KEY, false);
verifyVertexIndexContains(managementSystem, "Database.managedType");
}
@@ -146,13 +147,13 @@ public class GraphBackedSearchIndexerTest {
}
private void verifySystemCompositeIndex(TitanManagement managementSystem, String indexName, boolean isUnique) {
- TitanGraphIndex guidIndex = managementSystem.getGraphIndex(indexName);
- assertNotNull(guidIndex);
- assertTrue(guidIndex.isCompositeIndex());
+ TitanGraphIndex systemIndex = managementSystem.getGraphIndex(indexName);
+ assertNotNull(systemIndex);
+ assertTrue(systemIndex.isCompositeIndex());
if (isUnique) {
- assertTrue(guidIndex.isUnique());
+ assertTrue(systemIndex.isUnique());
} else {
- assertFalse(guidIndex.isUnique());
+ assertFalse(systemIndex.isUnique());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
index 8fbcdbc..fa48c0e 100755
--- a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
+++ b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
@@ -78,7 +78,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -91,7 +91,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -104,7 +104,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
|
| },
@@ -118,7 +118,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| }
| ]
@@ -174,7 +174,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -187,7 +187,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -200,7 +200,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -213,7 +213,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| }
| ]
@@ -253,7 +253,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -266,7 +266,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -279,7 +279,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -292,7 +292,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| }
| ]
@@ -331,7 +331,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -344,7 +344,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| }
| ]
@@ -380,7 +380,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -393,7 +393,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -406,7 +406,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -419,7 +419,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
@@ -516,7 +516,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -529,7 +529,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -542,7 +542,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -555,7 +555,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
@@ -606,7 +606,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -619,7 +619,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -632,7 +632,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -645,7 +645,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
@@ -781,7 +781,7 @@ class GremlinTest extends BaseGremlinTest {
@Test def testArith {
val r = QueryProcessor.evaluate(_class("DB").where(id("name").`=`(string("Reporting"))).
select(id("name"), id("createTime") + int(1)), g, gp)
- validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _src1.name, (_src1.createTime + 1) as (_src1.createTime + 1)\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct3\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_src1.name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"(_src1.createTime + 1)\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"
$typeName$\":\"__tempQueryResultStruct3\",\n \"(_src1.createTime + 1)\":1501,\n \"_src1.name\":\"Reporting\"\n }\n ]\n}")
+ validateJson(r, "{\n \"query\":\"DB where (name = \\\"Reporting\\\") as _src1 select _src1.name as _src1.name, (_src1.createTime + 1) as (_src1.createTime + 1)\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct3\",\n \"attributeDefinitions\":[\n {\n \"name\":\"_src1.name\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"(_src1.createTime + 1)\",\n \"dataTypeName\":\"int\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n
\"$typeName$\":\"__tempQueryResultStruct3\",\n \"(_src1.createTime + 1)\":1501,\n \"_src1.name\":\"Reporting\"\n }\n ]\n}")
}
@Test def testComparisonLogical {
@@ -806,7 +806,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -819,7 +819,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -832,7 +832,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| },
| {
@@ -845,7 +845,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite": false,
| "isUnique": false,
- | "isIndexable": true,
+ | "isIndexable": false,
| "reverseAttributeName": null
| }
| ]
@@ -872,7 +872,7 @@ class GremlinTest extends BaseGremlinTest {
where((isTrait("Dimension"))).
select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp
)
- validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct5\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\
n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
+ validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct5\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n
]\n },\n \"rows\":[\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct5\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
}
@Test def testJoinAndSelect2 {
@@ -881,7 +881,7 @@ class GremlinTest extends BaseGremlinTest {
.or(id("name").`=`(string("Reporting")))).field("Table").as("tab")
.select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp
)
- validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\"_
_tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
+ validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) or (name = \\\"Reporting\\\") Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct6\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$typeName$\":\
"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct6\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
}
@Test def testJoinAndSelect3 {
@@ -891,7 +891,7 @@ class GremlinTest extends BaseGremlinTest {
.or(id("db1").hasField("owner"))).field("Table").as("tab")
.select(id("db1").field("name").as("dbName"), id("tab").field("name").as("tabName")), g, gp
)
- validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n
\"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
+ validateJson(r, "{\n \"query\":\"DB as db1 where (db1.createTime > 0) and (db1.name = \\\"Reporting\\\") or db1 has owner Table as tab select db1.name as dbName, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"__tempQueryResultStruct7\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n
\"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"sales_fact\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Sales\",\n \"tabName\":\"customer_dim\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_daily_mv\"\n },\n {\n \"$typeName$\":\"__tempQueryResultStruct7\",\n \"dbName\":\"Reporting\",\n \"tabName\":\"sales_fact_monthly_mv\"\n }\n ]\n}")
}
@Test def testJoinAndSelect4 {
@@ -900,7 +900,7 @@ class GremlinTest extends BaseGremlinTest {
where((isTrait("Dimension"))).
select(id("db1").as("dbO"), id("tab").field("name").as("tabName")), g, gp
)
- validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1 as dbO, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbO\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":true,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$ty
peName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
+ validateJson(r, "{\n \"query\":\"DB as db1 where (name = \\\"Sales\\\") Table as tab where DB as db1 where (name = \\\"Sales\\\") Table as tab is Dimension as _src1 select db1 as dbO, tab.name as tabName\",\n \"dataType\":{\n \"typeName\":\"\",\n \"attributeDefinitions\":[\n {\n \"name\":\"dbO\",\n \"dataTypeName\":\"DB\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n },\n {\n \"name\":\"tabName\",\n \"dataTypeName\":\"string\",\n \"multiplicity\":{\n \"lower\":0,\n \"upper\":1,\n \"isUnique\":false\n },\n \"isComposite\":false,\n \"isUnique\":false,\n \"isIndexable\":false,\n \"reverseAttributeName\":null\n }\n ]\n },\n \"rows\":[\n {\n \"$
typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"product_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"time_dim\"\n },\n {\n \"$typeName$\":\"\",\n \"dbO\":{\n \"$typeName$\":\"DB\",\n \"version\":0\n },\n \"tabName\":\"customer_dim\"\n }\n ]\n}")
}
@Test def testArrayComparision {
@@ -924,7 +924,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
@@ -963,7 +963,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
@@ -1003,7 +1003,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -1016,7 +1016,7 @@ class GremlinTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
index 021a8d2..c8b635a 100755
--- a/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
+++ b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala
@@ -81,7 +81,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -94,7 +94,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -107,7 +107,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -120,7 +120,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
@@ -213,7 +213,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -226,7 +226,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -239,7 +239,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -252,7 +252,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
@@ -365,7 +365,7 @@ class LineageQueryTest extends BaseGremlinTest {
},
"isComposite":false,
"isUnique":false,
- "isIndexable":true,
+ "isIndexable":false,
"reverseAttributeName":null
},
{
@@ -378,7 +378,7 @@ class LineageQueryTest extends BaseGremlinTest {
},
"isComposite":false,
"isUnique":false,
- "isIndexable":true,
+ "isIndexable":false,
"reverseAttributeName":null
}
]
@@ -434,7 +434,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -447,7 +447,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -460,7 +460,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| },
| {
@@ -473,7 +473,7 @@ class LineageQueryTest extends BaseGremlinTest {
| },
| "isComposite":false,
| "isUnique":false,
- | "isIndexable":true,
+ | "isIndexable":false,
| "reverseAttributeName":null
| }
| ]
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/91072c10/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
index 29c3450..f556223 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
@@ -38,7 +38,7 @@ public final class AttributeDefinition {
public AttributeDefinition(String name, String dataTypeName, Multiplicity multiplicity, boolean isComposite,
String reverseAttributeName) {
- this(name, dataTypeName, multiplicity, isComposite, false, true, reverseAttributeName);
+ this(name, dataTypeName, multiplicity, isComposite, false, false, reverseAttributeName);
}
[2/2] incubator-atlas git commit: ATLAS-1097 Fix a potential NPE
issue flagged by Coverity scan (mneethiraj via shwethags)
Posted by sh...@apache.org.
ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/89f25d41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/89f25d41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/89f25d41
Branch: refs/heads/master
Commit: 89f25d411897dbd35a2baf7a96a0cf137c273f4a
Parents: d671b12
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Fri Aug 5 16:25:03 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Fri Aug 5 18:39:07 2016 +0530
----------------------------------------------------------------------
.../main/java/org/apache/atlas/hive/hook/HiveHook.java | 12 ++++++++----
release-log.txt | 1 +
2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f25d41/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 7905bcf..14359c5 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -773,10 +773,14 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
SortedMap<WriteEntity, Referenceable> hiveOutputsMap) throws HiveException {
HiveOperation op = eventContext.getOperation();
if (isCreateOp(eventContext)) {
- Table outTable = getEntityByType(sortedHiveOutputs, Type.TABLE).getTable();
- //refresh table
- outTable = dgiBridge.hiveClient.getTable(outTable.getDbName(), outTable.getTableName());
- return HiveMetaStoreBridge.getTableProcessQualifiedName(dgiBridge.getClusterName(), outTable);
+ Entity entity = getEntityByType(sortedHiveOutputs, Type.TABLE);
+
+ if (entity != null) {
+ Table outTable = entity.getTable();
+ //refresh table
+ outTable = dgiBridge.hiveClient.getTable(outTable.getDbName(), outTable.getTableName());
+ return HiveMetaStoreBridge.getTableProcessQualifiedName(dgiBridge.getClusterName(), outTable);
+ }
}
StringBuilder buffer = new StringBuilder(op.getOperationName());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f25d41/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 102ca16..f6daa1c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)
ATLAS-1090 UI: Multi-Select Tagging. (Kalyanikashikar via kevalbhatt)
ATLAS-1092 Add Table.CreateTime to process qualified Name for all hive_process (sumasai via shwethags)
ATLAS-1096 Modify HveMetaStoreBridge.import to use getEntity instead of DSL (sumasai via shwethags)