You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by jn...@apache.org on 2017/01/26 21:05:18 UTC
[1/2] incubator-atlas git commit: ATLAS-1114: Performance
improvements for create/update entity (2 of 2)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 6cd681194 -> 89f706096
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
index 51be64c..9ea71ad 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
@@ -95,6 +95,12 @@ public abstract class BaseResourceIT {
@BeforeClass
public void setUp() throws Exception {
+ //set high timeouts so that tests do not fail due to read timeouts while you
+ //are stepping through the code in a debugger
+ ApplicationProperties.get().setProperty("atlas.client.readTimeoutMSecs", "100000000");
+ ApplicationProperties.get().setProperty("atlas.client.connectTimeoutMSecs", "100000000");
+
+
Configuration configuration = ApplicationProperties.get();
atlasUrls = configuration.getStringArray(ATLAS_REST_ADDRESS);
@@ -221,13 +227,18 @@ public abstract class BaseResourceIT {
try {
if (!update) {
entity = entitiesClientV2.createEntity(atlasEntity);
+ assertNotNull(entity);
+ assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
+ assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size() > 0);
+ return entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0);
} else {
entity = entitiesClientV2.updateEntity(atlasEntity);
+ assertNotNull(entity);
+ assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
+ assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
+ return entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).get(0);
}
- assertNotNull(entity);
- assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
- assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
- return entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).get(0);
+
} catch (AtlasServiceException e) {
LOG.error("Entity {} failed", update ? "update" : "creation", entity);
}
@@ -242,10 +253,20 @@ public abstract class BaseResourceIT {
return modifyEntity(atlasEntity, true);
}
- protected static final String DATABASE_TYPE = "hive_db";
- protected static final String HIVE_TABLE_TYPE = "hive_table";
- protected static final String COLUMN_TYPE = "hive_column";
- protected static final String HIVE_PROCESS_TYPE = "hive_process";
+ protected static final String DATABASE_TYPE_V2 = "hive_db_v2";
+ protected static final String HIVE_TABLE_TYPE_V2 = "hive_table_v2";
+ protected static final String COLUMN_TYPE_V2 = "hive_column_v2";
+ protected static final String HIVE_PROCESS_TYPE_V2 = "hive_process_v2";
+
+ protected static final String DATABASE_TYPE = "hive_db_v1";
+ protected static final String HIVE_TABLE_TYPE = "hive_table_v1";
+ protected static final String COLUMN_TYPE = "hive_column_v1";
+ protected static final String HIVE_PROCESS_TYPE = "hive_process_v1";
+
+ protected static final String DATABASE_TYPE_BUILTIN = "hive_db";
+ protected static final String HIVE_TABLE_TYPE_BUILTIN = "hive_table";
+ protected static final String COLUMN_TYPE_BUILTIN = "hive_column";
+ protected static final String HIVE_PROCESS_TYPE_BUILTIN = "hive_process";
protected void createTypeDefinitionsV1() throws Exception {
HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil
@@ -323,7 +344,7 @@ public abstract class BaseResourceIT {
protected void createTypeDefinitionsV2() throws Exception {
AtlasEntityDef dbClsTypeDef = AtlasTypeUtil.createClassTypeDef(
- DATABASE_TYPE,
+ DATABASE_TYPE_V2,
null,
AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"),
AtlasTypeUtil.createRequiredAttrDef(DESCRIPTION, "string"),
@@ -332,7 +353,7 @@ public abstract class BaseResourceIT {
AtlasTypeUtil.createOptionalAttrDef("createTime", "int"));
AtlasEntityDef columnClsDef = AtlasTypeUtil
- .createClassTypeDef(COLUMN_TYPE, null,
+ .createClassTypeDef(COLUMN_TYPE_V2, null,
AtlasTypeUtil.createOptionalAttrDef(NAME, "string"),
AtlasTypeUtil.createOptionalAttrDef("dataType", "string"),
AtlasTypeUtil.createOptionalAttrDef("comment", "string"));
@@ -348,20 +369,21 @@ public abstract class BaseResourceIT {
));
AtlasEntityDef tblClsDef = AtlasTypeUtil
- .createClassTypeDef(HIVE_TABLE_TYPE,
+ .createClassTypeDef(HIVE_TABLE_TYPE_V2,
ImmutableSet.of("DataSet"),
AtlasTypeUtil.createOptionalAttrDef("owner", "string"),
AtlasTypeUtil.createOptionalAttrDef("createTime", "long"),
AtlasTypeUtil.createOptionalAttrDef("lastAccessTime", "date"),
AtlasTypeUtil.createOptionalAttrDef("temporary", "boolean"),
- AtlasTypeUtil.createRequiredAttrDef("db", DATABASE_TYPE),
- AtlasTypeUtil.createRequiredAttrDef("columns", DataTypes.arrayTypeName(COLUMN_TYPE)),
+ AtlasTypeUtil.createRequiredAttrDef("db", DATABASE_TYPE_V2),
+ //some tests don't set the columns field or set it to null...
+ AtlasTypeUtil.createOptionalAttrDef("columns", DataTypes.arrayTypeName(COLUMN_TYPE_V2)),
AtlasTypeUtil.createOptionalAttrDef("tableType", "tableType"),
AtlasTypeUtil.createOptionalAttrDef("serde1", "serdeType"),
AtlasTypeUtil.createOptionalAttrDef("serde2", "serdeType"));
AtlasEntityDef loadProcessClsDef = AtlasTypeUtil
- .createClassTypeDef(HIVE_PROCESS_TYPE,
+ .createClassTypeDef(HIVE_PROCESS_TYPE_V2,
ImmutableSet.of("Process"),
AtlasTypeUtil.createOptionalAttrDef("userName", "string"),
AtlasTypeUtil.createOptionalAttrDef("startTime", "int"),
@@ -415,7 +437,7 @@ public abstract class BaseResourceIT {
return RandomStringUtils.randomAlphabetic(1) + RandomStringUtils.randomAlphanumeric(9);
}
- protected Referenceable createHiveTableInstanceV1(String dbName, String tableName, Id dbId) throws Exception {
+ protected Referenceable createHiveTableInstanceBuiltIn(String dbName, String tableName, Id dbId) throws Exception {
Map<String, Object> values = new HashMap<>();
values.put(NAME, dbName);
values.put(DESCRIPTION, "foo database");
@@ -426,7 +448,7 @@ public abstract class BaseResourceIT {
values.put("location", "/tmp");
Referenceable databaseInstance = new Referenceable(dbId._getId(), dbId.getTypeName(), values);
Referenceable tableInstance =
- new Referenceable(HIVE_TABLE_TYPE, "classification", "pii", "phi", "pci", "sox", "sec", "finance");
+ new Referenceable(HIVE_TABLE_TYPE_BUILTIN, "classification", "pii", "phi", "pci", "sox", "sec", "finance");
tableInstance.set(NAME, tableName);
tableInstance.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName);
tableInstance.set("db", databaseInstance);
@@ -458,7 +480,7 @@ public abstract class BaseResourceIT {
protected AtlasEntityWithAssociations createHiveTableInstanceV2(AtlasEntity databaseInstance, String tableName) throws Exception {
AtlasEntityWithAssociations tableInstance =
- new AtlasEntityWithAssociations(HIVE_TABLE_TYPE);
+ new AtlasEntityWithAssociations(HIVE_TABLE_TYPE_V2);
tableInstance.setClassifications(
Arrays.asList(new AtlasClassification("classification"),
new AtlasClassification("pii"),
@@ -497,29 +519,34 @@ public abstract class BaseResourceIT {
return tableInstance;
}
+ protected Referenceable createHiveDBInstanceBuiltIn(String dbName) {
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
+ databaseInstance.set(NAME, dbName);
+ databaseInstance.set(QUALIFIED_NAME, dbName);
+ databaseInstance.set(CLUSTER_NAME, randomString());
+ databaseInstance.set(DESCRIPTION, "foo database");
+ return databaseInstance;
+ }
+
protected Referenceable createHiveDBInstanceV1(String dbName) {
Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
databaseInstance.set(NAME, dbName);
- databaseInstance.set(QUALIFIED_NAME, dbName);
- databaseInstance.set(CLUSTER_NAME, randomString());
databaseInstance.set(DESCRIPTION, "foo database");
return databaseInstance;
}
protected AtlasEntity createHiveDBInstanceV2(String dbName) {
- AtlasEntity atlasEntity = new AtlasEntity(DATABASE_TYPE);
+ AtlasEntity atlasEntity = new AtlasEntity(DATABASE_TYPE_V2);
atlasEntity.setAttribute(NAME, dbName);
- atlasEntity.setAttribute(QUALIFIED_NAME, dbName);
atlasEntity.setAttribute(DESCRIPTION, "foo database");
- atlasEntity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName);
atlasEntity.setAttribute("owner", "user1");
- atlasEntity.setAttribute(CLUSTER_NAME, "cl1");
- atlasEntity.setAttribute("parameters", Collections.EMPTY_MAP);
- atlasEntity.setAttribute("location", "/tmp");
+ atlasEntity.setAttribute("locationUri", "/tmp");
+ atlasEntity.setAttribute("createTime",1000);
return atlasEntity;
}
+
public interface Predicate {
/**
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/web/resources/DataSetLineageJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/DataSetLineageJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/DataSetLineageJerseyResourceIT.java
index 8334e4f..ee4057d 100644
--- a/webapp/src/test/java/org/apache/atlas/web/resources/DataSetLineageJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/DataSetLineageJerseyResourceIT.java
@@ -149,8 +149,7 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
LOG.info("JsonRow - {}", row);
Assert.assertNotNull(row.getString("name"));
Assert.assertNotNull(row.getString("comment"));
- Assert.assertNotNull(row.getString("type"));
- Assert.assertEquals(row.getString("$typeName$"), "hive_column");
+ Assert.assertEquals(row.getString("$typeName$"), "hive_column_v1");
}
}
@@ -168,8 +167,7 @@ public class DataSetLineageJerseyResourceIT extends BaseResourceIT {
LOG.info("JsonRow - {}", row);
Assert.assertNotNull(row.getString("name"));
Assert.assertNotNull(row.getString("comment"));
- Assert.assertNotNull(row.getString("type"));
- Assert.assertEquals(row.getString("$typeName$"), "hive_column");
+ Assert.assertEquals(row.getString("$typeName$"), "hive_column_v1");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
index 2bbe10a..8b51d1b 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityDiscoveryJerseyResourceIT.java
@@ -56,12 +56,12 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
super.setUp();
dbName = "db" + randomString();
createTypes();
- createInstance(createHiveDBInstanceV1(dbName));
+ createInstance(createHiveDBInstanceBuiltIn(dbName));
}
@Test
public void testSearchByDSL() throws Exception {
- String dslQuery = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
+ String dslQuery = "from "+ DATABASE_TYPE_BUILTIN + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(dslQuery);
assertNotNull(searchResult);
@@ -73,7 +73,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
assertEquals(entities.size(), 1);
AtlasEntityHeaderWithAssociations dbEntity = entities.get(0);
- assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
+ assertEquals(dbEntity.getTypeName(), DATABASE_TYPE_BUILTIN);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.ACTIVE);
assertNotNull(dbEntity.getGuid());
@@ -83,7 +83,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchDSLLimits() throws Exception {
- String dslQuery = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
+ String dslQuery = "from "+ DATABASE_TYPE_BUILTIN + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(dslQuery);
assertNotNull(searchResult);
@@ -124,7 +124,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchUsingDSL() throws Exception {
- String query = "from "+ DATABASE_TYPE + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
+ String query = "from "+ DATABASE_TYPE_BUILTIN + " " + QUALIFIED_NAME + "=\"" + dbName + "\"";
AtlasSearchResult searchResult = discoveryClientV2.dslSearch(query);
assertNotNull(searchResult);
@@ -135,7 +135,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
assertEquals(entities.size(), 1);
AtlasEntityHeaderWithAssociations dbEntity = entities.get(0);
- assertEquals(dbEntity.getTypeName(), DATABASE_TYPE);
+ assertEquals(dbEntity.getTypeName(), DATABASE_TYPE_BUILTIN);
assertEquals(dbEntity.getDisplayText(), dbName);
assertEquals(dbEntity.getStatus(), Status.ACTIVE);
@@ -166,7 +166,7 @@ public class EntityDiscoveryJerseyResourceIT extends BaseResourceIT {
AtlasFullTextResult result = fullTextResults.get(0);
assertNotNull(result.getEntity());
- assertEquals(result.getEntity().getTypeName(), DATABASE_TYPE);
+ assertEquals(result.getEntity().getTypeName(), DATABASE_TYPE_BUILTIN);
assertNotNull(result.getScore());
//API works without limit and offset
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
index f084053..9724c4c 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
@@ -85,10 +85,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
private final String DATABASE_NAME = "db" + randomString();
private final String TABLE_NAME = "table" + randomString();
- private static final String ENTITIES = "api/atlas/entities";
private static final String TRAITS = "traits";
- private static final String TRAIT_DEFINITION = "traitDefinitions";
-
private Referenceable tableInstance;
private Id tableId;
private Id dbId;
@@ -103,7 +100,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
super.setUp();
createTypeDefinitionsV1();
- Referenceable HiveDBInstance = createHiveDBInstanceV1(DATABASE_NAME);
+ Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(DATABASE_NAME);
dbId = createInstance(HiveDBInstance);
List<NotificationConsumer<EntityNotification>> consumers =
@@ -114,7 +111,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmitEntity() throws Exception {
- tableInstance = createHiveTableInstanceV1(DATABASE_NAME, TABLE_NAME, dbId);
+ tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, TABLE_NAME, dbId);
tableId = createInstance(tableInstance);
final String guid = tableId._getId();
@@ -127,7 +124,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testRequestUser() throws Exception {
- Referenceable entity = new Referenceable(DATABASE_TYPE);
+ Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
entity.set("name", dbName);
entity.set(QUALIFIED_NAME, dbName);
@@ -157,7 +154,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
//API should accept single entity (or jsonarray of entities)
public void testSubmitSingleEntity() throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
databaseInstance.set("name", dbName);
databaseInstance.set(QUALIFIED_NAME, dbName);
@@ -181,9 +178,9 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testEntityDeduping() throws Exception {
- final Referenceable db = new Referenceable(DATABASE_TYPE);
+ final Referenceable db = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
- Referenceable HiveDBInstance = createHiveDBInstanceV1(dbName);
+ Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(dbName);
Id dbIdReference = createInstance(HiveDBInstance);
final String dbId = dbIdReference._getId();
@@ -196,7 +193,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
}
});
- JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
+ JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 1);
//create entity again shouldn't create another instance with same unique attribute value
@@ -214,15 +211,15 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
//expected timeout
}
- results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
+ results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 1);
//Test the same across references
- Referenceable table = new Referenceable(HIVE_TABLE_TYPE);
+ Referenceable table = new Referenceable(HIVE_TABLE_TYPE_BUILTIN);
final String tableName = randomString();
- Referenceable tableInstance = createHiveTableInstanceV1(DATABASE_NAME, tableName, dbIdReference);
+ Referenceable tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, tableName, dbIdReference);
atlasClientV1.createEntity(tableInstance);
- results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
+ results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 1);
}
@@ -272,7 +269,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test(dataProvider = "invalidAttrValues")
public void testEntityInvalidValue(String value) throws Exception {
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
databaseInstance.set("name", randomString());
databaseInstance.set("description", value);
@@ -286,7 +283,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testGetEntityByAttribute() throws Exception {
- Referenceable db1 = new Referenceable(DATABASE_TYPE);
+ Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set(NAME, dbName);
db1.set(DESCRIPTION, randomString());
@@ -298,15 +295,15 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
createInstance(db1);
//get entity by attribute
- Referenceable referenceable = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
- Assert.assertEquals(referenceable.getTypeName(), DATABASE_TYPE);
+ Referenceable referenceable = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
+ Assert.assertEquals(referenceable.getTypeName(), DATABASE_TYPE_BUILTIN);
Assert.assertEquals(referenceable.get(QUALIFIED_NAME), dbName);
}
@Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
try {
- Referenceable tableInstance = createHiveTableInstanceV1("db" + randomString(), "table" + randomString(), dbId);
+ Referenceable tableInstance = createHiveTableInstanceBuiltIn("db" + randomString(), "table" + randomString(), dbId);
tableInstance.set("lastAccessTime", "2014-07-11");
tableId = createInstance(tableInstance);
Assert.fail("Was expecting an exception here ");
@@ -336,7 +333,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
}
- String currentTime = String.valueOf(new DateTime() );
+ String currentTime = String.valueOf(new DateTime());
addProperty(guid, "createTime", currentTime);
response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.GET_ENTITY, null, guid);
@@ -368,7 +365,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testAddReferenceProperty() throws Exception {
//Create new db instance
- Referenceable databaseInstance = new Referenceable(DATABASE_TYPE);
+ Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
databaseInstance.set(NAME, dbName);
databaseInstance.set(QUALIFIED_NAME, dbName);
@@ -420,7 +417,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetEntityList() throws Exception {
- List<String> entities = atlasClientV1.listEntities(HIVE_TABLE_TYPE);
+ List<String> entities = atlasClientV1.listEntities(HIVE_TABLE_TYPE_BUILTIN);
Assert.assertNotNull(entities);
Assert.assertTrue(entities.contains(tableId._getId()));
}
@@ -630,7 +627,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testUTF8() throws Exception {
- String classType = random();
+ //Type names cannot be arbitrary UTF8 characters. See org.apache.atlas.type.AtlasTypeUtil#validateType()
+ String classType = randomString();
String attrName = random();
String attrValue = random();
@@ -651,6 +649,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.assertEquals(getReferenceable.get(attrName), attrValue);
}
+
@Test(dependsOnMethods = "testSubmitEntity")
public void testPartialUpdate() throws Exception {
String colName = "col1"+randomString();
@@ -664,11 +663,11 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
values.put("owner", "user1");
values.put("position", 0);
values.put("description", "col1");
- values.put("table", null);
+ values.put("table", tableId ); //table is a required reference, can't be null
- Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
+ Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values);
columns.add(ref);
- Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
+ Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, new HashMap<String, Object>() {{
put("columns", columns);
}});
@@ -685,14 +684,14 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
//Update by unique attribute
values.put("type", "int");
- ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
+ ref = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values);
columns.set(0, ref);
- tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
+ tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, new HashMap<String, Object>() {{
put("columns", columns);
}});
LOG.debug("Updating entity= {}", tableUpdated);
- entityResult = atlasClientV1.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
+ entityResult = atlasClientV1.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
(String) tableInstance.get(QUALIFIED_NAME), tableUpdated);
assertEquals(entityResult.getUpdateEntities().size(), 2);
assertEquals(entityResult.getUpdateEntities().get(0), tableId._getId());
@@ -716,7 +715,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
values1.put("owner", "user1");
values1.put("position", 0);
values1.put("description", "col3");
- values1.put("table", null);
+ values1.put("table", tableId);
Map<String, Object> values2 = new HashMap<>();
@@ -727,10 +726,10 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
values2.put("owner", "user2");
values2.put("position", 1);
values2.put("description", "col4");
- values2.put("table", null);
+ values2.put("table", tableId);
- Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values1);
- Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values2);
+ Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values1);
+ Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values2);
columns.add(ref1);
columns.add(ref2);
tableInstance.set("columns", columns);
@@ -774,7 +773,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntitiesViaRestApi() throws Exception {
// Create 2 database entities
- Referenceable db1 = new Referenceable(DATABASE_TYPE);
+ Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set(NAME, dbName);
db1.set(DESCRIPTION, randomString());
@@ -785,7 +784,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
db1.set("location", "/tmp");
Id db1Id = createInstance(db1);
- Referenceable db2 = new Referenceable(DATABASE_TYPE);
+ Referenceable db2 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName2 = randomString();
db2.set(NAME, dbName2);
db2.set(QUALIFIED_NAME, dbName2);
@@ -818,7 +817,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntitiesViaClientApi() throws Exception {
// Create 2 database entities
- Referenceable db1 = new Referenceable(DATABASE_TYPE);
+ Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set("name", dbName);
db1.set("description", randomString());
@@ -828,7 +827,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
db1.set("parameters", Collections.EMPTY_MAP);
db1.set("location", "/tmp");
Id db1Id = createInstance(db1);
- Referenceable db2 = new Referenceable(DATABASE_TYPE);
+ Referenceable db2 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName2 = randomString();
db2.set("name", dbName2);
db2.set(QUALIFIED_NAME, dbName2);
@@ -844,7 +843,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
// Delete the database entities
List<String> deletedGuidsList =
atlasClientV1.deleteEntities(db1Id._getId(), db2Id._getId()).getDeletedEntities();
- // Verify that deleteEntities() response has database entity guids
+ // Verify that deleteEntities() response has database entity guids
Assert.assertEquals(deletedGuidsList.size(), 2);
Assert.assertTrue(deletedGuidsList.contains(db1Id._getId()));
Assert.assertTrue(deletedGuidsList.contains(db2Id._getId()));
@@ -859,7 +858,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntityByUniqAttribute() throws Exception {
// Create database entity
- Referenceable db1 = new Referenceable(DATABASE_TYPE);
+ Referenceable db1 = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = randomString();
db1.set(NAME, dbName);
db1.set(QUALIFIED_NAME, dbName);
@@ -873,7 +872,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Id db1Id = createInstance(db1);
// Delete the database entity
- List<String> deletedGuidsList = atlasClientV1.deleteEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName).getDeletedEntities();
+ List<String> deletedGuidsList = atlasClientV1.deleteEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName).getDeletedEntities();
// Verify that deleteEntities() response has database entity guids
Assert.assertEquals(deletedGuidsList.size(), 1);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java
index 74338fd..d9bffe9 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityV2JerseyResourceIT.java
@@ -71,16 +71,10 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
private final String DATABASE_NAME = "db" + randomString();
private final String TABLE_NAME = "table" + randomString();
- private static final String TRAITS = "traits";
- private static final String TRAIT_DEFINITION = "traitDefinitions";
-
private String traitName;
private AtlasEntity dbEntity;
- private AtlasEntityHeader dbEntityHeader;
private AtlasEntityWithAssociations tableEntity;
- private AtlasEntityHeader tableEntityHeader;
-
@Inject
private NotificationInterface notificationInterface;
private NotificationConsumer<EntityNotification> notificationConsumer;
@@ -99,7 +93,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmitEntity() throws Exception {
- TypeUtils.Pair dbAndTable = createDBAndTable(DATABASE_NAME, TABLE_NAME);
+ TypeUtils.Pair dbAndTable = createDBAndTable();
assertNotNull(dbAndTable);
assertNotNull(dbAndTable.left);
assertNotNull(dbAndTable.right);
@@ -107,18 +101,18 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testRequestUser() throws Exception {
- AtlasEntity hiveDBInstanceV2 = createHiveDB(DATABASE_NAME);
+ AtlasEntity hiveDBInstanceV2 = createHiveDB(randomString());
List<EntityAuditEvent> events = atlasClientV1.getEntityAuditEvents(hiveDBInstanceV2.getGuid(), (short) 10);
- assertTrue(events.size() > 1);
+ assertEquals(events.size(), 1);
assertEquals(events.get(0).getUser(), "admin");
}
@Test
public void testEntityDeduping() throws Exception {
- JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, DATABASE_NAME));
+ JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_V2, DATABASE_NAME));
assertEquals(results.length(), 1);
- final AtlasEntity hiveDBInstanceV2 = createHiveDB(DATABASE_NAME);
+ final AtlasEntity hiveDBInstanceV2 = createHiveDB();
// Do the notification thing here
waitForNotification(notificationConsumer, MAX_WAIT_TIME, new NotificationPredicate() {
@Override
@@ -128,7 +122,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
});
- results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, DATABASE_NAME));
+ results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_V2, DATABASE_NAME));
assertEquals(results.length(), 1);
//Test the same across references
@@ -139,7 +133,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
EntityMutationResponse entity = entitiesClientV2.createEntity(hiveTableInstanceV2);
assertNotNull(entity);
assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
- results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, DATABASE_NAME));
+ results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_V2, DATABASE_NAME));
assertEquals(results.length(), 1);
}
@@ -205,7 +199,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dataProvider = "invalidAttrValues")
public void testEntityInvalidValue(String value) throws Exception {
- AtlasEntity databaseInstance = new AtlasEntity(DATABASE_TYPE);
+ AtlasEntity databaseInstance = new AtlasEntity(DATABASE_TYPE_V2);
String dbName = randomString();
databaseInstance.setAttribute("name", dbName);
databaseInstance.setAttribute("description", value);
@@ -216,18 +210,18 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testGetEntityByAttribute() throws Exception {
- AtlasEntity hiveDB = createHiveDB(DATABASE_NAME);
- String qualifiedName = (String) hiveDB.getAttribute(QUALIFIED_NAME);
+ AtlasEntity hiveDB = createHiveDB();
+ String qualifiedName = (String) hiveDB.getAttribute(NAME);
//get entity by attribute
- AtlasEntity byAttribute = entitiesClientV2.getEntityByAttribute(DATABASE_TYPE, QUALIFIED_NAME, qualifiedName);
- assertEquals(byAttribute.getTypeName(), DATABASE_TYPE);
- assertEquals(byAttribute.getAttribute(QUALIFIED_NAME), qualifiedName);
+ AtlasEntity byAttribute = entitiesClientV2.getEntityByAttribute(DATABASE_TYPE_V2, NAME, qualifiedName);
+ assertEquals(byAttribute.getTypeName(), DATABASE_TYPE_V2);
+ assertEquals(byAttribute.getAttribute(NAME), qualifiedName);
}
@Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
AtlasEntity hiveDBInstance = createHiveDBInstanceV2("db" + randomString());
- AtlasEntityHeader entity = createEntity(hiveDBInstance);
+ createEntity(hiveDBInstance);
AtlasEntity tableInstance = createHiveTableInstanceV2(hiveDBInstance, "table" + randomString());
tableInstance.setAttribute("lastAccessTime", "2014-07-11");
@@ -239,9 +233,9 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
public void testAddProperty() throws Exception {
//add property
String description = "bar table - new desc";
- addProperty(tableEntity.getGuid(), "description", description);
+ addProperty(createHiveTable().getGuid(), "description", description);
- AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
+ AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entityByGuid);
entityByGuid.setAttribute("description", description);
@@ -260,9 +254,9 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
String currentTime = String.valueOf(new DateTime());
- addProperty(tableEntity.getGuid(), "createTime", currentTime);
+ addProperty(createHiveTable().getGuid(), "createTime", currentTime);
- entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
+ entityByGuid = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entityByGuid);
}
@@ -271,7 +265,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
// FIXME: Behavior has changed between v1 and v2
//add property
// try {
- addProperty(tableEntity.getGuid(), "description", null);
+ addProperty(createHiveTable().getGuid(), "description", null);
// Assert.fail("Expected AtlasServiceException");
// } catch(AtlasServiceException e) {
// Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
@@ -329,7 +323,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testGetTraitNames() throws Exception {
- AtlasClassifications classifications = entitiesClientV2.getClassifications(tableEntity.getGuid());
+ AtlasClassifications classifications = entitiesClientV2.getClassifications(createHiveTable().getGuid());
assertNotNull(classifications);
assertTrue(classifications.getList().size() > 0);
assertEquals(classifications.getList().size(), 8);
@@ -337,7 +331,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testCommonAttributes() throws Exception{
- AtlasEntity entity = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
+ AtlasEntity entity = entitiesClientV2.getEntityByGuid(createHiveTable().getGuid());
Assert.assertNotNull(entity.getStatus());
Assert.assertNotNull(entity.getVersion());
Assert.assertNotNull(entity.getCreatedBy());
@@ -356,11 +350,10 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
}
private AtlasEntity createHiveDB() {
- if (dbEntity != null) {
- return dbEntity;
- } else {
- return createHiveDB(DATABASE_NAME);
+ if (dbEntity == null) {
+ dbEntity = createHiveDB(DATABASE_NAME);
}
+ return dbEntity;
}
private AtlasEntity createHiveDB(String dbName) {
@@ -369,23 +362,21 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
assertNotNull(entityHeader);
assertNotNull(entityHeader.getGuid());
hiveDBInstanceV2.setGuid(entityHeader.getGuid());
- dbEntity = hiveDBInstanceV2;
- dbEntityHeader = entityHeader;
return hiveDBInstanceV2;
}
- private TypeUtils.Pair<AtlasEntity, AtlasEntityWithAssociations> createDBAndTable(String dbName, String tableName) throws Exception {
- AtlasEntity dbInstanceV2 = createHiveDB(dbName);
- AtlasEntityWithAssociations hiveTableInstanceV2 = createHiveTable(dbInstanceV2, tableName);
+ private TypeUtils.Pair<AtlasEntity, AtlasEntityWithAssociations> createDBAndTable() throws Exception {
+ AtlasEntity dbInstanceV2 = createHiveDB();
+ AtlasEntityWithAssociations hiveTableInstanceV2 = createHiveTable();
return TypeUtils.Pair.of(dbInstanceV2, hiveTableInstanceV2);
}
private AtlasEntityWithAssociations createHiveTable() throws Exception {
- if (tableEntity != null) {
- return tableEntity;
- } else {
- return createHiveTable(createHiveDB(), TABLE_NAME);
+ if (tableEntity == null) {
+ tableEntity = createHiveTable(createHiveDB(), TABLE_NAME);
}
+ return tableEntity;
+
}
private AtlasEntityWithAssociations createHiveTable(AtlasEntity dbInstanceV2, String tableName) throws Exception {
@@ -396,7 +387,6 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
hiveTableInstanceV2.setGuid(createdHeader.getGuid());
entitiesClientV2.addClassifications(createdHeader.getGuid(), hiveTableInstanceV2.getClassifications());
tableEntity = hiveTableInstanceV2;
- tableEntityHeader = createdHeader;
return hiveTableInstanceV2;
}
@@ -409,9 +399,9 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
typesDef.getClassificationDefs().add(piiTrait);
createType(typesDef);
- entitiesClientV2.addClassifications(tableEntity.getGuid(), ImmutableList.of(new AtlasClassification(piiTrait.getName())));
+ entitiesClientV2.addClassifications(createHiveTable().getGuid(), ImmutableList.of(new AtlasClassification(piiTrait.getName())));
- assertEntityAudit(tableEntity.getGuid(), EntityAuditEvent.EntityAuditAction.TAG_ADD);
+ assertEntityAudit(createHiveTable().getGuid(), EntityAuditEvent.EntityAuditAction.TAG_ADD);
}
@Test(dependsOnMethods = "testSubmitEntity")
@@ -426,13 +416,14 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
AtlasClassificationDef classificationByName = typedefClientV2.getClassificationByName(traitName);
assertNotNull(classificationByName);
- assertEquals(tableEntity.getClassifications().size(), 7);
+ AtlasEntityWithAssociations hiveTable = createHiveTable();
+ assertEquals(hiveTable.getClassifications().size(), 7);
AtlasClassification piiClassification = new AtlasClassification(piiTrait.getName());
- entitiesClientV2.addClassifications(tableEntity.getGuid(), Lists.newArrayList(piiClassification));
+ entitiesClientV2.addClassifications(hiveTable.getGuid(), Lists.newArrayList(piiClassification));
- AtlasClassifications classifications = entitiesClientV2.getClassifications(tableEntity.getGuid());
+ AtlasClassifications classifications = entitiesClientV2.getClassifications(hiveTable.getGuid());
assertNotNull(classifications);
assertTrue(classifications.getList().size() > 0);
assertEquals(classifications.getList().size(), 8);
@@ -452,7 +443,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
AtlasClassification traitInstance = new AtlasClassification(traitName);
traitInstance.setAttribute("type", "SSN");
- final String guid = tableEntity.getGuid();
+ final String guid = createHiveTable().getGuid();
entitiesClientV2.addClassifications(guid, ImmutableList.of(traitInstance));
// verify the response
@@ -474,8 +465,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(expectedExceptions = AtlasServiceException.class)
public void testAddTraitWithNoRegistration() throws Exception {
final String traitName = "PII_Trait" + randomString();
- AtlasClassificationDef piiTrait =
- AtlasTypeUtil.createTraitTypeDef(traitName, ImmutableSet.<String>of());
+ AtlasTypeUtil.createTraitTypeDef(traitName, ImmutableSet.<String>of());
AtlasClassification traitInstance = new AtlasClassification(traitName);
@@ -484,7 +474,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testAddTrait")
public void testDeleteTrait() throws Exception {
- final String guid = tableEntity.getGuid();
+ final String guid = createHiveTable().getGuid();
try {
entitiesClientV2.deleteClassification(guid, traitName);
@@ -512,7 +502,7 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test(dependsOnMethods = "testSubmitEntity")
public void testDeleteExistentTraitNonExistentForEntity() throws Exception {
- final String guid = tableEntity.getGuid();
+ final String guid = createHiveTable().getGuid();
final String traitName = "PII_Trait" + randomString();
AtlasClassificationDef piiTrait = AtlasTypeUtil
.createTraitTypeDef(traitName, ImmutableSet.<String>of(),
@@ -562,43 +552,44 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
final List<AtlasEntity> columns = new ArrayList<>();
Map<String, Object> values = new HashMap<>();
values.put("name", "col1");
- values.put(QUALIFIED_NAME, "qualifiedName.col1");
+ values.put(NAME, "qualifiedName.col1");
values.put("type", "string");
values.put("comment", "col1 comment");
- AtlasEntity ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values);
+ AtlasEntity ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values);
columns.add(ref);
+ AtlasEntityWithAssociations hiveTable = createHiveTable();
+ AtlasEntityWithAssociations tableUpdated = hiveTable;
- AtlasEntityWithAssociations tableUpdated = tableEntity;
- tableEntity.setAttribute("columns", columns);
+ hiveTable.setAttribute("columns", columns);
LOG.debug("Updating entity= " + tableUpdated);
- EntityMutationResponse updateResult = entitiesClientV2.updateEntity(tableEntity.getGuid(), tableUpdated);
+ EntityMutationResponse updateResult = entitiesClientV2.updateEntity(hiveTable.getGuid(), tableUpdated);
assertNotNull(updateResult);
assertNotNull(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
- AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
+ AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
assertNotNull(entityByGuid);
- List<AtlasEntity> columns1 = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
+ entityByGuid.getAttribute("columns");
//Update by unique attribute
values.put("type", "int");
- ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values);
+ ref = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values);
columns.set(0, ref);
- tableUpdated = tableEntity;
+ tableUpdated = hiveTable;
tableUpdated.setAttribute("columns", columns);
LOG.debug("Updating entity= " + tableUpdated);
- EntityMutationResponse updateResponse = entitiesClientV2.updateEntityByAttribute(BaseResourceIT.HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
- (String) tableEntity.getAttribute("name"), tableUpdated);
+ EntityMutationResponse updateResponse = entitiesClientV2.updateEntityByAttribute(BaseResourceIT.HIVE_TABLE_TYPE_V2, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
+ (String) hiveTable.getAttribute("name"), tableUpdated);
assertNotNull(updateResponse);
assertNotNull(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
assertTrue(updateResponse.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0);
- entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
+ entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
assertNotNull(entityByGuid);
- columns1 = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
+ entityByGuid.getAttribute("columns");
}
@Test(dependsOnMethods = "testSubmitEntity")
@@ -606,27 +597,31 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
final List<AtlasEntity> columns = new ArrayList<>();
Map<String, Object> values1 = new HashMap<>();
values1.put("name", "col3");
- values1.put(QUALIFIED_NAME, "qualifiedName.col3");
+ values1.put(NAME, "qualifiedName.col3");
values1.put("type", "string");
values1.put("comment", "col3 comment");
Map<String, Object> values2 = new HashMap<>();
values2.put("name", "col4");
- values2.put(QUALIFIED_NAME, "qualifiedName.col4");
+ values2.put(NAME, "qualifiedName.col4");
values2.put("type", "string");
values2.put("comment", "col4 comment");
- AtlasEntity ref1 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values1);
- AtlasEntity ref2 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE, values2);
+ AtlasEntity ref1 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values1);
+ AtlasEntity ref2 = new AtlasEntity(BaseResourceIT.COLUMN_TYPE_V2, values2);
columns.add(ref1);
columns.add(ref2);
- tableEntity.setAttribute("columns", columns);
- EntityMutationResponse updateEntityResult = entitiesClientV2.updateEntity(tableEntity);
+ AtlasEntityWithAssociations hiveTable = createHiveTable();
+ hiveTable.setAttribute("columns", columns);
+ EntityMutationResponse updateEntityResult = entitiesClientV2.updateEntity(hiveTable);
assertNotNull(updateEntityResult);
assertNotNull(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE));
- assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size(), 3);
+ assertNotNull(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE));
+ //2 columns are being created, and 1 hiveTable is being updated
+ assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size(), 1);
+ assertEquals(updateEntityResult.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size(), 2);
- AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(tableEntity.getGuid());
+ AtlasEntity entityByGuid = entitiesClientV2.getEntityByGuid(hiveTable.getGuid());
List<AtlasEntity> refs = (List<AtlasEntity>) entityByGuid.getAttribute("columns");
assertEquals(refs.size(), 2);
}
@@ -634,17 +629,17 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
@Test
public void testDeleteEntities() throws Exception {
// Create 2 database entities
- AtlasEntity db1 = new AtlasEntity(DATABASE_TYPE);
+ AtlasEntity db1 = new AtlasEntity(DATABASE_TYPE_V2);
String dbName1 = randomString();
db1.setAttribute("name", dbName1);
- db1.setAttribute(QUALIFIED_NAME, dbName1);
+ db1.setAttribute(NAME, dbName1);
db1.setAttribute("clusterName", randomString());
db1.setAttribute("description", randomString());
AtlasEntityHeader entity1Header = createEntity(db1);
- AtlasEntity db2 = new AtlasEntity(DATABASE_TYPE);
+ AtlasEntity db2 = new AtlasEntity(DATABASE_TYPE_V2);
String dbName2 = randomString();
db2.setAttribute("name", dbName2);
- db2.setAttribute(QUALIFIED_NAME, dbName2);
+ db2.setAttribute(NAME, dbName2);
db2.setAttribute("clusterName", randomString());
db2.setAttribute("description", randomString());
AtlasEntityHeader entity2Header = createEntity(db2);
@@ -664,10 +659,10 @@ public class EntityV2JerseyResourceIT extends BaseResourceIT {
public void testDeleteEntityByUniqAttribute() throws Exception {
// Create database entity
AtlasEntity hiveDB = createHiveDB(DATABASE_NAME + random());
- String qualifiedName = (String) hiveDB.getAttribute(QUALIFIED_NAME);
+ String qualifiedName = (String) hiveDB.getAttribute(NAME);
// Delete the database entity
- EntityMutationResponse deleteResponse = entitiesClientV2.deleteEntityByAttribute(DATABASE_TYPE, QUALIFIED_NAME, qualifiedName);
+ EntityMutationResponse deleteResponse = entitiesClientV2.deleteEntityByAttribute(DATABASE_TYPE_V2, NAME, qualifiedName);
// Verify that deleteEntities() response has database entity guids
assertNotNull(deleteResponse);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
index b004cb5..318df60 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java
@@ -66,7 +66,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchByDSL() throws Exception {
- String dslQuery = "from "+ DATABASE_TYPE + " qualifiedName=\"" + dbName + "\"";
+ String dslQuery = "from "+ DATABASE_TYPE + " name=\"" + dbName + "\"";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", dslQuery);
JSONObject response = atlasClientV1.callAPIWithQueryParams(AtlasClient.API.SEARCH_DSL, queryParams);
@@ -89,7 +89,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
public void testSearchDSLLimits() throws Exception {
//search without new parameters of limit and offset should work
- String dslQuery = "from "+ DATABASE_TYPE + " qualifiedName=\"" + dbName + "\"";
+ String dslQuery = "from "+ DATABASE_TYPE + " name=\"" + dbName + "\"";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", dslQuery);
JSONObject response = atlasClientV1.callAPIWithQueryParams(AtlasClient.API.SEARCH_DSL, queryParams);
@@ -146,7 +146,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchUsingGremlin() throws Exception {
- String query = "g.V.has('type', 'hive_db').toList()";
+ String query = "g.V.has('type', '" + BaseResourceIT.HIVE_TABLE_TYPE + "').toList()";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", query);
@@ -162,7 +162,7 @@ public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT {
@Test
public void testSearchUsingDSL() throws Exception {
//String query = "from dsl_test_type";
- String query = "from "+ DATABASE_TYPE + " qualifiedName=\"" + dbName +"\"";
+ String query = "from "+ DATABASE_TYPE + " name=\"" + dbName +"\"";
MultivaluedMap<String, String> queryParams = new MultivaluedMapImpl();
queryParams.add("query", query);
JSONObject response = atlasClientV1.callAPIWithQueryParams(AtlasClient.API.SEARCH, queryParams);
[2/2] incubator-atlas git commit: ATLAS-1114: Performance
improvements for create/update entity (2 of 2)
Posted by jn...@apache.org.
ATLAS-1114: Performance improvements for create/update entity (2 of 2)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/89f70609
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/89f70609
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/89f70609
Branch: refs/heads/master
Commit: 89f70609645fa53cb5492035c49c2aa4b99031e9
Parents: 6cd6811
Author: Jeff Hagelberg <jn...@us.ibm.com>
Authored: Thu Jan 26 16:03:24 2017 -0500
Committer: Jeff Hagelberg <jn...@us.ibm.com>
Committed: Thu Jan 26 16:04:51 2017 -0500
----------------------------------------------------------------------
.../audit/InMemoryEntityAuditRepository.java | 14 +-
.../atlas/repository/graph/DeleteHandler.java | 3 +-
.../atlas/repository/graph/FullTextMapper.java | 22 +-
.../graph/GraphBackedMetadataRepository.java | 55 +++--
.../atlas/repository/graph/GraphHelper.java | 209 ++++++++++++++--
.../graph/TypedInstanceToGraphMapper.java | 247 ++++++++++++-------
.../repository/graph/VertexLookupContext.java | 166 +++++++++++++
.../atlas/services/DefaultMetadataService.java | 4 +
.../apache/atlas/util/AttributeValueMap.java | 60 +++++
.../org/apache/atlas/util/IndexedInstance.java | 62 +++++
.../atlas/repository/graph/GraphHelperTest.java | 106 ++++++--
.../test/resources/atlas-application.properties | 1 +
.../atlas/web/resources/EntityResource.java | 2 +-
.../notification/EntityNotificationIT.java | 26 +-
.../NotificationHookConsumerIT.java | 42 ++--
.../atlas/web/resources/BaseResourceIT.java | 77 ++++--
.../DataSetLineageJerseyResourceIT.java | 6 +-
.../EntityDiscoveryJerseyResourceIT.java | 14 +-
.../web/resources/EntityJerseyResourceIT.java | 79 +++---
.../web/resources/EntityV2JerseyResourceIT.java | 151 ++++++------
.../MetadataDiscoveryJerseyResourceIT.java | 8 +-
21 files changed, 997 insertions(+), 357 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
index 50a007b..86f1af7 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java
@@ -18,17 +18,17 @@
package org.apache.atlas.repository.audit;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.EntityAuditEvent;
-
-import com.google.inject.Singleton;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.EntityAuditEvent;
+
+import com.google.inject.Singleton;
+
/**
* Entity audit repository where audit events are stored in-memory. Used only for integration tests
*/
@@ -50,8 +50,10 @@ public class InMemoryEntityAuditRepository implements EntityAuditRepository {
}
}
+ //synchronized to avoid concurrent modification exception that occurs if events are added
+ //while we are iterating through the map
@Override
- public List<EntityAuditEvent> listEvents(String entityId, String startKey, short maxResults)
+ public synchronized List<EntityAuditEvent> listEvents(String entityId, String startKey, short maxResults)
throws AtlasException {
List<EntityAuditEvent> events = new ArrayList<>();
String myStartKey = startKey;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
index 9eb086f..4973a33 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
@@ -22,6 +22,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
@@ -70,7 +71,7 @@ public abstract class DeleteHandler {
* @param instanceVertices
* @throws AtlasException
*/
- public void deleteEntities(List<AtlasVertex> instanceVertices) throws AtlasException {
+ public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasException {
RequestContext requestContext = RequestContext.get();
Set<AtlasVertex> deletionCandidateVertices = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
index 911b1ad..5be8d0b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
@@ -17,10 +17,15 @@
*/
package org.apache.atlas.repository.graph;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumValue;
@@ -29,23 +34,22 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class FullTextMapper {
private static final Logger LOG = LoggerFactory.getLogger(FullTextMapper.class);
private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
+ private final TypedInstanceToGraphMapper typedInstanceToGraphMapper;
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private static final String FULL_TEXT_DELIMITER = " ";
private final Map<String, ITypedReferenceableInstance> instanceCache;
- FullTextMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
+ FullTextMapper(TypedInstanceToGraphMapper typedInstanceToGraphMapper,
+ GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
+ this.typedInstanceToGraphMapper = typedInstanceToGraphMapper;
instanceCache = new HashMap<>();
}
@@ -126,8 +130,12 @@ public class FullTextMapper {
case CLASS:
if (followReferences) {
- String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
- AtlasVertex refVertex = graphHelper.getVertexForGUID(refGuid);
+ Id refId = ((ITypedReferenceableInstance) value).getId();
+ String refGuid = refId._getId();
+ AtlasVertex refVertex = typedInstanceToGraphMapper.lookupVertex(refId);
+ if(refVertex == null) {
+ refVertex = graphHelper.getVertexForGUID(refGuid);
+ }
return mapRecursive(refVertex, false);
}
break;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index 0c80aed..be02891 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -19,9 +19,11 @@
package org.apache.atlas.repository.graph;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
@@ -306,7 +308,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
}
-
+
AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
@@ -331,7 +333,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
}
-
+
private void updateTraits(AtlasVertex instanceVertex, List<String> traitNames) {
// remove the key
instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
@@ -357,8 +359,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
entitiesUpdated);
RequestContext requestContext = RequestContext.get();
- return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
- requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
+ return createEntityResultFromContext(requestContext);
} catch (AtlasException e) {
throw new RepositoryException(e);
}
@@ -375,13 +376,14 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity);
RequestContext requestContext = RequestContext.get();
- return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
- requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
+ return createEntityResultFromContext(requestContext);
} catch (AtlasException e) {
throw new RepositoryException(e);
}
}
+
+
@Override
@GraphTransaction
public AtlasClient.EntityResult deleteEntities(List<String> guids) throws RepositoryException {
@@ -390,32 +392,41 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
throw new IllegalArgumentException("guids must be non-null and non-empty");
}
- List<AtlasVertex> vertices = new ArrayList<>(guids.size());
- for (String guid : guids) {
- if (guid == null) {
- LOG.warn("deleteEntities: Ignoring null guid");
- continue;
- }
- try {
- AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
- vertices.add(instanceVertex);
- } catch (EntityNotFoundException e) {
- // Entity does not exist - treat as non-error, since the caller
- // wanted to delete the entity and it's already gone.
- LOG.info("Deletion request ignored for non-existent entity with guid {}", guid);
+ // Retrieve vertices for requested guids.
+ Map<String, AtlasVertex> vertices = graphHelper.getVerticesForGUIDs(guids);
+ Collection<AtlasVertex> deletionCandidates = vertices.values();
+
+ if(LOG.isDebugEnabled()) {
+ for(String guid : guids) {
+ if(! vertices.containsKey(guid)) {
+ // Entity does not exist - treat as non-error, since the caller
+ // wanted to delete the entity and it's already gone.
+ LOG.debug("Deletion request ignored for non-existent entity with guid " + guid);
+ }
}
}
+ if (deletionCandidates.isEmpty()) {
+ LOG.info("No deletion candidate entities were found for guids %s", guids);
+ return new AtlasClient.EntityResult(Collections.<String>emptyList(), Collections.<String>emptyList(), Collections.<String>emptyList());
+ }
+
try {
- deleteHandler.deleteEntities(vertices);
+ deleteHandler.deleteEntities(deletionCandidates);
}
catch (AtlasException e) {
throw new RepositoryException(e);
}
RequestContext requestContext = RequestContext.get();
- return new AtlasClient.EntityResult(requestContext.getCreatedEntityIds(),
- requestContext.getUpdatedEntityIds(), requestContext.getDeletedEntityIds());
+ return createEntityResultFromContext(requestContext);
+ }
+
+ private AtlasClient.EntityResult createEntityResultFromContext(RequestContext requestContext) {
+ return new AtlasClient.EntityResult(
+ requestContext.getCreatedEntityIds(),
+ requestContext.getUpdatedEntityIds(),
+ requestContext.getDeletedEntityIds());
}
public AtlasGraph getGraph() {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index c4bb61b..ce8d4c7 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -18,9 +18,20 @@
package org.apache.atlas.repository.graph;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.Stack;
+import java.util.UUID;
+
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
@@ -51,13 +62,17 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.atlas.util.AttributeValueMap;
+import org.apache.atlas.util.IndexedInstance;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
/**
* Utility class for graph operations.
@@ -402,7 +417,7 @@ public final class GraphHelper {
* Gets the value of a property that is stored in the graph as a single property value. If
* a multi-property such as {@link Constants#TRAIT_NAMES_PROPERTY_KEY} or {@link Constants#SUPER_TYPES_PROPERTY_KEY}
* is used, an exception will be thrown.
- *
+ *
* @param element
* @param propertyName
* @param clazz
@@ -414,7 +429,7 @@ public final class GraphHelper {
if (LOG.isDebugEnabled()) {
LOG.debug("Reading property {} from {}", actualPropertyName, string(element));
}
-
+
return element.getProperty(actualPropertyName, clazz);
}
@@ -442,8 +457,8 @@ public final class GraphHelper {
}
return edge.getProperty(actualPropertyName, Object.class);
- }
-
+ }
+
private static <T extends AtlasElement> String string(T element) {
if (element instanceof AtlasVertex) {
return string((AtlasVertex) element);
@@ -452,10 +467,10 @@ public final class GraphHelper {
}
return element.toString();
}
-
+
/**
* Adds an additional value to a multi-property.
- *
+ *
* @param vertex
* @param propertyName
* @param value
@@ -516,6 +531,60 @@ public final class GraphHelper {
return findVertex(Constants.GUID_PROPERTY_KEY, guid);
}
+
+ /**
+ * Finds the Vertices that correspond to the given property values. Property
+ * values that are not found in the graph will not be in the map.
+ *
+ * @return propertyValue to AtlasVertex map with the result.
+ */
+ private Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String> values)
+ throws RepositoryException {
+
+ if(values.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ Collection<String> nonNullValues = new HashSet<>(values.size());
+
+ for(String value : values) {
+ if(value != null) {
+ nonNullValues.add(value);
+ }
+ }
+
+ //create graph query that finds vertices with the guids
+ AtlasGraphQuery query = graph.query();
+ query.in(property, nonNullValues);
+ Iterable<AtlasVertex> results = query.vertices();
+
+ Map<String, AtlasVertex> result = new HashMap<>(values.size());
+ //Process the result, using the guidToIndexMap to figure out where
+ //each vertex should go in the result list.
+ for(AtlasVertex vertex : results) {
+ if(vertex.exists()) {
+ String propertyValue = vertex.getProperty(property, String.class);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Found a vertex {} with {} = {}", string(vertex), property, propertyValue);
+ }
+ result.put(propertyValue, vertex);
+ }
+ }
+ return result;
+ }
+
+
+ /**
+ * Finds the Vertices that correspond to the given GUIDs. GUIDs
+ * that are not found in the graph will not be in the map.
+ *
+ * @return GUID to AtlasVertex map with the result.
+ */
+ public Map<String, AtlasVertex> getVerticesForGUIDs(List<String> guids)
+ throws RepositoryException {
+
+ return getVerticesForPropertyValues(Constants.GUID_PROPERTY_KEY, guids);
+ }
+
public static String getQualifiedNameForMapKey(String prefix, String key) {
return prefix + "." + key;
}
@@ -638,6 +707,112 @@ public final class GraphHelper {
}
/**
+ * Finds vertices that match at least one unique attribute of the instances specified. The AtlasVertex at a given index in the result corresponds
+ * to the IReferencableInstance at that same index that was passed in. The number of elements in the resultant list is guaranteed to match the
+ * number of instances that were passed in. If no vertex is found for a given instance, that entry will be null in the resultant list.
+ *
+ *
+ * @param classType
+ * @param instancesForClass
+ * @return
+ * @throws AtlasException
+ */
+ public List<AtlasVertex> getVerticesForInstancesByUniqueAttribute(ClassType classType, List<? extends IReferenceableInstance> instancesForClass) throws AtlasException {
+
+ //For each attribute, need to figure out what values to search for and which instance(s)
+ //those values correspond to.
+ Map<String, AttributeValueMap> map = new HashMap<String, AttributeValueMap>();
+
+ for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
+ if (attributeInfo.isUnique) {
+ String propertyKey = getQualifiedFieldName(classType, attributeInfo.name);
+ AttributeValueMap mapForAttribute = new AttributeValueMap();
+ for(int idx = 0; idx < instancesForClass.size(); idx++) {
+ IReferenceableInstance instance = instancesForClass.get(idx);
+ Object value = instance.get(attributeInfo.name);
+ mapForAttribute.put(value, instance, idx);
+ }
+ map.put(propertyKey, mapForAttribute);
+ }
+ }
+
+ AtlasVertex[] result = new AtlasVertex[instancesForClass.size()];
+ if(map.isEmpty()) {
+ //no unique attributes
+ return Arrays.asList(result);
+ }
+
+ //construct gremlin query
+ AtlasGraphQuery query = graph.query();
+
+ query.has(Constants.ENTITY_TYPE_PROPERTY_KEY, classType.getName());
+ query.has(Constants.STATE_PROPERTY_KEY,Id.EntityState.ACTIVE.name());
+
+ List<AtlasGraphQuery> orChildren = new ArrayList<AtlasGraphQuery>();
+
+
+ //build up an or expression to find vertices which match at least one of the unique attribute constraints
+ //For each unique attributes, we add a within clause to match vertices that have a value of that attribute
+ //that matches the value in some instance.
+ for(Map.Entry<String, AttributeValueMap> entry : map.entrySet()) {
+ AtlasGraphQuery orChild = query.createChildQuery();
+ String propertyName = entry.getKey();
+ AttributeValueMap valueMap = entry.getValue();
+ Set<Object> values = valueMap.getAttributeValues();
+ if(values.size() == 1) {
+ orChild.has(propertyName, values.iterator().next());
+ }
+ else if(values.size() > 1) {
+ orChild.in(propertyName, values);
+ }
+ orChildren.add(orChild);
+ }
+
+ if(orChildren.size() == 1) {
+ AtlasGraphQuery child = orChildren.get(0);
+ query.addConditionsFrom(child);
+ }
+ else if(orChildren.size() > 1) {
+ query.or(orChildren);
+ }
+
+ Iterable<AtlasVertex> queryResult = query.vertices();
+
+
+ for(AtlasVertex matchingVertex : queryResult) {
+ Collection<IndexedInstance> matches = getInstancesForVertex(map, matchingVertex);
+ for(IndexedInstance wrapper : matches) {
+ result[wrapper.getIndex()]= matchingVertex;
+ }
+ }
+ return Arrays.asList(result);
+ }
+
+ //finds the instance(s) that correspond to the given vertex
+ private Collection<IndexedInstance> getInstancesForVertex(Map<String, AttributeValueMap> map, AtlasVertex foundVertex) {
+
+ //loop through the unique attributes. For each attribute, check to see if the vertex property that
+ //corresponds to that attribute has a value from one or more of the instances that were passed in.
+
+ for(Map.Entry<String, AttributeValueMap> entry : map.entrySet()) {
+
+ String propertyName = entry.getKey();
+ AttributeValueMap valueMap = entry.getValue();
+
+ Object vertexValue = foundVertex.getProperty(propertyName, Object.class);
+
+ Collection<IndexedInstance> instances = valueMap.get(vertexValue);
+ if(instances != null && instances.size() > 0) {
+ //return first match. Let the underling graph determine if this is a problem
+ //(i.e. if the other unique attributes change be changed safely to match what
+ //the user requested).
+ return instances;
+ }
+ //try another attribute
+ }
+ return Collections.emptyList();
+ }
+ /**
* Guid and AtlasVertex combo
*/
public static class VertexInfo {
@@ -779,9 +954,9 @@ public final class GraphHelper {
public static ITypedReferenceableInstance getTypedReferenceableInstance(TypeSystem typeSystem, Referenceable entityInstance)
throws AtlasException {
final String entityTypeName = ParamChecker.notEmpty(entityInstance.getTypeName(), "Entity type cannot be null");
-
+
ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
-
+
//Both assigned id and values are required for full update
//classtype.convert() will remove values if id is assigned. So, set temp id, convert and
// then replace with original id
@@ -839,7 +1014,7 @@ public final class GraphHelper {
}
}
-
+
public static void dumpToLog(final AtlasGraph<?,?> graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
@@ -951,7 +1126,7 @@ public final class GraphHelper {
}
return null;
}
-
+
public static boolean elementExists(AtlasElement v) {
return v != null && v.exists();
}
@@ -972,12 +1147,12 @@ public final class GraphHelper {
public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) throws AtlasException {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
- instanceVertex.setListProperty(actualPropertyName, value);
+ instanceVertex.setListProperty(actualPropertyName, value);
}
-
+
public static List<String> getListProperty(AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
- return instanceVertex.getListProperty(actualPropertyName);
+ return instanceVertex.getListProperty(actualPropertyName);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
index 4e55bbc..bae8b2a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
@@ -17,7 +17,19 @@
*/
package org.apache.atlas.repository.graph;
-import com.google.inject.Inject;
+import static org.apache.atlas.repository.graph.GraphHelper.string;
+
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
@@ -47,18 +59,9 @@ import org.apache.atlas.utils.MD5Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.atlas.repository.graph.GraphHelper.string;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.inject.Inject;
public final class TypedInstanceToGraphMapper {
@@ -86,40 +89,42 @@ public final class TypedInstanceToGraphMapper {
void mapTypedInstanceToGraph(Operation operation, ITypedReferenceableInstance... typedInstances)
throws AtlasException {
+
RequestContext requestContext = RequestContext.get();
+ Collection<IReferenceableInstance> allNewInstances = new ArrayList<>();
for (ITypedReferenceableInstance typedInstance : typedInstances) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding/updating entity {}", typedInstance);
- }
+ allNewInstances.addAll(walkClassInstances(typedInstance));
+ }
- Collection<IReferenceableInstance> newInstances = walkClassInstances(typedInstance);
- TypeUtils.Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> instancesPair =
- createVerticesAndDiscoverInstances(newInstances);
- List<ITypedReferenceableInstance> entitiesToCreate = instancesPair.left;
- List<ITypedReferenceableInstance> entitiesToUpdate = instancesPair.right;
- FullTextMapper fulltextMapper = new FullTextMapper(graphToTypedInstanceMapper);
- switch (operation) {
- case CREATE:
- List<String> ids = addOrUpdateAttributesAndTraits(operation, entitiesToCreate);
- addFullTextProperty(entitiesToCreate, fulltextMapper);
- requestContext.recordEntityCreate(ids);
- break;
+ TypeUtils.Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> instancesPair =
+ createVerticesAndDiscoverInstances(allNewInstances);
- case UPDATE_FULL:
- case UPDATE_PARTIAL:
- ids = addOrUpdateAttributesAndTraits(Operation.CREATE, entitiesToCreate);
- requestContext.recordEntityCreate(ids);
- ids = addOrUpdateAttributesAndTraits(operation, entitiesToUpdate);
- requestContext.recordEntityUpdate(ids);
+ List<ITypedReferenceableInstance> entitiesToCreate = instancesPair.left;
+ List<ITypedReferenceableInstance> entitiesToUpdate = instancesPair.right;
- addFullTextProperty(entitiesToCreate, fulltextMapper);
- addFullTextProperty(entitiesToUpdate, fulltextMapper);
- break;
+ FullTextMapper fulltextMapper = new FullTextMapper(this, graphToTypedInstanceMapper);
+ switch (operation) {
+ case CREATE:
+ List<String> ids = addOrUpdateAttributesAndTraits(operation, entitiesToCreate);
+ addFullTextProperty(entitiesToCreate, fulltextMapper);
+ requestContext.recordEntityCreate(ids);
+ break;
- default:
- throw new UnsupportedOperationException("Not handled - " + operation);
- }
+ case UPDATE_FULL:
+ case UPDATE_PARTIAL:
+ ids = addOrUpdateAttributesAndTraits(Operation.CREATE, entitiesToCreate);
+ requestContext.recordEntityCreate(ids);
+ ids = addOrUpdateAttributesAndTraits(operation, entitiesToUpdate);
+ requestContext.recordEntityUpdate(ids);
+
+ addFullTextProperty(entitiesToCreate, fulltextMapper);
+ addFullTextProperty(entitiesToUpdate, fulltextMapper);
+ break;
+
+ default:
+ throw new UnsupportedOperationException("Not handled - " + operation);
}
+
}
private Collection<IReferenceableInstance> walkClassInstances(ITypedReferenceableInstance typedInstance)
@@ -257,68 +262,112 @@ public final class TypedInstanceToGraphMapper {
List<ITypedReferenceableInstance> instancesToCreate = new ArrayList<>();
List<ITypedReferenceableInstance> instancesToUpdate = new ArrayList<>();
- for (IReferenceableInstance instance : instances) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Discovering instance to create/update for {}", instance.toShortString());
- }
+ Map<Id,AtlasVertex> foundVertices = findExistingVertices(instances);
+ //cache all the ids
+ idToVertexMap.putAll(foundVertices);
- ITypedReferenceableInstance newInstance;
+ Set<Id> processedIds = new HashSet<>();
+ for(IReferenceableInstance instance : instances) {
Id id = instance.getId();
+ if(processedIds.contains(id)) {
+ continue;
+ }
- if (!idToVertexMap.containsKey(id)) {
- AtlasVertex instanceVertex;
- if (id.isAssigned()) { // has a GUID
- if (LOG.isDebugEnabled()) {
- LOG.debug("Instance has an assigned id {}", instance.getId()._getId());
- }
+ AtlasVertex instanceVertex = foundVertices.get(id);
+ ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
- instanceVertex = graphHelper.getVertexForGUID(id.id);
- if (!(instance instanceof ReferenceableInstance)) {
- throw new IllegalStateException(
- String.format("%s is not of type ITypedReferenceableInstance", instance.toShortString()));
- }
- newInstance = (ITypedReferenceableInstance) instance;
- instancesToUpdate.add(newInstance);
-
- } else {
- //Check if there is already an instance with the same unique attribute value
- ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
- instanceVertex = graphHelper.getVertexForInstanceByUniqueAttribute(classType, instance);
-
- //no entity with the given unique attribute, create new
- if (instanceVertex == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Creating new vertex for instance {}", instance.toShortString());
- }
+ if(instanceVertex == null) {
- newInstance = classType.convert(instance, Multiplicity.REQUIRED);
- instanceVertex = graphHelper.createVertexWithIdentity(newInstance, classType.getAllSuperTypeNames());
- instancesToCreate.add(newInstance);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Creating new vertex for instance {}", instance.toShortString());
+ }
- //Map only unique attributes for cases of circular references
- mapInstanceToVertex(newInstance, instanceVertex, classType.fieldMapping().fields, true, Operation.CREATE);
+ ITypedReferenceableInstance newInstance = classType.convert(instance, Multiplicity.REQUIRED);
+ instanceVertex = graphHelper.createVertexWithIdentity(newInstance, classType.getAllSuperTypeNames());
+ instancesToCreate.add(newInstance);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Re-using existing vertex {} for instance {}", string(instanceVertex), instance.toShortString());
- }
+ //Map only unique attributes for cases of circular references
+ mapInstanceToVertex(newInstance, instanceVertex, classType.fieldMapping().fields, true, Operation.CREATE);
+ idToVertexMap.put(id, instanceVertex);
- if (!(instance instanceof ReferenceableInstance)) {
- throw new IllegalStateException(
- String.format("%s is not of type ITypedReferenceableInstance", instance.toShortString()));
- }
- newInstance = (ITypedReferenceableInstance) instance;
- instancesToUpdate.add(newInstance);
- }
+ }
+ else {
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Re-using existing vertex {} for instance {}", string(instanceVertex), instance.toShortString());
}
- //Set the id in the new instance
- idToVertexMap.put(id, instanceVertex);
+ if (!(instance instanceof ITypedReferenceableInstance)) {
+ throw new IllegalStateException(
+ String.format("%s is not of type ITypedReferenceableInstance", instance.toShortString()));
+ }
+ ITypedReferenceableInstance existingInstance = (ITypedReferenceableInstance) instance;
+ instancesToUpdate.add(existingInstance);
}
+ processedIds.add(id);
+
}
return TypeUtils.Pair.of(instancesToCreate, instancesToUpdate);
}
+ private Map<Id,AtlasVertex> findExistingVertices(Collection<IReferenceableInstance> instances) throws AtlasException {
+
+ VertexLookupContext context = new VertexLookupContext(this);
+ Map<Id,AtlasVertex> result = new HashMap<>();
+
+ for(IReferenceableInstance instance : instances) {
+ context.addInstance(instance);
+ }
+
+ List<Id> instancesToLoad = new ArrayList<>(context.getInstancesToLoadByGuid());
+ List<String> guidsToLoad = Lists.transform(instancesToLoad, new Function<Id,String>() {
+
+ @Override
+ public String apply(Id instance) {
+ Id id = getExistingId(instance);
+ return id.id;
+ }
+
+ });
+
+ Map<String, AtlasVertex> instanceVertices = graphHelper.getVerticesForGUIDs(guidsToLoad);
+
+ List<String> missingGuids = new ArrayList<>();
+ for(int i = 0 ; i < instancesToLoad.size(); i++) {
+
+ String guid = guidsToLoad.get(i);
+ AtlasVertex instanceVertex = instanceVertices.get(guid);
+ if(instanceVertex == null) {
+ missingGuids.add(guid);
+ continue;
+ }
+
+ Id instance = instancesToLoad.get(i);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Found vertex {} for instance {}", string(instanceVertex), instance);
+ }
+ result.put(instance, instanceVertex);
+ }
+
+ if(missingGuids.size() > 0) {
+ throw new EntityNotFoundException("Could not find entities in the repository with the following GUIDs: " + missingGuids);
+ }
+
+ for(Map.Entry<ClassType,List<IReferenceableInstance>> entry : context.getInstancesToLoadByUniqueAttribute().entrySet()) {
+ ClassType type = entry.getKey();
+ List<IReferenceableInstance> instancesForClass = entry.getValue();
+ List<AtlasVertex> correspondingVertices = graphHelper.getVerticesForInstancesByUniqueAttribute(type, instancesForClass);
+ for(int i = 0; i < instancesForClass.size(); i++) {
+ IReferenceableInstance inst = instancesForClass.get(i);
+ AtlasVertex vertex = correspondingVertices.get(i);
+ result.put(getExistingId(inst), vertex);
+ }
+ }
+
+ return result;
+ }
+
+
private void addFullTextProperty(List<ITypedReferenceableInstance> instances, FullTextMapper fulltextMapper) throws AtlasException {
for (ITypedReferenceableInstance typedInstance : instances) { // Traverse
AtlasVertex instanceVertex = getClassVertex(typedInstance);
@@ -356,9 +405,9 @@ public final class TypedInstanceToGraphMapper {
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
-
+
List<Object> currentElements = GraphHelper.getArrayElementsProperty(elementType, instanceVertex, propertyName);
-
+
List<Object> newElementsCreated = new ArrayList<>();
if (!newAttributeEmpty) {
@@ -582,6 +631,7 @@ public final class TypedInstanceToGraphMapper {
// add a new vertex for the struct or trait instance
AtlasVertex structInstanceVertex = graphHelper.createVertexWithoutIdentity(structInstance.getTypeName(), null,
Collections.<String>emptySet()); // no super types for struct type
+
if (LOG.isDebugEnabled()) {
LOG.debug("created vertex {} for struct {} value {}", string(structInstanceVertex), attributeInfo.name,
structInstance.toShortString());
@@ -649,21 +699,24 @@ public final class TypedInstanceToGraphMapper {
return graphHelper.getOrCreateEdge(instanceVertex, toVertex, edgeLabel);
}
- private AtlasVertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
- AtlasVertex referenceVertex = null;
+ private <V,E> AtlasVertex<V,E> getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
+ AtlasVertex<V,E> referenceVertex = null;
Id id = null;
if (typedReference != null) {
- id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
- if (id.isAssigned()) {
+ id = getExistingId(typedReference);
+ referenceVertex = idToVertexMap.get(id);
+ if(referenceVertex == null && id.isAssigned()) {
referenceVertex = graphHelper.getVertexForGUID(id.id);
- } else {
- referenceVertex = idToVertexMap.get(id);
}
}
return referenceVertex;
}
+ Id getExistingId(IReferenceableInstance instance) {
+ return instance instanceof Id ? (Id) instance : instance.getId();
+ }
+
private Id getId(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
if (typedReference == null) {
throw new IllegalArgumentException("typedReference must be non-null");
@@ -768,4 +821,8 @@ public final class TypedInstanceToGraphMapper {
GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
}
+
+ public AtlasVertex lookupVertex(Id refId) {
+ return idToVertexMap.get(refId);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/repository/graph/VertexLookupContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/VertexLookupContext.java b/repository/src/main/java/org/apache/atlas/repository/graph/VertexLookupContext.java
new file mode 100644
index 0000000..dd90be6
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/VertexLookupContext.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graph;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+
+/**
+ * Helper class for TypedInstanceGraphMapper. Determines which instances
+ * should be loaded by GUID and which ones should be loaded by unique attribute.
+ * In addition, it sorts the instances that should be loaded by unique
+ * attribute by class.
+ *
+ */
+public class VertexLookupContext {
+
+ private final TypedInstanceToGraphMapper mapper;
+
+ private static final TypeSystem typeSystem = TypeSystem.getInstance();
+
+ private Map<ClassType,List<IReferenceableInstance>> instancesWithoutGuids = new HashMap<>();
+ private Set<Id> guidsToLookup = new HashSet<>();
+
+
+ /**
+ * @param typedInstanceToGraphMapper
+ */
+ VertexLookupContext(TypedInstanceToGraphMapper typedInstanceToGraphMapper) {
+ mapper = typedInstanceToGraphMapper;
+ }
+
+ /**
+ * Adds an instance to be loaded.
+ *
+ */
+ public void addInstance(IReferenceableInstance instance) throws AtlasException {
+
+ ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
+ ITypedReferenceableInstance newInstance = classType.convert(instance, Multiplicity.REQUIRED);
+ findReferencedInstancesToPreLoad(newInstance);
+ Id id = instance.getId();
+ if(mapper.lookupVertex(id) == null) {
+ if(id.isAssigned()) {
+ guidsToLookup.add(id);
+ }
+ else {
+ addToClassMap(classType, instance);
+ }
+ }
+ }
+
+ /**
+ * Returns the instances that should be loaded by unique attribute, sorted by
+ * class.
+ *
+ */
+ public Map<ClassType,List<IReferenceableInstance>> getInstancesToLoadByUniqueAttribute() {
+ return instancesWithoutGuids;
+ }
+
+ /**
+ * Returns the Ids of the instance that should be loaded by GUID
+ *
+ * @return
+ */
+ public Set<Id> getInstancesToLoadByGuid() {
+ return guidsToLookup;
+ }
+
+ private void addToClassMap(ClassType classType, IReferenceableInstance instance) throws AtlasException {
+
+ List<IReferenceableInstance> toUpdate = instancesWithoutGuids.get(classType);
+ if(toUpdate == null) {
+ toUpdate = new ArrayList<>();
+ instancesWithoutGuids.put(classType, toUpdate);
+ }
+ toUpdate.add(instance);
+ }
+
+ private void findReferencedInstancesToPreLoad(ITypedReferenceableInstance newInstance) throws AtlasException {
+ //pre-load vertices for reference fields
+ for(AttributeInfo info : newInstance.fieldMapping().fields.values()) {
+
+ if(info.dataType().getTypeCategory() == TypeCategory.CLASS) {
+ ITypedReferenceableInstance newAttributeValue = (ITypedReferenceableInstance)newInstance.get(info.name);
+ addAdditionalInstance(newAttributeValue);
+ }
+
+ if(info.dataType().getTypeCategory() == TypeCategory.ARRAY) {
+ IDataType elementType = ((DataTypes.ArrayType) info.dataType()).getElemType();
+ if(elementType.getTypeCategory() == TypeCategory.CLASS) {
+ List<ITypedReferenceableInstance> newElements = (List) newInstance.get(info.name);
+ addAdditionalInstances(newElements);
+ }
+ }
+
+ if(info.dataType().getTypeCategory() == TypeCategory.MAP) {
+ IDataType elementType = ((DataTypes.MapType) info.dataType()).getValueType();
+ if(elementType.getTypeCategory() == TypeCategory.CLASS) {
+ Map<Object, ITypedReferenceableInstance> newAttribute =
+ (Map<Object, ITypedReferenceableInstance>) newInstance.get(info.name);
+
+ if(newAttribute != null) {
+ addAdditionalInstances(newAttribute.values());
+ }
+ }
+ }
+ }
+ }
+
+ private void addAdditionalInstance(ITypedReferenceableInstance instance) {
+
+ if(instance == null) {
+ return;
+ }
+
+ Id id = mapper.getExistingId(instance);
+ if(! id.isAssigned()) {
+ return;
+ }
+ guidsToLookup.add(id);
+ }
+
+
+
+ private void addAdditionalInstances(Collection<ITypedReferenceableInstance> newElements) {
+ if(newElements != null) {
+ for(ITypedReferenceableInstance instance: newElements) {
+ addAdditionalInstance(instance);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 35a489f..b14531f 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -367,6 +367,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private void validateUniqueAttribute(String entityType, String attributeName) throws AtlasException {
ClassType type = typeSystem.getDataType(ClassType.class, entityType);
AttributeInfo attribute = type.fieldMapping().fields.get(attributeName);
+ if(attribute == null) {
+ throw new IllegalArgumentException(
+ String.format("%s is not an attribute in %s", attributeName, entityType));
+ }
if (!attribute.isUnique) {
throw new IllegalArgumentException(
String.format("%s.%s is not a unique attribute", entityType, attributeName));
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/util/AttributeValueMap.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/util/AttributeValueMap.java b/repository/src/main/java/org/apache/atlas/util/AttributeValueMap.java
new file mode 100644
index 0000000..6e22604
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/util/AttributeValueMap.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.util;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+
+/**
+ * Map of attribute values to a collection of IndexedInstances with that attribute value.
+ *
+ * @see GraphHelper#getVerticesForInstancesByUniqueAttributes
+ *
+ */
+public class AttributeValueMap {
+
+ //need collection in case they are adding the same entity twice?
+ private Map<Object,Collection<IndexedInstance>> valueMap_ = new HashMap<>();
+
+ public void put(Object value, IReferenceableInstance instance, int index) {
+ IndexedInstance wrapper = new IndexedInstance(instance, index);
+ Collection<IndexedInstance> existingValues = valueMap_.get(value);
+ if(existingValues == null) {
+ //only expect 1 value
+ existingValues = new HashSet<>(1);
+ valueMap_.put(value, existingValues);
+ }
+ existingValues.add(wrapper);
+ }
+
+ public Collection<IndexedInstance> get(Object value) {
+ return valueMap_.get(value);
+ }
+
+
+ public Set<Object> getAttributeValues() {
+ return valueMap_.keySet();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/main/java/org/apache/atlas/util/IndexedInstance.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/util/IndexedInstance.java b/repository/src/main/java/org/apache/atlas/util/IndexedInstance.java
new file mode 100644
index 0000000..60ec8cc
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/util/IndexedInstance.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.util;
+
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+
+/**
+ * Data structure that stores an IReferenceableInstance and its location within
+ * a list.
+ *
+ * @see GraphHelper#getVerticesForInstancesByUniqueAttributes
+ */
+public class IndexedInstance {
+
+ private final IReferenceableInstance instance_;
+ private final int index_;
+
+ public IndexedInstance(IReferenceableInstance instance, int index) {
+ super();
+ this.instance_ = instance;
+ this.index_ = index;
+ }
+
+ public IReferenceableInstance getInstance() {
+ return instance_;
+ }
+
+ public int getIndex() {
+ return index_;
+ }
+
+ @Override
+ public int hashCode() {
+ return instance_.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if(!(other instanceof IndexedInstance)) {
+ return false;
+ }
+ IndexedInstance otherInstance = (IndexedInstance)other;
+ return instance_.equals(otherInstance.getInstance());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
index a7dc13d..f5a6a05 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphHelperTest.java
@@ -18,15 +18,42 @@
package org.apache.atlas.repository.graph;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.inject.Inject;
+
+import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphHelper.VertexInfo;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.exception.TypeNotFoundException;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
+import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -34,20 +61,6 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import javax.inject.Inject;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-
@Guice(modules = RepositoryMetadataModule.class)
public class GraphHelperTest {
@@ -69,6 +82,9 @@ public class GraphHelperTest {
}
@Inject
+ private MetadataService metadataService;
+
+ @Inject
private GraphBackedMetadataRepository repositoryService;
private TypeSystem typeSystem;
@@ -82,7 +98,12 @@ public class GraphHelperTest {
typeSystem.reset();
new GraphBackedSearchIndexer(typeRegistry);
-
+ TypesDef typesDef = TestUtils.defineHiveTypes();
+ try {
+ metadataService.getTypeDefinition(TestUtils.TABLE_TYPE);
+ } catch (TypeNotFoundException e) {
+ metadataService.createType(TypesSerialization.toJson(typesDef));
+ }
TestUtils.defineDeptEmployeeTypes(typeSystem);
}
@@ -92,6 +113,43 @@ public class GraphHelperTest {
}
@Test
+ public void testGetInstancesByUniqueAttributes() throws Exception {
+
+ GraphHelper helper = GraphHelper.getInstance();
+ List<ITypedReferenceableInstance> instances = new ArrayList<>();
+ List<String> guids = new ArrayList<>();
+ TypeSystem ts = TypeSystem.getInstance();
+ ClassType dbType = ts.getDataType(ClassType.class, TestUtils.DATABASE_TYPE);
+
+ for(int i = 0; i < 10; i++) {
+ Referenceable db = TestUtils.createDBEntity();
+ String guid = createInstance(db);
+ ITypedReferenceableInstance instance = convert(db, dbType);
+ instances.add(instance);
+ guids.add(guid);
+ }
+
+ //lookup vertices via getVertexForInstanceByUniqueAttributes
+ List<AtlasVertex> vertices = helper.getVerticesForInstancesByUniqueAttribute(dbType, instances);
+ assertEquals(instances.size(), vertices.size());
+ //assert vertex matches the vertex we get through getVertexForGUID
+ for(int i = 0; i < instances.size(); i++) {
+ String guid = guids.get(i);
+ AtlasVertex foundVertex = vertices.get(i);
+ AtlasVertex expectedVertex = helper.getVertexForGUID(guid);
+ assertEquals(foundVertex, expectedVertex);
+ }
+ }
+ @Test
+ public void testGetVerticesForGUIDSWithDuplicates() throws Exception {
+ ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(TypeSystem.getInstance());
+ List<String> result = repositoryService.createEntities(hrDept);
+ String guid = result.get(0);
+ Map<String, AtlasVertex> verticesForGUIDs = GraphHelper.getInstance().getVerticesForGUIDs(Arrays.asList(guid, guid));
+ Assert.assertEquals(verticesForGUIDs.size(), 1);
+ Assert.assertTrue(verticesForGUIDs.containsKey(guid));
+ }
+ @Test
public void testGetCompositeGuidsAndVertices() throws Exception {
ITypedReferenceableInstance hrDept = TestUtils.createDeptEg1(typeSystem);
List<String> createdGuids = repositoryService.createEntities(hrDept);
@@ -144,4 +202,22 @@ public class GraphHelperTest {
assertFalse(iterator.hasNext());
assertFalse(iterator.hasNext());
}
+
+ private ITypedReferenceableInstance convert(Referenceable instance, ClassType type) throws AtlasException {
+
+ return type.convert(instance, Multiplicity.REQUIRED);
+ }
+
+ private String createInstance(Referenceable entity) throws Exception {
+ TestUtils.resetRequestContext();
+
+ String entityjson = InstanceSerialization.toJson(entity, true);
+ JSONArray entitiesJson = new JSONArray();
+ entitiesJson.put(entityjson);
+ List<String> guids = metadataService.createEntities(entitiesJson.toString());
+ if (guids != null && guids.size() > 0) {
+ return guids.get(guids.size() - 1);
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/typesystem/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/typesystem/src/test/resources/atlas-application.properties b/typesystem/src/test/resources/atlas-application.properties
index 108630b..0e6bc41 100644
--- a/typesystem/src/test/resources/atlas-application.properties
+++ b/typesystem/src/test/resources/atlas-application.properties
@@ -77,6 +77,7 @@ atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
######### Hive Lineage Configs #########
## Schema
atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns
+atlas.lineage.schema.query.hive_table_v1=hive_table_v1 where __guid='%s'\, columns
######### Notification Configs #########
atlas.notification.embedded=true
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
index 17c8237..9ca684d 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
@@ -144,7 +144,7 @@ public class EntityResource {
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (ValueConversionException ve) {
LOG.error("Unable to persist entity instance due to a deserialization error entityDef={}", entityJson, ve);
- throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
+ throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause() != null ? ve.getCause() : ve, Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
index 1774611..29be942 100644
--- a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
+++ b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
@@ -51,8 +51,6 @@ import static org.testng.Assert.assertTrue;
@Guice(modules = NotificationModule.class)
public class EntityNotificationIT extends BaseResourceIT {
- private static final String ENTITIES = "api/atlas/entities";
- private static final String TRAITS = "traits";
private final String DATABASE_NAME = "db" + randomString();
private final String TABLE_NAME = "table" + randomString();
@Inject
@@ -66,7 +64,7 @@ public class EntityNotificationIT extends BaseResourceIT {
public void setUp() throws Exception {
super.setUp();
createTypeDefinitionsV1();
- Referenceable HiveDBInstance = createHiveDBInstanceV1(DATABASE_NAME);
+ Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(DATABASE_NAME);
dbId = createInstance(HiveDBInstance);
List<NotificationConsumer<EntityNotification>> consumers =
@@ -77,13 +75,13 @@ public class EntityNotificationIT extends BaseResourceIT {
@Test
public void testCreateEntity() throws Exception {
- Referenceable tableInstance = createHiveTableInstanceV1(DATABASE_NAME, TABLE_NAME, dbId);
+ Referenceable tableInstance = createHiveTableInstanceBuiltIn(DATABASE_NAME, TABLE_NAME, dbId);
tableId = createInstance(tableInstance);
final String guid = tableId._getId();
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
- newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE, guid));
+ newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE_BUILTIN, guid));
}
@Test(dependsOnMethods = "testCreateEntity")
@@ -96,29 +94,29 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.updateEntityAttribute(guid, property, newValue);
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
- newNotificationPredicate(EntityNotification.OperationType.ENTITY_UPDATE, HIVE_TABLE_TYPE, guid));
+ newNotificationPredicate(EntityNotification.OperationType.ENTITY_UPDATE, HIVE_TABLE_TYPE_BUILTIN, guid));
}
@Test
public void testDeleteEntity() throws Exception {
final String tableName = "table-" + randomString();
final String dbName = "db-" + randomString();
- Referenceable HiveDBInstance = createHiveDBInstanceV1(dbName);
+ Referenceable HiveDBInstance = createHiveDBInstanceBuiltIn(dbName);
Id dbId = createInstance(HiveDBInstance);
- Referenceable tableInstance = createHiveTableInstanceV1(dbName, tableName, dbId);
+ Referenceable tableInstance = createHiveTableInstanceBuiltIn(dbName, tableName, dbId);
final Id tableId = createInstance(tableInstance);
final String guid = tableId._getId();
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
- newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE, guid));
+ newNotificationPredicate(EntityNotification.OperationType.ENTITY_CREATE, HIVE_TABLE_TYPE_BUILTIN, guid));
final String name = (String) tableInstance.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
- atlasClientV1.deleteEntity(HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
+ atlasClientV1.deleteEntity(HIVE_TABLE_TYPE_BUILTIN, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
waitForNotification(notificationConsumer, MAX_WAIT_TIME,
- newNotificationPredicate(EntityNotification.OperationType.ENTITY_DELETE, HIVE_TABLE_TYPE, guid));
+ newNotificationPredicate(EntityNotification.OperationType.ENTITY_DELETE, HIVE_TABLE_TYPE_BUILTIN, guid));
}
@Test(dependsOnMethods = "testCreateEntity")
@@ -141,7 +139,7 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.addTrait(guid, traitInstance);
EntityNotification entityNotification = waitForNotification(notificationConsumer, MAX_WAIT_TIME,
- newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE, guid));
+ newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE_BUILTIN, guid));
IReferenceableInstance entity = entityNotification.getEntity();
assertTrue(entity.getTraits().contains(traitName));
@@ -166,7 +164,7 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.addTrait(guid, traitInstance);
entityNotification = waitForNotification(notificationConsumer, MAX_WAIT_TIME,
- newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE, guid));
+ newNotificationPredicate(EntityNotification.OperationType.TRAIT_ADD, HIVE_TABLE_TYPE_BUILTIN, guid));
allTraits = entityNotification.getAllTraits();
allTraitNames = new LinkedList<>();
@@ -187,7 +185,7 @@ public class EntityNotificationIT extends BaseResourceIT {
atlasClientV1.deleteTrait(guid, traitName);
EntityNotification entityNotification = waitForNotification(notificationConsumer, MAX_WAIT_TIME,
- newNotificationPredicate(EntityNotification.OperationType.TRAIT_DELETE, HIVE_TABLE_TYPE, guid));
+ newNotificationPredicate(EntityNotification.OperationType.TRAIT_DELETE, HIVE_TABLE_TYPE_BUILTIN, guid));
assertFalse(entityNotification.getEntity().getTraits().contains(traitName));
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/89f70609/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
index 4a3db88..1c2cdc6 100644
--- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
+++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
@@ -68,7 +68,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
new Referenceable(randomString())));
//send valid message
- final Referenceable entity = new Referenceable(DATABASE_TYPE);
+ final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
@@ -79,7 +79,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
- JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE, entity.get(NAME)));
+ JSONArray results = searchByDSL(String.format("%s where name='%s'", DATABASE_TYPE_BUILTIN, entity.get(NAME)));
return results.length() == 1;
}
});
@@ -87,7 +87,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
@Test
public void testCreateEntity() throws Exception {
- final Referenceable entity = new Referenceable(DATABASE_TYPE);
+ final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
@@ -99,13 +99,13 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
- JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, entity.get(QUALIFIED_NAME)));
+ JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, entity.get(QUALIFIED_NAME)));
return results.length() == 1;
}
});
//Assert that user passed in hook message is used in audit
- Referenceable instance = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, (String) entity.get(QUALIFIED_NAME));
+ Referenceable instance = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, (String) entity.get(QUALIFIED_NAME));
List<EntityAuditEvent> events =
atlasClientV1.getEntityAuditEvents(instance.getId()._getId(), (short) 1);
assertEquals(events.size(), 1);
@@ -114,7 +114,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
@Test
public void testUpdateEntityPartial() throws Exception {
- final Referenceable entity = new Referenceable(DATABASE_TYPE);
+ final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
@@ -123,26 +123,26 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
atlasClientV1.createEntity(entity);
- final Referenceable newEntity = new Referenceable(DATABASE_TYPE);
+ final Referenceable newEntity = new Referenceable(DATABASE_TYPE_BUILTIN);
newEntity.set("owner", randomString());
sendHookMessage(
- new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE, QUALIFIED_NAME, dbName, newEntity));
+ new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity));
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
- Referenceable localEntity = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
+ Referenceable localEntity = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
return (localEntity.get("owner") != null && localEntity.get("owner").equals(newEntity.get("owner")));
}
});
//Its partial update and un-set fields are not updated
- Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
+ Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
assertEquals(actualEntity.get(DESCRIPTION), entity.get(DESCRIPTION));
}
@Test
public void testUpdatePartialUpdatingQualifiedName() throws Exception {
- final Referenceable entity = new Referenceable(DATABASE_TYPE);
+ final Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
@@ -151,29 +151,29 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
atlasClientV1.createEntity(entity);
- final Referenceable newEntity = new Referenceable(DATABASE_TYPE);
+ final Referenceable newEntity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String newName = "db" + randomString();
newEntity.set(QUALIFIED_NAME, newName);
sendHookMessage(
- new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE, QUALIFIED_NAME, dbName, newEntity));
+ new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity));
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
- JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, newName));
+ JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, newName));
return results.length() == 1;
}
});
//no entity with the old qualified name
- JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, dbName));
+ JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, dbName));
assertEquals(results.length(), 0);
}
@Test
public void testDeleteByQualifiedName() throws Exception {
- Referenceable entity = new Referenceable(DATABASE_TYPE);
+ Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
@@ -183,7 +183,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
final String dbId = atlasClientV1.createEntity(entity).get(0);
sendHookMessage(
- new HookNotification.EntityDeleteRequest(TEST_USER, DATABASE_TYPE, QUALIFIED_NAME, dbName));
+ new HookNotification.EntityDeleteRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName));
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
@@ -195,7 +195,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
@Test
public void testUpdateEntityFullUpdate() throws Exception {
- Referenceable entity = new Referenceable(DATABASE_TYPE);
+ Referenceable entity = new Referenceable(DATABASE_TYPE_BUILTIN);
final String dbName = "db" + randomString();
entity.set(NAME, dbName);
entity.set(DESCRIPTION, randomString());
@@ -204,7 +204,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
atlasClientV1.createEntity(entity);
- final Referenceable newEntity = new Referenceable(DATABASE_TYPE);
+ final Referenceable newEntity = new Referenceable(DATABASE_TYPE_BUILTIN);
newEntity.set(NAME, randomString());
newEntity.set(DESCRIPTION, randomString());
newEntity.set("owner", randomString());
@@ -216,12 +216,12 @@ public class NotificationHookConsumerIT extends BaseResourceIT {
waitFor(MAX_WAIT_TIME, new Predicate() {
@Override
public boolean evaluate() throws Exception {
- JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE, newEntity.get(QUALIFIED_NAME)));
+ JSONArray results = searchByDSL(String.format("%s where qualifiedName='%s'", DATABASE_TYPE_BUILTIN, newEntity.get(QUALIFIED_NAME)));
return results.length() == 1;
}
});
- Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE, QUALIFIED_NAME, dbName);
+ Referenceable actualEntity = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName);
assertEquals(actualEntity.get(DESCRIPTION), newEntity.get(DESCRIPTION));
assertEquals(actualEntity.get("owner"), newEntity.get("owner"));
}