You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/11/12 18:14:24 UTC
[15/42] atlas git commit: ATLAS-2251: Remove TypeSystem and related
implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
deleted file mode 100644
index e885b8c..0000000
--- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
+++ /dev/null
@@ -1,1381 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.service;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.Inject;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.CreateUpdateEntitiesResult;
-import org.apache.atlas.EntityAuditEvent;
-import org.apache.atlas.RequestContext;
-import org.apache.atlas.TestModules;
-import org.apache.atlas.TestUtils;
-import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.listener.ChangedTypeDefs;
-import org.apache.atlas.listener.EntityChangeListener;
-import org.apache.atlas.listener.TypeDefChangeListener;
-import org.apache.atlas.model.legacy.EntityResult;
-import org.apache.atlas.query.QueryParams;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.audit.EntityAuditRepository;
-import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
-import org.apache.atlas.repository.audit.HBaseTestUtils;
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.services.DefaultMetadataService;
-import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.type.AtlasTypeUtil;
-import org.apache.atlas.typesystem.IReferenceableInstance;
-import org.apache.atlas.typesystem.IStruct;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.Struct;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.exception.EntityNotFoundException;
-import org.apache.atlas.typesystem.exception.TypeNotFoundException;
-import org.apache.atlas.typesystem.json.InstanceSerialization;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumValue;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.ValueConversionException;
-import org.apache.atlas.typesystem.types.cache.TypeCache;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.atlas.utils.ParamChecker;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.testng.Assert;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import static org.apache.atlas.TestUtils.*;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.testng.Assert.*;
-
-@Guice(modules = TestModules.TestOnlyModule.class)
-public class DefaultMetadataServiceTest {
- @Inject
- private MetadataService metadataService;
-
- private TypeDefChangeListener typeDefChangeListener;
-
- @Inject
- private EntityAuditRepository auditRepository;
-
- @Inject
- private GraphBackedDiscoveryService discoveryService;
-
- private Referenceable db = createDBEntity();
-
- private Referenceable table;
-
- private Id tableId;
-
- private final String NAME = "name";
-
-
- @BeforeTest
- public void setUp() throws Exception {
-
- typeDefChangeListener = (DefaultMetadataService)metadataService;
- metadataService = TestUtils.addSessionCleanupWrapper(metadataService);
-
- if (auditRepository instanceof HBaseBasedAuditRepository) {
- HBaseTestUtils.startCluster();
- ((HBaseBasedAuditRepository) auditRepository).start();
- }
- TestUtils.resetRequestContext();
- RequestContext.get().setUser("testuser");
-
- TypesDef typesDef = TestUtils.defineHiveTypes();
- try {
- metadataService.getTypeDefinition(TestUtils.TABLE_TYPE);
- } catch (TypeNotFoundException e) {
- metadataService.createType(TypesSerialization.toJson(typesDef));
- }
-
- String dbGUid = TestUtils.createInstance(metadataService, db);
- table = createTableEntity(dbGUid);
- String tableGuid = TestUtils.createInstance(metadataService, table);
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- table = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- tableId = new Id(tableGuid, 0, TestUtils.TABLE_TYPE);
- }
-
- @AfterTest
- public void shutdown() throws Exception {
- try {
- TypeSystem.getInstance().reset();
-
- if (auditRepository instanceof HBaseBasedAuditRepository) {
- ((HBaseBasedAuditRepository) auditRepository).stop();
- HBaseTestUtils.stopCluster();
- }
- }
- finally {
- AtlasGraphProvider.cleanup();
- }
- }
- private EntityResult updateInstance(Referenceable entity) throws Exception {
- RequestContext.createContext();
- ParamChecker.notNull(entity, "Entity");
- ParamChecker.notNull(entity.getId(), "Entity");
- String entityjson = InstanceSerialization.toJson(entity, true);
- JSONArray entitiesJson = new JSONArray();
- entitiesJson.put(entityjson);
- return metadataService.updateEntities(entitiesJson.toString()).getEntityResult();
- }
-
- @Test(expectedExceptions = TypeNotFoundException.class)
- public void testCreateEntityWithUnknownDatatype() throws Exception {
- Referenceable entity = new Referenceable("Unknown datatype");
- String dbName = TestUtils.randomString(10);
- entity.set(NAME, dbName);
- entity.set("description", "us db");
- TestUtils.createInstance(metadataService, entity);
- Assert.fail(TypeNotFoundException.class.getSimpleName() + " was expected but none thrown.");
- }
-
- @Test
- public void testCreateEntityWithUniqueAttribute() throws Exception {
- //name is the unique attribute
- Referenceable entity = createDBEntity();
- String id = TestUtils.createInstance(metadataService, entity);
- assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.ENTITY_CREATE);
-
- //using the same name should succeed, but not create another entity
- String newId = TestUtils.createInstance(metadataService, entity);
- assertNull(newId);
-
- //Same entity, but different qualified name should succeed
- entity.set(NAME, TestUtils.randomString());
- newId = TestUtils.createInstance(metadataService, entity);
- Assert.assertNotEquals(newId, id);
- }
-
- @Test
- //Titan doesn't allow some reserved chars in property keys. Verify that atlas encodes these
- //See GraphHelper.encodePropertyKey()
- public void testSpecialCharacters() throws Exception {
- //Verify that type can be created with reserved characters in typename, attribute name
- String strAttrName = randomStrWithReservedChars();
- String arrayAttrName = randomStrWithReservedChars();
- String mapAttrName = randomStrWithReservedChars();
- HierarchicalTypeDefinition<ClassType> typeDefinition =
- createClassTypeDef("test_type_"+ TestUtils.randomString(10), ImmutableSet.<String>of(),
- createOptionalAttrDef(strAttrName, DataTypes.STRING_TYPE),
- new AttributeDefinition(arrayAttrName, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(mapAttrName,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null));
- metadataService.createType(TypesSerialization.toJson(typeDefinition, false));
-
- //verify that entity can be created with reserved characters in string value, array value and map key and value
- Referenceable entity = new Referenceable(typeDefinition.typeName);
- entity.set(strAttrName, randomStrWithReservedChars());
- entity.set(arrayAttrName, new ArrayList<String>() {{ add(randomStrWithReservedChars()); }});
- entity.set(mapAttrName, new HashMap<String, String>() {{
- put(randomStrWithReservedChars(), randomStrWithReservedChars());
- }});
- String id = createInstance(metadataService, entity);
-
- //Verify that get entity definition returns actual values with reserved characters
- Referenceable instance =
- InstanceSerialization.fromJsonReferenceable(metadataService.getEntityDefinitionJson(id), true);
- assertReferenceableEquals(instance, entity);
-
- //Verify that search with reserved characters works - for string attribute
- String query =
- String.format("`%s` where `%s` = '%s'", typeDefinition.typeName, strAttrName, entity.get(strAttrName));
- String responseJson = discoveryService.searchByDSL(query, new QueryParams(1, 0));
- JSONObject response = new JSONObject(responseJson);
- assertEquals(response.getJSONArray("rows").length(), 1);
- }
-
- //equals excluding the id
- private void assertReferenceableEquals(Referenceable actual, Referenceable expected) {
- List<String> traits = actual.getTraits();
- Map<String, IStruct> traitsMap = new HashMap<>();
- for (String trait : traits) {
- traitsMap.put(trait, actual.getTrait(trait));
- }
-
- Referenceable newActual = new Referenceable(expected.getId(), actual.getTypeName(), actual.getValuesMap(),
- traits, traitsMap);
- assertEquals(newActual, expected);
- }
-
- private String randomStrWithReservedChars() {
- return randomString() + "\"${}%";
- }
-
- @Test
- public void testAddDeleteTrait() throws Exception {
- Referenceable entity = createDBEntity();
- String id = TestUtils.createInstance(metadataService, entity);
-
- //add trait
- Struct tag = new Struct(TestUtils.PII);
- metadataService.addTrait(id, InstanceSerialization.toJson(tag, true));
-
- List<String> traits = metadataService.getTraitNames(id);
- assertEquals(traits.size(), 1);
- assertEquals(traits.get(0), PII);
-
- //getTrait
- IStruct traitDefinition = metadataService.getTraitDefinition(id, PII);
- Assert.assertNotNull(traitDefinition);
- assertEquals(traitDefinition.getValuesMap().size(), 0);
-
- //delete trait
- metadataService.deleteTrait(id, PII);
- traits = metadataService.getTraitNames(id);
- assertEquals(traits.size(), 0);
-
- //add trait again
- metadataService.addTrait(id, InstanceSerialization.toJson(tag, true));
-
- traits = metadataService.getTraitNames(id);
- assertEquals(traits.size(), 1);
- assertEquals(traits.get(0), PII);
- }
-
- @Test
- public void testEntityAudit() throws Exception {
- //create entity
- Referenceable entity = createDBEntity();
- String id = TestUtils.createInstance(metadataService, entity);
- assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.ENTITY_CREATE);
-
- Struct tag = new Struct(TestUtils.PII);
- metadataService.addTrait(id, InstanceSerialization.toJson(tag, true));
- assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.TAG_ADD);
-
- metadataService.deleteTrait(id, TestUtils.PII);
- assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.TAG_DELETE);
-
- metadataService.updateEntityAttributeByGuid(id, "description", "new description");
- assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.ENTITY_UPDATE);
-
- metadataService.deleteEntities(Arrays.asList(id));
- assertAuditEvents(id, EntityAuditEvent.EntityAuditAction.ENTITY_DELETE);
- }
-
- private EntityResult deleteEntities(String... guids) throws AtlasException {
- RequestContext.createContext();
- return metadataService.deleteEntities(Arrays.asList(guids));
- }
-
- private void assertAuditEvents(String id, EntityAuditEvent.EntityAuditAction expectedAction) throws Exception {
- List<EntityAuditEvent> events =
- auditRepository.listEvents(id, null, (short) 10);
- for (EntityAuditEvent event : events) {
- if (event.getAction() == expectedAction) {
- return;
- }
- }
- fail("Expected audit action " + expectedAction);
- }
-
- private void assertAuditEvents(String entityId, int numEvents) throws Exception {
- List<EntityAuditEvent> events = metadataService.getAuditEvents(entityId, null, (short) numEvents);
- assertNotNull(events);
- assertEquals(events.size(), numEvents);
- }
-
- @Test
- public void testCreateEntityWithUniqueAttributeWithReference() throws Exception {
- Referenceable db = createDBEntity();
- String dbId = TestUtils.createInstance(metadataService, db);
-
- //Assert that there is just 1 audit events and thats for entity create
- assertAuditEvents(dbId, 1);
- assertAuditEvents(dbId, EntityAuditEvent.EntityAuditAction.ENTITY_CREATE);
-
- Referenceable table = new Referenceable(TestUtils.TABLE_TYPE);
- table.set(NAME, TestUtils.randomString());
- table.set("description", "random table");
- table.set("type", "type");
- table.set("tableType", "MANAGED");
- table.set("database", new Id(dbId, 0, TestUtils.DATABASE_TYPE));
- table.set("databaseComposite", db);
- TestUtils.createInstance(metadataService, table);
-
- //table create should re-use the db instance created earlier
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- Referenceable actualDb = (Referenceable) tableDefinition.get("databaseComposite");
- assertEquals(actualDb.getId().id, dbId);
-
- //Assert that as part table create, db is not created and audit event is not added to db
- assertAuditEvents(dbId, 1);
- }
-
- @Test
- public void testUpdateEntityByUniqueAttribute() throws Exception {
- final List<String> colNameList = ImmutableList.of("col1", "col2");
- Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
- put("columnNames", colNameList);
- }});
- metadataService.updateEntityByUniqueAttribute(table.getTypeName(), NAME, (String) table.get(NAME),
- tableUpdated);
-
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- List<String> actualColumns = (List) tableDefinition.get("columnNames");
- assertEquals(actualColumns, colNameList);
- }
-
- @Test
- public void testUpdateEntityWithMap() throws Exception {
- final Map<String, Struct> partsMap = new HashMap<>();
- partsMap.put("part0", new Struct(TestUtils.PARTITION_STRUCT_TYPE,
- new HashMap<String, Object>() {{
- put(NAME, "test");
- }}));
-
- table.set("partitionsMap", partsMap);
-
- updateInstance(table);
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertTrue(partsMap.get("part0").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")));
-
- //update map - add a map key
- partsMap.put("part1", new Struct(TestUtils.PARTITION_STRUCT_TYPE,
- new HashMap<String, Object>() {{
- put(NAME, "test1");
- }}));
- table.set("partitionsMap", partsMap);
-
- updateInstance(table);
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
- Assert.assertTrue(partsMap.get("part1").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part1")));
-
- //update map - remove a key and add another key
- partsMap.remove("part0");
- partsMap.put("part2", new Struct(TestUtils.PARTITION_STRUCT_TYPE,
- new HashMap<String, Object>() {{
- put(NAME, "test2");
- }}));
- table.set("partitionsMap", partsMap);
-
- updateInstance(table);
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
- Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
- Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
-
- //update struct value for existing map key
- Struct partition2 = partsMap.get("part2");
- partition2.set(NAME, "test2Updated");
- updateInstance(table);
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
- Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
- Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
-
- //Test map pointing to a class
- final Map<String, Referenceable> columnsMap = new HashMap<>();
- Referenceable col0Type = new Referenceable(TestUtils.COLUMN_TYPE,
- new HashMap<String, Object>() {{
- put(NAME, "test1");
- put("type", "string");
- }});
-
- columnsMap.put("col0", col0Type);
-
- Referenceable col1Type = new Referenceable(TestUtils.COLUMN_TYPE,
- new HashMap<String, Object>() {{
- put(NAME, "test2");
- put("type", "string");
- }});
-
- columnsMap.put("col1", col1Type);
- table.set(TestUtils.COLUMNS_MAP, columnsMap);
- updateInstance(table);
- verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columnsMap, TestUtils.COLUMNS_MAP);
-
- //Swap elements
- columnsMap.clear();
- columnsMap.put("col0", col1Type);
- columnsMap.put("col1", col0Type);
-
- table.set(TestUtils.COLUMNS_MAP, columnsMap);
- updateInstance(table);
- verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columnsMap, TestUtils.COLUMNS_MAP);
-
- //Drop the first key and change the class type as well to col0
- columnsMap.clear();
- columnsMap.put("col0", col0Type);
-
- table.set(TestUtils.COLUMNS_MAP, columnsMap);
- updateInstance(table);
- verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columnsMap, TestUtils.COLUMNS_MAP);
-
- //Clear state
- table.setNull(TestUtils.COLUMNS_MAP);
- updateInstance(table);
- verifyMapUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), null, TestUtils.COLUMNS_MAP);
- }
-
- private void verifyMapUpdates(String typeName, String uniqAttrName, String uniqAttrValue,
- Map<String, Referenceable> expectedMap, String mapAttrName) throws AtlasException {
- String json =
- metadataService.getEntityDefinition(typeName, uniqAttrName, uniqAttrValue);
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(json, true);
- Map<String, Referenceable> actualMap = (Map<String, Referenceable>) tableDefinition.get(mapAttrName);
-
- if (expectedMap == null && actualMap != null) {
- //all are marked as deleted in case of soft delete
- for (String key : actualMap.keySet()) {
- assertEquals(actualMap.get(key).getId().state, Id.EntityState.DELETED);
- }
- } else if(expectedMap == null) {
- //hard delete case
- assertNull(actualMap);
- } else {
- assertTrue(actualMap.size() >= expectedMap.size());
-
- for (String key : expectedMap.keySet()) {
- assertTrue(actualMap.get(key).equalsContents(expectedMap.get(key)));
- }
-
- //rest of the keys are marked as deleted
- List<String> extraKeys = new ArrayList<>(actualMap.keySet());
- extraKeys.removeAll(expectedMap.keySet());
- for (String key : extraKeys) {
- assertEquals(actualMap.get(key).getId().getState(), Id.EntityState.DELETED);
- }
- }
- }
-
- @Test
- public void testUpdateEntityAddAndUpdateArrayAttr() throws Exception {
- //Update entity, add new array attribute
- //add array of primitives
- final List<String> colNameList = ImmutableList.of("col1", "col2");
- Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
- put("columnNames", colNameList);
- }});
- metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
-
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- List<String> actualColumns = (List) tableDefinition.get("columnNames");
- assertEquals(actualColumns, colNameList);
-
- //update array of primitives
- final List<String> updatedColNameList = ImmutableList.of("col2", "col3");
- tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
- put("columnNames", updatedColNameList);
- }});
- metadataService.updateEntityPartialByGuid(tableId.getId()._getId(), tableUpdated);
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- actualColumns = (List) tableDefinition.get("columnNames");
- assertEquals(actualColumns, updatedColNameList);
- }
-
- private EntityResult updateEntityPartial(String guid, Referenceable entity) throws AtlasException {
- RequestContext.createContext();
- return metadataService.updateEntityPartialByGuid(guid, entity).getEntityResult();
- }
-
- @Test
- public void testUpdateEntityArrayOfClass() throws Exception {
- //test array of class with id
- final List<Referenceable> columns = new ArrayList<>();
- Map<String, Object> values = new HashMap<>();
- values.put(NAME, "col1");
- values.put("type", "type");
- Referenceable col1 = new Referenceable(TestUtils.COLUMN_TYPE, values);
- columns.add(col1);
- Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
- put(COLUMNS_ATTR_NAME, columns);
- }});
-
- EntityResult entityResult = updateEntityPartial(tableId._getId(), tableUpdated);
- assertEquals(entityResult.getCreatedEntities().size(), 1); //col1 created
- assertEquals(entityResult.getUpdateEntities().size(), 1); //table updated
- assertEquals(entityResult.getUpdateEntities().get(0), tableId._getId());
- verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
-
- //Partial update. Add col2 But also update col1
- Map<String, Object> valuesCol5 = new HashMap<>();
- valuesCol5.put(NAME, "col2");
- valuesCol5.put("type", "type");
- Referenceable col2 = new Referenceable(TestUtils.COLUMN_TYPE, valuesCol5);
- //update col1
- col1.set("type", "type1");
- columns.add(col2);
-
- tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
- put(COLUMNS_ATTR_NAME, columns);
- }});
- entityResult = updateEntityPartial(tableId._getId(), tableUpdated);
- assertEquals(entityResult.getCreatedEntities().size(), 1); //col2 created
- assertEquals(entityResult.getUpdateEntities().size(), 2); //table, col1 updated
-
- verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
-
- //Complete update. Add array elements - col3,col4
- Map<String, Object> values1 = new HashMap<>();
- values1.put(NAME, "col3");
- values1.put("type", "type");
- Referenceable col3 = new Referenceable(TestUtils.COLUMN_TYPE, values1);
- columns.add(col3);
-
- Map<String, Object> values2 = new HashMap<>();
- values2.put(NAME, "col4");
- values2.put("type", "type");
- Referenceable col4 = new Referenceable(TestUtils.COLUMN_TYPE, values2);
- columns.add(col4);
-
- table.set(COLUMNS_ATTR_NAME, columns);
- entityResult = updateInstance(table);
- assertEquals(entityResult.getCreatedEntities().size(), 2); //col3, col4 created
-
- verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
-
- //Swap elements
- columns.clear();
- columns.add(col4);
- columns.add(col3);
-
- table.set(COLUMNS_ATTR_NAME, columns);
- entityResult = updateInstance(table);
- assertEquals(entityResult.getDeletedEntities().size(), 2); //col1, col2 are deleted
- verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
-
- //drop a single column
- columns.clear();
- columns.add(col3);
-
- table.set(COLUMNS_ATTR_NAME, columns);
- entityResult = updateInstance(table);
- assertEquals(entityResult.getDeletedEntities().size(), 1); //col4 deleted
- verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
-
- //Remove a class reference/Id and insert another reference
- //Also covers isComposite case since columns is a composite
- values.clear();
- columns.clear();
-
- values.put(NAME, "col5");
- values.put("type", "type");
- Referenceable col5 = new Referenceable(TestUtils.COLUMN_TYPE, values);
- columns.add(col5);
- table.set(COLUMNS_ATTR_NAME, columns);
- entityResult = updateInstance(table);
- assertEquals(entityResult.getCreatedEntities().size(), 1); //col5 created
- assertEquals(entityResult.getDeletedEntities().size(), 1); //col3 deleted
-
- verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), columns, COLUMNS_ATTR_NAME);
-
- //Update array column to null
- table.setNull(COLUMNS_ATTR_NAME);
- entityResult = updateInstance(table);
- assertEquals(entityResult.getDeletedEntities().size(), 1);
- verifyArrayUpdates(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME), null, COLUMNS_ATTR_NAME);
- }
-
- private void verifyArrayUpdates(String typeName, String uniqAttrName, String uniqAttrValue,
- List<Referenceable> expectedArray, String arrAttrName) throws AtlasException {
- String json = metadataService.getEntityDefinition(typeName, uniqAttrName, uniqAttrValue);
- Referenceable entityDefinition = InstanceSerialization.fromJsonReferenceable(json, true);
- List<Referenceable> actualArray = (List<Referenceable>) entityDefinition.get(arrAttrName);
- if (expectedArray == null && actualArray != null) {
- //all are marked as deleted in case of soft delete
- for (Referenceable referenceable : actualArray) {
- assertEquals(referenceable.getId().state, Id.EntityState.DELETED);
- }
- } else if(expectedArray == null) {
- //hard delete case
- assertNull(actualArray);
- } else {
- int index;
- for (index = 0; index < expectedArray.size(); index++) {
- Assert.assertTrue(actualArray.get(index).equalsContents(expectedArray.get(index)));
- }
-
- //Rest of the entities in the list are marked as deleted
- for (; index < actualArray.size(); index++) {
- assertEquals(actualArray.get(index).getId().state, Id.EntityState.DELETED);
- }
- }
- }
-
- @Test
- public void testStructs() throws Exception {
- Struct serdeInstance = new Struct(TestUtils.SERDE_TYPE);
- serdeInstance.set(NAME, "serde1Name");
- serdeInstance.set("serde", "test");
- serdeInstance.set("description", "testDesc");
- table.set("serde1", serdeInstance);
-
- String newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- Assert.assertNotNull(tableDefinition.get("serde1"));
- Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
-
- //update struct attribute
- serdeInstance.set("serde", "testUpdated");
- updateInstance(table);
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
-
- //set to null
- serdeInstance.setNull("description");
- updateInstance(table);
- tableDefinitionJson =
- metadataService.getEntityDefinitionJson(tableId._getId());
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- Assert.assertNull(((Struct) tableDefinition.get("serde1")).get("description"));
- }
-
-
- @Test
- public void testCreateEntityWithReferenceableHavingIdNoValue() throws Exception {
- //ATLAS-383 Test
- Referenceable sdReferenceable = new Referenceable(TestUtils.STORAGE_DESC_TYPE);
- sdReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, TestUtils.randomString());
- sdReferenceable.set("compressed", "false");
- sdReferenceable.set("location", "hdfs://tmp/hive-user");
- String sdGuid = TestUtils.createInstance(metadataService, sdReferenceable);
-
- Referenceable sdRef2 = new Referenceable(sdGuid, TestUtils.STORAGE_DESC_TYPE, null);
-
- Referenceable partRef = new Referenceable(TestUtils.PARTITION_CLASS_TYPE);
- partRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "part-unique");
- partRef.set("values", ImmutableList.of("2014-10-01"));
- partRef.set("table", table);
- partRef.set("sd", sdRef2);
-
- String partGuid = TestUtils.createInstance(metadataService, partRef);
- Assert.assertNotNull(partGuid);
- }
-
- @Test
- public void testClassUpdate() throws Exception {
- //Create new db instance
- final Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE);
- databaseInstance.set(NAME, TestUtils.randomString());
- databaseInstance.set("description", "new database");
-
- String dbId = TestUtils.createInstance(metadataService, databaseInstance);
-
- /*Update reference property with Id */
- metadataService.updateEntityAttributeByGuid(tableId._getId(), "database", dbId);
-
- String tableDefinitionJson =
- metadataService.getEntityDefinitionJson(tableId._getId());
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- assertEquals(dbId, (((Id) tableDefinition.get("database"))._getId()));
-
- /* Update with referenceable - TODO - Fails . Need to fix this */
- /*final String dbName = TestUtils.randomString();
- final Referenceable databaseInstance2 = new Referenceable(TestUtils.DATABASE_TYPE);
- databaseInstance2.set(NAME, dbName);
- databaseInstance2.set("description", "new database 2");
-
- Referenceable updateTable = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
- put("database", databaseInstance2);
- }});
- metadataService.updateEntityAttributeByGuid(tableId._getId(), updateTable);
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(tableId._getId());
- Referenceable tableDefinitionActual = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- String dbDefJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, NAME, dbName);
- Referenceable dbDef = InstanceSerialization.fromJsonReferenceable(dbDefJson, true);
-
- Assert.assertNotEquals(dbId, (((Id) tableDefinitionActual.get("database"))._getId()));
- Assert.assertEquals(dbDef.getObjectId()._getId(), (((Id) tableDefinitionActual.get("database"))._getId())); */
-
- }
-
- @Test
- public void testArrayOfStructs() throws Exception {
- //Add array of structs
- TestUtils.dumpGraph(TestUtils.getGraph());
-
- final Struct partition1 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
- partition1.set(NAME, "part1");
-
- final Struct partition2 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
- partition2.set(NAME, "part2");
-
- List<Struct> partitions = new ArrayList<Struct>(){{ add(partition1); add(partition2); }};
- table.set("partitions", partitions);
-
- String newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertNotNull(tableDefinition.get("partitions"));
- List<Struct> partitionsActual = (List<Struct>) tableDefinition.get("partitions");
- assertPartitions(partitionsActual, partitions);
-
- //add a new element to array of struct
- final Struct partition3 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
- partition3.set(NAME, "part3");
- partitions.add(partition3);
- table.set("partitions", partitions);
- newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertNotNull(tableDefinition.get("partitions"));
- partitionsActual = (List<Struct>) tableDefinition.get("partitions");
- assertPartitions(partitionsActual, partitions);
-
- //remove one of the struct values
- partitions.remove(1);
- table.set("partitions", partitions);
- newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertNotNull(tableDefinition.get("partitions"));
- partitionsActual = (List<Struct>) tableDefinition.get("partitions");
- assertPartitions(partitionsActual, partitions);
-
- //Update struct value within array of struct
- partitions.get(0).set(NAME, "part4");
- newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertNotNull(tableDefinition.get("partitions"));
- partitionsActual = (List<Struct>) tableDefinition.get("partitions");
- assertPartitions(partitionsActual, partitions);
-
- //add a repeated element to array of struct
- final Struct partition4 = new Struct(TestUtils.PARTITION_STRUCT_TYPE);
- partition4.set(NAME, "part4");
- partitions.add(partition4);
- table.set("partitions", partitions);
- newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertNotNull(tableDefinition.get("partitions"));
- partitionsActual = (List<Struct>) tableDefinition.get("partitions");
- assertPartitions(partitionsActual, partitions);
-
-
- // Remove all elements. Should set array attribute to null
- partitions.clear();
- newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- Assert.assertNull(tableDefinition.get("partitions"));
- }
-
- private void assertPartitions(List<Struct> partitionsActual, List<Struct> partitions) {
- assertEquals(partitionsActual.size(), partitions.size());
- for (int index = 0; index < partitions.size(); index++) {
- assertTrue(partitionsActual.get(index).equalsContents(partitions.get(index)));
- }
- }
-
- @Test(expectedExceptions = ValueConversionException.class)
- public void testCreateRequiredAttrNull() throws Exception {
- //Update required attribute
-
- Referenceable tableEntity = new Referenceable(TABLE_TYPE);
- tableEntity.set(NAME, "table_" + TestUtils.randomString());
-
- TestUtils.createInstance(metadataService, tableEntity);
- Assert.fail("Expected exception while creating with required attribute null");
- }
-
- @Test(expectedExceptions = ValueConversionException.class)
- public void testUpdateRequiredAttrToNull() throws Exception {
- //Update required attribute
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- assertEquals(tableDefinition.get("description"), "random table");
- table.setNull("description");
-
- updateInstance(table);
- Assert.fail("Expected exception while updating required attribute to null");
- }
-
- @Test
- public void testCheckOptionalAttrValueRetention() throws Exception {
-
- Referenceable entity = createDBEntity();
-
- String dbId = TestUtils.createInstance(metadataService, entity);
-
- entity = getEntity(dbId);
-
- //The optional boolean attribute should have a non-null value
- final String isReplicatedAttr = "isReplicated";
- final String paramsAttr = "parameters";
- Assert.assertNotNull(entity.get(isReplicatedAttr));
- Assert.assertEquals(entity.get(isReplicatedAttr), Boolean.FALSE);
- Assert.assertNull(entity.get(paramsAttr));
-
- //Update to true
- entity.set(isReplicatedAttr, Boolean.TRUE);
- //Update array
- final HashMap<String, String> params = new HashMap<String, String>() {{ put("param1", "val1"); put("param2", "val2"); }};
- entity.set(paramsAttr, params);
- //Complete update
- updateInstance(entity);
-
- entity = getEntity(dbId);
-
- Assert.assertNotNull(entity.get(isReplicatedAttr));
- Assert.assertEquals(entity.get(isReplicatedAttr), Boolean.TRUE);
- Assert.assertEquals(entity.get(paramsAttr), params);
-
- //Complete update without setting the attribute
- Referenceable newEntity = createDBEntity();
- //Reset name to the current DB name
- newEntity.set(NAME, entity.get(NAME));
- updateInstance(newEntity);
-
- entity = getEntity(dbId);
- Assert.assertNotNull(entity.get(isReplicatedAttr));
- Assert.assertEquals(entity.get(isReplicatedAttr), Boolean.TRUE);
- Assert.assertEquals(entity.get(paramsAttr), params);
- }
-
- private Referenceable getEntity(String guid) throws AtlasException {
- String entityJson = metadataService.getEntityDefinitionJson(guid);
- Assert.assertNotNull(entityJson);
- return InstanceSerialization.fromJsonReferenceable(entityJson, true);
- }
-
- @Test
- public void testUpdateOptionalAttrToNull() throws Exception {
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
-
- //Update optional Attribute
- Assert.assertNotNull(tableDefinition.get("created"));
- //Update optional attribute
- table.setNull("created");
-
- String newtableId = updateInstance(table).getUpdateEntities().get(0);
- assertEquals(newtableId, tableId._getId());
-
- tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- Assert.assertNull(tableDefinition.get("created"));
- }
-
- @Test
- public void testCreateEntityWithEnum ()throws Exception {
- String tableDefinitionJson =
- metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, NAME, (String) table.get(NAME));
- Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- EnumValue tableType = (EnumValue) tableDefinition.get("tableType");
-
- assertEquals(tableType, new EnumValue("MANAGED", 1));
- }
-
- @Test
- public void testGetEntityByUniqueAttribute() throws Exception {
- Referenceable entity = createDBEntity();
- TestUtils.createInstance(metadataService, entity);
-
- //get entity by valid qualified name
- String entityJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, NAME,
- (String) entity.get(NAME));
- Assert.assertNotNull(entityJson);
- Referenceable referenceable = InstanceSerialization.fromJsonReferenceable(entityJson, true);
- assertEquals(referenceable.get(NAME), entity.get(NAME));
-
- //get entity by invalid qualified name
- try {
- metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, NAME, "random");
- Assert.fail("Expected EntityNotFoundException");
- } catch (EntityNotFoundException e) {
- //expected
- }
-
- //get entity by non-unique attribute
- try {
- metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "description",
- (String) entity.get("description"));
- Assert.fail("Expected IllegalArgumentException");
- } catch (IllegalArgumentException e) {
- //expected
- }
- }
-
- @Test
- public void testDeleteEntities() throws Exception {
- // Create a table entity, with 3 composite column entities
- Referenceable dbEntity = createDBEntity();
- String dbGuid = TestUtils.createInstance(metadataService, dbEntity);
- Referenceable table1Entity = createTableEntity(dbGuid);
- Referenceable col1 = createColumnEntity();
- Referenceable col2 = createColumnEntity();
- Referenceable col3 = createColumnEntity();
- table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3));
- TestUtils.createInstance(metadataService, table1Entity);
-
- // Retrieve the table entities from the repository,
- // to get their guids and the composite column guids.
- String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE,
- NAME, (String)table1Entity.get(NAME));
- Assert.assertNotNull(entityJson);
- table1Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true);
- List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) table1Entity.get(COLUMNS_ATTR_NAME);
-
- // Register an EntityChangeListener to verify the notification mechanism
- // is working for deleteEntities().
- EntitiesChangeListener listener = new EntitiesChangeListener();
- metadataService.registerListener(listener);
-
- //Delete one column
- String columnId = table1Columns.get(0).getId()._getId();
- EntityResult entityResult = deleteEntities(columnId);
- //column is deleted and table is updated
- assertEquals(entityResult.getDeletedEntities().get(0), columnId);
- assertEquals(entityResult.getUpdateEntities().get(0), table1Entity.getId()._getId());
-
- //verify listener was called for updates and deletes
- assertEquals(entityResult.getDeletedEntities(), listener.getDeletedEntities());
- assertEquals(entityResult.getUpdateEntities(), listener.getUpdatedEntities());
-
- // Delete the table entities. The deletion should cascade
- // to their composite columns.
- entityResult = deleteEntities(table1Entity.getId()._getId());
-
- // Verify that deleteEntities() response has guids for tables and their composite columns.
- Assert.assertTrue(entityResult.getDeletedEntities().contains(table1Entity.getId()._getId()));
- Assert.assertTrue(entityResult.getDeletedEntities().contains(table1Columns.get(1).getId()._getId()));
- Assert.assertTrue(entityResult.getDeletedEntities().contains(table1Columns.get(2).getId()._getId()));
-
- // Verify that tables and their composite columns have been deleted from the repository.
- assertEntityDeleted(TABLE_TYPE, NAME, table1Entity.get(NAME));
- assertEntityDeleted(COLUMN_TYPE, NAME, col2.get(NAME));
- assertEntityDeleted(COLUMN_TYPE, NAME, col3.get(NAME));
-
- // Verify that the listener was notified about the deleted entities.
- List<String> deletedEntitiesFromListener = listener.getDeletedEntities();
- Assert.assertNotNull(deletedEntitiesFromListener);
- assertEquals(deletedEntitiesFromListener.size(), entityResult.getDeletedEntities().size());
- Assert.assertTrue(deletedEntitiesFromListener.containsAll(entityResult.getDeletedEntities()));
- }
-
- private void assertEntityDeleted(String typeName, String attributeName, Object attributeValue)
- throws AtlasException {
- try {
- metadataService.getEntityDefinition(typeName, attributeName, (String) attributeValue);
- fail("Expected EntityNotFoundException");
- } catch(EntityNotFoundException e) {
- //expected
- }
- }
-
- @Test
- public void testDeleteEntityByUniqueAttribute() throws Exception {
- // Create a table entity, with 3 composite column entities
- Referenceable dbEntity = createDBEntity();
- String dbGuid = TestUtils.createInstance(metadataService, dbEntity);
- Referenceable table1Entity = createTableEntity(dbGuid);
- Referenceable col1 = createColumnEntity();
- Referenceable col2 = createColumnEntity();
- Referenceable col3 = createColumnEntity();
- table1Entity.set(COLUMNS_ATTR_NAME, ImmutableList.of(col1, col2, col3));
- TestUtils.createInstance(metadataService, table1Entity);
-
- // to get their guids and the composite column guids.
- String entityJson = metadataService.getEntityDefinition(TestUtils.TABLE_TYPE,
- NAME, (String) table1Entity.get(NAME));
- Assert.assertNotNull(entityJson);
- table1Entity = InstanceSerialization.fromJsonReferenceable(entityJson, true);
- List<IReferenceableInstance> table1Columns = (List<IReferenceableInstance>) table1Entity.get(COLUMNS_ATTR_NAME);
-
- // Register an EntityChangeListener to verify the notification mechanism
- // is working for deleteEntityByUniqueAttribute().
- EntitiesChangeListener listener = new EntitiesChangeListener();
- metadataService.registerListener(listener);
-
- // Delete the table entities. The deletion should cascade
- // to their composite columns.
- List<String> deletedGuids = metadataService.deleteEntityByUniqueAttribute(TestUtils.TABLE_TYPE, NAME,
- (String) table1Entity.get(NAME)).getDeletedEntities();
-
- // Verify that deleteEntities() response has guids for tables and their composite columns.
- Assert.assertTrue(deletedGuids.contains(table1Entity.getId()._getId()));
- for (IReferenceableInstance column : table1Columns) {
- Assert.assertTrue(deletedGuids.contains(column.getId()._getId()));
- }
-
- // Verify that tables and their composite columns have been deleted from the repository.
- // Verify that tables and their composite columns have been deleted from the repository.
- assertEntityDeleted(TABLE_TYPE, NAME, table1Entity.get(NAME));
- assertEntityDeleted(COLUMN_TYPE, NAME, col1.get(NAME));
- assertEntityDeleted(COLUMN_TYPE, NAME, col2.get(NAME));
- assertEntityDeleted(COLUMN_TYPE, NAME, col3.get(NAME));
-
- // Verify that the listener was notified about the deleted entities.
- List<String> deletedEntitiesFromListener = listener.getDeletedEntities();
- Assert.assertNotNull(deletedEntitiesFromListener);
- assertEquals(deletedEntitiesFromListener.size(), deletedGuids.size());
- Assert.assertTrue(deletedEntitiesFromListener.containsAll(deletedGuids));
- }
-
- @Test
- public void testTypeUpdateFailureShouldRollBack() throws AtlasException, JSONException {
- String typeName = TestUtils.randomString(10);
- HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef(
- typeName, ImmutableSet.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE));
- TypesDef typesDef = new TypesDef(typeDef, false);
- JSONObject type = metadataService.createType(TypesSerialization.toJson(typesDef));
- Assert.assertNotNull(type.get(AtlasClient.TYPES));
-
- HierarchicalTypeDefinition<ClassType> updatedTypeDef = TypesUtil.createClassTypeDef(
- typeName, ImmutableSet.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE),
- TypesUtil.createRequiredAttrDef("test_type_invalid_attribute$", DataTypes.STRING_TYPE));
- TypesDef updatedTypesDef = new TypesDef(updatedTypeDef, false);
-
- try {
- metadataService.updateType(TypesSerialization.toJson(updatedTypesDef));
- fail("Expected AtlasException");
- } catch (AtlasException e) {
- //expected
- }
-
- //type definition should reflect old type
- String typeDefinition = metadataService.getTypeDefinition(typeName);
- typesDef = TypesSerialization.fromJson(typeDefinition);
- assertEquals(typesDef.classTypes().head().attributeDefinitions.length, 1);
- }
-
- @Test
- public void testTypeWithDotsCreationShouldNotBeCreated() throws AtlasException, JSONException {
- String typeName = "test_.v1_type_XXXX";
- HierarchicalTypeDefinition<ClassType> typeDef = TypesUtil.createClassTypeDef(
- typeName, ImmutableSet.<String>of(),
- TypesUtil.createUniqueRequiredAttrDef("test_type_attribute", DataTypes.STRING_TYPE));
- TypesDef typesDef = new TypesDef(typeDef, false);
-
- try {
- metadataService.createType(TypesSerialization.toJson(typesDef));
- fail("Expected IllegalArgumentException");
- } catch (IllegalArgumentException e) {
- assertTrue (e.getCause().getMessage().contains(AtlasTypeUtil.getInvalidTypeNameErrorMessage()), e.getCause().getMessage());
- }
- }
-
- @Test
- public void testAuditEventsInvalidParams() throws Exception {
- //entity id can't be null
- try {
- metadataService.getAuditEvents(null, "key", (short) 10);
- fail("expected IllegalArgumentException");
- } catch(IllegalArgumentException e) {
- //expected IllegalArgumentException
- assertEquals(e.getMessage(), "entity id cannot be null");
- }
-
- //entity id can't be empty
- try {
- metadataService.getAuditEvents("", "key", (short) 10);
- fail("expected IllegalArgumentException");
- } catch(IllegalArgumentException e) {
- //expected IllegalArgumentException
- assertEquals(e.getMessage(), "entity id cannot be empty");
- }
-
- //start key can be null
- metadataService.getAuditEvents("id", null, (short) 10);
-
- //start key can't be emoty
- try {
- metadataService.getAuditEvents("id", "", (short) 10);
- fail("expected IllegalArgumentException");
- } catch(IllegalArgumentException e) {
- //expected IllegalArgumentException
- assertEquals(e.getMessage(), "start key cannot be empty");
- }
-
- //number of results can't be > max value
- try {
- metadataService.getAuditEvents("id", "key", (short) 10000);
- fail("expected IllegalArgumentException");
- } catch(IllegalArgumentException e) {
- //expected IllegalArgumentException
- assertEquals(e.getMessage(), "count should be <= 1000, current value 10000");
- }
-
- //number of results can't be <= 0
- try {
- metadataService.getAuditEvents("id", "key", (short) -1);
- fail("expected IllegalArgumentException");
- } catch(IllegalArgumentException e) {
- //expected IllegalArgumentException
- assertEquals(e.getMessage(), "count should be > 0, current value -1");
- }
- }
-
- @Test
- public void testOnChangeRefresh() {
- try {
- List<String> beforeChangeTypeNames = new ArrayList<>();
- beforeChangeTypeNames.addAll(metadataService.getTypeNames(new HashMap<TypeCache.TYPE_FILTER, String>()));
-
- typeDefChangeListener.onChange(new ChangedTypeDefs());
-
- List<String> afterChangeTypeNames = new ArrayList<>();
- afterChangeTypeNames.addAll(metadataService.getTypeNames(new HashMap<TypeCache.TYPE_FILTER, String>()));
-
- Collections.sort(beforeChangeTypeNames);
- Collections.sort(afterChangeTypeNames);
- assertEquals(afterChangeTypeNames, beforeChangeTypeNames);
- } catch (AtlasBaseException e) {
- fail("Should've succeeded", e);
- } catch (AtlasException e) {
- fail("getTypeNames should've succeeded", e);
- }
- }
-
- @Test
- // ATLAS-2092: Concurrent edge label creation leads to inconsistency
- // This test tries entity creation in parallel and ensures that the edges with the same label actually get created
- public void testConcurrentCalls() {
- final HierarchicalTypeDefinition<ClassType> refType =
- createClassTypeDef(randomString(), ImmutableSet.<String>of());
- HierarchicalTypeDefinition<ClassType> type =
- createClassTypeDef(randomString(), ImmutableSet.<String>of(),
- new AttributeDefinition("ref", refType.typeName, Multiplicity.OPTIONAL, true, null));
- try {
- metadataService.createType(TypesSerialization.toJson(refType, false));
- metadataService.createType(TypesSerialization.toJson(type, false));
-
- String refId1 = createBasicEntity(refType);
- String refId2 = createBasicEntity(refType);
-
- // Add referenced entity for edge creation
- final Referenceable instance1 = new Referenceable(type.typeName);
- instance1.set("ref", new Referenceable(refId1, refType.typeName, null));
-
- // Add referenced entity for edge creation
- final Referenceable instance2 = new Referenceable(type.typeName);
- instance2.set("ref", new Referenceable(refId2, refType.typeName, null));
-
- ExecutorService executor = Executors.newFixedThreadPool(3);
- List<Future<Object>> futures = new ArrayList<>();
- // Try parallel creation of both the entities
- futures.add(executor.submit(new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return createEntity(instance1);
- }
- }));
- futures.add(executor.submit(new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return createEntity(instance2);
- }
- }));
- futures.add(executor.submit(new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return discoveryService.searchByDSL(TestUtils.TABLE_TYPE, new QueryParams(10, 0));
- }
- }));
-
- try {
- String id1 = (String) futures.get(0).get();
- String id2 = (String) futures.get(1).get();
- futures.get(2).get();
- executor.shutdown();
-
- assertNotNull(id1);
- assertNotNull(id2);
-
- boolean validated1 = assertEdge(id1, type.typeName);
- boolean validated2 = assertEdge(id2, type.typeName);
- assertTrue(validated1 && validated2);
- } catch (InterruptedException | ExecutionException e) {
- fail("Parallel entity creation failed", e);
- }
- } catch (AtlasException e) {
- fail("Type/Entity creation failed", e);
- }
- }
-
- private String createBasicEntity(final HierarchicalTypeDefinition<ClassType> refType) throws AtlasException {
- String json = InstanceSerialization.toJson(new Referenceable(refType.typeName), false);
- CreateUpdateEntitiesResult entities = metadataService.createEntities("[" + json + "]");
- return entities.getCreatedEntities().get(0);
- }
-
- private String createEntity(final Referenceable referenceable) throws AtlasException {
- String json = InstanceSerialization.toJson(referenceable, false);
- CreateUpdateEntitiesResult entities = metadataService.createEntities("[" + json + "]");
- return entities.getCreatedEntities().get(0);
- }
-
- private boolean assertEdge(String id, String typeName) throws AtlasException {
- AtlasGraph graph = TestUtils.getGraph();
- Iterable<AtlasVertex> vertices = graph.query().has(Constants.GUID_PROPERTY_KEY, id).vertices();
- AtlasVertex AtlasVertex = vertices.iterator().next();
- Iterable<AtlasEdge> edges = AtlasVertex.getEdges(AtlasEdgeDirection.OUT, Constants.INTERNAL_PROPERTY_KEY_PREFIX + typeName + ".ref");
- if (edges.iterator().hasNext()) {
- ITypedReferenceableInstance entity = metadataService.getEntityDefinition(id);
- assertNotNull(entity.get("ref"));
- return true;
- }
- return false;
- }
-
- private static class EntitiesChangeListener implements EntityChangeListener {
- private List<String> deletedEntities = new ArrayList<>();
- private List<String> updatedEntities = new ArrayList<>();
-
- @Override
- public void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities, boolean isImport)
- throws AtlasException {
- }
-
- @Override
- public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities, boolean isImport)
- throws AtlasException {
- updatedEntities.clear();
- for (ITypedReferenceableInstance entity : entities) {
- updatedEntities.add(entity.getId()._getId());
- }
- }
-
- @Override
- public void onTraitsAdded(ITypedReferenceableInstance entity, Collection<? extends IStruct> traits)
- throws AtlasException {
- }
-
- @Override
- public void onTraitsDeleted(ITypedReferenceableInstance entity, Collection<String> traitNames)
- throws AtlasException {
- }
-
- @Override
- public void onTraitsUpdated(ITypedReferenceableInstance entity, Collection<? extends IStruct> traits)
- throws AtlasException {
- }
-
- @Override
- public void onEntitiesDeleted(Collection<ITypedReferenceableInstance> entities, boolean isImport)
- throws AtlasException {
- deletedEntities.clear();
- for (ITypedReferenceableInstance entity : entities) {
- deletedEntities.add(entity.getId()._getId());
- }
- }
-
- public List<String> getDeletedEntities() {
- return deletedEntities;
- }
-
- public List<String> getUpdatedEntities() {
- return updatedEntities;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
deleted file mode 100644
index 5c37b9e..0000000
--- a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.service;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.TestModules;
-import org.apache.atlas.TestUtils;
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.apache.atlas.repository.typestore.ITypeStore;
-import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
-import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUpdateException;
-import org.apache.atlas.typesystem.types.cache.TypeCache;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import javax.inject.Inject;
-
-
-/**
- * Verify MetadataService type operations trigger StoreBackedTypeCache to load non-cached types from the store.
- * StoreBackedTypeCacheTestModule Guice module sets Atlas configuration
- * to use {@link StoreBackedTypeCache} as the TypeCache implementation class.
- */
-@Guice(modules = TestModules.StoreBackedTypeCacheTestModule.class)
-public class StoreBackedTypeCacheMetadataServiceTest
-{
- @Inject
- private MetadataService metadataService;
-
- @Inject
- private ITypeStore typeStore;
-
- @Inject
- TypeCache typeCache;
-
- private StoreBackedTypeCache storeBackedTypeCache;
-
- private TypeSystem ts;
-
- @BeforeClass
- public void oneTimeSetup() throws Exception {
- Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
- storeBackedTypeCache = (StoreBackedTypeCache) typeCache;
-
- ts = TypeSystem.getInstance();
- ts.reset();
-
- // Populate the type store for testing.
- TestUtils.defineDeptEmployeeTypes(ts);
- TestUtils.createHiveTypes(ts);
- ImmutableList<String> typeNames = ts.getTypeNames();
- typeStore.store(ts, typeNames);
- }
-
- @BeforeMethod
- public void setUp() throws Exception {
- ts.reset();
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- TypeSystem.getInstance().reset();
- AtlasGraphProvider.cleanup();
- }
-
- @Test
- public void testGetTypeDefinition() throws Exception {
- // Cache should be empty
- Assert.assertFalse(storeBackedTypeCache.isCachedInMemory("Manager"));
-
- // Type lookup on MetadataService should cause Manager type to be loaded from the type store
- // and cached.
- Assert.assertNotNull(metadataService.getTypeDefinition("Manager"));
- Assert.assertTrue(storeBackedTypeCache.isCachedInMemory("Manager"));
- }
-
- @Test
- public void testValidUpdateType() throws Exception {
- // Cache should be empty
- Assert.assertFalse(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
-
- TypesDef typesDef = TestUtils.defineHiveTypes();
- String json = TypesSerialization.toJson(typesDef);
-
- // Update types with same definition, which should succeed.
- metadataService.updateType(json);
-
- // hive_table type should now be cached.
- Assert.assertTrue(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
- }
-
- @Test
- public void testInvalidUpdateType() throws Exception {
- // Cache should be empty
- Assert.assertFalse(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
-
- HierarchicalTypeDefinition<ClassType> classTypeDef = TypesUtil.createClassTypeDef(TestUtils.TABLE_TYPE, ImmutableSet.<String>of(),
- new AttributeDefinition("attr1", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null));
- String json = TypesSerialization.toJson(classTypeDef, false);
-
- // Try to update the type with disallowed changes. Should fail with TypeUpdateException.
- try {
- metadataService.updateType(json);
- Assert.fail(TypeUpdateException.class.getSimpleName() + " was expected but none thrown");
- }
- catch(TypeUpdateException e) {
- // good
- }
-
- // hive_table type should now be cached.
- Assert.assertTrue(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java b/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java
deleted file mode 100644
index 725aa43..0000000
--- a/repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.util;
-
-import org.apache.atlas.query.QueryParams;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotSame;
-
-/**
- * Tests hashcode/equals behavior of CompiledQueryCacheKey
- *
- *
- */
-public class CompiledQueryCacheKeyTest {
-
- @Test
- public void testNoQueryParams() {
-
-
- CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1");
- CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1");
- CompiledQueryCacheKey e3 = new CompiledQueryCacheKey("query 2");
-
- assertKeysEqual(e1, e2);
- assertKeysDifferent(e2, e3);
- }
-
-
- @Test
- public void testWithQueryParams() {
-
- CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
- CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
- CompiledQueryCacheKey e3 = new CompiledQueryCacheKey("query 2", new QueryParams(10,10));
-
- assertKeysEqual(e1, e2);
- assertKeysDifferent(e2, e3);
- }
-
- @Test
- public void testOnlyQueryParamsDifferent() {
-
-
- CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
- CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1", new QueryParams(20,10));
-
- assertKeysDifferent(e1, e2);
- }
-
- @Test
- public void testOnlyDslDifferent() {
-
-
- CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
- CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 2", new QueryParams(10,10));
-
- assertKeysDifferent(e1, e2);
- }
-
-
- @Test
- public void testMixOfQueryParamsAndNone() {
-
-
- CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
- CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1");
-
- assertKeysDifferent(e1, e2);
- }
-
-
- private void assertKeysEqual(CompiledQueryCacheKey e1, CompiledQueryCacheKey e2) {
-
- assertEquals(e1.hashCode(), e2.hashCode());
- assertEquals(e1, e2);
- assertEquals(e2, e1);
- }
-
- private void assertKeysDifferent(CompiledQueryCacheKey e1, CompiledQueryCacheKey e2) {
-
- assertNotSame(e1.hashCode(), e2.hashCode());
- assertNotSame(e1, e2);
- assertNotSame(e2, e1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/test/java/org/apache/atlas/utils/HiveModel.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/utils/HiveModel.java b/repository/src/test/java/org/apache/atlas/utils/HiveModel.java
deleted file mode 100644
index 4fc0473..0000000
--- a/repository/src/test/java/org/apache/atlas/utils/HiveModel.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.utils;
-
-import org.apache.atlas.TestUtils;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.Struct;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.TypeSystem;
-
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-
-/**
- * Allows easy creation of entities for classes in the hive test model.
- *
- */
-public class HiveModel {
-
- public static class StructInstance {
-
- public String getTypeName() {
- return getClass().getSimpleName();
- }
-
- public Struct toStruct() throws Exception {
-
- Struct entity = new Struct(getTypeName());
- addDeclaredFields(getClass(), entity);
- return entity;
- }
-
- protected void addDeclaredFields(Class clazz, Struct r) throws Exception {
-
- for (Field f : clazz.getDeclaredFields()) {
-
- if (Modifier.isTransient(f.getModifiers())) {
- continue;
- }
- String fieldName = f.getName();
-
- f.setAccessible(true);
- Object value = f.get(this);
-
- if (value instanceof List) {
-
- List listValue = (List) value;
- List toSet = new ArrayList(listValue.size());
- for (Object listItem : listValue) {
- Object toAdd = null;
- toAdd = convertValue(listItem);
- toSet.add(toAdd);
- }
- r.set(fieldName, toSet);
- } else {
-
- Object converted = convertValue(value);
- r.set(fieldName, converted);
- }
- }
-
- if (clazz != StructInstance.class) {
- addDeclaredFields(clazz.getSuperclass(), r);
- }
- }
-
- private Object convertValue(Object toConvert) throws Exception {
-
- if (toConvert instanceof ClassInstance) {
- return ((ClassInstance) toConvert).toReferenceable();
- }
- if (toConvert instanceof StructInstance) {
- return ((StructInstance) toConvert).toStruct();
- } else {
- return toConvert;
- }
- }
- }
-
- public static class ClassInstance<T> extends StructInstance {
-
- private transient final Id guid;
- private transient List<String> traits = new ArrayList();
-
- public T withTrait(String name) {
- traits.add(name);
- return getInstance();
- }
-
- public T withTraits(List<String> names) {
- traits.addAll(names);
- return getInstance();
- }
-
- public T getInstance() {
- return (T) this;
- }
-
- public ClassInstance() {
- guid = new Id(getTypeName());
- }
-
- public Referenceable toReferenceable() throws Exception {
-
- String[] traitArray = new String[traits.size()];
- traitArray = traits.toArray(traitArray);
- Referenceable entity = new Referenceable(getTypeName(), traitArray);
- entity.replaceWithNewId(guid);
- addDeclaredFields(getClass(), entity);
-
- return entity;
- }
-
- public List<ITypedReferenceableInstance> getTypedReferencebles() throws Exception {
-
- List<ITypedReferenceableInstance> result = new ArrayList();
- for (ClassInstance containedInstance : getAllInstances()) {
- Referenceable entity = containedInstance.toReferenceable();
- ClassType type = TypeSystem.getInstance().getDataType(ClassType.class, entity.getTypeName());
- ITypedReferenceableInstance converted = type.convert(entity, Multiplicity.REQUIRED);
- result.add(converted);
- }
- return result;
- }
-
- protected List<ClassInstance> getAllInstances() {
-
- return (List) Collections.singletonList(this);
- }
-
- public Id getId() {
- return guid;
- }
- }
-
- public static class NamedInstance<T> extends ClassInstance<T> {
-
- private final String name;
-
- public NamedInstance(String name) {
- super();
- this.name = name;
- }
- }
-
- public static class HiveOrder extends StructInstance {
-
- private String col;
- private int order;
-
- public HiveOrder(String col, int order) {
- super();
- this.col = col;
- this.order = order;
- }
-
- }
-
- public static class DB extends NamedInstance<DB> {
-
- private String owner;
- private int createTime;
- private String clusterName;
-
- public DB(String name, String owner, int createTime, String clusterName) {
- super(name);
- this.owner = owner;
- this.createTime = createTime;
- this.clusterName = clusterName;
- }
- }
-
- public static class StorageDescriptor extends ClassInstance<StorageDescriptor> {
-
- private String inputFormat;
- private String outputFormat;
- private List<HiveOrder> sortCols;
-
- public StorageDescriptor(String inputFormat, String outputFormat, List<HiveOrder> sortCols) {
- super();
- this.inputFormat = inputFormat;
- this.outputFormat = outputFormat;
- this.sortCols = sortCols;
- }
- }
-
- public static class Column extends NamedInstance<Column> {
-
- private String type;
- private StorageDescriptor sd;
-
- public Column(String name, String type) {
- super(name);
- this.type = type;
- }
-
- public void setStorageDescriptor(StorageDescriptor sd) {
- this.sd = sd;
- }
- }
-
- public static class Table extends NamedInstance<Table> {
-
- private DB db;
- private Date created;
- private StorageDescriptor sd;
- private transient List<Column> colDefs;
-
- public Table(String name, DB db, StorageDescriptor sd, List<Column> colDefs) {
- this(name, db, sd, new Date(TestUtils.TEST_DATE_IN_LONG), colDefs);
- }
-
- public Table(String name, DB db, StorageDescriptor sd, Date created, List<Column> colDefs) {
-
- super(name);
- this.colDefs = colDefs;
- this.db = db;
- this.sd = sd;
- this.created = created;
- for (Column col : colDefs) {
- col.setStorageDescriptor(sd);
- }
- }
-
- public List<Column> getColumns() {
- return colDefs;
- }
-
- @Override
- protected List<ClassInstance> getAllInstances() {
-
- List<ClassInstance> result = new ArrayList(colDefs.size() + 2);
- result.add(sd);
- result.addAll(colDefs);
- result.add(this);
- return result;
- }
- }
-
- public static class Partition extends ClassInstance<Partition> {
-
- private List<String> values;
- private Table table;
-
- public Partition(List<String> values, Table table) {
-
- super();
- this.values = values;
- this.table = table;
- }
-
- }
-
- public static class LoadProcess extends NamedInstance<LoadProcess> {
-
- private List<Table> inputTables;
- private Table outputTable;
-
- public LoadProcess(String name, List<Table> inputTables, Table outputTable) {
- super(name);
- this.inputTables = inputTables;
- this.outputTable = outputTable;
- }
-
- }
-
- public static class View extends NamedInstance<View> {
-
- private DB db;
- private List<Table> inputTables;
-
- public View(String name, DB db, List<Table> inputTables) {
- super(name);
- this.db = db;
- this.inputTables = inputTables;
- }
-
- }
-
-}