You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/01/20 08:11:13 UTC
incubator-atlas git commit: ATLAS-106 Store createTimestamp and
modified timestamp separately for an entity (dkantor via shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master d519ae8cf -> 64de52336
ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/64de5233
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/64de5233
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/64de5233
Branch: refs/heads/master
Commit: 64de5233600b2f4a837415d148a3c3aa7234c7a4
Parents: d519ae8
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Wed Jan 20 12:33:05 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Wed Jan 20 12:40:51 2016 +0530
----------------------------------------------------------------------
release-log.txt | 1 +
.../org/apache/atlas/repository/Constants.java | 2 +
.../graph/GraphBackedMetadataRepository.java | 4 +-
.../graph/TypedInstanceToGraphMapper.java | 4 +-
.../GraphBackedDiscoveryServiceTest.java | 47 +++++++++++++++++++-
...kedMetadataRepositoryDeleteEntitiesTest.java | 26 +++++------
.../GraphBackedMetadataRepositoryTest.java | 41 +++++++++++++++++
7 files changed, 106 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index e4346d0..52c0584 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags)
ATLAS-433 Fix checkstyle issues for common and notification module (shwethags)
ATLAS-183 Add a Hook in Storm to post the topology metadata (svenkat,yhemanth via shwethags)
ATLAS-370 Implement deleteEntities at repository level (dkantor via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java
index d5d3b3c..1b4cb36 100755
--- a/repository/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java
@@ -57,6 +57,8 @@ public final class Constants {
public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
+ public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp";
+
/**
* search backing index name.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index d2f6103..e002545 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -224,7 +224,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
// update the traits in entity once adding trait instance is successful
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
-
+ GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
+
} catch (RepositoryException e) {
throw e;
} catch (Exception e) {
@@ -285,6 +286,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
for (String traitName : traitNames) {
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
}
+ GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
index dda2b5c..8668b46 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
@@ -21,7 +21,6 @@ import com.thinkaurelius.titan.core.SchemaViolationException;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
-
import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
@@ -166,8 +165,9 @@ public final class TypedInstanceToGraphMapper {
if (Operation.CREATE.equals(operation)) {
//TODO - Handle Trait updates
addTraits(typedInstance, instanceVertex, classType);
+ } else if (Operation.UPDATE_FULL.equals(operation) || Operation.UPDATE_PARTIAL.equals(operation)) {
+ GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.valueOf(System.currentTimeMillis()));
}
-
return getId(typedInstance)._getId();
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index 74813a1..bb4e61f 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -24,10 +24,12 @@ import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
+import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
@@ -43,6 +45,10 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
@@ -71,6 +77,13 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
repositoryService.createEntities(hrDept2);
+
+ ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
+ Id janeGuid = jane.getId();
+ ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
+ ITypedReferenceableInstance instance = personType.createInstance(janeGuid);
+ instance.set("orgLevel", "L1");
+ repositoryService.updateEntities(instance);
}
@AfterClass
@@ -115,15 +128,45 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testRawSearch1() throws Exception {
// Query for all Vertices in Graph
Object r = discoveryService.searchByGremlin("g.V.toList()");
+ Assert.assertTrue(r instanceof List);
+ List<Map<String, Object>> resultList = (List<Map<String, Object>>) r;
+ Assert.assertTrue(resultList.size() > 0);
System.out.println("search result = " + r);
// Query for all Vertices of a Type
- r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Department'}.toList()");
+ r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Department'}.toList()");
+ Assert.assertTrue(r instanceof List);
+ resultList = (List<Map<String, Object>>) r;
+ Assert.assertTrue(resultList.size() > 0);
System.out.println("search result = " + r);
// Property Query: list all Person names
- r = discoveryService.searchByGremlin("g.V.filter{it.typeName == 'Person'}.'Person.name'.toList()");
+ r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.ENTITY_TYPE_PROPERTY_KEY + " == 'Person'}.'Person.name'.toList()");
+ Assert.assertTrue(r instanceof List);
+ resultList = (List<Map<String, Object>>) r;
+ Assert.assertTrue(resultList.size() > 0);
System.out.println("search result = " + r);
+ List<Object> names = new ArrayList<Object>(resultList.size());
+ for (Map<String, Object> vertexProps : resultList) {
+ names.addAll(vertexProps.values());
+ }
+ for (String name : Arrays.asList("John", "Max")) {
+ Assert.assertTrue(names.contains(name));
+ }
+
+ // Query for all Vertices modified after 01/01/2015 00:00:00 GMT
+ r = discoveryService.searchByGremlin("g.V.filter{it." + Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY + " > 1420070400000}.toList()");
+ Assert.assertTrue(r instanceof List);
+ resultList = (List<Map<String, Object>>) r;
+ Assert.assertTrue(resultList.size() > 0);
+ for (Map<String, Object> vertexProps : resultList) {
+ Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(object);
+ Long timestampAsLong = Long.valueOf((String)object);
+ Assert.assertTrue(timestampAsLong > 1420070400000L);
+ object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(object);
+ }
}
@DataProvider(name = "dslQueriesProvider")
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
index e63f6d3..e199913 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteEntitiesTest.java
@@ -18,11 +18,9 @@
package org.apache.atlas.repository.graph;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.inject.Inject;
-
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
@@ -40,9 +38,9 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.util.TitanCleanup;
-import com.tinkerpop.blueprints.Vertex;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
/**
* Test for GraphBackedMetadataRepository.deleteEntities
@@ -102,7 +100,8 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
Assert.assertTrue(refValue instanceof List);
List<Object> employees = (List<Object>)refValue;
Assert.assertEquals(employees.size(), 4);
- List<String> employeeGuids = new ArrayList<String>(4);
+
+ List<String> employeeGuids = new ArrayList(4);
for (Object listValue : employees) {
Assert.assertTrue(listValue instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance employee = (ITypedReferenceableInstance) listValue;
@@ -123,6 +122,7 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
for (String employeeGuid : employeeGuids) {
verifyEntityDoesNotExist(employeeGuid);
}
+
// Verify all Person.address struct vertices were removed.
vertexCount = countVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, "Address");
Assert.assertEquals(vertexCount, 0);
@@ -159,11 +159,11 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
List<String> guids = repositoryService.createEntities(hrDept2);
Assert.assertNotNull(guids);
Assert.assertEquals(guids.size(), 5);
+
List<String> entityList = repositoryService.getEntityList("Department");
Assert.assertNotNull(entityList);
Assert.assertEquals(entityList.size(), 1);
- String hrDeptGuid = entityList.get(0);
- return hrDeptGuid;
+ return entityList.get(0);
}
private int countVertices(String propertyName, Object value) {
@@ -176,12 +176,10 @@ public class GraphBackedMetadataRepositoryDeleteEntitiesTest {
}
private void verifyEntityDoesNotExist(String hrDeptGuid) throws RepositoryException {
-
try {
repositoryService.getEntityDefinition(hrDeptGuid);
Assert.fail("EntityNotFoundException was expected but none thrown");
- }
- catch(EntityNotFoundException e) {
+ } catch(EntityNotFoundException e) {
// good
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/64de5233/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
index a23556b..4083dc5 100755
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
@@ -24,6 +24,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
+
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
@@ -52,9 +53,11 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
+
import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
+
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -237,6 +240,9 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testGetTraitNames")
public void testAddTrait() throws Exception {
final String aGUID = getGUID();
+ Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
+ Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNull(modificationTimestampPreUpdate);
List<String> traitNames = repositoryService.getTraitNames(aGUID);
System.out.println("traitNames = " + traitNames);
@@ -254,6 +260,11 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertEquals(traitNames.size(), 2);
Assert.assertTrue(traitNames.contains(TestUtils.PII));
Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION));
+
+ // Verify modification timestamp was updated.
+ GraphHelper.getInstance().getVertexForGUID(aGUID);
+ Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(modificationTimestampPostUpdate);
}
@Test(dependsOnMethods = "testAddTrait")
@@ -301,6 +312,9 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testAddTrait")
public void testDeleteTrait() throws Exception {
final String aGUID = getGUID();
+ Vertex vertex = GraphHelper.getInstance().getVertexForGUID(aGUID);
+ Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(modificationTimestampPreUpdate);
List<String> traitNames = repositoryService.getTraitNames(aGUID);
Assert.assertEquals(traitNames.size(), 3);
@@ -315,6 +329,12 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertEquals(traitNames.size(), 2);
Assert.assertTrue(traitNames.contains(TestUtils.CLASSIFICATION));
Assert.assertFalse(traitNames.contains(TestUtils.PII));
+
+ // Verify modification timestamp was updated.
+ GraphHelper.getInstance().getVertexForGUID(aGUID);
+ Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(modificationTimestampPostUpdate);
+ Assert.assertTrue(modificationTimestampPostUpdate > modificationTimestampPreUpdate);
}
@Test(expectedExceptions = RepositoryException.class)
@@ -479,8 +499,16 @@ public class GraphBackedMetadataRepositoryTest {
public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John");
Id johnGuid = john.getId();
+
ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max");
String maxGuid = max.getId()._getId();
+ Vertex vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
+ Long creationTimestamp = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(creationTimestamp);
+
+ Long modificationTimestampPreUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNull(modificationTimestampPreUpdate);
+
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
Id janeGuid = jane.getId();
@@ -497,6 +525,13 @@ public class GraphBackedMetadataRepositoryTest {
ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId());
+
+ // Verify modification timestamp was updated.
+ vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
+ Long modificationTimestampPostUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(modificationTimestampPostUpdate);
+ Assert.assertTrue(creationTimestamp < modificationTimestampPostUpdate);
+
// Update max's mentor reference to jane.
instance = personType.createInstance(max.getId());
instance.set("mentor", janeGuid);
@@ -508,6 +543,12 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
+
+ // Verify modification timestamp was updated.
+ vertex = GraphHelper.getInstance().getVertexForGUID(maxGuid);
+ Long modificationTimestampPost2ndUpdate = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ Assert.assertNotNull(modificationTimestampPost2ndUpdate);
+ Assert.assertTrue(modificationTimestampPostUpdate < modificationTimestampPost2ndUpdate);
}
private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {